From e66820bd874a15c2e86052335fef864869ad9f02 Mon Sep 17 00:00:00 2001 From: Pepijn van Oort Date: Wed, 22 Mar 2023 14:41:41 +0100 Subject: [PATCH 01/66] test --- FarmmapsDataDownload/DataDownloadInput.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/FarmmapsDataDownload/DataDownloadInput.json b/FarmmapsDataDownload/DataDownloadInput.json index a675294..84d35e8 100644 --- a/FarmmapsDataDownload/DataDownloadInput.json +++ b/FarmmapsDataDownload/DataDownloadInput.json @@ -1,6 +1,6 @@ [ { - "UseCreatedCropfield": true, + "UseCreatedCropfield": false, "outputFileName": "TestData", "fieldName": "TestField", "DownloadFolder": "Downloads", //"C:\\workdir\\groenmonitor\\", // "Downloads", -> if you just put "Downloads" the program will download to somewhere in ..\FarmMapsApiClient_WURtest\FarmmapsDataDownload\bin\Debug\netcoreapp3.1\Downloads\ From 32138bef80a64d721ccf456c14bf067f9c4389fe Mon Sep 17 00:00:00 2001 From: abel Date: Wed, 22 Mar 2023 14:48:13 +0100 Subject: [PATCH 02/66] blah test --- FarmmapsApiSamples.sln | 10 +- FarmmapsKPI/FarmmapsKPI.csproj | 12 ++ FarmmapsKPI/KPIApplication.cs | 199 +++++++++++++++++++++++++++ FarmmapsKPI/KPIInput.json | 34 +++++ FarmmapsKPI/KPIService.cs | 29 ++++ FarmmapsKPI/Models/CropRecordings.cs | 10 ++ FarmmapsKPI/Models/KPIInput.cs | 28 ++++ FarmmapsKPI/Models/Settings.cs | 11 ++ FarmmapsKPI/Program.cs | 21 +++ 9 files changed, 352 insertions(+), 2 deletions(-) create mode 100644 FarmmapsKPI/FarmmapsKPI.csproj create mode 100644 FarmmapsKPI/KPIApplication.cs create mode 100644 FarmmapsKPI/KPIInput.json create mode 100644 FarmmapsKPI/KPIService.cs create mode 100644 FarmmapsKPI/Models/CropRecordings.cs create mode 100644 FarmmapsKPI/Models/KPIInput.cs create mode 100644 FarmmapsKPI/Models/Settings.cs create mode 100644 FarmmapsKPI/Program.cs diff --git a/FarmmapsApiSamples.sln b/FarmmapsApiSamples.sln index e44d74d..49b0a34 100644 --- a/FarmmapsApiSamples.sln +++ b/FarmmapsApiSamples.sln @@ -1,7 +1,7 @@  Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio Version 16 -VisualStudioVersion = 16.0.30320.27 +# Visual Studio Version 17 +VisualStudioVersion = 17.4.33122.133 MinimumVisualStudioVersion = 10.0.40219.1 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FarmmapsNbs", "FarmmapsNbs\FarmmapsNbs.csproj", "{E08EF7E9-F09E-42D8-825C-164E458C78F4}" EndProject @@ -31,6 +31,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Secrets", "Secrets\Secrets. EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FarmmapsCleanUp", "FarmmapsCleanUp\FarmmapsCleanUp.csproj", "{5E4387F9-5953-4A9B-BCA5-DF3964EED3CB}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "FarmmapsKPI", "FarmmapsKPI\FarmmapsKPI.csproj", "{14575235-9867-4CE5-A22F-3F9FE002FF42}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -81,6 +83,10 @@ Global {5E4387F9-5953-4A9B-BCA5-DF3964EED3CB}.Debug|Any CPU.Build.0 = Debug|Any CPU {5E4387F9-5953-4A9B-BCA5-DF3964EED3CB}.Release|Any CPU.ActiveCfg = Release|Any CPU {5E4387F9-5953-4A9B-BCA5-DF3964EED3CB}.Release|Any CPU.Build.0 = Release|Any CPU + {14575235-9867-4CE5-A22F-3F9FE002FF42}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {14575235-9867-4CE5-A22F-3F9FE002FF42}.Debug|Any CPU.Build.0 = Debug|Any CPU + {14575235-9867-4CE5-A22F-3F9FE002FF42}.Release|Any CPU.ActiveCfg = Release|Any CPU + {14575235-9867-4CE5-A22F-3F9FE002FF42}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/FarmmapsKPI/FarmmapsKPI.csproj b/FarmmapsKPI/FarmmapsKPI.csproj new file mode 100644 index 0000000..bdd33b9 --- /dev/null +++ b/FarmmapsKPI/FarmmapsKPI.csproj @@ -0,0 +1,12 @@ + + + + Exe + netcoreapp3.1 + + + + + + + diff --git a/FarmmapsKPI/KPIApplication.cs b/FarmmapsKPI/KPIApplication.cs new file mode 100644 index 0000000..0e8bac8 --- /dev/null +++ b/FarmmapsKPI/KPIApplication.cs @@ -0,0 +1,199 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Threading.Tasks; +using FarmmapsApi; +using FarmmapsApi.Models; +using FarmmapsApi.Services; +using FarmmapsKPI.Models; +using Microsoft.Extensions.Logging; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using static FarmmapsApiSamples.Constants; + +namespace FarmmapsKPI +{ + public class KPIApplication : IApplication + { + //private const string DownloadFolder = "Downloads"; + private const string SettingsFile = "settings.json"; + + private readonly ILogger _logger; + private readonly FarmmapsApiService _farmmapsApiService; + private readonly KPIService _dataDownloadService; + private readonly GeneralService _generalService; + + private Settings _settings; + + public KPIApplication(ILogger logger, FarmmapsApiService farmmapsApiService, + GeneralService generalService, KPIService dataDownloadService) + { + _logger = logger; + _farmmapsApiService = farmmapsApiService; + _generalService = generalService; + _dataDownloadService = dataDownloadService; + } + + public async Task RunAsync() + { + var fieldsInputJson = File.ReadAllText("KPIInput.json"); + + List fieldsInputs = JsonConvert.DeserializeObject>(fieldsInputJson); + + // !! this call is needed the first time an api is called with a fresh clientid and secret !! + await _farmmapsApiService.GetCurrentUserCodeAsync(); + var roots = await _farmmapsApiService.GetCurrentUserRootsAsync(); + + foreach (var input in fieldsInputs) + { + try + { + await Process(roots, input); + } + catch (Exception ex) + { + _logger.LogError(ex.Message); + } + } + } + + private async Task Process(List roots, KPIInput input) + { + string downloadFolder = input.DownloadFolder; + if (string.IsNullOrEmpty(downloadFolder)) { + downloadFolder = "Downloads"; + } + if (!Directory.Exists(downloadFolder)) + Directory.CreateDirectory(downloadFolder); + + // !!specify if you are using an already created cropfield: + bool useCreatedCropfield = input.UseCreatedCropfield; + var cropYear = input.CropYear; + var fieldName = input.fieldName; + //bool storeSatelliteStatistics = input.StoreSatelliteStatisticsSingleImage; + //bool storeSatelliteStatisticsCropYear = input.StoreSatelliteStatisticsCropYear; + //List SatelliteBands = new List(1) { input.SatelliteBand }; + //string headerLineStats = $"FieldName,satelliteDate,satelliteBand,max,min,mean,mode,median,stddev,minPlus,curtosis,maxMinus,skewness,variance,populationCount,variationCoefficient,confidenceIntervalLow, confidenceIntervalHigh,confidenceIntervalErrorMargin" + Environment.NewLine; + + + string settingsfile = $"Settings_{fieldName}.json"; + + LoadSettings(settingsfile); + + var uploadedRoot = roots.SingleOrDefault(r => r.Name == "USER_IN"); + if (uploadedRoot == null) + { + _logger.LogError("Could not find a needed root item"); + return; + } + + var myDriveRoot = roots.SingleOrDefault(r => r.Name == "USER_FILES"); + if (myDriveRoot == null) + { + _logger.LogError("Could not find a needed root item"); + return; + } + + // Use already created cropfield or create new one + Item cropfieldItem; + if (useCreatedCropfield == false || string.IsNullOrEmpty(_settings.CropfieldItemCode)) + { + _logger.LogInformation("Creating cropfield"); + cropfieldItem = await _generalService.CreateCropfieldItemAsync(myDriveRoot.Code, + $"DataCropfield {input.OutputFileName}", cropYear, input.GeometryJson.ToString(Formatting.None)); + _settings.CropfieldItemCode = cropfieldItem.Code; + SaveSettings(settingsfile); + } + else + { + _logger.LogInformation("Cropfield already exists, trying to get it"); + cropfieldItem = await _farmmapsApiService.GetItemAsync(_settings.CropfieldItemCode); + } + + //Get croprecordings + if (input.GetCropRecordings) + { + var crprecItem = input.CrprecItem; + _logger.LogInformation($"Trying to get crop recordings of croprecording: {crprecItem}"); + + var cropRec = await _farmmapsApiService.GetItemChildrenAsync(crprecItem, CROPREC_ITEMTYPE); + if (cropRec == null) + { + _logger.LogError("Something went wrong while obtaining the croprecordings"); + return; + } + + var cropRecPath = Path.Combine(downloadFolder, $"croprecordings_{crprecItem}.json"); + _logger.LogInformation($"Found {cropRec.Count} crop recordings"); + var count = 0; + await Task.Delay(500); + foreach (var item in cropRec) + { + Console.WriteLine($"Crop recording #{count}: {item.Name}"); + File.AppendAllText(cropRecPath, item.Data +Environment.NewLine); + count++; + } + _logger.LogInformation($"Downloaded file {cropRecPath}"); + } + else + { + //todo set croprecordings from file + } + + // Get KPI data + + //_logger.LogInformation("Calculate KPI map for field"); + //var KPIItem = await _generalService.RunKPITask(cropfieldItem); + var KPIItem = null; + if (KPIItem == null) + { + _logger.LogError("Something went wrong while obtaining the KPI map"); + return; + } + + //_logger.LogInformation("Downloading KPI map"); + //await _farmmapsApiService.DownloadItemAsync(KPIItem.Code, + // Path.Combine(downloadFolder, $"{input.OutputFileName}_KPI.zip")); + + + + + + } + + // Functions to save previously created cropfields + private void LoadSettings(string file) + { + if (File.Exists(file)) + { + var jsonText = File.ReadAllText(file); + _settings = JsonConvert.DeserializeObject(jsonText); + } + else + { + _settings = new Settings(); + } + } + + private void SaveSettings(string file) + { + if (_settings == null) + return; + + var json = JsonConvert.SerializeObject(_settings); + File.WriteAllText(file, json); + } + private void SaveInfo(string file) + { + if (_settings == null) + return; + + var json = JsonConvert.SerializeObject(_settings); + File.WriteAllText(file, json); + + } + + } +} diff --git a/FarmmapsKPI/KPIInput.json b/FarmmapsKPI/KPIInput.json new file mode 100644 index 0000000..84d35e8 --- /dev/null +++ b/FarmmapsKPI/KPIInput.json @@ -0,0 +1,34 @@ +[ + { + "UseCreatedCropfield": false, + "outputFileName": "TestData", + "fieldName": "TestField", + "DownloadFolder": "Downloads", //"C:\\workdir\\groenmonitor\\", // "Downloads", -> if you just put "Downloads" the program will download to somewhere in ..\FarmMapsApiClient_WURtest\FarmmapsDataDownload\bin\Debug\netcoreapp3.1\Downloads\ + "GetCropRecordings": true, + "CrprecItem": "...", //item code of de crop recording parrent - can be found by opening the crop recording page of a field. + "GetShadowData": false, + "GetSatelliteData": false, + "SatelliteBand": "wdvi", // "natural", "ndvi" or "wdvi" + "StoreSatelliteStatisticsSingleImage": false, + "StoreSatelliteStatisticsCropYear": false, + "GetVanDerSatData": false, + "StoreVanDerSatStatistics": false, + "CropYear": 2020, + "geometryJson": { + "type": "Polygon", + "coordinates": [ + [ + [ 4.960707146896585, 52.800583669708487 ], + [ 4.960645975538824, 52.800470217610922 ], + [ 4.962140695752897, 52.799177147194797 ], + [ 4.967523821195745, 52.801502400041208 ], + [ 4.966336768950911, 52.802543735879809 ], + [ 4.961711880764330, 52.801009996856429 ], + [ 4.960707146896585, 52.800583669708487 ] + ] + ] + } + } + + +] \ No newline at end of file diff --git a/FarmmapsKPI/KPIService.cs b/FarmmapsKPI/KPIService.cs new file mode 100644 index 0000000..ce88acd --- /dev/null +++ b/FarmmapsKPI/KPIService.cs @@ -0,0 +1,29 @@ +using System; +using System.Globalization; +using System.Threading.Tasks; +using FarmmapsApi.Models; +using FarmmapsApi.Services; +using FarmmapsKPI.Models; +using Microsoft.Extensions.Logging; +using static FarmmapsApi.Extensions; +using static FarmmapsApiSamples.Constants; + +namespace FarmmapsKPI +{ + public class KPIService + { + private readonly ILogger _logger; + private readonly FarmmapsApiService _farmmapsApiService; + private readonly GeneralService _generalService; + + public KPIService(ILogger logger, FarmmapsApiService farmmapsApiService, + GeneralService generalService) + { + _logger = logger; + _farmmapsApiService = farmmapsApiService; + _generalService = generalService; + } + + + } +} \ No newline at end of file diff --git a/FarmmapsKPI/Models/CropRecordings.cs b/FarmmapsKPI/Models/CropRecordings.cs new file mode 100644 index 0000000..61e667f --- /dev/null +++ b/FarmmapsKPI/Models/CropRecordings.cs @@ -0,0 +1,10 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace FarmmapsKPI.Models +{ + internal class CropRecordings + { + } +} diff --git a/FarmmapsKPI/Models/KPIInput.cs b/FarmmapsKPI/Models/KPIInput.cs new file mode 100644 index 0000000..63e5c68 --- /dev/null +++ b/FarmmapsKPI/Models/KPIInput.cs @@ -0,0 +1,28 @@ +using System; +using Newtonsoft.Json.Linq; + +namespace FarmmapsKPI.Models +{ + public class KPIInput + { + public bool UseCreatedCropfield { get; set; } + public string File { get; set; } + public string InputVariable { get; set; } + public string OutputFileName { get; set; } + public string DownloadFolder { get; set; } + public int CropYear { get; set; } + public JObject GeometryJson { get; set; } + public string InputLayerName { get; set; } + public string fieldName { get; set; } + public bool GetSatelliteData { get; set; } + public bool GetVanDerSatData { get; set; } + public string SatelliteBand { get; set; } + public bool StoreSatelliteStatisticsSingleImage { get; set; } + public bool StoreSatelliteStatisticsCropYear { get; set; } + public bool StoreVanDerSatStatistics { get; set; } + public bool GetShadowData { get; set; } + public bool GetCropRecordings { get; set; } + public string CrprecItem { get; set; } + + } +} \ No newline at end of file diff --git a/FarmmapsKPI/Models/Settings.cs b/FarmmapsKPI/Models/Settings.cs new file mode 100644 index 0000000..bd7ff29 --- /dev/null +++ b/FarmmapsKPI/Models/Settings.cs @@ -0,0 +1,11 @@ +namespace FarmmapsKPI +{ + public class Settings + { + public string CropfieldItemCode { get; set; } + //public string SatelliteTaskCode { get; set; } + //public string VanDerSatTaskCode { get; set; } + //public string WatBalTaskCode { get; set; } + + } +} \ No newline at end of file diff --git a/FarmmapsKPI/Program.cs b/FarmmapsKPI/Program.cs new file mode 100644 index 0000000..9e8e737 --- /dev/null +++ b/FarmmapsKPI/Program.cs @@ -0,0 +1,21 @@ +using System.Threading.Tasks; +using FarmmapsApi; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; + +namespace FarmmapsKPI +{ + class Program : FarmmapsProgram + { + private static async Task Main(string[] args) + { + await new Program().Start(args); + } + + protected override void Configure(IServiceCollection serviceCollection) + { + serviceCollection.AddLogging() + .AddTransient(); + } + } +} From 7dda21dd77c425c1560894f8c83211e457caf08c Mon Sep 17 00:00:00 2001 From: abel Date: Fri, 31 Mar 2023 11:02:06 +0200 Subject: [PATCH 03/66] probeersel om een runKPItask te maken --- FarmmapsApi/Constants.cs | 1 + FarmmapsApi/Services/GeneralService.cs | 35 ++++++++++++++++++++++++++ FarmmapsKPI/KPIApplication.cs | 3 +-- FarmmapsKPI/KPIInput.json | 4 +-- FarmmapsKPI/Models/KPIInput.cs | 12 ++++----- 5 files changed, 45 insertions(+), 10 deletions(-) diff --git a/FarmmapsApi/Constants.cs b/FarmmapsApi/Constants.cs index 6db165b..027d98b 100644 --- a/FarmmapsApi/Constants.cs +++ b/FarmmapsApi/Constants.cs @@ -27,5 +27,6 @@ namespace FarmmapsApiSamples public const string SHADOW_TASK = "vnd.farmmaps.task.shadow"; public const string AHN_TASK = "vnd.farmmaps.task.ahn"; public const string WATBAL_TASK = "vnd.farmmaps.task.watbal"; + public const string KPI_TASK = "vnd.farmmaps.task.kpi"; // dus nieuwe taak om de KPIs te berekenen } } diff --git a/FarmmapsApi/Services/GeneralService.cs b/FarmmapsApi/Services/GeneralService.cs index 7027985..3d542ef 100644 --- a/FarmmapsApi/Services/GeneralService.cs +++ b/FarmmapsApi/Services/GeneralService.cs @@ -277,6 +277,41 @@ namespace FarmmapsApi.Services return bofekItem; } + public async Task RunKPITask(Item cropfieldItem) // dit is dus nieuw om de KPI task te runnen + { + var taskmapRequest = new TaskRequest { TaskType = KPI_TASK }; + + string itemTaskCode = await _farmmapsApiService.QueueTaskAsync(cropfieldItem.Code, taskmapRequest); + + await PollTask(TimeSpan.FromSeconds(5), async (tokenSource) => { + var itemTaskStatus = await _farmmapsApiService.GetTaskStatusAsync(cropfieldItem.Code, itemTaskCode); + _logger.LogInformation($"Waiting on retreiving KPI data; status: {itemTaskStatus.State}"); + if (itemTaskStatus.IsFinished) + tokenSource.Cancel(); + }); + + var itemTask = await _farmmapsApiService.GetTaskStatusAsync(cropfieldItem.Code, itemTaskCode); + if (itemTask.State == ItemTaskState.Error) + { + _logger.LogError($"Something went wrong with task execution: {itemTask.Message}"); + return null; + } + + //hier nog definieren waar in de hierarchie een KPI item is? + + ////the BOFEK data is a child of the cropfield + //var itemName = "bofek"; + //var bofekItem = await FindChildItemAsync(cropfieldItem.Code, + // SHAPE_PROCESSED_ITEMTYPE, itemName); + //if (bofekItem == null) + //{ + // _logger.LogError("Could not find the BOFEK data as a child item under the cropfield"); + // return null; + //} + + return bofekItem; + } + public async Task RunAhnTask(Item cropfieldItem) { var taskmapRequest = new TaskRequest { TaskType = AHN_TASK }; diff --git a/FarmmapsKPI/KPIApplication.cs b/FarmmapsKPI/KPIApplication.cs index 0e8bac8..7a32a0e 100644 --- a/FarmmapsKPI/KPIApplication.cs +++ b/FarmmapsKPI/KPIApplication.cs @@ -146,8 +146,7 @@ namespace FarmmapsKPI //_logger.LogInformation("Calculate KPI map for field"); //var KPIItem = await _generalService.RunKPITask(cropfieldItem); - var KPIItem = null; - if (KPIItem == null) + if ((object)null == null) { _logger.LogError("Something went wrong while obtaining the KPI map"); return; diff --git a/FarmmapsKPI/KPIInput.json b/FarmmapsKPI/KPIInput.json index 84d35e8..b51d3b6 100644 --- a/FarmmapsKPI/KPIInput.json +++ b/FarmmapsKPI/KPIInput.json @@ -1,10 +1,10 @@ [ { - "UseCreatedCropfield": false, + "UseCreatedCropfield": true, "outputFileName": "TestData", "fieldName": "TestField", "DownloadFolder": "Downloads", //"C:\\workdir\\groenmonitor\\", // "Downloads", -> if you just put "Downloads" the program will download to somewhere in ..\FarmMapsApiClient_WURtest\FarmmapsDataDownload\bin\Debug\netcoreapp3.1\Downloads\ - "GetCropRecordings": true, + "GetCropRecordings": false, "CrprecItem": "...", //item code of de crop recording parrent - can be found by opening the crop recording page of a field. "GetShadowData": false, "GetSatelliteData": false, diff --git a/FarmmapsKPI/Models/KPIInput.cs b/FarmmapsKPI/Models/KPIInput.cs index 63e5c68..d21f594 100644 --- a/FarmmapsKPI/Models/KPIInput.cs +++ b/FarmmapsKPI/Models/KPIInput.cs @@ -14,12 +14,12 @@ namespace FarmmapsKPI.Models public JObject GeometryJson { get; set; } public string InputLayerName { get; set; } public string fieldName { get; set; } - public bool GetSatelliteData { get; set; } - public bool GetVanDerSatData { get; set; } - public string SatelliteBand { get; set; } - public bool StoreSatelliteStatisticsSingleImage { get; set; } - public bool StoreSatelliteStatisticsCropYear { get; set; } - public bool StoreVanDerSatStatistics { get; set; } + //public bool GetSatelliteData { get; set; } + // public bool GetVanDerSatData { get; set; } + // public string SatelliteBand { get; set; } + // public bool StoreSatelliteStatisticsSingleImage { get; set; } + // public bool StoreSatelliteStatisticsCropYear { get; set; } + //public bool StoreVanDerSatStatistics { get; set; } public bool GetShadowData { get; set; } public bool GetCropRecordings { get; set; } public string CrprecItem { get; set; } From be22d631bce4a6aa8f6f4c8073f17d2f7cbdf083 Mon Sep 17 00:00:00 2001 From: abel Date: Fri, 31 Mar 2023 11:13:51 +0200 Subject: [PATCH 04/66] geupdate runKPIitemtask, helaas nog geen KPI data gevonden als child item --- FarmmapsApi/Services/GeneralService.cs | 20 ++++++++++---------- FarmmapsKPI/KPIApplication.cs | 4 ++-- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/FarmmapsApi/Services/GeneralService.cs b/FarmmapsApi/Services/GeneralService.cs index 3d542ef..8fde298 100644 --- a/FarmmapsApi/Services/GeneralService.cs +++ b/FarmmapsApi/Services/GeneralService.cs @@ -299,17 +299,17 @@ namespace FarmmapsApi.Services //hier nog definieren waar in de hierarchie een KPI item is? - ////the BOFEK data is a child of the cropfield - //var itemName = "bofek"; - //var bofekItem = await FindChildItemAsync(cropfieldItem.Code, - // SHAPE_PROCESSED_ITEMTYPE, itemName); - //if (bofekItem == null) - //{ - // _logger.LogError("Could not find the BOFEK data as a child item under the cropfield"); - // return null; - //} + //the kpi data is a child of the cropfield + var itemname = "KPI"; + var KPIItem = await FindChildItemAsync(cropfieldItem.Code, + CROPFIELD_ITEMTYPE, itemname); //hier moet ik dus verwijzen naar waar het KPIItem zit? ik gok dat het een cropfield item type is... + if (KPIItem == null) + { + _logger.LogError("could not find the bofek data as a child item under the cropfield"); + return null; + } - return bofekItem; + return KPIItem; } public async Task RunAhnTask(Item cropfieldItem) { diff --git a/FarmmapsKPI/KPIApplication.cs b/FarmmapsKPI/KPIApplication.cs index 7a32a0e..3130c48 100644 --- a/FarmmapsKPI/KPIApplication.cs +++ b/FarmmapsKPI/KPIApplication.cs @@ -144,8 +144,8 @@ namespace FarmmapsKPI // Get KPI data - //_logger.LogInformation("Calculate KPI map for field"); - //var KPIItem = await _generalService.RunKPITask(cropfieldItem); + _logger.LogInformation("Calculate KPI map for field"); + var KPIItem = await _generalService.RunKPITask(cropfieldItem);// hier dus verwijzen naar de KPI task if ((object)null == null) { _logger.LogError("Something went wrong while obtaining the KPI map"); From b7bedf182ccabc053a7f895bc39415d51abc1fb5 Mon Sep 17 00:00:00 2001 From: abel Date: Mon, 3 Apr 2023 13:03:10 +0200 Subject: [PATCH 05/66] aggegrate KPI = false toegevoegd aan inputs --- FarmmapsApi/Constants.cs | 2 ++ FarmmapsApi/Services/GeneralService.cs | 6 +++--- FarmmapsKPI/KPIService.cs | 12 +++++++++++- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/FarmmapsApi/Constants.cs b/FarmmapsApi/Constants.cs index 027d98b..51bc178 100644 --- a/FarmmapsApi/Constants.cs +++ b/FarmmapsApi/Constants.cs @@ -13,6 +13,8 @@ namespace FarmmapsApiSamples public const string GEOJSON_ITEMTYPE = "vnd.farmmaps.itemtype.geojson"; public const string BLIGHT_ITEMTYPE = "vnd.farmmaps.itemtype.blight"; public const string CROPREC_ITEMTYPE = "vnd.farmmaps.itemtype.crprec.operation"; + public const string CROPSCHEME_ITEMTYPE = "vnd.farmmaps.itemtype.croppingscheme"; // deze toegevoegd, misschien is het type van de KPI task wel een croppingscheme + public const string VRANBS_TASK = "vnd.farmmaps.task.vranbs"; public const string VRAHERBICIDE_TASK = "vnd.farmmaps.task.vraherbicide"; diff --git a/FarmmapsApi/Services/GeneralService.cs b/FarmmapsApi/Services/GeneralService.cs index 8fde298..80ee8fe 100644 --- a/FarmmapsApi/Services/GeneralService.cs +++ b/FarmmapsApi/Services/GeneralService.cs @@ -300,12 +300,12 @@ namespace FarmmapsApi.Services //hier nog definieren waar in de hierarchie een KPI item is? //the kpi data is a child of the cropfield - var itemname = "KPI"; + var itemName = "kpi"; var KPIItem = await FindChildItemAsync(cropfieldItem.Code, - CROPFIELD_ITEMTYPE, itemname); //hier moet ik dus verwijzen naar waar het KPIItem zit? ik gok dat het een cropfield item type is... + CROPFIELD_ITEMTYPE, itemName); //hier moet ik dus verwijzen naar waar het KPIItem zit? ik gok dat het een cropfield item type is.-> of onderdeel van een croppingscheme? net toegevoegd. if (KPIItem == null) { - _logger.LogError("could not find the bofek data as a child item under the cropfield"); + _logger.LogError("could not find the KPI data as a child item under the cropfield"); return null; } diff --git a/FarmmapsKPI/KPIService.cs b/FarmmapsKPI/KPIService.cs index ce88acd..846bce7 100644 --- a/FarmmapsKPI/KPIService.cs +++ b/FarmmapsKPI/KPIService.cs @@ -23,7 +23,17 @@ namespace FarmmapsKPI _farmmapsApiService = farmmapsApiService; _generalService = generalService; } + // zoiets kan ik overwegen als ik de itemtype niet kan vinden + //public async Task CreateTargetKPIItem(Item cropfieldItem) + //{ + // var itemRequest = new ItemRequest() + // { + // ParentCode = cropfieldItem.ParentCode, + // ItemType = USERINPUT_ITEMTYPE, + // Name = "kpi" + // }; + // return await _farmmapsApiService.CreateItemAsync(itemRequest); + //} - } } \ No newline at end of file From 8f8f857bf08bcfc9f3575e0e3e2ab9b54401e44d Mon Sep 17 00:00:00 2001 From: abel Date: Mon, 3 Apr 2023 13:05:00 +0200 Subject: [PATCH 06/66] ook in in de andere KPIinout toegevoegd --- FarmmapsKPI/KPIInput.json | 1 + 1 file changed, 1 insertion(+) diff --git a/FarmmapsKPI/KPIInput.json b/FarmmapsKPI/KPIInput.json index b51d3b6..17322ee 100644 --- a/FarmmapsKPI/KPIInput.json +++ b/FarmmapsKPI/KPIInput.json @@ -14,6 +14,7 @@ "GetVanDerSatData": false, "StoreVanDerSatStatistics": false, "CropYear": 2020, + "ProcessAggegrateKpi": "false", // toegevoegd, want alleen veld level KPIs voor nu. "geometryJson": { "type": "Polygon", "coordinates": [ From 3d7bf186e7f5e940df9e19bd35db20570874d332 Mon Sep 17 00:00:00 2001 From: abel Date: Thu, 6 Apr 2023 16:11:37 +0200 Subject: [PATCH 07/66] de properties voor de KPI task worden nu correct doorgegeven denk ik --- FarmmapsApi/Services/GeneralService.cs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/FarmmapsApi/Services/GeneralService.cs b/FarmmapsApi/Services/GeneralService.cs index 80ee8fe..5cfd6a5 100644 --- a/FarmmapsApi/Services/GeneralService.cs +++ b/FarmmapsApi/Services/GeneralService.cs @@ -279,7 +279,9 @@ namespace FarmmapsApi.Services public async Task RunKPITask(Item cropfieldItem) // dit is dus nieuw om de KPI task te runnen { - var taskmapRequest = new TaskRequest { TaskType = KPI_TASK }; + var taskmapRequest = new TaskRequest { TaskType = KPI_TASK }; //hier KPI request van maken + taskmapRequest.attributes["processAggregateKpi"] = "false"; // zo de properties meegeven?, moet dit niet bij de KPIinput? + taskmapRequest.attributes["year"] = "2022"; string itemTaskCode = await _farmmapsApiService.QueueTaskAsync(cropfieldItem.Code, taskmapRequest); @@ -299,7 +301,7 @@ namespace FarmmapsApi.Services //hier nog definieren waar in de hierarchie een KPI item is? - //the kpi data is a child of the cropfield + //the kpi data is a child of the cropfield --> is dit wel zo? var itemName = "kpi"; var KPIItem = await FindChildItemAsync(cropfieldItem.Code, CROPFIELD_ITEMTYPE, itemName); //hier moet ik dus verwijzen naar waar het KPIItem zit? ik gok dat het een cropfield item type is.-> of onderdeel van een croppingscheme? net toegevoegd. From 55f00da9d08adb731e4c451c53c3e18d5d88f371 Mon Sep 17 00:00:00 2001 From: abel Date: Fri, 7 Apr 2023 14:33:49 +0200 Subject: [PATCH 08/66] KPI items ophalen voor een cropfield werkt nu. wel dus een bestaand cropfield ingeven bij de settings in de BIN. Alle KPIs worden opgehaald na een minuut wachten. --- FarmmapsApi/Constants.cs | 2 +- FarmmapsApi/Services/GeneralService.cs | 16 +++++----------- FarmmapsKPI/KPIApplication.cs | 2 +- 3 files changed, 7 insertions(+), 13 deletions(-) diff --git a/FarmmapsApi/Constants.cs b/FarmmapsApi/Constants.cs index 51bc178..613dfde 100644 --- a/FarmmapsApi/Constants.cs +++ b/FarmmapsApi/Constants.cs @@ -14,7 +14,7 @@ namespace FarmmapsApiSamples public const string BLIGHT_ITEMTYPE = "vnd.farmmaps.itemtype.blight"; public const string CROPREC_ITEMTYPE = "vnd.farmmaps.itemtype.crprec.operation"; public const string CROPSCHEME_ITEMTYPE = "vnd.farmmaps.itemtype.croppingscheme"; // deze toegevoegd, misschien is het type van de KPI task wel een croppingscheme - + public const string KPI_ITEM = "vnd.farmmaps.itemtype.kpi.data"; public const string VRANBS_TASK = "vnd.farmmaps.task.vranbs"; public const string VRAHERBICIDE_TASK = "vnd.farmmaps.task.vraherbicide"; diff --git a/FarmmapsApi/Services/GeneralService.cs b/FarmmapsApi/Services/GeneralService.cs index 5cfd6a5..8215260 100644 --- a/FarmmapsApi/Services/GeneralService.cs +++ b/FarmmapsApi/Services/GeneralService.cs @@ -277,7 +277,7 @@ namespace FarmmapsApi.Services return bofekItem; } - public async Task RunKPITask(Item cropfieldItem) // dit is dus nieuw om de KPI task te runnen + public async Task> GetKpiItemsForCropField(Item cropfieldItem) // dit is dus nieuw om de KPI task te runnen { var taskmapRequest = new TaskRequest { TaskType = KPI_TASK }; //hier KPI request van maken taskmapRequest.attributes["processAggregateKpi"] = "false"; // zo de properties meegeven?, moet dit niet bij de KPIinput? @@ -298,20 +298,14 @@ namespace FarmmapsApi.Services _logger.LogError($"Something went wrong with task execution: {itemTask.Message}"); return null; } + await Task.Delay(60000); //wacht hier een minuut tot de KPIs berekend zijn //hier nog definieren waar in de hierarchie een KPI item is? - //the kpi data is a child of the cropfield --> is dit wel zo? - var itemName = "kpi"; - var KPIItem = await FindChildItemAsync(cropfieldItem.Code, - CROPFIELD_ITEMTYPE, itemName); //hier moet ik dus verwijzen naar waar het KPIItem zit? ik gok dat het een cropfield item type is.-> of onderdeel van een croppingscheme? net toegevoegd. - if (KPIItem == null) - { - _logger.LogError("could not find the KPI data as a child item under the cropfield"); - return null; - } + //the kpi data is a child of the cropfield --> is dit wel zo? is kpi ook de itemName? - return KPIItem; + return await _farmmapsApiService.GetItemChildrenAsync(cropfieldItem.Code, KPI_ITEM); + } public async Task RunAhnTask(Item cropfieldItem) { diff --git a/FarmmapsKPI/KPIApplication.cs b/FarmmapsKPI/KPIApplication.cs index 3130c48..c772e19 100644 --- a/FarmmapsKPI/KPIApplication.cs +++ b/FarmmapsKPI/KPIApplication.cs @@ -145,7 +145,7 @@ namespace FarmmapsKPI // Get KPI data _logger.LogInformation("Calculate KPI map for field"); - var KPIItem = await _generalService.RunKPITask(cropfieldItem);// hier dus verwijzen naar de KPI task + var KPIItem = await _generalService.GetKpiItemsForCropField(cropfieldItem);// hier dus verwijzen naar de KPI task if ((object)null == null) { _logger.LogError("Something went wrong while obtaining the KPI map"); From 7149086ce29e1f914c2ebbea1a126cdc78b22433 Mon Sep 17 00:00:00 2001 From: "Abel Hoeven, van" Date: Tue, 25 Apr 2023 12:00:08 +0200 Subject: [PATCH 09/66] KPI items opslaan werkt. alle KPI items worden nu naar een JSON file in de dowload folder geschreven --- FarmmapsKPI/KPIApplication.cs | 36 ++++++++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/FarmmapsKPI/KPIApplication.cs b/FarmmapsKPI/KPIApplication.cs index c772e19..af93856 100644 --- a/FarmmapsKPI/KPIApplication.cs +++ b/FarmmapsKPI/KPIApplication.cs @@ -127,13 +127,13 @@ namespace FarmmapsKPI var cropRecPath = Path.Combine(downloadFolder, $"croprecordings_{crprecItem}.json"); _logger.LogInformation($"Found {cropRec.Count} crop recordings"); - var count = 0; + var count1 = 0; await Task.Delay(500); foreach (var item in cropRec) { - Console.WriteLine($"Crop recording #{count}: {item.Name}"); + Console.WriteLine($"Crop recording #{count1}: {item.Name}"); File.AppendAllText(cropRecPath, item.Data +Environment.NewLine); - count++; + count1++; } _logger.LogInformation($"Downloaded file {cropRecPath}"); } @@ -142,15 +142,33 @@ namespace FarmmapsKPI //todo set croprecordings from file } - // Get KPI data + // Get KPI data for saving it in a file - _logger.LogInformation("Calculate KPI map for field"); - var KPIItem = await _generalService.GetKpiItemsForCropField(cropfieldItem);// hier dus verwijzen naar de KPI task - if ((object)null == null) + + _logger.LogInformation($"Trying to get the cropfielditem: {cropfieldItem}"); + var KPIItem = await _generalService.GetKpiItemsForCropField(cropfieldItem); + + + var KPIItemPath = Path.Combine(downloadFolder, $"KPIItems_{KPIItem}.json"); + _logger.LogInformation($"Found {KPIItem.Count} KPI items"); + var count = 0; + await Task.Delay(50); + foreach (var item in KPIItem) { - _logger.LogError("Something went wrong while obtaining the KPI map"); - return; + Console.WriteLine($"KPI item #{count}: {item.Name}"); + File.AppendAllText(KPIItemPath, item.Data + Environment.NewLine); + count++; } + _logger.LogInformation($"Downloaded file {KPIItemPath}"); + + //////////// + //_logger.LogInformation("Calculate KPI map for field"); + //var KPIItem = await _generalService.GetKpiItemsForCropField(cropfieldItem);// hier dus verwijzen naar de KPI task + //if ((object)null == null) + //{ + // _logger.LogError("Something went wrong while obtaining the KPI map"); + // return; + // } //_logger.LogInformation("Downloading KPI map"); //await _farmmapsApiService.DownloadItemAsync(KPIItem.Code, From 8bb5c6f581154d75433c5820e6bdedafbfb5894c Mon Sep 17 00:00:00 2001 From: "Abel Hoeven, van" Date: Tue, 25 Apr 2023 16:14:43 +0200 Subject: [PATCH 10/66] Input aangepast, nu kan er van de gegeven input een cropfield wordern gemaakt, en hier kunnen vervolgens KPIs over berekend worden. Volgende stap is het strippen van de KPIinput om te kijken wat het minimale is dat nodig is om de kpis te berekenen. --- FarmmapsKPI/KPIApplication.cs | 4 ++-- FarmmapsKPI/KPIInput.json | 2 +- FarmmapsKPI/Models/KPIInput.cs | 1 + 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/FarmmapsKPI/KPIApplication.cs b/FarmmapsKPI/KPIApplication.cs index af93856..3ec2434 100644 --- a/FarmmapsKPI/KPIApplication.cs +++ b/FarmmapsKPI/KPIApplication.cs @@ -98,11 +98,11 @@ namespace FarmmapsKPI // Use already created cropfield or create new one Item cropfieldItem; - if (useCreatedCropfield == false || string.IsNullOrEmpty(_settings.CropfieldItemCode)) + if (useCreatedCropfield == true || string.IsNullOrEmpty(_settings.CropfieldItemCode)) { _logger.LogInformation("Creating cropfield"); cropfieldItem = await _generalService.CreateCropfieldItemAsync(myDriveRoot.Code, - $"DataCropfield {input.OutputFileName}", cropYear, input.GeometryJson.ToString(Formatting.None)); + $"DataCropfield {input.OutputFileName}", cropYear, input.GeometryJson.ToString(Formatting.None), input.Data.ToString(Formatting.None)); _settings.CropfieldItemCode = cropfieldItem.Code; SaveSettings(settingsfile); } diff --git a/FarmmapsKPI/KPIInput.json b/FarmmapsKPI/KPIInput.json index 17322ee..da10356 100644 --- a/FarmmapsKPI/KPIInput.json +++ b/FarmmapsKPI/KPIInput.json @@ -1,6 +1,6 @@ [ { - "UseCreatedCropfield": true, + "UseCreatedCropfield": false, "outputFileName": "TestData", "fieldName": "TestField", "DownloadFolder": "Downloads", //"C:\\workdir\\groenmonitor\\", // "Downloads", -> if you just put "Downloads" the program will download to somewhere in ..\FarmMapsApiClient_WURtest\FarmmapsDataDownload\bin\Debug\netcoreapp3.1\Downloads\ diff --git a/FarmmapsKPI/Models/KPIInput.cs b/FarmmapsKPI/Models/KPIInput.cs index d21f594..99314b0 100644 --- a/FarmmapsKPI/Models/KPIInput.cs +++ b/FarmmapsKPI/Models/KPIInput.cs @@ -12,6 +12,7 @@ namespace FarmmapsKPI.Models public string DownloadFolder { get; set; } public int CropYear { get; set; } public JObject GeometryJson { get; set; } + public JObject Data { get; set; } public string InputLayerName { get; set; } public string fieldName { get; set; } //public bool GetSatelliteData { get; set; } From 59f9ef474333b2660cca486f881028812177bca3 Mon Sep 17 00:00:00 2001 From: "Abel Hoeven, van" Date: Tue, 25 Apr 2023 16:14:43 +0200 Subject: [PATCH 11/66] nu kan er van de gegeven input een cropfield wordern gemaakt, en hier kunnen vervolgens KPIs over berekend worden. Volgende stap is het strippen van de KPIinput om te kijken wat het minimale is dat nodig is om de kpis te berekenen. verder nu eem aantal inputs die niet nodig waren verwijdertInput aangepast, --- FarmmapsKPI/KPIApplication.cs | 8 +++----- FarmmapsKPI/KPIInput.json | 2 +- FarmmapsKPI/Models/KPIInput.cs | 7 +------ 3 files changed, 5 insertions(+), 12 deletions(-) diff --git a/FarmmapsKPI/KPIApplication.cs b/FarmmapsKPI/KPIApplication.cs index af93856..5bdad37 100644 --- a/FarmmapsKPI/KPIApplication.cs +++ b/FarmmapsKPI/KPIApplication.cs @@ -96,13 +96,13 @@ namespace FarmmapsKPI return; } - // Use already created cropfield or create new one + // Use already created cropfield or create new one, added a Data input, with field specific data for the KPI calculation Item cropfieldItem; - if (useCreatedCropfield == false || string.IsNullOrEmpty(_settings.CropfieldItemCode)) + if (useCreatedCropfield == true || string.IsNullOrEmpty(_settings.CropfieldItemCode)) { _logger.LogInformation("Creating cropfield"); cropfieldItem = await _generalService.CreateCropfieldItemAsync(myDriveRoot.Code, - $"DataCropfield {input.OutputFileName}", cropYear, input.GeometryJson.ToString(Formatting.None)); + $"DataCropfield {input.OutputFileName}", cropYear, input.GeometryJson.ToString(Formatting.None), input.Data.ToString(Formatting.None)); _settings.CropfieldItemCode = cropfieldItem.Code; SaveSettings(settingsfile); } @@ -143,8 +143,6 @@ namespace FarmmapsKPI } // Get KPI data for saving it in a file - - _logger.LogInformation($"Trying to get the cropfielditem: {cropfieldItem}"); var KPIItem = await _generalService.GetKpiItemsForCropField(cropfieldItem); diff --git a/FarmmapsKPI/KPIInput.json b/FarmmapsKPI/KPIInput.json index 17322ee..da10356 100644 --- a/FarmmapsKPI/KPIInput.json +++ b/FarmmapsKPI/KPIInput.json @@ -1,6 +1,6 @@ [ { - "UseCreatedCropfield": true, + "UseCreatedCropfield": false, "outputFileName": "TestData", "fieldName": "TestField", "DownloadFolder": "Downloads", //"C:\\workdir\\groenmonitor\\", // "Downloads", -> if you just put "Downloads" the program will download to somewhere in ..\FarmMapsApiClient_WURtest\FarmmapsDataDownload\bin\Debug\netcoreapp3.1\Downloads\ diff --git a/FarmmapsKPI/Models/KPIInput.cs b/FarmmapsKPI/Models/KPIInput.cs index d21f594..bcb35b8 100644 --- a/FarmmapsKPI/Models/KPIInput.cs +++ b/FarmmapsKPI/Models/KPIInput.cs @@ -12,14 +12,9 @@ namespace FarmmapsKPI.Models public string DownloadFolder { get; set; } public int CropYear { get; set; } public JObject GeometryJson { get; set; } + public JObject Data { get; set; } public string InputLayerName { get; set; } public string fieldName { get; set; } - //public bool GetSatelliteData { get; set; } - // public bool GetVanDerSatData { get; set; } - // public string SatelliteBand { get; set; } - // public bool StoreSatelliteStatisticsSingleImage { get; set; } - // public bool StoreSatelliteStatisticsCropYear { get; set; } - //public bool StoreVanDerSatStatistics { get; set; } public bool GetShadowData { get; set; } public bool GetCropRecordings { get; set; } public string CrprecItem { get; set; } From 8c5a0aed95492d1f3c1f8a315a4264f673cdeeba Mon Sep 17 00:00:00 2001 From: "Abel Hoeven, van" Date: Tue, 2 May 2023 10:55:24 +0200 Subject: [PATCH 12/66] de niet verplichte inputs verwijderd van de input file. Wel wordt er dus de input file gebruikt die in de C:\Users\hoeve065\Documents\hoofdmap_werk\project_KB34MAST nutrienten DB\API04.03\FarmMapsApiClient_KB34_MAST\FarmmapsKPI\bin\Debug\netcoreapp3.1\KPIInput.json staat, en niet bij het KPI programma. --- FarmmapsKPI/KPIInput.json | 61 +++++++++++++++++++++++++-------------- 1 file changed, 40 insertions(+), 21 deletions(-) diff --git a/FarmmapsKPI/KPIInput.json b/FarmmapsKPI/KPIInput.json index da10356..9d9aef9 100644 --- a/FarmmapsKPI/KPIInput.json +++ b/FarmmapsKPI/KPIInput.json @@ -1,35 +1,54 @@ [ { - "UseCreatedCropfield": false, + "UseCreatedCropfield": true, "outputFileName": "TestData", - "fieldName": "TestField", - "DownloadFolder": "Downloads", //"C:\\workdir\\groenmonitor\\", // "Downloads", -> if you just put "Downloads" the program will download to somewhere in ..\FarmMapsApiClient_WURtest\FarmmapsDataDownload\bin\Debug\netcoreapp3.1\Downloads\ + //"fieldName": "aardappelveld_test", + //"DownloadFolder": "Downloads", //"C:\\workdir\\groenmonitor\\", // "Downloads", -> if you just put "Downloads" the program will download to somewhere in ..\FarmMapsApiClient_WURtest\FarmmapsDataDownload\bin\Debug\netcoreapp3.1\Downloads\ "GetCropRecordings": false, - "CrprecItem": "...", //item code of de crop recording parrent - can be found by opening the crop recording page of a field. - "GetShadowData": false, - "GetSatelliteData": false, - "SatelliteBand": "wdvi", // "natural", "ndvi" or "wdvi" - "StoreSatelliteStatisticsSingleImage": false, - "StoreSatelliteStatisticsCropYear": false, - "GetVanDerSatData": false, - "StoreVanDerSatStatistics": false, - "CropYear": 2020, - "ProcessAggegrateKpi": "false", // toegevoegd, want alleen veld level KPIs voor nu. + "CropYear": 2022, + "dataDate": "2022-04-15T00:00:00Z", + "dataEndDate": "2022-09-15T00:00:00Z", + "data": { + //"area": 4.22, + "final": true, + //"soilCode": "5", + "soilName": "Loam", + "cropTypeCode": "1010101", + "cropTypeName": "Potato", + //"rootDepthMax": 45, + //"emergenceDate": "2022-05-16T00:00:00", + "productionPurposeCode": "003" + //"productionPurposeName": "consumption" + }, + //"ProcessAggegrateKpi": "false", // toegevoegd, want alleen veld level KPIs voor nu. "geometryJson": { "type": "Polygon", "coordinates": [ [ - [ 4.960707146896585, 52.800583669708487 ], - [ 4.960645975538824, 52.800470217610922 ], - [ 4.962140695752897, 52.799177147194797 ], - [ 4.967523821195745, 52.801502400041208 ], - [ 4.966336768950911, 52.802543735879809 ], - [ 4.961711880764330, 52.801009996856429 ], - [ 4.960707146896585, 52.800583669708487 ] + [ + 5.5945257993548765, + 52.57080744107003 + ], + [ + 5.598645994070678, + 52.571540800206236 + ], + [ + 5.599381743127071, + 52.57012773140724 + ], + [ + 5.595408698222548, + 52.56968054825188 + ], + [ + 5.5945257993548765, + 52.57080744107003 + ] ] ] } } - + ] \ No newline at end of file From 9be9eb31451e8328d15fe3330bcdaf81aa41ca86 Mon Sep 17 00:00:00 2001 From: Pepijn van Oort Date: Tue, 13 Jun 2023 12:05:30 +0200 Subject: [PATCH 13/66] update TargetFramework of all projects to netcoreapp3.1. Except FarmmapsAPi & Secrets which remain 2.1 --- FarmmapsDataDownload/FarmmapsDataDownload.csproj | 2 +- FarmmapsHaulmkilling/FarmmapsHaulmkilling.csproj | 2 +- FarmmapsHerbicide/FarmmapsHerbicide.csproj | 2 +- FarmmapsKPI/FarmmapsKPI.csproj | 6 ++++++ FarmmapsZonering/FarmmapsZonering.csproj | 4 ++-- Secrets/Secrets.csproj | 2 +- 6 files changed, 12 insertions(+), 6 deletions(-) diff --git a/FarmmapsDataDownload/FarmmapsDataDownload.csproj b/FarmmapsDataDownload/FarmmapsDataDownload.csproj index 22b003c..173a795 100644 --- a/FarmmapsDataDownload/FarmmapsDataDownload.csproj +++ b/FarmmapsDataDownload/FarmmapsDataDownload.csproj @@ -2,7 +2,7 @@ Exe - netcoreapp3.0 + netcoreapp3.1 diff --git a/FarmmapsHaulmkilling/FarmmapsHaulmkilling.csproj b/FarmmapsHaulmkilling/FarmmapsHaulmkilling.csproj index d4c7f60..1c75ad4 100644 --- a/FarmmapsHaulmkilling/FarmmapsHaulmkilling.csproj +++ b/FarmmapsHaulmkilling/FarmmapsHaulmkilling.csproj @@ -2,7 +2,7 @@ Exe - netcoreapp3.0 + netcoreapp3.1 diff --git a/FarmmapsHerbicide/FarmmapsHerbicide.csproj b/FarmmapsHerbicide/FarmmapsHerbicide.csproj index ac5e7c3..32610e1 100644 --- a/FarmmapsHerbicide/FarmmapsHerbicide.csproj +++ b/FarmmapsHerbicide/FarmmapsHerbicide.csproj @@ -2,7 +2,7 @@ Exe - netcoreapp3.0 + netcoreapp3.1 diff --git a/FarmmapsKPI/FarmmapsKPI.csproj b/FarmmapsKPI/FarmmapsKPI.csproj index bdd33b9..debe75f 100644 --- a/FarmmapsKPI/FarmmapsKPI.csproj +++ b/FarmmapsKPI/FarmmapsKPI.csproj @@ -9,4 +9,10 @@ + + + Always + + + diff --git a/FarmmapsZonering/FarmmapsZonering.csproj b/FarmmapsZonering/FarmmapsZonering.csproj index 7bc9ac4..1a23724 100644 --- a/FarmmapsZonering/FarmmapsZonering.csproj +++ b/FarmmapsZonering/FarmmapsZonering.csproj @@ -1,8 +1,8 @@ - + Exe - netcoreapp3.0 + netcoreapp3.1 diff --git a/Secrets/Secrets.csproj b/Secrets/Secrets.csproj index f62f090..7ee51c4 100644 --- a/Secrets/Secrets.csproj +++ b/Secrets/Secrets.csproj @@ -1,7 +1,7 @@  - netstandard1.0 + netstandard2.1 From b99191b7b135ee6f1acc0372bbeed8f6e773109f Mon Sep 17 00:00:00 2001 From: "Abel Hoeven, van" Date: Tue, 27 Jun 2023 12:58:36 +0200 Subject: [PATCH 14/66] niewste versie --- FarmmapsApi/FarmmapsApi.csproj | 2 ++ FarmmapsApiSamples.sln | 52 +++++++++++++++++++++++++++++++++- FarmmapsKPI/FarmmapsKPI.csproj | 4 ++- FarmmapsKPI/KPIApplication.cs | 2 +- FarmmapsKPI/KPIInput.json | 3 +- Secrets/Secrets.csproj | 1 + 6 files changed, 60 insertions(+), 4 deletions(-) diff --git a/FarmmapsApi/FarmmapsApi.csproj b/FarmmapsApi/FarmmapsApi.csproj index 20b6873..2c961f0 100644 --- a/FarmmapsApi/FarmmapsApi.csproj +++ b/FarmmapsApi/FarmmapsApi.csproj @@ -2,6 +2,8 @@ netstandard2.1 + x64 + AnyCPU;x64 diff --git a/FarmmapsApiSamples.sln b/FarmmapsApiSamples.sln index 49b0a34..751b624 100644 --- a/FarmmapsApiSamples.sln +++ b/FarmmapsApiSamples.sln @@ -31,62 +31,112 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Secrets", "Secrets\Secrets. EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FarmmapsCleanUp", "FarmmapsCleanUp\FarmmapsCleanUp.csproj", "{5E4387F9-5953-4A9B-BCA5-DF3964EED3CB}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "FarmmapsKPI", "FarmmapsKPI\FarmmapsKPI.csproj", "{14575235-9867-4CE5-A22F-3F9FE002FF42}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FarmmapsKPI", "FarmmapsKPI\FarmmapsKPI.csproj", "{14575235-9867-4CE5-A22F-3F9FE002FF42}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU + Debug|x64 = Debug|x64 Release|Any CPU = Release|Any CPU + Release|x64 = Release|x64 EndGlobalSection GlobalSection(ProjectConfigurationPlatforms) = postSolution {E08EF7E9-F09E-42D8-825C-164E458C78F4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {E08EF7E9-F09E-42D8-825C-164E458C78F4}.Debug|Any CPU.Build.0 = Debug|Any CPU + {E08EF7E9-F09E-42D8-825C-164E458C78F4}.Debug|x64.ActiveCfg = Debug|Any CPU + {E08EF7E9-F09E-42D8-825C-164E458C78F4}.Debug|x64.Build.0 = Debug|Any CPU {E08EF7E9-F09E-42D8-825C-164E458C78F4}.Release|Any CPU.ActiveCfg = Release|Any CPU {E08EF7E9-F09E-42D8-825C-164E458C78F4}.Release|Any CPU.Build.0 = Release|Any CPU + {E08EF7E9-F09E-42D8-825C-164E458C78F4}.Release|x64.ActiveCfg = Release|Any CPU + {E08EF7E9-F09E-42D8-825C-164E458C78F4}.Release|x64.Build.0 = Release|Any CPU {1FA9E50B-F45E-4534-953A-37C783D03C74}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {1FA9E50B-F45E-4534-953A-37C783D03C74}.Debug|Any CPU.Build.0 = Debug|Any CPU + {1FA9E50B-F45E-4534-953A-37C783D03C74}.Debug|x64.ActiveCfg = Debug|x64 + {1FA9E50B-F45E-4534-953A-37C783D03C74}.Debug|x64.Build.0 = Debug|x64 {1FA9E50B-F45E-4534-953A-37C783D03C74}.Release|Any CPU.ActiveCfg = Release|Any CPU {1FA9E50B-F45E-4534-953A-37C783D03C74}.Release|Any CPU.Build.0 = Release|Any CPU + {1FA9E50B-F45E-4534-953A-37C783D03C74}.Release|x64.ActiveCfg = Release|x64 + {1FA9E50B-F45E-4534-953A-37C783D03C74}.Release|x64.Build.0 = Release|x64 {731A88CD-9DC4-4969-86F2-2315830A6998}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {731A88CD-9DC4-4969-86F2-2315830A6998}.Debug|Any CPU.Build.0 = Debug|Any CPU + {731A88CD-9DC4-4969-86F2-2315830A6998}.Debug|x64.ActiveCfg = Debug|Any CPU + {731A88CD-9DC4-4969-86F2-2315830A6998}.Debug|x64.Build.0 = Debug|Any CPU {731A88CD-9DC4-4969-86F2-2315830A6998}.Release|Any CPU.ActiveCfg = Release|Any CPU {731A88CD-9DC4-4969-86F2-2315830A6998}.Release|Any CPU.Build.0 = Release|Any CPU + {731A88CD-9DC4-4969-86F2-2315830A6998}.Release|x64.ActiveCfg = Release|Any CPU + {731A88CD-9DC4-4969-86F2-2315830A6998}.Release|x64.Build.0 = Release|Any CPU {DFA89D0B-5400-4374-B824-8367B76B4B6E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {DFA89D0B-5400-4374-B824-8367B76B4B6E}.Debug|Any CPU.Build.0 = Debug|Any CPU + {DFA89D0B-5400-4374-B824-8367B76B4B6E}.Debug|x64.ActiveCfg = Debug|Any CPU + {DFA89D0B-5400-4374-B824-8367B76B4B6E}.Debug|x64.Build.0 = Debug|Any CPU {DFA89D0B-5400-4374-B824-8367B76B4B6E}.Release|Any CPU.ActiveCfg = Release|Any CPU {DFA89D0B-5400-4374-B824-8367B76B4B6E}.Release|Any CPU.Build.0 = Release|Any CPU + {DFA89D0B-5400-4374-B824-8367B76B4B6E}.Release|x64.ActiveCfg = Release|Any CPU + {DFA89D0B-5400-4374-B824-8367B76B4B6E}.Release|x64.Build.0 = Release|Any CPU {AAFAB03A-6F5C-4D91-991F-867B7898F981}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {AAFAB03A-6F5C-4D91-991F-867B7898F981}.Debug|Any CPU.Build.0 = Debug|Any CPU + {AAFAB03A-6F5C-4D91-991F-867B7898F981}.Debug|x64.ActiveCfg = Debug|Any CPU + {AAFAB03A-6F5C-4D91-991F-867B7898F981}.Debug|x64.Build.0 = Debug|Any CPU {AAFAB03A-6F5C-4D91-991F-867B7898F981}.Release|Any CPU.ActiveCfg = Release|Any CPU {AAFAB03A-6F5C-4D91-991F-867B7898F981}.Release|Any CPU.Build.0 = Release|Any CPU + {AAFAB03A-6F5C-4D91-991F-867B7898F981}.Release|x64.ActiveCfg = Release|Any CPU + {AAFAB03A-6F5C-4D91-991F-867B7898F981}.Release|x64.Build.0 = Release|Any CPU {892E0932-5D11-4A37-979E-CEDB39C2E181}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {892E0932-5D11-4A37-979E-CEDB39C2E181}.Debug|Any CPU.Build.0 = Debug|Any CPU + {892E0932-5D11-4A37-979E-CEDB39C2E181}.Debug|x64.ActiveCfg = Debug|Any CPU + {892E0932-5D11-4A37-979E-CEDB39C2E181}.Debug|x64.Build.0 = Debug|Any CPU {892E0932-5D11-4A37-979E-CEDB39C2E181}.Release|Any CPU.ActiveCfg = Release|Any CPU {892E0932-5D11-4A37-979E-CEDB39C2E181}.Release|Any CPU.Build.0 = Release|Any CPU + {892E0932-5D11-4A37-979E-CEDB39C2E181}.Release|x64.ActiveCfg = Release|Any CPU + {892E0932-5D11-4A37-979E-CEDB39C2E181}.Release|x64.Build.0 = Release|Any CPU {91A58C4A-4A80-4079-B43D-9B851206194F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {91A58C4A-4A80-4079-B43D-9B851206194F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {91A58C4A-4A80-4079-B43D-9B851206194F}.Debug|x64.ActiveCfg = Debug|Any CPU + {91A58C4A-4A80-4079-B43D-9B851206194F}.Debug|x64.Build.0 = Debug|Any CPU {91A58C4A-4A80-4079-B43D-9B851206194F}.Release|Any CPU.ActiveCfg = Release|Any CPU {91A58C4A-4A80-4079-B43D-9B851206194F}.Release|Any CPU.Build.0 = Release|Any CPU + {91A58C4A-4A80-4079-B43D-9B851206194F}.Release|x64.ActiveCfg = Release|Any CPU + {91A58C4A-4A80-4079-B43D-9B851206194F}.Release|x64.Build.0 = Release|Any CPU {32ED9500-AAAB-4030-9C7A-F611A85DF890}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {32ED9500-AAAB-4030-9C7A-F611A85DF890}.Debug|Any CPU.Build.0 = Debug|Any CPU + {32ED9500-AAAB-4030-9C7A-F611A85DF890}.Debug|x64.ActiveCfg = Debug|Any CPU + {32ED9500-AAAB-4030-9C7A-F611A85DF890}.Debug|x64.Build.0 = Debug|Any CPU {32ED9500-AAAB-4030-9C7A-F611A85DF890}.Release|Any CPU.ActiveCfg = Release|Any CPU {32ED9500-AAAB-4030-9C7A-F611A85DF890}.Release|Any CPU.Build.0 = Release|Any CPU + {32ED9500-AAAB-4030-9C7A-F611A85DF890}.Release|x64.ActiveCfg = Release|Any CPU + {32ED9500-AAAB-4030-9C7A-F611A85DF890}.Release|x64.Build.0 = Release|Any CPU {772DBDCD-9FAA-40A7-8551-2C1620C4AB67}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {772DBDCD-9FAA-40A7-8551-2C1620C4AB67}.Debug|Any CPU.Build.0 = Debug|Any CPU + {772DBDCD-9FAA-40A7-8551-2C1620C4AB67}.Debug|x64.ActiveCfg = Debug|Any CPU + {772DBDCD-9FAA-40A7-8551-2C1620C4AB67}.Debug|x64.Build.0 = Debug|Any CPU {772DBDCD-9FAA-40A7-8551-2C1620C4AB67}.Release|Any CPU.ActiveCfg = Release|Any CPU {772DBDCD-9FAA-40A7-8551-2C1620C4AB67}.Release|Any CPU.Build.0 = Release|Any CPU + {772DBDCD-9FAA-40A7-8551-2C1620C4AB67}.Release|x64.ActiveCfg = Release|Any CPU + {772DBDCD-9FAA-40A7-8551-2C1620C4AB67}.Release|x64.Build.0 = Release|Any CPU {C4EE5ECA-253A-4B71-9F67-D231AC4517D6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {C4EE5ECA-253A-4B71-9F67-D231AC4517D6}.Debug|Any CPU.Build.0 = Debug|Any CPU + {C4EE5ECA-253A-4B71-9F67-D231AC4517D6}.Debug|x64.ActiveCfg = Debug|x64 + {C4EE5ECA-253A-4B71-9F67-D231AC4517D6}.Debug|x64.Build.0 = Debug|x64 {C4EE5ECA-253A-4B71-9F67-D231AC4517D6}.Release|Any CPU.ActiveCfg = Release|Any CPU {C4EE5ECA-253A-4B71-9F67-D231AC4517D6}.Release|Any CPU.Build.0 = Release|Any CPU + {C4EE5ECA-253A-4B71-9F67-D231AC4517D6}.Release|x64.ActiveCfg = Release|x64 + {C4EE5ECA-253A-4B71-9F67-D231AC4517D6}.Release|x64.Build.0 = Release|x64 {5E4387F9-5953-4A9B-BCA5-DF3964EED3CB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {5E4387F9-5953-4A9B-BCA5-DF3964EED3CB}.Debug|Any CPU.Build.0 = Debug|Any CPU + {5E4387F9-5953-4A9B-BCA5-DF3964EED3CB}.Debug|x64.ActiveCfg = Debug|Any CPU + {5E4387F9-5953-4A9B-BCA5-DF3964EED3CB}.Debug|x64.Build.0 = Debug|Any CPU {5E4387F9-5953-4A9B-BCA5-DF3964EED3CB}.Release|Any CPU.ActiveCfg = Release|Any CPU {5E4387F9-5953-4A9B-BCA5-DF3964EED3CB}.Release|Any CPU.Build.0 = Release|Any CPU + {5E4387F9-5953-4A9B-BCA5-DF3964EED3CB}.Release|x64.ActiveCfg = Release|Any CPU + {5E4387F9-5953-4A9B-BCA5-DF3964EED3CB}.Release|x64.Build.0 = Release|Any CPU {14575235-9867-4CE5-A22F-3F9FE002FF42}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {14575235-9867-4CE5-A22F-3F9FE002FF42}.Debug|Any CPU.Build.0 = Debug|Any CPU + {14575235-9867-4CE5-A22F-3F9FE002FF42}.Debug|x64.ActiveCfg = Debug|x64 + {14575235-9867-4CE5-A22F-3F9FE002FF42}.Debug|x64.Build.0 = Debug|x64 {14575235-9867-4CE5-A22F-3F9FE002FF42}.Release|Any CPU.ActiveCfg = Release|Any CPU {14575235-9867-4CE5-A22F-3F9FE002FF42}.Release|Any CPU.Build.0 = Release|Any CPU + {14575235-9867-4CE5-A22F-3F9FE002FF42}.Release|x64.ActiveCfg = Release|x64 + {14575235-9867-4CE5-A22F-3F9FE002FF42}.Release|x64.Build.0 = Release|x64 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/FarmmapsKPI/FarmmapsKPI.csproj b/FarmmapsKPI/FarmmapsKPI.csproj index debe75f..e2a7293 100644 --- a/FarmmapsKPI/FarmmapsKPI.csproj +++ b/FarmmapsKPI/FarmmapsKPI.csproj @@ -1,8 +1,10 @@ - + Exe netcoreapp3.1 + x64 + AnyCPU;x64 diff --git a/FarmmapsKPI/KPIApplication.cs b/FarmmapsKPI/KPIApplication.cs index 5bdad37..9050b77 100644 --- a/FarmmapsKPI/KPIApplication.cs +++ b/FarmmapsKPI/KPIApplication.cs @@ -142,7 +142,7 @@ namespace FarmmapsKPI //todo set croprecordings from file } - // Get KPI data for saving it in a file + // Get KPI data for saving it in a file, here the generalsedrvice is called to get the KPI data _logger.LogInformation($"Trying to get the cropfielditem: {cropfieldItem}"); var KPIItem = await _generalService.GetKpiItemsForCropField(cropfieldItem); diff --git a/FarmmapsKPI/KPIInput.json b/FarmmapsKPI/KPIInput.json index c14c9af..01ddca5 100644 --- a/FarmmapsKPI/KPIInput.json +++ b/FarmmapsKPI/KPIInput.json @@ -5,6 +5,7 @@ //"fieldName": "aardappelveld_test", //"DownloadFolder": "Downloads", //"C:\\workdir\\groenmonitor\\", // "Downloads", -> if you just put "Downloads" the program will download to somewhere in ..\FarmMapsApiClient_WURtest\FarmmapsDataDownload\bin\Debug\netcoreapp3.1\Downloads\ "GetCropRecordings": false, + //cropfield code. toevoegen "CropYear": 2022, "dataDate": "2022-04-15T00:00:00Z", "dataEndDate": "2022-09-15T00:00:00Z", @@ -13,7 +14,7 @@ "final": true, //"soilCode": "5", "soilName": "Loam", - "cropTypeCode": "1010101", + "cropTypeCode": "1010101", // make a table/list with possible inputs. "cropTypeName": "Potato", //"rootDepthMax": 45, //"emergenceDate": "2022-05-16T00:00:00", diff --git a/Secrets/Secrets.csproj b/Secrets/Secrets.csproj index 7ee51c4..3713f53 100644 --- a/Secrets/Secrets.csproj +++ b/Secrets/Secrets.csproj @@ -2,6 +2,7 @@ netstandard2.1 + AnyCPU;x64 From 8a770a176249214e94e0d7a67b1ed781c4dcb55d Mon Sep 17 00:00:00 2001 From: Pepijn van Oort Date: Tue, 27 Jun 2023 14:08:27 +0200 Subject: [PATCH 15/66] handles existing cropfielditemcode --- FarmmapsApi/Services/GeneralService.cs | 4 +--- FarmmapsKPI/KPIApplication.cs | 22 +++++++++++++++------- FarmmapsKPI/KPIInput.json | 5 +++-- FarmmapsKPI/Models/KPIInput.cs | 1 + 4 files changed, 20 insertions(+), 12 deletions(-) diff --git a/FarmmapsApi/Services/GeneralService.cs b/FarmmapsApi/Services/GeneralService.cs index 8215260..c292508 100644 --- a/FarmmapsApi/Services/GeneralService.cs +++ b/FarmmapsApi/Services/GeneralService.cs @@ -299,11 +299,9 @@ namespace FarmmapsApi.Services return null; } await Task.Delay(60000); //wacht hier een minuut tot de KPIs berekend zijn - + //PO20230627 Je zou hier ook om de 10 sec eens kunnen kijken of we al zo ver zijn? Iets in trant van while KPI_ITEM is null? //hier nog definieren waar in de hierarchie een KPI item is? - //the kpi data is a child of the cropfield --> is dit wel zo? is kpi ook de itemName? - return await _farmmapsApiService.GetItemChildrenAsync(cropfieldItem.Code, KPI_ITEM); } diff --git a/FarmmapsKPI/KPIApplication.cs b/FarmmapsKPI/KPIApplication.cs index 9050b77..546e6a6 100644 --- a/FarmmapsKPI/KPIApplication.cs +++ b/FarmmapsKPI/KPIApplication.cs @@ -98,18 +98,27 @@ namespace FarmmapsKPI // Use already created cropfield or create new one, added a Data input, with field specific data for the KPI calculation Item cropfieldItem; - if (useCreatedCropfield == true || string.IsNullOrEmpty(_settings.CropfieldItemCode)) + //1 useCreatedCropfield = false -> get new + //2 useCreatedCropfield = true && CropfieldItemCode = "" or absent -> read from settings json + //2 useCreatedCropfield = true && CropfieldItemCode like "deb48a74c5b54299bb852f17288010e9" in KPIinput -> use this one + + if (useCreatedCropfield == false || string.IsNullOrEmpty(_settings.CropfieldItemCode)) { - _logger.LogInformation("Creating cropfield"); + _logger.LogInformation("Creating cropfield, writting to settings file"); cropfieldItem = await _generalService.CreateCropfieldItemAsync(myDriveRoot.Code, $"DataCropfield {input.OutputFileName}", cropYear, input.GeometryJson.ToString(Formatting.None), input.Data.ToString(Formatting.None)); _settings.CropfieldItemCode = cropfieldItem.Code; SaveSettings(settingsfile); } + else if (string.IsNullOrEmpty(input.CropfieldItemCode)) + { + _logger.LogInformation("CropfieldItemCode not in json input, reading from settings json"); + cropfieldItem = await _farmmapsApiService.GetItemAsync(_settings.CropfieldItemCode); + } else { - _logger.LogInformation("Cropfield already exists, trying to get it"); - cropfieldItem = await _farmmapsApiService.GetItemAsync(_settings.CropfieldItemCode); + _logger.LogInformation("CropfieldItemCode not in json input, reading from settings json"); + cropfieldItem = await _farmmapsApiService.GetItemAsync(input.CropfieldItemCode); } //Get croprecordings @@ -143,11 +152,10 @@ namespace FarmmapsKPI } // Get KPI data for saving it in a file, here the generalsedrvice is called to get the KPI data - _logger.LogInformation($"Trying to get the cropfielditem: {cropfieldItem}"); + _logger.LogInformation($"Trying to get the cropfielditem: {cropfieldItem.Code}"); var KPIItem = await _generalService.GetKpiItemsForCropField(cropfieldItem); - - var KPIItemPath = Path.Combine(downloadFolder, $"KPIItems_{KPIItem}.json"); + var KPIItemPath = Path.Combine(downloadFolder, $"KPIItems_{cropfieldItem.Code}.json"); _logger.LogInformation($"Found {KPIItem.Count} KPI items"); var count = 0; await Task.Delay(50); diff --git a/FarmmapsKPI/KPIInput.json b/FarmmapsKPI/KPIInput.json index 01ddca5..d9aeb63 100644 --- a/FarmmapsKPI/KPIInput.json +++ b/FarmmapsKPI/KPIInput.json @@ -1,6 +1,7 @@ [ { - "UseCreatedCropfield": false, + "UseCreatedCropfield": true, + "CropfieldItemCode": "38870d5ff9b54b12877b5b01018b6fec", "outputFileName": "TestData", //"fieldName": "aardappelveld_test", //"DownloadFolder": "Downloads", //"C:\\workdir\\groenmonitor\\", // "Downloads", -> if you just put "Downloads" the program will download to somewhere in ..\FarmMapsApiClient_WURtest\FarmmapsDataDownload\bin\Debug\netcoreapp3.1\Downloads\ @@ -14,7 +15,7 @@ "final": true, //"soilCode": "5", "soilName": "Loam", - "cropTypeCode": "1010101", // make a table/list with possible inputs. + "cropTypeCode": "1010101", // make a table/list with possible inputs. "cropTypeName": "Potato", //"rootDepthMax": 45, //"emergenceDate": "2022-05-16T00:00:00", diff --git a/FarmmapsKPI/Models/KPIInput.cs b/FarmmapsKPI/Models/KPIInput.cs index bcb35b8..c48a808 100644 --- a/FarmmapsKPI/Models/KPIInput.cs +++ b/FarmmapsKPI/Models/KPIInput.cs @@ -6,6 +6,7 @@ namespace FarmmapsKPI.Models public class KPIInput { public bool UseCreatedCropfield { get; set; } + public string CropfieldItemCode { get; set; } public string File { get; set; } public string InputVariable { get; set; } public string OutputFileName { get; set; } From f04cc239d522a7bce258f6779629dc6a2da9ce09 Mon Sep 17 00:00:00 2001 From: Pepijn van Oort Date: Wed, 4 Oct 2023 10:43:44 +0200 Subject: [PATCH 16/66] Extra note to Clone the git repository to a local drive. DON'T clone to a network drive like "My Documents". Latter may cause authentication problems. --- README.MD | 1 + 1 file changed, 1 insertion(+) diff --git a/README.MD b/README.MD index 36f4d61..ae6519b 100644 --- a/README.MD +++ b/README.MD @@ -1,6 +1,7 @@ ##### NOT PRODUCTION READY CODE, JUST AN EXAMPLE Put your clientId and clientSecret or your Farmmaps-account data (username and password) in a newly created appsettings.secrets.json file inside the root of the Secrets project. +Clone the git repository to a local drive. DON'T clone to a network drive like "My Documents". Latter may cause authentication problems. **appsettings.secrets.json** ``` From fedd3630758133b563fbe0d71c8a61bd4b5bec0b Mon Sep 17 00:00:00 2001 From: Pepijn van Oort Date: Wed, 4 Oct 2023 17:21:05 +0200 Subject: [PATCH 17/66] in KPIApplication possible to (1) add an operation, e.g. applying fertilizer and (2) add a cropfield characteristic containing cropyield. --- FarmmapsApi/Constants.cs | 8 +- FarmmapsApi/Services/GeneralService.cs | 76 ++++++++++++- FarmmapsKPI/KPIApplication.cs | 144 ++++++++++++++++--------- FarmmapsKPI/KPIInput.json | 50 ++++++--- FarmmapsKPI/Models/KPIInput.cs | 14 ++- FarmmapsKPI/Models/Settings.cs | 6 +- 6 files changed, 221 insertions(+), 77 deletions(-) diff --git a/FarmmapsApi/Constants.cs b/FarmmapsApi/Constants.cs index 613dfde..6b37e24 100644 --- a/FarmmapsApi/Constants.cs +++ b/FarmmapsApi/Constants.cs @@ -12,9 +12,12 @@ namespace FarmmapsApiSamples public const string SHAPE_ITEMTYPE = "vnd.farmmaps.itemtype.shape"; public const string GEOJSON_ITEMTYPE = "vnd.farmmaps.itemtype.geojson"; public const string BLIGHT_ITEMTYPE = "vnd.farmmaps.itemtype.blight"; - public const string CROPREC_ITEMTYPE = "vnd.farmmaps.itemtype.crprec.operation"; + public const string CROPREC_ITEMTYPE = "vnd.farmmaps.itemtype.crprec"; + public const string CROPOP_ITEMTYPE = "vnd.farmmaps.itemtype.crprec.operation"; + public const string CROPCHAR_ITEMTYPE = "vnd.farmmaps.itemtype.edicrop.characteristic"; public const string CROPSCHEME_ITEMTYPE = "vnd.farmmaps.itemtype.croppingscheme"; // deze toegevoegd, misschien is het type van de KPI task wel een croppingscheme - public const string KPI_ITEM = "vnd.farmmaps.itemtype.kpi.data"; + //public const string KPI_ITEM = "vnd.farmmaps.itemtype.kpi.data"; //PO20231004: originally with .data + public const string KPI_ITEM = "vnd.farmmaps.itemtype.kpi.data.container"; //PO20231004: originally without .container public const string VRANBS_TASK = "vnd.farmmaps.task.vranbs"; public const string VRAHERBICIDE_TASK = "vnd.farmmaps.task.vraherbicide"; @@ -26,6 +29,7 @@ namespace FarmmapsApiSamples public const string TASKMAP_TASK = "vnd.farmmaps.task.taskmap"; public const string WORKFLOW_TASK = "vnd.farmmaps.task.workflow"; public const string BOFEK_TASK = "vnd.farmmaps.task.bofek"; + public const string CROPREC_TASK = "vnd.farmmaps.task.crprec"; public const string SHADOW_TASK = "vnd.farmmaps.task.shadow"; public const string AHN_TASK = "vnd.farmmaps.task.ahn"; public const string WATBAL_TASK = "vnd.farmmaps.task.watbal"; diff --git a/FarmmapsApi/Services/GeneralService.cs b/FarmmapsApi/Services/GeneralService.cs index c292508..a5184e8 100644 --- a/FarmmapsApi/Services/GeneralService.cs +++ b/FarmmapsApi/Services/GeneralService.cs @@ -37,6 +37,35 @@ namespace FarmmapsApi.Services return await _farmmapsApiService.CreateItemAsync(cropfieldItemRequest); } + public async Task CreateOperationItemAsync(string cropRecordingItemCode, string data = "{}") + { + JObject jdata = JObject.Parse(data); + string name = string.Format($"CrpRec Operation, {jdata.GetValue("name")}"); + + ItemRequest operationItemRequest = new ItemRequest() + { + ParentCode = cropRecordingItemCode, + ItemType = CROPOP_ITEMTYPE, + Name = name, + Data = jdata, + }; + + return await _farmmapsApiService.CreateItemAsync(operationItemRequest); + } + public async Task CreateCropfieldCharacteristicItemAsync(string cropfieldItemCode, string data = "{}") + { + string name = "Cropfield characteristic"; + + ItemRequest cropfieldCharactericsticItemRequest = new ItemRequest() + { + ParentCode = cropfieldItemCode, + ItemType = CROPCHAR_ITEMTYPE, + Name = name, + Data = JObject.Parse(data), + }; + + return await _farmmapsApiService.CreateItemAsync(cropfieldCharactericsticItemRequest); + } public async Task UploadDataAsync(UserRoot root, string itemType, string filePath, string itemName, string geoJsonString = null) { var startUpload = DateTime.UtcNow.AddSeconds(-3); @@ -247,6 +276,39 @@ namespace FarmmapsApi.Services return dataItem; } + public async Task RunCropRecordingTask(Item cropfieldItem) + { + var cropRecordingRequest = new TaskRequest { TaskType = CROPREC_TASK }; + + string itemTaskCode = await _farmmapsApiService.QueueTaskAsync(cropfieldItem.Code, cropRecordingRequest); + + await PollTask(TimeSpan.FromSeconds(5), async (tokenSource) => { + var itemTaskStatus = await _farmmapsApiService.GetTaskStatusAsync(cropfieldItem.Code, itemTaskCode); + _logger.LogInformation($"Waiting on RunCropRecordingTask; status: {itemTaskStatus.State}"); + if (itemTaskStatus.IsFinished) + tokenSource.Cancel(); + }); + + var itemTask = await _farmmapsApiService.GetTaskStatusAsync(cropfieldItem.Code, itemTaskCode); + if (itemTask.State == ItemTaskState.Error) + { + _logger.LogError($"Something went wrong with task execution: {itemTask.Message}"); + return null; + } + + //the CropRecording data is a child of the cropfield + var itemName = "Crprec"; + var cropRecordingItem = await FindChildItemAsync(cropfieldItem.Code, + CROPREC_ITEMTYPE, itemName); + if (cropRecordingItem == null) + { + _logger.LogError("Could not find the CropRecording data as a child item under the cropfield"); + return null; + } + + return cropRecordingItem; + } + public async Task RunBofekTask(Item cropfieldItem) { var taskmapRequest = new TaskRequest { TaskType = BOFEK_TASK }; @@ -279,11 +341,12 @@ namespace FarmmapsApi.Services public async Task> GetKpiItemsForCropField(Item cropfieldItem) // dit is dus nieuw om de KPI task te runnen { - var taskmapRequest = new TaskRequest { TaskType = KPI_TASK }; //hier KPI request van maken - taskmapRequest.attributes["processAggregateKpi"] = "false"; // zo de properties meegeven?, moet dit niet bij de KPIinput? - taskmapRequest.attributes["year"] = "2022"; + var kpiRequest = new TaskRequest { TaskType = KPI_TASK }; + kpiRequest.attributes["processAggregateKpi"] = "false"; + int year = cropfieldItem.DataDate.Value.Year; + kpiRequest.attributes["year"] = year.ToString(); - string itemTaskCode = await _farmmapsApiService.QueueTaskAsync(cropfieldItem.Code, taskmapRequest); + string itemTaskCode = await _farmmapsApiService.QueueTaskAsync(cropfieldItem.Code, kpiRequest); await PollTask(TimeSpan.FromSeconds(5), async (tokenSource) => { var itemTaskStatus = await _farmmapsApiService.GetTaskStatusAsync(cropfieldItem.Code, itemTaskCode); @@ -299,10 +362,13 @@ namespace FarmmapsApi.Services return null; } await Task.Delay(60000); //wacht hier een minuut tot de KPIs berekend zijn + //PO20230627 Je zou hier ook om de 10 sec eens kunnen kijken of we al zo ver zijn? Iets in trant van while KPI_ITEM is null? //hier nog definieren waar in de hierarchie een KPI item is? + var allChildren = await _farmmapsApiService.GetItemChildrenAsync(cropfieldItem.Code); + var kpiItems = await _farmmapsApiService.GetItemChildrenAsync(cropfieldItem.Code, KPI_ITEM); - return await _farmmapsApiService.GetItemChildrenAsync(cropfieldItem.Code, KPI_ITEM); + return kpiItems; } diff --git a/FarmmapsKPI/KPIApplication.cs b/FarmmapsKPI/KPIApplication.cs index 546e6a6..301a3e3 100644 --- a/FarmmapsKPI/KPIApplication.cs +++ b/FarmmapsKPI/KPIApplication.cs @@ -22,18 +22,18 @@ namespace FarmmapsKPI private readonly ILogger _logger; private readonly FarmmapsApiService _farmmapsApiService; - private readonly KPIService _dataDownloadService; + private readonly KPIService _kpiService; private readonly GeneralService _generalService; private Settings _settings; public KPIApplication(ILogger logger, FarmmapsApiService farmmapsApiService, - GeneralService generalService, KPIService dataDownloadService) + GeneralService generalService, KPIService kpiService) { _logger = logger; _farmmapsApiService = farmmapsApiService; _generalService = generalService; - _dataDownloadService = dataDownloadService; + _kpiService = kpiService; } public async Task RunAsync() @@ -70,6 +70,9 @@ namespace FarmmapsKPI // !!specify if you are using an already created cropfield: bool useCreatedCropfield = input.UseCreatedCropfield; + bool useCreatedCropRecording = input.UseCreatedCropRecording; + bool useCreatedOperations = input.UseCreatedOperation; + bool useCreatedCropfieldCharacteristic = input.UseCreatedCropfieldCharacteristic; var cropYear = input.CropYear; var fieldName = input.fieldName; //bool storeSatelliteStatistics = input.StoreSatelliteStatisticsSingleImage; @@ -98,63 +101,119 @@ namespace FarmmapsKPI // Use already created cropfield or create new one, added a Data input, with field specific data for the KPI calculation Item cropfieldItem; - //1 useCreatedCropfield = false -> get new + //1 useCreatedCropfield = false -> create new //2 useCreatedCropfield = true && CropfieldItemCode = "" or absent -> read from settings json - //2 useCreatedCropfield = true && CropfieldItemCode like "deb48a74c5b54299bb852f17288010e9" in KPIinput -> use this one - + //3 useCreatedCropfield = true && CropfieldItemCode like "deb48a74c5b54299bb852f17288010e9" in KPIinput -> use this one if (useCreatedCropfield == false || string.IsNullOrEmpty(_settings.CropfieldItemCode)) { _logger.LogInformation("Creating cropfield, writting to settings file"); cropfieldItem = await _generalService.CreateCropfieldItemAsync(myDriveRoot.Code, - $"DataCropfield {input.OutputFileName}", cropYear, input.GeometryJson.ToString(Formatting.None), input.Data.ToString(Formatting.None)); + $"{input.OutputFileName}", cropYear, input.GeometryJson.ToString(Formatting.None), input.DataCropfield.ToString(Formatting.None)); _settings.CropfieldItemCode = cropfieldItem.Code; SaveSettings(settingsfile); } else if (string.IsNullOrEmpty(input.CropfieldItemCode)) { - _logger.LogInformation("CropfieldItemCode not in json input, reading from settings json"); + _logger.LogInformation("reading CropfieldItemCode from settings file"); cropfieldItem = await _farmmapsApiService.GetItemAsync(_settings.CropfieldItemCode); } else { - _logger.LogInformation("CropfieldItemCode not in json input, reading from settings json"); + _logger.LogInformation("reading CropfieldItemCode from KPIinput.json"); cropfieldItem = await _farmmapsApiService.GetItemAsync(input.CropfieldItemCode); } - //Get croprecordings - if (input.GetCropRecordings) + + // Use already created croprecording or create new one, added a Data input, with field specific data for the KPI calculation + Item crprecItem; + //1 useCreatedCropRecording = false -> create new + //2 useCreatedCropRecording = true && CropRecordingItemCode = "" or absent -> read from settings json + //3 useCreatedCropRecording = true && CropRecordingItemCode like "deb48a74c5b54299bb852f17288010e9" in KPIinput -> use this one + if (useCreatedCropRecording == false || string.IsNullOrEmpty(_settings.CropRecordingItemCode)) { - var crprecItem = input.CrprecItem; - _logger.LogInformation($"Trying to get crop recordings of croprecording: {crprecItem}"); - - var cropRec = await _farmmapsApiService.GetItemChildrenAsync(crprecItem, CROPREC_ITEMTYPE); - if (cropRec == null) - { - _logger.LogError("Something went wrong while obtaining the croprecordings"); - return; - } - - var cropRecPath = Path.Combine(downloadFolder, $"croprecordings_{crprecItem}.json"); - _logger.LogInformation($"Found {cropRec.Count} crop recordings"); - var count1 = 0; - await Task.Delay(500); - foreach (var item in cropRec) - { - Console.WriteLine($"Crop recording #{count1}: {item.Name}"); - File.AppendAllText(cropRecPath, item.Data +Environment.NewLine); - count1++; - } - _logger.LogInformation($"Downloaded file {cropRecPath}"); + _logger.LogInformation("RunCropRecordingTask ..."); + crprecItem = await _generalService.RunCropRecordingTask(cropfieldItem); + _settings.CropRecordingItemCode = crprecItem.Code; + SaveSettings(settingsfile); + } + else if (string.IsNullOrEmpty(input.CropfieldItemCode)) + { + _logger.LogInformation("reading CropRecordingItemCode from settings file"); + crprecItem = await _farmmapsApiService.GetItemAsync(_settings.CropRecordingItemCode); } else { - //todo set croprecordings from file + _logger.LogInformation("reading CropRecordingItemCode from KPIinput.json"); + crprecItem = await _farmmapsApiService.GetItemAsync(input.CropRecordingItemCode); } + // Use already created operation or create new one, added a Data input, with field specific data for the KPI calculation + Item crpOperationItem; + //1 useCreatedOperation = false -> create new + //2 useCreatedOperation = true && OperationItemCode = "" or absent -> read from settings json + //3 useCreatedOperation = true && OperationItemCode like "deb48a74c5b54299bb852f17288010e9" in KPIinput -> use this one + if (useCreatedOperations == false || string.IsNullOrEmpty(_settings.OperationItemCode)) + { + _logger.LogInformation("CreateOperationItemAsync ..."); + crpOperationItem = await _generalService.CreateOperationItemAsync(crprecItem.Code,input.DataOperation.ToString(Formatting.None)); + _settings.OperationItemCode = crpOperationItem.Code; + SaveSettings(settingsfile); + } + else if (string.IsNullOrEmpty(input.CropfieldItemCode)) + { + _logger.LogInformation("reading OperationItemCode from settings file"); + crpOperationItem = await _farmmapsApiService.GetItemAsync(_settings.OperationItemCode); + } + else + { + _logger.LogInformation("reading OperationItemCode from KPIinput.json"); + crpOperationItem = await _farmmapsApiService.GetItemAsync(input.OperationItemCode); + } + + // The cropfieldCharacteristicItem is used to enter crop yields + // So once we have added an operation for fertilizer application and a crop yield, then KPIapp can calculate + // Nutrient balance. + // Use already created cropfieldCharacteristicItem or create new one, added a Data input, with field specific data for the KPI calculation + Item cropfieldCharacteristicItem; + //1 useCreatedCropfieldCharacteristic = false -> create new + //2 useCreatedCropfieldCharacteristic = true && CropfieldCharacteristicItemCode = "" or absent -> read from settings json + //3 useCreatedCropfieldCharacteristic = true && CropfieldCharacteristicItemCode like "deb48a74c5b54299bb852f17288010e9" in KPIinput -> use this one + if (useCreatedCropfieldCharacteristic == false) + { + _logger.LogInformation("CreateCropfieldCharacteristicItemAsync ..."); + cropfieldCharacteristicItem = await _generalService.CreateCropfieldCharacteristicItemAsync(cropfieldItem.Code, input.DataCropfieldCharacteristic.ToString(Formatting.None)); + _settings.CropfieldCharacteristicItemCode = cropfieldCharacteristicItem.Code; + SaveSettings(settingsfile); + } + else if (string.IsNullOrEmpty(input.CropfieldCharacteristicItemCode)) + { + _logger.LogInformation("reading OperationItemCode from settings file"); + cropfieldCharacteristicItem = await _farmmapsApiService.GetItemAsync(_settings.CropfieldCharacteristicItemCode); + } + else + { + _logger.LogInformation("reading CropfieldCharacteristicItemCode from KPIinput.json"); + cropfieldCharacteristicItem = await _farmmapsApiService.GetItemAsync(input.CropfieldCharacteristicItemCode); + } + + // Inspect the children. If all is well, cropfield will have one crprec and one edicrop.characteristic + // And the crprec will have 0-many operations as children + // And the Data of an operation will have specification of how much fertilizer was applied + // And crprec can have multiple operations + // Note existing cropfields and croprecordings keep existing properties you added in previous runs + // (unless you deleted them) + // So ech time you run with "UseCreatedCropfield": true & "UseCreatedCropRecording": false -> a new recording will be added to the existing cropfield + // So ech time you run with "UseCreatedCropfield": true & "useCreatedCropfieldCharacteristic": false -> a new CropfieldCharacteristic will be added to the existing cropfield + // So ech time you run with "UseCreatedCropRecording": true & "UseCreatedOperation": false -> a new operation will be added to the existing crop recording + var cropfieldChildren = await _farmmapsApiService.GetItemChildrenAsync(cropfieldItem.Code); + var crprecChildren = await _farmmapsApiService.GetItemChildrenAsync(crprecItem.Code); + + //Now get the KPIs for this cropfield, mounted with operations & cropyield // Get KPI data for saving it in a file, here the generalsedrvice is called to get the KPI data - _logger.LogInformation($"Trying to get the cropfielditem: {cropfieldItem.Code}"); + _logger.LogInformation($"GetKpiItemsForCropField({cropfieldItem.Code})"); var KPIItem = await _generalService.GetKpiItemsForCropField(cropfieldItem); + //Download KPI's into a json output file for this specific cropfield (with unique cropfieldItem.Code) var KPIItemPath = Path.Combine(downloadFolder, $"KPIItems_{cropfieldItem.Code}.json"); _logger.LogInformation($"Found {KPIItem.Count} KPI items"); var count = 0; @@ -167,23 +226,6 @@ namespace FarmmapsKPI } _logger.LogInformation($"Downloaded file {KPIItemPath}"); - //////////// - //_logger.LogInformation("Calculate KPI map for field"); - //var KPIItem = await _generalService.GetKpiItemsForCropField(cropfieldItem);// hier dus verwijzen naar de KPI task - //if ((object)null == null) - //{ - // _logger.LogError("Something went wrong while obtaining the KPI map"); - // return; - // } - - //_logger.LogInformation("Downloading KPI map"); - //await _farmmapsApiService.DownloadItemAsync(KPIItem.Code, - // Path.Combine(downloadFolder, $"{input.OutputFileName}_KPI.zip")); - - - - - } // Functions to save previously created cropfields diff --git a/FarmmapsKPI/KPIInput.json b/FarmmapsKPI/KPIInput.json index d9aeb63..d76e98e 100644 --- a/FarmmapsKPI/KPIInput.json +++ b/FarmmapsKPI/KPIInput.json @@ -1,16 +1,8 @@ [ { "UseCreatedCropfield": true, - "CropfieldItemCode": "38870d5ff9b54b12877b5b01018b6fec", - "outputFileName": "TestData", - //"fieldName": "aardappelveld_test", - //"DownloadFolder": "Downloads", //"C:\\workdir\\groenmonitor\\", // "Downloads", -> if you just put "Downloads" the program will download to somewhere in ..\FarmMapsApiClient_WURtest\FarmmapsDataDownload\bin\Debug\netcoreapp3.1\Downloads\ - "GetCropRecordings": false, - //cropfield code. toevoegen - "CropYear": 2022, - "dataDate": "2022-04-15T00:00:00Z", - "dataEndDate": "2022-09-15T00:00:00Z", - "data": { + "CropfieldItemCode": "", + "dataCropfield": { //"area": 4.22, "final": true, //"soilCode": "5", @@ -22,6 +14,39 @@ "productionPurposeCode": "003" //"productionPurposeName": "consumption" }, + "UseCreatedCropRecording": true, + "CropRecordingItemCode": "", + "dataCropRecording": {}, //not yet used + "UseCreatedOperation": true, + "OperationItemCode": "", + "dataOperation": { + "n": "92", + "to": "2022-05-23T12:34:00", + "area": "0.08", //?! + "from": "2022-05-23T11:34:00", + "name": "Kunstmest strooien", + "unit": "kg/ha", + "method": "70400", + "status": "3", + "product": "7360", + "quantity": "200", + "unitCode": "KGMHAR", //?! + "contractor": false, + "designator": "Kunstmest strooien", + "operationCode": "7" + }, + "UseCreatedCropfieldCharacteristic": false, + "CropfieldCharacteristicItemCode": "", + "DataCropfieldCharacteristic": { + "code": "860619", //PO20231004: so what does this code mean? Can we see the code list somewhere? + "label": "cropyield", + "value": "48.01" + }, + //"DownloadFolder": "Downloads", //"C:\\workdir\\groenmonitor\\", // "Downloads", -> if you just put "Downloads" the program will download to somewhere in ..\FarmMapsApiClient_WURtest\FarmmapsDataDownload\bin\Debug\netcoreapp3.1\Downloads\ + "CropYear": 2022, + "dataDate": "2022-04-15T00:00:00Z", + "dataEndDate": "2022-09-15T00:00:00Z", + "fieldName": "aardappelveld_test", //"ProcessAggegrateKpi": "false", // toegevoegd, want alleen veld level KPIs voor nu. "geometryJson": { "type": "Polygon", @@ -49,8 +74,7 @@ ] ] ] - } + }, + "outputFileName": "TestData" } - - ] \ No newline at end of file diff --git a/FarmmapsKPI/Models/KPIInput.cs b/FarmmapsKPI/Models/KPIInput.cs index c48a808..fb832fb 100644 --- a/FarmmapsKPI/Models/KPIInput.cs +++ b/FarmmapsKPI/Models/KPIInput.cs @@ -7,18 +7,26 @@ namespace FarmmapsKPI.Models { public bool UseCreatedCropfield { get; set; } public string CropfieldItemCode { get; set; } + public JObject DataCropfield { get; set; } + public bool UseCreatedCropRecording { get; set; } + public string CropRecordingItemCode { get; set; } + public JObject DataCropRecording { get; set; } + public bool UseCreatedOperation { get; set; } + public string OperationItemCode { get; set; } + public JObject DataOperation { get; set; } + public bool UseCreatedCropfieldCharacteristic { get; set; } + public string CropfieldCharacteristicItemCode { get; set; } + public JObject DataCropfieldCharacteristic { get; set; } + public string File { get; set; } public string InputVariable { get; set; } public string OutputFileName { get; set; } public string DownloadFolder { get; set; } public int CropYear { get; set; } public JObject GeometryJson { get; set; } - public JObject Data { get; set; } public string InputLayerName { get; set; } public string fieldName { get; set; } public bool GetShadowData { get; set; } - public bool GetCropRecordings { get; set; } - public string CrprecItem { get; set; } } } \ No newline at end of file diff --git a/FarmmapsKPI/Models/Settings.cs b/FarmmapsKPI/Models/Settings.cs index bd7ff29..f6ed630 100644 --- a/FarmmapsKPI/Models/Settings.cs +++ b/FarmmapsKPI/Models/Settings.cs @@ -3,9 +3,9 @@ public class Settings { public string CropfieldItemCode { get; set; } - //public string SatelliteTaskCode { get; set; } - //public string VanDerSatTaskCode { get; set; } - //public string WatBalTaskCode { get; set; } + public string CropRecordingItemCode { get; set; } + public string OperationItemCode { get; set; } + public string CropfieldCharacteristicItemCode { get; set; } } } \ No newline at end of file From 1cc36422a4dd12f1d4f6fad9e165a8486c82e72a Mon Sep 17 00:00:00 2001 From: Pepijn van Oort Date: Wed, 11 Oct 2023 17:00:25 +0200 Subject: [PATCH 18/66] start & end date & geometry added to operations item and to cropfield characteristics item --- FarmmapsApi/Services/GeneralService.cs | 17 ++++++++++++++--- FarmmapsKPI/KPIApplication.cs | 6 +++--- FarmmapsKPI/KPIInput.json | 6 +++--- 3 files changed, 20 insertions(+), 9 deletions(-) diff --git a/FarmmapsApi/Services/GeneralService.cs b/FarmmapsApi/Services/GeneralService.cs index a5184e8..8581455 100644 --- a/FarmmapsApi/Services/GeneralService.cs +++ b/FarmmapsApi/Services/GeneralService.cs @@ -37,8 +37,11 @@ namespace FarmmapsApi.Services return await _farmmapsApiService.CreateItemAsync(cropfieldItemRequest); } - public async Task CreateOperationItemAsync(string cropRecordingItemCode, string data = "{}") + public async Task CreateOperationItemAsync(string cropRecordingItemCode, int year, + string fieldGeomJson, string data = "{}") { + // not sure if here we also need to specify DataDate, DataEndDate & Geometry. Do it just in case + var currentYear = new DateTime(year, 1, 1); JObject jdata = JObject.Parse(data); string name = string.Format($"CrpRec Operation, {jdata.GetValue("name")}"); @@ -47,21 +50,29 @@ namespace FarmmapsApi.Services ParentCode = cropRecordingItemCode, ItemType = CROPOP_ITEMTYPE, Name = name, + DataDate = currentYear, + DataEndDate = currentYear.AddYears(1).AddDays(-1), Data = jdata, + Geometry = JObject.Parse(fieldGeomJson) }; return await _farmmapsApiService.CreateItemAsync(operationItemRequest); } - public async Task CreateCropfieldCharacteristicItemAsync(string cropfieldItemCode, string data = "{}") + public async Task CreateCropfieldCharacteristicItemAsync(string cropfieldItemCode, int year, + string fieldGeomJson, string data = "{}") { + // not sure if here we also need to specify DataDate, DataEndDate & Geometry. Do it just in case string name = "Cropfield characteristic"; - + var currentYear = new DateTime(year, 1, 1); ItemRequest cropfieldCharactericsticItemRequest = new ItemRequest() { ParentCode = cropfieldItemCode, ItemType = CROPCHAR_ITEMTYPE, Name = name, + DataDate = currentYear, + DataEndDate = currentYear.AddYears(1).AddDays(-1), Data = JObject.Parse(data), + Geometry = JObject.Parse(fieldGeomJson) }; return await _farmmapsApiService.CreateItemAsync(cropfieldCharactericsticItemRequest); diff --git a/FarmmapsKPI/KPIApplication.cs b/FarmmapsKPI/KPIApplication.cs index 301a3e3..7fd90b8 100644 --- a/FarmmapsKPI/KPIApplication.cs +++ b/FarmmapsKPI/KPIApplication.cs @@ -155,7 +155,7 @@ namespace FarmmapsKPI if (useCreatedOperations == false || string.IsNullOrEmpty(_settings.OperationItemCode)) { _logger.LogInformation("CreateOperationItemAsync ..."); - crpOperationItem = await _generalService.CreateOperationItemAsync(crprecItem.Code,input.DataOperation.ToString(Formatting.None)); + crpOperationItem = await _generalService.CreateOperationItemAsync(crprecItem.Code, cropYear, input.GeometryJson.ToString(Formatting.None), input.DataOperation.ToString(Formatting.None)); _settings.OperationItemCode = crpOperationItem.Code; SaveSettings(settingsfile); } @@ -181,7 +181,7 @@ namespace FarmmapsKPI if (useCreatedCropfieldCharacteristic == false) { _logger.LogInformation("CreateCropfieldCharacteristicItemAsync ..."); - cropfieldCharacteristicItem = await _generalService.CreateCropfieldCharacteristicItemAsync(cropfieldItem.Code, input.DataCropfieldCharacteristic.ToString(Formatting.None)); + cropfieldCharacteristicItem = await _generalService.CreateCropfieldCharacteristicItemAsync(cropfieldItem.Code, cropYear, input.GeometryJson.ToString(Formatting.None), input.DataCropfieldCharacteristic.ToString(Formatting.None)); _settings.CropfieldCharacteristicItemCode = cropfieldCharacteristicItem.Code; SaveSettings(settingsfile); } @@ -210,7 +210,7 @@ namespace FarmmapsKPI //Now get the KPIs for this cropfield, mounted with operations & cropyield // Get KPI data for saving it in a file, here the generalsedrvice is called to get the KPI data - _logger.LogInformation($"GetKpiItemsForCropField({cropfieldItem.Code})"); + _logger.LogInformation($"GetKpiItemsForCropField('{cropfieldItem.Code}')"); var KPIItem = await _generalService.GetKpiItemsForCropField(cropfieldItem); //Download KPI's into a json output file for this specific cropfield (with unique cropfieldItem.Code) diff --git a/FarmmapsKPI/KPIInput.json b/FarmmapsKPI/KPIInput.json index d76e98e..c34b7f6 100644 --- a/FarmmapsKPI/KPIInput.json +++ b/FarmmapsKPI/KPIInput.json @@ -1,6 +1,6 @@ [ { - "UseCreatedCropfield": true, + "UseCreatedCropfield": false, "CropfieldItemCode": "", "dataCropfield": { //"area": 4.22, @@ -14,10 +14,10 @@ "productionPurposeCode": "003" //"productionPurposeName": "consumption" }, - "UseCreatedCropRecording": true, + "UseCreatedCropRecording": false, "CropRecordingItemCode": "", "dataCropRecording": {}, //not yet used - "UseCreatedOperation": true, + "UseCreatedOperation": false, "OperationItemCode": "", "dataOperation": { "n": "92", From 4102ed628eb1ce84f2aa4d83941ea9ae001f2b15 Mon Sep 17 00:00:00 2001 From: Pepijn van Oort Date: Fri, 13 Oct 2023 16:37:50 +0200 Subject: [PATCH 19/66] farmmapsKPI receives minimum necessary input and writes relevant KPI output --- FarmmapsApi/Constants.cs | 2 +- FarmmapsApi/Services/GeneralService.cs | 25 +++- FarmmapsKPI/KPIApplication.cs | 197 ++++++++++++++++++------- FarmmapsKPI/KPIInput.json | 152 ++++++++++++++----- FarmmapsKPI/KPIdefinitions.csv | 8 + FarmmapsKPI/Models/KPIInput.cs | 16 +- FarmmapsKPI/Models/Settings.cs | 2 +- 7 files changed, 285 insertions(+), 117 deletions(-) create mode 100644 FarmmapsKPI/KPIdefinitions.csv diff --git a/FarmmapsApi/Constants.cs b/FarmmapsApi/Constants.cs index 6b37e24..e747493 100644 --- a/FarmmapsApi/Constants.cs +++ b/FarmmapsApi/Constants.cs @@ -17,7 +17,7 @@ namespace FarmmapsApiSamples public const string CROPCHAR_ITEMTYPE = "vnd.farmmaps.itemtype.edicrop.characteristic"; public const string CROPSCHEME_ITEMTYPE = "vnd.farmmaps.itemtype.croppingscheme"; // deze toegevoegd, misschien is het type van de KPI task wel een croppingscheme //public const string KPI_ITEM = "vnd.farmmaps.itemtype.kpi.data"; //PO20231004: originally with .data - public const string KPI_ITEM = "vnd.farmmaps.itemtype.kpi.data.container"; //PO20231004: originally without .container + public const string KPICONTAINER_ITEM = "vnd.farmmaps.itemtype.kpi.data.container"; //PO20231004: originally without .container public const string VRANBS_TASK = "vnd.farmmaps.task.vranbs"; public const string VRAHERBICIDE_TASK = "vnd.farmmaps.task.vraherbicide"; diff --git a/FarmmapsApi/Services/GeneralService.cs b/FarmmapsApi/Services/GeneralService.cs index 8581455..6b452a0 100644 --- a/FarmmapsApi/Services/GeneralService.cs +++ b/FarmmapsApi/Services/GeneralService.cs @@ -350,9 +350,9 @@ namespace FarmmapsApi.Services return bofekItem; } - public async Task> GetKpiItemsForCropField(Item cropfieldItem) // dit is dus nieuw om de KPI task te runnen + public async Task> GetKpiItemsForCropField(Item cropfieldItem) { - var kpiRequest = new TaskRequest { TaskType = KPI_TASK }; + TaskRequest kpiRequest = new TaskRequest { TaskType = KPI_TASK }; kpiRequest.attributes["processAggregateKpi"] = "false"; int year = cropfieldItem.DataDate.Value.Year; kpiRequest.attributes["year"] = year.ToString(); @@ -372,15 +372,24 @@ namespace FarmmapsApi.Services _logger.LogError($"Something went wrong with task execution: {itemTask.Message}"); return null; } - await Task.Delay(60000); //wacht hier een minuut tot de KPIs berekend zijn + await Task.Delay(6000); // wait 6000 secs for task to be completed - //PO20230627 Je zou hier ook om de 10 sec eens kunnen kijken of we al zo ver zijn? Iets in trant van while KPI_ITEM is null? - //hier nog definieren waar in de hierarchie een KPI item is? - var allChildren = await _farmmapsApiService.GetItemChildrenAsync(cropfieldItem.Code); - var kpiItems = await _farmmapsApiService.GetItemChildrenAsync(cropfieldItem.Code, KPI_ITEM); + //After the task is completed we have 1 kpiContainerItem with a code and with no data + //Because the data are in the children of this kpiContainerItem. The container will have a list of children each with an "id" aand data, + //The ids' are "a1", "b1", "b2", "c1","d1","d3","d5" + //with following meanings: + //| A1 | Opbrengst | Yield | + //| B1 | Stikstofoverschot | Nitrogen surplus | + //| B2 | Fosfaatoverschot | Phosphate surplus | + //| C1 | Effectieve Organischestof aanvoer| Effective Organic Matter supply | + //| D1 | Gebruik bestrijdingsmiddelen | Use of pesticides| + //| D3 | Gewasdiversiteit(randdichtheid) | Crop diversity(edge density) | + //| D5 | Percentage rustgewassen | Percentage of rest crops | + List kpiContainerItem = await _farmmapsApiService.GetItemChildrenAsync(cropfieldItem.Code, KPICONTAINER_ITEM); + string kpiContainerItemCode = kpiContainerItem[0].Code; + List kpiItems = await _farmmapsApiService.GetItemChildrenAsync(kpiContainerItemCode); return kpiItems; - } public async Task RunAhnTask(Item cropfieldItem) { diff --git a/FarmmapsKPI/KPIApplication.cs b/FarmmapsKPI/KPIApplication.cs index 7fd90b8..b9f7e69 100644 --- a/FarmmapsKPI/KPIApplication.cs +++ b/FarmmapsKPI/KPIApplication.cs @@ -46,21 +46,45 @@ namespace FarmmapsKPI await _farmmapsApiService.GetCurrentUserCodeAsync(); var roots = await _farmmapsApiService.GetCurrentUserRootsAsync(); + //Where to write the output + string downloadFolder = fieldsInputs[0].DownloadFolder; + if (string.IsNullOrEmpty(downloadFolder)) + { + downloadFolder = "Downloads"; + } + if (!Directory.Exists(downloadFolder)) + Directory.CreateDirectory(downloadFolder); + + //Write the same info to a single csv file. Note this means existing file will be overwritten! + StreamWriter sw; string KPIItemPathCsv = Path.Combine(downloadFolder, "KPIItems.csv"); + List headerList = new List { "parentName", "area_ha", "cropTypeCode", "cropTypeName", "KPIid", "KPIvariable", "KPIvalue", "KPIunit", "KPItargetvalue", "KPIthresholdValue" }; + //Create a new csv file. Means if existing then overwritten !!! + sw = new StreamWriter(KPIItemPathCsv); + sw.WriteLine(string.Join(",", headerList)); + + //Now loop through the list of cropfields in KPIinput.json and get the KPI's for each cropfield foreach (var input in fieldsInputs) { try { - await Process(roots, input); + await Process(roots, input, sw); } catch (Exception ex) { _logger.LogError(ex.Message); } } + + //Close the csv file, write message to screen + sw.Close(); + _logger.LogInformation($"Done! Written all KPI for all fields in 'KPIinput.json' to output file '{KPIItemPathCsv}'"); } - private async Task Process(List roots, KPIInput input) + private async Task Process(List roots, KPIInput input, StreamWriter sw) { + KPIOutput kpio; + KPIOutput kpioPrevious = null; + string downloadFolder = input.DownloadFolder; if (string.IsNullOrEmpty(downloadFolder)) { downloadFolder = "Downloads"; @@ -69,20 +93,13 @@ namespace FarmmapsKPI Directory.CreateDirectory(downloadFolder); // !!specify if you are using an already created cropfield: - bool useCreatedCropfield = input.UseCreatedCropfield; - bool useCreatedCropRecording = input.UseCreatedCropRecording; - bool useCreatedOperations = input.UseCreatedOperation; - bool useCreatedCropfieldCharacteristic = input.UseCreatedCropfieldCharacteristic; - var cropYear = input.CropYear; - var fieldName = input.fieldName; - //bool storeSatelliteStatistics = input.StoreSatelliteStatisticsSingleImage; - //bool storeSatelliteStatisticsCropYear = input.StoreSatelliteStatisticsCropYear; - //List SatelliteBands = new List(1) { input.SatelliteBand }; - //string headerLineStats = $"FieldName,satelliteDate,satelliteBand,max,min,mean,mode,median,stddev,minPlus,curtosis,maxMinus,skewness,variance,populationCount,variationCoefficient,confidenceIntervalLow, confidenceIntervalHigh,confidenceIntervalErrorMargin" + Environment.NewLine; - + bool useExistingCropfieldWithChildren = input.UseExistingCropfieldWithChildren; + int cropYear = input.CropYear; + string fieldName = input.fieldName; + string fieldGeom = input.GeometryJson.ToString(Formatting.None); + //Settings string settingsfile = $"Settings_{fieldName}.json"; - LoadSettings(settingsfile); var uploadedRoot = roots.SingleOrDefault(r => r.Name == "USER_IN"); @@ -101,14 +118,14 @@ namespace FarmmapsKPI // Use already created cropfield or create new one, added a Data input, with field specific data for the KPI calculation Item cropfieldItem; - //1 useCreatedCropfield = false -> create new - //2 useCreatedCropfield = true && CropfieldItemCode = "" or absent -> read from settings json - //3 useCreatedCropfield = true && CropfieldItemCode like "deb48a74c5b54299bb852f17288010e9" in KPIinput -> use this one - if (useCreatedCropfield == false || string.IsNullOrEmpty(_settings.CropfieldItemCode)) + //1 useExistingCropfieldWithChildren = false -> create new + //2 useExistingCropfieldWithChildren = true && input.CropfieldItemCode = "" or absent -> read from settings json + //3 useExistingCropfieldWithChildren = true && input.CropfieldItemCode like "deb48a74c5b54299bb852f17288010e9" in KPIinput -> use this one + if (useExistingCropfieldWithChildren == false) { _logger.LogInformation("Creating cropfield, writting to settings file"); cropfieldItem = await _generalService.CreateCropfieldItemAsync(myDriveRoot.Code, - $"{input.OutputFileName}", cropYear, input.GeometryJson.ToString(Formatting.None), input.DataCropfield.ToString(Formatting.None)); + $"{fieldName}", cropYear, input.GeometryJson.ToString(Formatting.None), input.DataCropfield.ToString(Formatting.None)); _settings.CropfieldItemCode = cropfieldItem.Code; SaveSettings(settingsfile); } @@ -121,15 +138,16 @@ namespace FarmmapsKPI { _logger.LogInformation("reading CropfieldItemCode from KPIinput.json"); cropfieldItem = await _farmmapsApiService.GetItemAsync(input.CropfieldItemCode); + SaveSettings(settingsfile); } // Use already created croprecording or create new one, added a Data input, with field specific data for the KPI calculation Item crprecItem; - //1 useCreatedCropRecording = false -> create new - //2 useCreatedCropRecording = true && CropRecordingItemCode = "" or absent -> read from settings json - //3 useCreatedCropRecording = true && CropRecordingItemCode like "deb48a74c5b54299bb852f17288010e9" in KPIinput -> use this one - if (useCreatedCropRecording == false || string.IsNullOrEmpty(_settings.CropRecordingItemCode)) + //1 useExistingCropfieldWithChildren = false -> create new + //2 useExistingCropfieldWithChildren = true && input.CropRecordingItemCode = "" or absent -> read from settings json + //3 useExistingCropfieldWithChildren = true && input.CropRecordingItemCode like "deb48a74c5b54299bb852f17288010e9" in KPIinput -> use this one + if (useExistingCropfieldWithChildren == false) { _logger.LogInformation("RunCropRecordingTask ..."); crprecItem = await _generalService.RunCropRecordingTask(cropfieldItem); @@ -145,29 +163,55 @@ namespace FarmmapsKPI { _logger.LogInformation("reading CropRecordingItemCode from KPIinput.json"); crprecItem = await _farmmapsApiService.GetItemAsync(input.CropRecordingItemCode); + SaveSettings(settingsfile); } - // Use already created operation or create new one, added a Data input, with field specific data for the KPI calculation + // Use already created operations or create new one,s added a Data input, with field specific data for the KPI calculation + List crpOperationItems = new List { }; + List crpOperationItemCodes = new List { }; Item crpOperationItem; - //1 useCreatedOperation = false -> create new - //2 useCreatedOperation = true && OperationItemCode = "" or absent -> read from settings json - //3 useCreatedOperation = true && OperationItemCode like "deb48a74c5b54299bb852f17288010e9" in KPIinput -> use this one - if (useCreatedOperations == false || string.IsNullOrEmpty(_settings.OperationItemCode)) + string dataOperation; + string codeOperation; + //1 useExistingCropfieldWithChildren = false -> create new + //2 useExistingCropfieldWithChildren = true && input.OperationItemCode = "" or absent -> read from settings json + //3 useExistingCropfieldWithChildren = true && input.OperationItemCode like "deb48a74c5b54299bb852f17288010e9" in KPIinput -> use this one + if (useExistingCropfieldWithChildren == false) { - _logger.LogInformation("CreateOperationItemAsync ..."); - crpOperationItem = await _generalService.CreateOperationItemAsync(crprecItem.Code, cropYear, input.GeometryJson.ToString(Formatting.None), input.DataOperation.ToString(Formatting.None)); - _settings.OperationItemCode = crpOperationItem.Code; + for (int i = 0; i < input.DataOperations.Length; i++) + { + dataOperation = input.DataOperations[i].ToString(Formatting.None); + dynamic data = JObject.Parse(dataOperation); + _logger.LogInformation($"CreateOperationItemAsync ... for operation {i}: '{data.name}', {data.n} kg N/ha, on date '{data.from}'"); + crpOperationItem = await _generalService.CreateOperationItemAsync(crprecItem.Code, cropYear, fieldGeom, dataOperation); + crpOperationItems.Add(crpOperationItem); + crpOperationItemCodes.Add(crpOperationItem.Code); + } + _settings.OperationItemCodes = crpOperationItemCodes.ToArray(); SaveSettings(settingsfile); } else if (string.IsNullOrEmpty(input.CropfieldItemCode)) { _logger.LogInformation("reading OperationItemCode from settings file"); - crpOperationItem = await _farmmapsApiService.GetItemAsync(_settings.OperationItemCode); + for (int i = 0; i < _settings.OperationItemCodes.Length; i++) + { + codeOperation = _settings.OperationItemCodes[i]; + crpOperationItem = await _farmmapsApiService.GetItemAsync(codeOperation); + crpOperationItems.Add(crpOperationItem); + crpOperationItemCodes.Add(crpOperationItem.Code); + } } else { - _logger.LogInformation("reading OperationItemCode from KPIinput.json"); - crpOperationItem = await _farmmapsApiService.GetItemAsync(input.OperationItemCode); + _logger.LogInformation("reading OperationItemCodes from KPIinput.json"); + for (int i = 0; i < input.OperationItemCodes.Length; i++) + { + codeOperation = input.OperationItemCodes[i]; + crpOperationItem = await _farmmapsApiService.GetItemAsync(codeOperation); + crpOperationItems.Add(crpOperationItem); + crpOperationItemCodes.Add(crpOperationItem.Code); + } + _settings.OperationItemCodes = crpOperationItemCodes.ToArray(); + SaveSettings(settingsfile); } // The cropfieldCharacteristicItem is used to enter crop yields @@ -175,17 +219,17 @@ namespace FarmmapsKPI // Nutrient balance. // Use already created cropfieldCharacteristicItem or create new one, added a Data input, with field specific data for the KPI calculation Item cropfieldCharacteristicItem; - //1 useCreatedCropfieldCharacteristic = false -> create new - //2 useCreatedCropfieldCharacteristic = true && CropfieldCharacteristicItemCode = "" or absent -> read from settings json - //3 useCreatedCropfieldCharacteristic = true && CropfieldCharacteristicItemCode like "deb48a74c5b54299bb852f17288010e9" in KPIinput -> use this one - if (useCreatedCropfieldCharacteristic == false) + //1 useExistingCropfieldWithChildren = false -> create new + //2 useExistingCropfieldWithChildren = true && input.CropfieldCharacteristicItemCode = "" or absent -> read from settings json + //3 useExistingCropfieldWithChildren = true && input.CropfieldCharacteristicItemCode like "deb48a74c5b54299bb852f17288010e9" in KPIinput -> use this one + if (useExistingCropfieldWithChildren == false) { _logger.LogInformation("CreateCropfieldCharacteristicItemAsync ..."); - cropfieldCharacteristicItem = await _generalService.CreateCropfieldCharacteristicItemAsync(cropfieldItem.Code, cropYear, input.GeometryJson.ToString(Formatting.None), input.DataCropfieldCharacteristic.ToString(Formatting.None)); + cropfieldCharacteristicItem = await _generalService.CreateCropfieldCharacteristicItemAsync(cropfieldItem.Code, cropYear, fieldGeom, input.DataCropfieldCharacteristic.ToString(Formatting.None)); _settings.CropfieldCharacteristicItemCode = cropfieldCharacteristicItem.Code; SaveSettings(settingsfile); } - else if (string.IsNullOrEmpty(input.CropfieldCharacteristicItemCode)) + else if (string.IsNullOrEmpty(input.CropfieldItemCode)) { _logger.LogInformation("reading OperationItemCode from settings file"); cropfieldCharacteristicItem = await _farmmapsApiService.GetItemAsync(_settings.CropfieldCharacteristicItemCode); @@ -194,38 +238,77 @@ namespace FarmmapsKPI { _logger.LogInformation("reading CropfieldCharacteristicItemCode from KPIinput.json"); cropfieldCharacteristicItem = await _farmmapsApiService.GetItemAsync(input.CropfieldCharacteristicItemCode); + SaveSettings(settingsfile); } - // Inspect the children. If all is well, cropfield will have one crprec and one edicrop.characteristic - // And the crprec will have 0-many operations as children - // And the Data of an operation will have specification of how much fertilizer was applied - // And crprec can have multiple operations - // Note existing cropfields and croprecordings keep existing properties you added in previous runs - // (unless you deleted them) - // So ech time you run with "UseCreatedCropfield": true & "UseCreatedCropRecording": false -> a new recording will be added to the existing cropfield - // So ech time you run with "UseCreatedCropfield": true & "useCreatedCropfieldCharacteristic": false -> a new CropfieldCharacteristic will be added to the existing cropfield - // So ech time you run with "UseCreatedCropRecording": true & "UseCreatedOperation": false -> a new operation will be added to the existing crop recording + // Inspect the children and grandchildren. If all is well, cropfield will have: + // one crprec, with 0-many operations as children. And the Data of an operation will have specification of how much fertilizer was applied + // one edicrop.characteristic (with yield in the data) var cropfieldChildren = await _farmmapsApiService.GetItemChildrenAsync(cropfieldItem.Code); var crprecChildren = await _farmmapsApiService.GetItemChildrenAsync(crprecItem.Code); //Now get the KPIs for this cropfield, mounted with operations & cropyield // Get KPI data for saving it in a file, here the generalsedrvice is called to get the KPI data _logger.LogInformation($"GetKpiItemsForCropField('{cropfieldItem.Code}')"); - var KPIItem = await _generalService.GetKpiItemsForCropField(cropfieldItem); + List KPIItems = await _generalService.GetKpiItemsForCropField(cropfieldItem); + _logger.LogInformation($"Found {KPIItems.Count} KPI items"); //Download KPI's into a json output file for this specific cropfield (with unique cropfieldItem.Code) - var KPIItemPath = Path.Combine(downloadFolder, $"KPIItems_{cropfieldItem.Code}.json"); - _logger.LogInformation($"Found {KPIItem.Count} KPI items"); + string KPIItemPathJson = Path.Combine(downloadFolder, $"KPIItems_{cropfieldItem.Code}.json"); var count = 0; - await Task.Delay(50); - foreach (var item in KPIItem) + foreach (Item item in KPIItems) { - Console.WriteLine($"KPI item #{count}: {item.Name}"); - File.AppendAllText(KPIItemPath, item.Data + Environment.NewLine); + //Console.WriteLine($"KPI item #{count}: {item.Name}"); + File.AppendAllText(KPIItemPathJson, item.Data + Environment.NewLine); count++; } - _logger.LogInformation($"Downloaded file {KPIItemPath}"); + _logger.LogInformation($"Downloaded file {KPIItemPathJson}"); + //Write to the csv file that collects all KPI's for all the crop fields + List dataList; + foreach (Item item in KPIItems) + { + dataList = new List { }; + kpio = JsonConvert.DeserializeObject(item.Data.ToString()); + //Seems sometimes duplicate KPI items are returned. So check that here and only write if this kpio is different from previous + if (kpio != kpioPrevious) + { + dataList.Add(kpio.parentName); + dataList.Add(kpio.data.area); + dataList.Add(kpio.data.cropTypeCode); + dataList.Add(kpio.data.cropTypeName); + dataList.Add(kpio.id); + dataList.Add(kpio.quantity); + dataList.Add(kpio.value); + dataList.Add(kpio.unit); + dataList.Add(kpio.targetValue); + dataList.Add(kpio.thresholdValue); + sw.WriteLine(string.Join(",", dataList)); + } + kpioPrevious = kpio; + } + + //Clean up. Only newly created fields + //Look up instruction in Swagger / api / v1 / items /{ code} + if (useExistingCropfieldWithChildren == false && input.DeleteNewlyCreatedAfterCalc == true) + { + // Tested and found that if I delete the parent (cropfieldItem) then + // automatically deletes children (crprecItem, cropfieldCharacteristicItem, KPIcontainer) + // and automatically deletes grand children (crprecItem: operations, KPIcontainer: KPI items) + await _farmmapsApiService.DeleteItemAsync(cropfieldItem.Code); + //Do we need to wait here for the task to be completed? Or does it continue in the background? + //await Task.Delay(60000); // wait 60 secs for task to be completed + //Before the GetItem you will see cropfieldItem, crprecItem etc exist (with data). + //After the GetItem these cropfieldItem, crprecItem etc are null which means they have been succesfully deleted. + //// Check if below works, i.e. is item really deleted? + //cropfieldItem = await _farmmapsApiService.GetItemAsync(cropfieldItem.Code); + //// Check if croprecording has also been deleted. If not also delete it + //crprecItem = await _farmmapsApiService.GetItemAsync(crprecItem.Code); + //// Check if 1 operation has also been deleted. If not also delete it + //crpOperationItem = await _farmmapsApiService.GetItemAsync(crpOperationItemCodes[0]); + //// Check if 1 cropfieldCharacteristicItem has also been deleted. If not also delete it + //cropfieldCharacteristicItem = await _farmmapsApiService.GetItemAsync(cropfieldCharacteristicItem.Code); + } } // Functions to save previously created cropfields diff --git a/FarmmapsKPI/KPIInput.json b/FarmmapsKPI/KPIInput.json index c34b7f6..04b9b70 100644 --- a/FarmmapsKPI/KPIInput.json +++ b/FarmmapsKPI/KPIInput.json @@ -1,53 +1,39 @@ [ { - "UseCreatedCropfield": false, - "CropfieldItemCode": "", + "useExistingCropfieldWithChildren": false, + "deleteNewlyCreatedAfterCalc": true, + //Note when you run the FarmmapsKPI project, somewhere to your C:\git\FarmMapsApiClient_KB34_MAST\FarmmapsKPI\bin\Debug\netcoreapp3.1\ directory, files named like this are written: + //'Settings_aardappelveld_test_Potato_ZeroFertilizer.json', where 'aardappelveld_test_Potato_ZeroFertilizer.json' is your fieldName (see below) + //1 useExistingCropfieldWithChildren = false -> create new + //2 useExistingCropfieldWithChildren = true && CropfieldItemCode = "" or absent -> read fieldName (see below) -> read all codes from settings json file + //3 useExistingCropfieldWithChildren = true && CropfieldItemCode like "deb48a74c5b54299bb852f17288010e9" in KPIinput -> if you have in your account already an existing cropfield then use that + "CropfieldItemCode": "", // could contain for example this: "abae97f89f3c4ac08953b1b8bea9f076" if this is an exisiting CropfieldItemCode in your account. + //And then if useExistingCropfieldWithChildren = true, this would be used. "dataCropfield": { - //"area": 4.22, - "final": true, - //"soilCode": "5", - "soilName": "Loam", - "cropTypeCode": "1010101", // make a table/list with possible inputs. + //"area": 4.22, //not needed for KPI calculation, but shown here to know this is a possible property + "final": true, //always true + //"soilCode": "5", //not needed for KPI calculation, but shown here to know this is a possible property + //"soilName": "Loam", //not needed for KPI calculation, but shown here to know this is a possible property + "cropTypeCode": "1010101", "cropTypeName": "Potato", - //"rootDepthMax": 45, - //"emergenceDate": "2022-05-16T00:00:00", + //"rootDepthMax": 45, //not needed for KPI calculation, but shown here to know this is a possible property + //"emergenceDate": "2022-05-16T00:00:00", //not needed for KPI calculation, but shown here to know this is a possible property "productionPurposeCode": "003" - //"productionPurposeName": "consumption" + //"productionPurposeName": "consumption" //not needed for KPI calculation, but shown here to know this is a possible property }, - "UseCreatedCropRecording": false, - "CropRecordingItemCode": "", - "dataCropRecording": {}, //not yet used - "UseCreatedOperation": false, - "OperationItemCode": "", - "dataOperation": { - "n": "92", - "to": "2022-05-23T12:34:00", - "area": "0.08", //?! - "from": "2022-05-23T11:34:00", - "name": "Kunstmest strooien", - "unit": "kg/ha", - "method": "70400", - "status": "3", - "product": "7360", - "quantity": "200", - "unitCode": "KGMHAR", //?! - "contractor": false, - "designator": "Kunstmest strooien", - "operationCode": "7" - }, - "UseCreatedCropfieldCharacteristic": false, + "CropRecordingItemCode": "", // could contain for example this: "314aa1b6480b493ba28a5f55039a01d4" if this is the CropRecordingItemCode of the provided CropfieldItemCode. And then if useExistingCropfieldWithChildren = true, this would be used + "OperationItemCodes": [], // could contain for example this: ["4d1a6f28ea3f414180663baa4b08c60f"] or ["4d1a6f28ea3f414180663baa4b08c60f","21e08386f0454ad79acfebf78649d59b"] if these are the operations of the provided CropRecordingItemCode. And then if useExistingCropfieldWithChildren = true, these would be used + "dataOperations": [], //leaving the operations empty means no fertilizer is applied "CropfieldCharacteristicItemCode": "", "DataCropfieldCharacteristic": { "code": "860619", //PO20231004: so what does this code mean? Can we see the code list somewhere? "label": "cropyield", "value": "48.01" }, - //"DownloadFolder": "Downloads", //"C:\\workdir\\groenmonitor\\", // "Downloads", -> if you just put "Downloads" the program will download to somewhere in ..\FarmMapsApiClient_WURtest\FarmmapsDataDownload\bin\Debug\netcoreapp3.1\Downloads\ + //"DownloadFolder": "Downloads", //"C:\\hugoschrererdir\\kpidir\\", // "Downloads", -> if you just put "Downloads" the program will download to somewhere in ..\FarmMapsApiClient_KB34_MAST\FarmmapsKPI\bin\Debug\netcoreapp3.1\Downloads\ "CropYear": 2022, - "dataDate": "2022-04-15T00:00:00Z", - "dataEndDate": "2022-09-15T00:00:00Z", - "fieldName": "aardappelveld_test", - //"ProcessAggegrateKpi": "false", // toegevoegd, want alleen veld level KPIs voor nu. + "fieldName": "aardappelveld_test_Potato_ZeroFertilizer", + //geometryJson = polygon of location of the cropfield, coordinates in LON,LAT (LON: decimal degrees East, LAT: decimal degrees North) "geometryJson": { "type": "Polygon", "coordinates": [ @@ -74,7 +60,97 @@ ] ] ] + } + }, + { + "useExistingCropfieldWithChildren": false, + "CropfieldItemCode": "", + "dataCropfield": { + //"area": 4.22, //not needed for KPI calculation, but shown here to know this is a possible property + "final": true, //always true + //"soilCode": "5", //not needed for KPI calculation, but shown here to know this is a possible property + //"soilName": "Loam", //not needed for KPI calculation, but shown here to know this is a possible property + "cropTypeCode": "1010101", + "cropTypeName": "Potato", + //"rootDepthMax": 45, //not needed for KPI calculation, but shown here to know this is a possible property + //"emergenceDate": "2022-05-16T00:00:00", //not needed for KPI calculation, but shown here to know this is a possible property + "productionPurposeCode": "003" + //"productionPurposeName": "consumption" //not needed for KPI calculation, but shown here to know this is a possible property }, - "outputFileName": "TestData" + "CropRecordingItemCode": "", + "OperationItemCodes": [], + "dataOperations": [ + { + "area": "0.08", //?! + "contractor": false, + "designator": "Kunstmest strooien", + "from": "2022-05-23T11:34:00", + "method": "70400", + "n": "92", + "name": "Kunstmest strooien", + "operationCode": "7", + "product": "7360", + "quantity": "400", + "status": "3", + "to": "2022-05-23T12:34:00", + "unit": "kg/ha", + "unitCode": "KGMHAR" + }, + { + "k": "201.6", + "n": "144", + "p": "60.8", + "to": "2022-04-19T09:19:00", + "area": "0.08", //?!, + "from": "2022-04-19T08:19:00", + "name": "Injecteren", + "unit": "kg/ha", + "method": "70700", + "status": "3", + "product": "2329", + "quantity": "32000", + "unitCode": "KGMHAR", + "contractor": false, + "designator": "Injecteren", + "operationCode": "7" + } + ], + "CropfieldCharacteristicItemCode": "", + "DataCropfieldCharacteristic": { + "code": "860619", //PO20231004: so what does this code mean? Can we see the code list somewhere? + "label": "cropyield", + "value": "48.01" + }, + //"DownloadFolder": "Downloads", //"C:\\hugoschrererdir\\kpidir\\", // "Downloads", -> if you just put "Downloads" the program will download to somewhere in ..\FarmMapsApiClient_WURtest\FarmmapsDataDownload\bin\Debug\netcoreapp3.1\Downloads\ + "CropYear": 2022, + "fieldName": "aardappelveld_test_Potato_cattlemanure144kgnha_plusUrea92kgNha", + "geometryJson": { + "type": "Polygon", + "coordinates": [ + [ + [ + 5.5945257993548765, + 52.57080744107003 + ], + [ + 5.598645994070678, + 52.571540800206236 + ], + [ + 5.599381743127071, + 52.57012773140724 + ], + [ + 5.595408698222548, + 52.56968054825188 + ], + [ + 5.5945257993548765, + 52.57080744107003 + ] + ] + ] + } } + ] \ No newline at end of file diff --git a/FarmmapsKPI/KPIdefinitions.csv b/FarmmapsKPI/KPIdefinitions.csv new file mode 100644 index 0000000..5d31cae --- /dev/null +++ b/FarmmapsKPI/KPIdefinitions.csv @@ -0,0 +1,8 @@ +KPIid,KPIvariable,Description +A1,yield,observed yield (user input) +B1,nitrogen,Nitrogen surplus = N fertilizer + N atmospheric deposition + N fixation by crop - N removal through harvested product. Calculated KPI internal model & parameters and from user input: yield and fertilizer applications +B2,phosphate,Phosphate surplus = P fertilizer - P removal through harvested product. Calculated KPI internal model & parameters and from user input: yield and fertilizer applications +C1,organic matter supply,Organic matter surplus = Organic matter from manure + Crop residues to soil - organic matter removal through harvested product. Calculated KPI internal model & parameters and from user input: yield and manure applications +D1,pesticides,? +,KPItargetvalue,target value as in benchmark value for same crop in same region +,KPIthresholdValue,threshold from ??? Ask farmmaps. Surplus nitrogen / phosphate / pesticides must not be above threshold. Surplus organic matter supply must be above threshold diff --git a/FarmmapsKPI/Models/KPIInput.cs b/FarmmapsKPI/Models/KPIInput.cs index fb832fb..2fef5d9 100644 --- a/FarmmapsKPI/Models/KPIInput.cs +++ b/FarmmapsKPI/Models/KPIInput.cs @@ -5,28 +5,20 @@ namespace FarmmapsKPI.Models { public class KPIInput { - public bool UseCreatedCropfield { get; set; } + public bool UseExistingCropfieldWithChildren { get; set; } + public bool DeleteNewlyCreatedAfterCalc { get; set; } public string CropfieldItemCode { get; set; } public JObject DataCropfield { get; set; } - public bool UseCreatedCropRecording { get; set; } public string CropRecordingItemCode { get; set; } public JObject DataCropRecording { get; set; } - public bool UseCreatedOperation { get; set; } - public string OperationItemCode { get; set; } - public JObject DataOperation { get; set; } - public bool UseCreatedCropfieldCharacteristic { get; set; } + public string[] OperationItemCodes { get; set; } + public JObject[] DataOperations { get; set; } public string CropfieldCharacteristicItemCode { get; set; } public JObject DataCropfieldCharacteristic { get; set; } - - public string File { get; set; } - public string InputVariable { get; set; } - public string OutputFileName { get; set; } public string DownloadFolder { get; set; } public int CropYear { get; set; } public JObject GeometryJson { get; set; } - public string InputLayerName { get; set; } public string fieldName { get; set; } - public bool GetShadowData { get; set; } } } \ No newline at end of file diff --git a/FarmmapsKPI/Models/Settings.cs b/FarmmapsKPI/Models/Settings.cs index f6ed630..aaaf1ca 100644 --- a/FarmmapsKPI/Models/Settings.cs +++ b/FarmmapsKPI/Models/Settings.cs @@ -4,7 +4,7 @@ { public string CropfieldItemCode { get; set; } public string CropRecordingItemCode { get; set; } - public string OperationItemCode { get; set; } + public string[] OperationItemCodes { get; set; } public string CropfieldCharacteristicItemCode { get; set; } } From 8771e47b289f70eed57710ab5ac95e4cb0d984cc Mon Sep 17 00:00:00 2001 From: Pepijn van Oort Date: Fri, 13 Oct 2023 16:38:27 +0200 Subject: [PATCH 20/66] TODO --- FarmmapsKPI/TODO.txt | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 FarmmapsKPI/TODO.txt diff --git a/FarmmapsKPI/TODO.txt b/FarmmapsKPI/TODO.txt new file mode 100644 index 0000000..f434c24 --- /dev/null +++ b/FarmmapsKPI/TODO.txt @@ -0,0 +1,22 @@ +TODO.txt + +Laat de farmmapsKPI zelf maar uitzoeken wat de n-p gehaltes zijn: +a. We willen als input opgeven we hebben 400 kg/ha urea gegeven, of 32000 kg/ha runderdrijfmest. + Daarvoor wel de juiste gewas code opgeven -> zie onder, FarmmapsEditeeltList 263 gewassen + Daarvoor wel de juiste meststof code opgeven -> zie onder, FarmmapsEditeeltList 022 meststoffen +b. Check of de operation data een element “n” bevat. Zoniet of als waarde 0, dan + i. Zoek onder de motorkap op: de omreken factor (bijv 46% -> 400*0.46 = 184 kg N/ha). In editeelt list + ii. Voeg die 184 toe aan de operation data +c. Reken N surplus uit (KPI item id b1) + +Nieuw project: FarmmapsEditeeltLists +a. Why? Because + i. This is (1) so that we can at any time retrieve the most recent up-to-date list + ii. so that you can look up crops and fertilizer types and fill in KPIinput.json correctly +b. Download full list of items from vnd.farmmaps.itemtype.codelist.cl263 +c. Write them to a csv file +d. In any case following lists: 127 operations; 022 meststoffen (fertilizer products); 263 gewassen; 405 soilcode + +Testing: in json input data, can certain fields be left out? Which are minimum fields needed? +Testing: effect of loation (e.g. different targetValue, different N deposition?) +Testing: productionPurposeCode: different result depending on starch or consumption potato (different N contents in tubers; different targetValues? From 1a9d5af5f6bfb3a32982174e1fcdc6166d81be80 Mon Sep 17 00:00:00 2001 From: Pepijn van Oort Date: Mon, 16 Oct 2023 13:55:12 +0200 Subject: [PATCH 21/66] don't write duplicate KPIOutput items to output csv --- FarmmapsKPI/KPIApplication.cs | 4 ++-- FarmmapsKPI/KPIInput.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/FarmmapsKPI/KPIApplication.cs b/FarmmapsKPI/KPIApplication.cs index b9f7e69..6b1eec3 100644 --- a/FarmmapsKPI/KPIApplication.cs +++ b/FarmmapsKPI/KPIApplication.cs @@ -83,7 +83,7 @@ namespace FarmmapsKPI private async Task Process(List roots, KPIInput input, StreamWriter sw) { KPIOutput kpio; - KPIOutput kpioPrevious = null; + KPIOutput kpioPrevious = new KPIOutput(); //creates a new empty string downloadFolder = input.DownloadFolder; if (string.IsNullOrEmpty(downloadFolder)) { @@ -271,7 +271,7 @@ namespace FarmmapsKPI dataList = new List { }; kpio = JsonConvert.DeserializeObject(item.Data.ToString()); //Seems sometimes duplicate KPI items are returned. So check that here and only write if this kpio is different from previous - if (kpio != kpioPrevious) + if (kpio.id != kpioPrevious.id) { dataList.Add(kpio.parentName); dataList.Add(kpio.data.area); diff --git a/FarmmapsKPI/KPIInput.json b/FarmmapsKPI/KPIInput.json index 04b9b70..46eff35 100644 --- a/FarmmapsKPI/KPIInput.json +++ b/FarmmapsKPI/KPIInput.json @@ -90,7 +90,7 @@ "name": "Kunstmest strooien", "operationCode": "7", "product": "7360", - "quantity": "400", + "quantity": "200", "status": "3", "to": "2022-05-23T12:34:00", "unit": "kg/ha", From 2d588026e24c1d76f16b8fdd4a9f756c873c96c3 Mon Sep 17 00:00:00 2001 From: Pepijn van Oort Date: Mon, 16 Oct 2023 15:34:07 +0200 Subject: [PATCH 22/66] also write totalNapplied to csv file with KPI outputs per cropfield --- FarmmapsApi/Services/GeneralService.cs | 19 ++++ FarmmapsKPI/KPIApplication.cs | 26 +++++ FarmmapsKPI/KPIInput.json | 152 ++++++++++++++++++++++++- FarmmapsKPI/KPIdefinitions.csv | 1 + 4 files changed, 195 insertions(+), 3 deletions(-) diff --git a/FarmmapsApi/Services/GeneralService.cs b/FarmmapsApi/Services/GeneralService.cs index 6b452a0..d95380c 100644 --- a/FarmmapsApi/Services/GeneralService.cs +++ b/FarmmapsApi/Services/GeneralService.cs @@ -44,6 +44,25 @@ namespace FarmmapsApi.Services var currentYear = new DateTime(year, 1, 1); JObject jdata = JObject.Parse(data); string name = string.Format($"CrpRec Operation, {jdata.GetValue("name")}"); + int code022; + string type022; + double quantity; + double applied_kgNha; + double ncontent = 0.0; // for now just any value + + //Is it a fertilizer application? + + //If the operation contains an element "n" then nothing, use that value (kg N/ha administred) + //Else: look up the N content for the code022, calculate "n" based on fertilizer amount (data) & content (cl022) and add applied_kgNha to the jdata + if (jdata.ContainsKey("n") == false) + { + quantity = jdata.GetValue("quantity").ToObject(); + code022 = jdata.GetValue("product").ToObject(); + //TODO: Now here look up this code022 in the cl022 and get the ncontent from that list. + //And check the unit in which the ncontent is expressed, e.g. % or kg/ton and check if it is not null + applied_kgNha = quantity * ncontent; + jdata.Add("n", applied_kgNha.ToString()); //all Data elements in Farmmaps code ar strings + }; ItemRequest operationItemRequest = new ItemRequest() { diff --git a/FarmmapsKPI/KPIApplication.cs b/FarmmapsKPI/KPIApplication.cs index 6b1eec3..5fec1ad 100644 --- a/FarmmapsKPI/KPIApplication.cs +++ b/FarmmapsKPI/KPIApplication.cs @@ -288,6 +288,32 @@ namespace FarmmapsKPI kpioPrevious = kpio; } + //Total N applied + double totalNapplied = 0.0; + double operationNapplied; + JObject opData; + for (int i = 0; i < crpOperationItemCodes.Count; i++) + { + codeOperation = crpOperationItemCodes[i]; + crpOperationItem = await _farmmapsApiService.GetItemAsync(codeOperation); + operationNapplied = crpOperationItem.Data.GetValue("n").ToObject(); + totalNapplied = totalNapplied + operationNapplied; + } + //Also add totalNapplied to the csv + dataList = new List { }; + //Seems sometimes duplicate KPI items are returned. So check that here and only write if this kpio is different from previous + dataList.Add(kpioPrevious.parentName); + dataList.Add(kpioPrevious.data.area); + dataList.Add(kpioPrevious.data.cropTypeCode); + dataList.Add(kpioPrevious.data.cropTypeName); + dataList.Add(""); + dataList.Add("totalNapplied"); + dataList.Add(totalNapplied.ToString()); + dataList.Add(kpioPrevious.unit); + dataList.Add(""); + dataList.Add(""); + sw.WriteLine(string.Join(",", dataList)); + //Clean up. Only newly created fields //Look up instruction in Swagger / api / v1 / items /{ code} if (useExistingCropfieldWithChildren == false && input.DeleteNewlyCreatedAfterCalc == true) diff --git a/FarmmapsKPI/KPIInput.json b/FarmmapsKPI/KPIInput.json index 46eff35..36922f0 100644 --- a/FarmmapsKPI/KPIInput.json +++ b/FarmmapsKPI/KPIInput.json @@ -64,6 +64,7 @@ }, { "useExistingCropfieldWithChildren": false, + "deleteNewlyCreatedAfterCalc": true, "CropfieldItemCode": "", "dataCropfield": { //"area": 4.22, //not needed for KPI calculation, but shown here to know this is a possible property @@ -85,11 +86,11 @@ "contractor": false, "designator": "Kunstmest strooien", "from": "2022-05-23T11:34:00", - "method": "70400", + "method": "70400", //refers to codelist 127 with operation types "n": "92", "name": "Kunstmest strooien", "operationCode": "7", - "product": "7360", + "product": "7360", //refers to codelist 022 with fertilizer types "quantity": "200", "status": "3", "to": "2022-05-23T12:34:00", @@ -151,6 +152,151 @@ ] ] } + }, + { + "useExistingCropfieldWithChildren": false, + "deleteNewlyCreatedAfterCalc": true, + "CropfieldItemCode": "", + "dataCropfield": { + //"area": 4.22, //not needed for KPI calculation, but shown here to know this is a possible property + "final": true, //always true + //"soilCode": "5", //not needed for KPI calculation, but shown here to know this is a possible property + //"soilName": "Loam", //not needed for KPI calculation, but shown here to know this is a possible property + "cropTypeCode": "1010101", + "cropTypeName": "Potato", + //"rootDepthMax": 45, //not needed for KPI calculation, but shown here to know this is a possible property + //"emergenceDate": "2022-05-16T00:00:00", //not needed for KPI calculation, but shown here to know this is a possible property + "productionPurposeCode": "003" + //"productionPurposeName": "consumption" //not needed for KPI calculation, but shown here to know this is a possible property + }, + "CropRecordingItemCode": "", + "OperationItemCodes": [], + "dataOperations": [ + { + "area": "0.08", //?! + "contractor": false, + "designator": "Kunstmest strooien", + "from": "2022-05-23T11:34:00", + "method": "70400", + "n": "92", + "name": "Kunstmest strooien", + "operationCode": "7", + "product": "7360", + "quantity": "200", + "status": "3", + "to": "2022-05-23T12:34:00", + "unit": "kg/ha", + "unitCode": "KGMHAR" + } + ], + "CropfieldCharacteristicItemCode": "", + "DataCropfieldCharacteristic": { + "code": "860619", //PO20231004: so what does this code mean? Can we see the code list somewhere? + "label": "cropyield", + "value": "48.01" + }, + //"DownloadFolder": "Downloads", //"C:\\hugoschrererdir\\kpidir\\", // "Downloads", -> if you just put "Downloads" the program will download to somewhere in ..\FarmMapsApiClient_WURtest\FarmmapsDataDownload\bin\Debug\netcoreapp3.1\Downloads\ + "CropYear": 2022, + "fieldName": "aardappelveld_test_Potato_Urea92kgNha", + "geometryJson": { + "type": "Polygon", + "coordinates": [ + [ + [ + 5.5945257993548765, + 52.57080744107003 + ], + [ + 5.598645994070678, + 52.571540800206236 + ], + [ + 5.599381743127071, + 52.57012773140724 + ], + [ + 5.595408698222548, + 52.56968054825188 + ], + [ + 5.5945257993548765, + 52.57080744107003 + ] + ] + ] + } + }, + { + "useExistingCropfieldWithChildren": false, + "deleteNewlyCreatedAfterCalc": true, + "CropfieldItemCode": "", + "dataCropfield": { + //"area": 4.22, //not needed for KPI calculation, but shown here to know this is a possible property + "final": true, //always true + //"soilCode": "5", //not needed for KPI calculation, but shown here to know this is a possible property + //"soilName": "Loam", //not needed for KPI calculation, but shown here to know this is a possible property + "cropTypeCode": "1010101", + "cropTypeName": "Potato", + //"rootDepthMax": 45, //not needed for KPI calculation, but shown here to know this is a possible property + //"emergenceDate": "2022-05-16T00:00:00", //not needed for KPI calculation, but shown here to know this is a possible property + "productionPurposeCode": "003" + //"productionPurposeName": "consumption" //not needed for KPI calculation, but shown here to know this is a possible property + }, + "CropRecordingItemCode": "", + "OperationItemCodes": [], + "dataOperations": [ + { + "area": "0.08", //?! + "contractor": false, + "designator": "Kunstmest strooien", + "from": "2022-05-23T11:34:00", + "method": "70400", + //"n": "92", + "name": "Kunstmest strooien", + "operationCode": "7", + "product": "7360", + "quantity": "200", + "status": "3", + "to": "2022-05-23T12:34:00", + "unit": "kg/ha", + "unitCode": "KGMHAR" + } + ], + "CropfieldCharacteristicItemCode": "", + "DataCropfieldCharacteristic": { + "code": "860619", //PO20231004: so what does this code mean? Can we see the code list somewhere? + "label": "cropyield", + "value": "48.01" + }, + //"DownloadFolder": "Downloads", //"C:\\hugoschrererdir\\kpidir\\", // "Downloads", -> if you just put "Downloads" the program will download to somewhere in ..\FarmMapsApiClient_WURtest\FarmmapsDataDownload\bin\Debug\netcoreapp3.1\Downloads\ + "CropYear": 2022, + "fieldName": "aardappelveld_test_Potato_Urea200kgha", + "geometryJson": { + "type": "Polygon", + "coordinates": [ + [ + [ + 5.5945257993548765, + 52.57080744107003 + ], + [ + 5.598645994070678, + 52.571540800206236 + ], + [ + 5.599381743127071, + 52.57012773140724 + ], + [ + 5.595408698222548, + 52.56968054825188 + ], + [ + 5.5945257993548765, + 52.57080744107003 + ] + ] + ] + } } - ] \ No newline at end of file diff --git a/FarmmapsKPI/KPIdefinitions.csv b/FarmmapsKPI/KPIdefinitions.csv index 5d31cae..b82cbc5 100644 --- a/FarmmapsKPI/KPIdefinitions.csv +++ b/FarmmapsKPI/KPIdefinitions.csv @@ -6,3 +6,4 @@ C1,organic matter supply,Organic matter surplus = Organic matter from manure + C D1,pesticides,? ,KPItargetvalue,target value as in benchmark value for same crop in same region ,KPIthresholdValue,threshold from ??? Ask farmmaps. Surplus nitrogen / phosphate / pesticides must not be above threshold. Surplus organic matter supply must be above threshold +,totalNapplied,"Not a KPI output, simply calculated from the input operations with their respective ""n"" in their ""data""" From d8c2ecb88bd47a3e2dd31cf8be3078d86b983182 Mon Sep 17 00:00:00 2001 From: Pepijn van Oort Date: Wed, 18 Oct 2023 17:32:38 +0200 Subject: [PATCH 23/66] new project to download latest version of editeelt codelists --- FarmmapsApiSamples.sln | 10 + FarmmapsDownloadCL/DownloadCLApplication.cs | 178 ++++++++++++++++++ FarmmapsDownloadCL/DownloadCLInput.json | 4 + FarmmapsDownloadCL/FarmmapsDownloadCL.csproj | 24 +++ FarmmapsDownloadCL/Models/CodelistsClasses.cs | 87 +++++++++ FarmmapsDownloadCL/Models/DownloadCLInput.cs | 12 ++ FarmmapsDownloadCL/Program.cs | 21 +++ 7 files changed, 336 insertions(+) create mode 100644 FarmmapsDownloadCL/DownloadCLApplication.cs create mode 100644 FarmmapsDownloadCL/DownloadCLInput.json create mode 100644 FarmmapsDownloadCL/FarmmapsDownloadCL.csproj create mode 100644 FarmmapsDownloadCL/Models/CodelistsClasses.cs create mode 100644 FarmmapsDownloadCL/Models/DownloadCLInput.cs create mode 100644 FarmmapsDownloadCL/Program.cs diff --git a/FarmmapsApiSamples.sln b/FarmmapsApiSamples.sln index 751b624..8f35ae1 100644 --- a/FarmmapsApiSamples.sln +++ b/FarmmapsApiSamples.sln @@ -33,6 +33,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FarmmapsCleanUp", "Farmmaps EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FarmmapsKPI", "FarmmapsKPI\FarmmapsKPI.csproj", "{14575235-9867-4CE5-A22F-3F9FE002FF42}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FarmmapsDownloadCL", "FarmmapsDownloadCL\FarmmapsDownloadCL.csproj", "{63E69101-D804-4DBC-B2E4-A33771CD5C5F}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -137,6 +139,14 @@ Global {14575235-9867-4CE5-A22F-3F9FE002FF42}.Release|Any CPU.Build.0 = Release|Any CPU {14575235-9867-4CE5-A22F-3F9FE002FF42}.Release|x64.ActiveCfg = Release|x64 {14575235-9867-4CE5-A22F-3F9FE002FF42}.Release|x64.Build.0 = Release|x64 + {63E69101-D804-4DBC-B2E4-A33771CD5C5F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {63E69101-D804-4DBC-B2E4-A33771CD5C5F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {63E69101-D804-4DBC-B2E4-A33771CD5C5F}.Debug|x64.ActiveCfg = Debug|x64 + {63E69101-D804-4DBC-B2E4-A33771CD5C5F}.Debug|x64.Build.0 = Debug|x64 + {63E69101-D804-4DBC-B2E4-A33771CD5C5F}.Release|Any CPU.ActiveCfg = Release|Any CPU + {63E69101-D804-4DBC-B2E4-A33771CD5C5F}.Release|Any CPU.Build.0 = Release|Any CPU + {63E69101-D804-4DBC-B2E4-A33771CD5C5F}.Release|x64.ActiveCfg = Release|x64 + {63E69101-D804-4DBC-B2E4-A33771CD5C5F}.Release|x64.Build.0 = Release|x64 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/FarmmapsDownloadCL/DownloadCLApplication.cs b/FarmmapsDownloadCL/DownloadCLApplication.cs new file mode 100644 index 0000000..9b53ea0 --- /dev/null +++ b/FarmmapsDownloadCL/DownloadCLApplication.cs @@ -0,0 +1,178 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Reflection; +using System.Threading.Tasks; +using FarmmapsApi; +using FarmmapsApi.Models; +using FarmmapsApi.Services; +using FarmmapsDownloadCL.Models; +using Microsoft.Extensions.Logging; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using static FarmmapsApiSamples.Constants; + +namespace FarmmapsDownloadCL +{ + public class DownloadCLApplication : IApplication + { + + private readonly ILogger _logger; + private readonly FarmmapsApiService _farmmapsApiService; + private readonly DownloadCLService _DownloadCLService; + private readonly GeneralService _generalService; + public readonly Dictionary> _dictCl; + string _itemcode; + public DownloadCLApplication(ILogger logger, FarmmapsApiService farmmapsApiService, + GeneralService generalService, DownloadCLService DownloadCLService) + { + _logger = logger; + _farmmapsApiService = farmmapsApiService; + _generalService = generalService; + _DownloadCLService = DownloadCLService; + CodelistsClasses clc = new CodelistsClasses(); + _itemcode = clc.itemcode; + _dictCl = clc.dictCl; + } + + public async Task RunAsync() + { + var fieldsInputJson = File.ReadAllText("DownloadCLInput.json"); + + + DownloadCLInput clInput = JsonConvert.DeserializeObject(fieldsInputJson); + + // !! this call is needed the first time an api is called with a fresh clientid and secret !! + await _farmmapsApiService.GetCurrentUserCodeAsync(); + var roots = await _farmmapsApiService.GetCurrentUserRootsAsync(); + + //Where to write the output + string downloadFolder = clInput.DownloadFolder; + if (string.IsNullOrEmpty(downloadFolder)) + { + downloadFolder = "Downloads"; + } + if (!Directory.Exists(downloadFolder)) + Directory.CreateDirectory(downloadFolder); + + //Get the most recent codelists + foreach (string codelist in clInput.codelists) + { + try + { + await Process(roots, codelist, downloadFolder); + } + catch (Exception ex) + { + _logger.LogError(ex.Message); + } + } + } + + private async Task Process(List roots, string codelistname, string downloadFolder) + { + List codelist; + string itemtype; + string className; + string body; + string header; + string value; + string[] aboutArray = null; + string[] headerArray = null; + PropertyInfo[] props; + PropertyInfo prop; + List dataList; + string clJson = Path.Combine(downloadFolder, $"{codelistname}.json"); + string clCsv = Path.Combine(downloadFolder, $"{codelistname}.csv"); + + try + { + itemtype = _dictCl[codelistname][0]; + className = _dictCl[codelistname][1]; + codelist = await _farmmapsApiService.GetItemChildrenAsync(_itemcode, itemtype); + + //Write full codelist in json format to clJson. + body = JsonConvert.SerializeObject(codelist); + File.WriteAllText(clJson, body); + _logger.LogInformation($"Downloaded file {clJson}"); + + //Write full list in csv format to clCsv. + StreamWriter sw = new StreamWriter(clCsv); + //Write metadata in top: when downloaded + sw.WriteLine($"Editeelt codelist {codelistname} downloaded on {DateTime.Now} with the FarmmapsDownloadCL application by Pepijn van Oort (WPR)"); + + //Generic, for any Codelist as long as it is also defined in FarmmapsDownloadCL.Models in + string typerequest = "FarmmapsDownloadCL.Models." + className + ", FarmmapsDownloadCL, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null"; + Type objType = Type.GetType(typerequest); + object clitem = Activator.CreateInstance(objType); + props = clitem.GetType().GetProperties(); + + //Lookup about info and write to top + foreach (PropertyInfo pi in props) + { + if (pi.Name == "about") + { + aboutArray = (string[])pi.GetValue(clitem); + foreach (string about in aboutArray) + { + //Add quotes + value = "\"" + about + "\""; + //Write each about to a separate line in the csv file + sw.WriteLine(value); + } + + } + } + //Write an empty line + sw.WriteLine(); + + //Look up the headerArray and write the header comma separated + int i = 0; + while (headerArray == null) + { + prop = props[i]; + if (prop.Name == "headers") + { + headerArray = (string[])prop.GetValue(clitem); + } + i++; + } + header = string.Join(",", headerArray); + sw.WriteLine(header); + + //Add quotes because a value for field 'description' could be '21% Cl, 16.5%CaO, 1330 g/l' and without the quotes that + //would show up as separate columns. So per definition we will add quotes, except if we have a fieldname in the listNotToQuote, then value for that fieldname will not be given quotes + //e.g. not code "3" but code 3. + List listNotToQuote = new List() { "code", "n", "p", "k", "quantity", "cropGroupCode", "cultivationGroupCode" }; + //Loop through all items in the codelist + foreach (Item item in codelist) + { + dataList = new List { }; + clitem = JsonConvert.DeserializeObject(item.Data.ToString(), objType); + + //Add values of the cLitem to the dataList in the same order as the header + foreach (string h in headerArray) + { + value = (string)clitem.GetType().GetProperty(h).GetValue(clitem, null); + if(listNotToQuote.Contains(h) == false) + value = "\"" +value + "\""; + + dataList.Add(value); + } + string dataLine = string.Join(",", dataList); + sw.WriteLine(string.Join(",", dataList)); + } + + sw.Close(); + _logger.LogInformation($"Downloaded file {clCsv}"); + + } + catch + { + _logger.LogWarning($"Missing lookup information on codelist {codelistname}"); + } + } + } +} diff --git a/FarmmapsDownloadCL/DownloadCLInput.json b/FarmmapsDownloadCL/DownloadCLInput.json new file mode 100644 index 0000000..8b5c9c6 --- /dev/null +++ b/FarmmapsDownloadCL/DownloadCLInput.json @@ -0,0 +1,4 @@ +{ + "codeLists": [ "CL022", "CL127", "CL263", "CL042" ] + //"DownloadFolder": "Downloads", //"C:\\hugoschrererdir\\kpidir\\", // "Downloads", -> if you just put "Downloads" the program will download to somewhere in ..\FarmMapsApiClient_WURtest\FarmmapsDataDownload\bin\Debug\netcoreapp3.1\Downloads\ +} \ No newline at end of file diff --git a/FarmmapsDownloadCL/FarmmapsDownloadCL.csproj b/FarmmapsDownloadCL/FarmmapsDownloadCL.csproj new file mode 100644 index 0000000..a4d4a51 --- /dev/null +++ b/FarmmapsDownloadCL/FarmmapsDownloadCL.csproj @@ -0,0 +1,24 @@ + + + + Exe + netcoreapp3.1 + x64 + AnyCPU;x64 + + + + + + + + + + + + + Always + + + + diff --git a/FarmmapsDownloadCL/Models/CodelistsClasses.cs b/FarmmapsDownloadCL/Models/CodelistsClasses.cs new file mode 100644 index 0000000..22f9c4a --- /dev/null +++ b/FarmmapsDownloadCL/Models/CodelistsClasses.cs @@ -0,0 +1,87 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace FarmmapsDownloadCL.Models +{ + public class CodelistsClasses + { + public CodelistsClasses() + { + this.itemcode = "938c0aec4efc46e88783fd029f17309e%3ASYSTEM_DATA"; + this.dictCl = new Dictionary>(); + //Add here once you have defined a new CLxxxitem below + //To add a new item, look it up in https://test.farmmaps.eu/swagger/index.html. Download the item you are interested in (like "vnd.farmmaps.itemtype.codelist.cl263") + //then below create a new CLxxxitem with fieldnames as you see them in the just downloaded new codelist + this.dictCl.Add("CL022", new List { "vnd.farmmaps.itemtype.codelist.cl022", "CL022item" }); + this.dictCl.Add("CL127", new List { "vnd.farmmaps.itemtype.codelist.cl127", "CL127item" }); + this.dictCl.Add("CL263", new List { "vnd.farmmaps.itemtype.codelist.cl263", "CL263item" }); + } + public string itemcode; + //Define here the codelist names, their item type and the class defining the contents of the data type (and defined in the Models.CodelistsClasses) + public Dictionary> dictCl; + } + + public class CL022item + { + public CL022item() + { + this.headers = new string[] { "codelist","code", "description", "type","k", "n","p","composition" }; + this.codelist = "CL022"; + this.about = new string[] { "EDI-Crop, coderingslijst meststoffen", "type MOR = organische meststof; MAN = anorganische meststof" }; + } + + public string[] headers { get; } + public string codelist { get; } + public string[] about { get; } + public string k { get; set; } + public string n { get; set; } + public string p { get; set; } + public string code { get; set; } + public string type { get; set; } + public string composition { get; set; } + public string description { get; set; } + + } + public class CL127item + { + public CL127item() + { + this.headers = new string[] { "codelist", "code", "description", "description_nl", "culturalPracticeLevel1", "culturalPracticeLevel2" }; + this.codelist = "CL127"; + } + + public string[] headers { get; } + public string codelist { get; } + public string code { get; set; } + public string description { get; set; } + public string description_nl { get; set; } + public string culturalPracticeLevel1 { get; set; } + public string culturalPracticeLevel2 { get; set; } + } + public class CL263item + { + public CL263item() + { + this.headers = new string[] { "codelist", "code", "eppoCode", "botanicName", "description", "description_fr", "description_nl", "cropGroupCode", "cropGroupName", "cultivationGroupCode", "cultivationGroupName" }; + this.codelist = "CL263"; + this.about = new string[] { "EDI-Crop, coderingslijst gewassoorten" }; + } + + public string[] headers { get; } + public string codelist { get; } + public string[] about { get; } + public string code { get; set; } + public string eppoCode { get; set; } + public string botanicName { get; set; } + public string description { get; set; } + public string description_nl { get; set; } + public string description_fr { get; set; } + public string cropGroupCode { get; set; } + public string cropGroupName { get; set; } + public string cultivationGroupCode { get; set; } + public string cultivationGroupName { get; set; } + public string composition { get; set; } + + } +} \ No newline at end of file diff --git a/FarmmapsDownloadCL/Models/DownloadCLInput.cs b/FarmmapsDownloadCL/Models/DownloadCLInput.cs new file mode 100644 index 0000000..a18acf6 --- /dev/null +++ b/FarmmapsDownloadCL/Models/DownloadCLInput.cs @@ -0,0 +1,12 @@ +using System; +using Newtonsoft.Json.Linq; + +namespace FarmmapsDownloadCL.Models +{ + public class DownloadCLInput + { + public string[] codelists { get; set; } + public string DownloadFolder { get; set; } + + } +} \ No newline at end of file diff --git a/FarmmapsDownloadCL/Program.cs b/FarmmapsDownloadCL/Program.cs new file mode 100644 index 0000000..0d2e87b --- /dev/null +++ b/FarmmapsDownloadCL/Program.cs @@ -0,0 +1,21 @@ +using System.Threading.Tasks; +using FarmmapsApi; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; + +namespace FarmmapsDownloadCL +{ + class Program : FarmmapsProgram + { + private static async Task Main(string[] args) + { + await new Program().Start(args); + } + + protected override void Configure(IServiceCollection serviceCollection) + { + serviceCollection.AddLogging() + .AddTransient(); + } + } +} From 546ce131e4537397eda7b55962155033ef6c9a76 Mon Sep 17 00:00:00 2001 From: Pepijn van Oort Date: Wed, 18 Oct 2023 17:33:32 +0200 Subject: [PATCH 24/66] also commit this file --- FarmmapsDownloadCL/DownloadCLService.cs | 27 +++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 FarmmapsDownloadCL/DownloadCLService.cs diff --git a/FarmmapsDownloadCL/DownloadCLService.cs b/FarmmapsDownloadCL/DownloadCLService.cs new file mode 100644 index 0000000..7441350 --- /dev/null +++ b/FarmmapsDownloadCL/DownloadCLService.cs @@ -0,0 +1,27 @@ +using System; +using System.Globalization; +using System.Threading.Tasks; +using FarmmapsApi.Models; +using FarmmapsApi.Services; +using FarmmapsDownloadCL.Models; +using Microsoft.Extensions.Logging; +using static FarmmapsApi.Extensions; +using static FarmmapsApiSamples.Constants; + +namespace FarmmapsDownloadCL +{ + public class DownloadCLService + { + private readonly ILogger _logger; + private readonly FarmmapsApiService _farmmapsApiService; + private readonly GeneralService _generalService; + + public DownloadCLService(ILogger logger, FarmmapsApiService farmmapsApiService, + GeneralService generalService) + { + _logger = logger; + _farmmapsApiService = farmmapsApiService; + _generalService = generalService; + } + } +} \ No newline at end of file From 90f285a0d1bb2e24143b3cc0241072368e215224 Mon Sep 17 00:00:00 2001 From: Pepijn van Oort Date: Wed, 18 Oct 2023 17:33:55 +0200 Subject: [PATCH 25/66] update todo list --- FarmmapsKPI/TODO.txt | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/FarmmapsKPI/TODO.txt b/FarmmapsKPI/TODO.txt index f434c24..cdc2529 100644 --- a/FarmmapsKPI/TODO.txt +++ b/FarmmapsKPI/TODO.txt @@ -9,14 +9,15 @@ b. Check of de operation data een element ii. Voeg die 184 toe aan de operation data c. Reken N surplus uit (KPI item id b1) -Nieuw project: FarmmapsEditeeltLists +Nieuw project: FarmmapsEditeeltLists DONE a. Why? Because i. This is (1) so that we can at any time retrieve the most recent up-to-date list ii. so that you can look up crops and fertilizer types and fill in KPIinput.json correctly -b. Download full list of items from vnd.farmmaps.itemtype.codelist.cl263 -c. Write them to a csv file -d. In any case following lists: 127 operations; 022 meststoffen (fertilizer products); 263 gewassen; 405 soilcode +b. Download full list of items from vnd.farmmaps.itemtype.codelist.cl263 DONE +c. Write them to a csv file DONE +d. In any case following lists: 127 operations DONE; 022 meststoffen (fertilizer products) DONE; 263 gewassen DONE; 405 soilcode NOT YET + in which list the productionPurposeCode? Testing: in json input data, can certain fields be left out? Which are minimum fields needed? -Testing: effect of loation (e.g. different targetValue, different N deposition?) +Testing: effect of location (e.g. different targetValue, different N deposition? Different crop N contents depending on soil?) Testing: productionPurposeCode: different result depending on starch or consumption potato (different N contents in tubers; different targetValues? From 7f3b87ae387457cc17191135adc1a42b015297b0 Mon Sep 17 00:00:00 2001 From: Pepijn van Oort Date: Wed, 18 Oct 2023 17:35:34 +0200 Subject: [PATCH 26/66] and of course also add & commit this one ... --- FarmmapsKPI/Models/KPIoutput.cs | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 FarmmapsKPI/Models/KPIoutput.cs diff --git a/FarmmapsKPI/Models/KPIoutput.cs b/FarmmapsKPI/Models/KPIoutput.cs new file mode 100644 index 0000000..5cf2acd --- /dev/null +++ b/FarmmapsKPI/Models/KPIoutput.cs @@ -0,0 +1,25 @@ +using System; +using Newtonsoft.Json.Linq; + +namespace FarmmapsKPI.Models +{ + public class KPIOutput + { + public string id { get; set; } + public KPIOutputData data { get; set; } + public string type { get; set; } + public string unit { get; set; } + public string value { get; set; } + public string duration { get; set; } + public string quantity { get; set; } + public string parentName { get; set; } + public string targetValue { get; set; } + public string thresholdValue { get; set; } + } + public class KPIOutputData + { + public string area { get; set; } + public string cropTypeCode { get; set; } + public string cropTypeName { get; set; } + } +} \ No newline at end of file From 6eb36b2b4f897097460231a97d2ab249c8d3a6d0 Mon Sep 17 00:00:00 2001 From: Pepijn van Oort Date: Mon, 23 Oct 2023 12:48:28 +0200 Subject: [PATCH 27/66] remove author name from code --- FarmmapsDownloadCL/DownloadCLApplication.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/FarmmapsDownloadCL/DownloadCLApplication.cs b/FarmmapsDownloadCL/DownloadCLApplication.cs index 9b53ea0..01e8d5f 100644 --- a/FarmmapsDownloadCL/DownloadCLApplication.cs +++ b/FarmmapsDownloadCL/DownloadCLApplication.cs @@ -101,7 +101,7 @@ namespace FarmmapsDownloadCL //Write full list in csv format to clCsv. StreamWriter sw = new StreamWriter(clCsv); //Write metadata in top: when downloaded - sw.WriteLine($"Editeelt codelist {codelistname} downloaded on {DateTime.Now} with the FarmmapsDownloadCL application by Pepijn van Oort (WPR)"); + sw.WriteLine($"Editeelt codelist {codelistname} downloaded on {DateTime.Now} with the FarmmapsDownloadCL application"); //Generic, for any Codelist as long as it is also defined in FarmmapsDownloadCL.Models in string typerequest = "FarmmapsDownloadCL.Models." + className + ", FarmmapsDownloadCL, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null"; From 6d903c3e9be91a5d630c132e4511febb0d56059c Mon Sep 17 00:00:00 2001 From: Pepijn van Oort Date: Tue, 24 Oct 2023 13:06:36 +0200 Subject: [PATCH 28/66] error catching and console input to ask which KPIinput file to use --- FarmmapsApi/Services/GeneralService.cs | 19 ---------- FarmmapsKPI/KPIApplication.cs | 50 +++++++++++++++++--------- FarmmapsKPI/KPIService.cs | 12 ------- 3 files changed, 34 insertions(+), 47 deletions(-) diff --git a/FarmmapsApi/Services/GeneralService.cs b/FarmmapsApi/Services/GeneralService.cs index d95380c..6b452a0 100644 --- a/FarmmapsApi/Services/GeneralService.cs +++ b/FarmmapsApi/Services/GeneralService.cs @@ -44,25 +44,6 @@ namespace FarmmapsApi.Services var currentYear = new DateTime(year, 1, 1); JObject jdata = JObject.Parse(data); string name = string.Format($"CrpRec Operation, {jdata.GetValue("name")}"); - int code022; - string type022; - double quantity; - double applied_kgNha; - double ncontent = 0.0; // for now just any value - - //Is it a fertilizer application? - - //If the operation contains an element "n" then nothing, use that value (kg N/ha administred) - //Else: look up the N content for the code022, calculate "n" based on fertilizer amount (data) & content (cl022) and add applied_kgNha to the jdata - if (jdata.ContainsKey("n") == false) - { - quantity = jdata.GetValue("quantity").ToObject(); - code022 = jdata.GetValue("product").ToObject(); - //TODO: Now here look up this code022 in the cl022 and get the ncontent from that list. - //And check the unit in which the ncontent is expressed, e.g. % or kg/ton and check if it is not null - applied_kgNha = quantity * ncontent; - jdata.Add("n", applied_kgNha.ToString()); //all Data elements in Farmmaps code ar strings - }; ItemRequest operationItemRequest = new ItemRequest() { diff --git a/FarmmapsKPI/KPIApplication.cs b/FarmmapsKPI/KPIApplication.cs index 5fec1ad..91f0a9e 100644 --- a/FarmmapsKPI/KPIApplication.cs +++ b/FarmmapsKPI/KPIApplication.cs @@ -38,7 +38,15 @@ namespace FarmmapsKPI public async Task RunAsync() { - var fieldsInputJson = File.ReadAllText("KPIInput.json"); + // var fieldsInputJson = File.ReadAllText("KPIInputChemie.json"); // hier gebleven, bad gateway?! + string fnKPIinput; + + Console.WriteLine("Type name of input json file. Example: KPIinput.json (in same directory as FarmmapsKPI.exe) or C:/temp/KPIinputChemieTmp.json"); + fnKPIinput = Console.ReadLine(); + if (string.IsNullOrEmpty(fnKPIinput)) + fnKPIinput = "KPIinput.json"; + + var fieldsInputJson = File.ReadAllText(fnKPIinput); List fieldsInputs = JsonConvert.DeserializeObject>(fieldsInputJson); @@ -55,8 +63,11 @@ namespace FarmmapsKPI if (!Directory.Exists(downloadFolder)) Directory.CreateDirectory(downloadFolder); + //Write the same info to a single csv file. Note this means existing file will be overwritten! - StreamWriter sw; string KPIItemPathCsv = Path.Combine(downloadFolder, "KPIItems.csv"); + StreamWriter sw; + string KPIItemCsv = Path.GetFileNameWithoutExtension(fnKPIinput) + "_Items.csv"; + string KPIItemPathCsv = Path.Combine(downloadFolder, KPIItemCsv); List headerList = new List { "parentName", "area_ha", "cropTypeCode", "cropTypeName", "KPIid", "KPIvariable", "KPIvalue", "KPIunit", "KPItargetvalue", "KPIthresholdValue" }; //Create a new csv file. Means if existing then overwritten !!! sw = new StreamWriter(KPIItemPathCsv); @@ -271,19 +282,22 @@ namespace FarmmapsKPI dataList = new List { }; kpio = JsonConvert.DeserializeObject(item.Data.ToString()); //Seems sometimes duplicate KPI items are returned. So check that here and only write if this kpio is different from previous - if (kpio.id != kpioPrevious.id) + if (kpio.id != null) { - dataList.Add(kpio.parentName); - dataList.Add(kpio.data.area); - dataList.Add(kpio.data.cropTypeCode); - dataList.Add(kpio.data.cropTypeName); - dataList.Add(kpio.id); - dataList.Add(kpio.quantity); - dataList.Add(kpio.value); - dataList.Add(kpio.unit); - dataList.Add(kpio.targetValue); - dataList.Add(kpio.thresholdValue); - sw.WriteLine(string.Join(",", dataList)); + if (kpio.id != kpioPrevious.id) + { + dataList.Add(kpio.parentName); + dataList.Add(kpio.data.area); + dataList.Add(kpio.data.cropTypeCode); + dataList.Add(kpio.data.cropTypeName); + dataList.Add(kpio.id); + dataList.Add(kpio.quantity); + dataList.Add(kpio.value); + dataList.Add(kpio.unit); + dataList.Add(kpio.targetValue); + dataList.Add(kpio.thresholdValue); + sw.WriteLine(string.Join(",", dataList)); + } } kpioPrevious = kpio; } @@ -296,8 +310,12 @@ namespace FarmmapsKPI { codeOperation = crpOperationItemCodes[i]; crpOperationItem = await _farmmapsApiService.GetItemAsync(codeOperation); - operationNapplied = crpOperationItem.Data.GetValue("n").ToObject(); - totalNapplied = totalNapplied + operationNapplied; + dynamic data = JObject.Parse(crpOperationItem.Data.ToString(Formatting.None)); + if (data.n != null) + { + operationNapplied = crpOperationItem.Data.GetValue("n").ToObject(); + totalNapplied = totalNapplied + operationNapplied; + } } //Also add totalNapplied to the csv dataList = new List { }; diff --git a/FarmmapsKPI/KPIService.cs b/FarmmapsKPI/KPIService.cs index 846bce7..4e0614c 100644 --- a/FarmmapsKPI/KPIService.cs +++ b/FarmmapsKPI/KPIService.cs @@ -23,17 +23,5 @@ namespace FarmmapsKPI _farmmapsApiService = farmmapsApiService; _generalService = generalService; } - // zoiets kan ik overwegen als ik de itemtype niet kan vinden - //public async Task CreateTargetKPIItem(Item cropfieldItem) - //{ - // var itemRequest = new ItemRequest() - // { - // ParentCode = cropfieldItem.ParentCode, - // ItemType = USERINPUT_ITEMTYPE, - // Name = "kpi" - // }; - // return await _farmmapsApiService.CreateItemAsync(itemRequest); - //} - } } \ No newline at end of file From 847ff75845575a2e95386b3d44b58d9a30f70d4d Mon Sep 17 00:00:00 2001 From: Pepijn van Oort Date: Tue, 24 Oct 2023 13:47:58 +0200 Subject: [PATCH 29/66] download codelist CL405, soil types --- FarmmapsDownloadCL/DownloadCLInput.json | 2 +- FarmmapsDownloadCL/Models/CodelistsClasses.cs | 22 ++++++++++++++++++- FarmmapsKPI/TODO.txt | 10 ++++----- Secrets/appsettings.json | 20 ++++++++--------- 4 files changed, 37 insertions(+), 17 deletions(-) diff --git a/FarmmapsDownloadCL/DownloadCLInput.json b/FarmmapsDownloadCL/DownloadCLInput.json index 8b5c9c6..0cfa53f 100644 --- a/FarmmapsDownloadCL/DownloadCLInput.json +++ b/FarmmapsDownloadCL/DownloadCLInput.json @@ -1,4 +1,4 @@ { - "codeLists": [ "CL022", "CL127", "CL263", "CL042" ] + "codeLists": [ "CL022", "CL127", "CL263", "CL405", "CL042" ] //"DownloadFolder": "Downloads", //"C:\\hugoschrererdir\\kpidir\\", // "Downloads", -> if you just put "Downloads" the program will download to somewhere in ..\FarmMapsApiClient_WURtest\FarmmapsDataDownload\bin\Debug\netcoreapp3.1\Downloads\ } \ No newline at end of file diff --git a/FarmmapsDownloadCL/Models/CodelistsClasses.cs b/FarmmapsDownloadCL/Models/CodelistsClasses.cs index 22f9c4a..b53615a 100644 --- a/FarmmapsDownloadCL/Models/CodelistsClasses.cs +++ b/FarmmapsDownloadCL/Models/CodelistsClasses.cs @@ -16,6 +16,7 @@ namespace FarmmapsDownloadCL.Models this.dictCl.Add("CL022", new List { "vnd.farmmaps.itemtype.codelist.cl022", "CL022item" }); this.dictCl.Add("CL127", new List { "vnd.farmmaps.itemtype.codelist.cl127", "CL127item" }); this.dictCl.Add("CL263", new List { "vnd.farmmaps.itemtype.codelist.cl263", "CL263item" }); + this.dictCl.Add("CL405", new List { "vnd.farmmaps.itemtype.codelist.cl405", "CL405item" }); } public string itemcode; //Define here the codelist names, their item type and the class defining the contents of the data type (and defined in the Models.CodelistsClasses) @@ -26,7 +27,7 @@ namespace FarmmapsDownloadCL.Models { public CL022item() { - this.headers = new string[] { "codelist","code", "description", "type","k", "n","p","composition" }; + this.headers = new string[] { "codelist", "code", "description", "type", "n", "p", "k", "composition" }; this.codelist = "CL022"; this.about = new string[] { "EDI-Crop, coderingslijst meststoffen", "type MOR = organische meststof; MAN = anorganische meststof" }; } @@ -84,4 +85,23 @@ namespace FarmmapsDownloadCL.Models public string composition { get; set; } } + public class CL405item + { + public CL405item() + { + this.headers = new string[] { "codelist", "code", "description", "description_fr", "description_nl", "hoofdgroep" }; + this.codelist = "CL405"; + this.about = new string[] { }; + } + + public string[] headers { get; } + public string codelist { get; } + public string[] about { get; } + public string code { get; set; } + public string hoofdgroep { get; set; } + public string description { get; set; } + public string description_fr { get; set; } + public string description_nl { get; set; } + + } } \ No newline at end of file diff --git a/FarmmapsKPI/TODO.txt b/FarmmapsKPI/TODO.txt index cdc2529..e51c71f 100644 --- a/FarmmapsKPI/TODO.txt +++ b/FarmmapsKPI/TODO.txt @@ -15,9 +15,9 @@ a. Why? Because ii. so that you can look up crops and fertilizer types and fill in KPIinput.json correctly b. Download full list of items from vnd.farmmaps.itemtype.codelist.cl263 DONE c. Write them to a csv file DONE -d. In any case following lists: 127 operations DONE; 022 meststoffen (fertilizer products) DONE; 263 gewassen DONE; 405 soilcode NOT YET - in which list the productionPurposeCode? +d. In any case following lists: 127 operations DONE; 022 meststoffen (fertilizer products) DONE; 263 gewassen DONE; 405 soilcode DONE + in which list the productionPurposeCode NOT YET? -Testing: in json input data, can certain fields be left out? Which are minimum fields needed? -Testing: effect of location (e.g. different targetValue, different N deposition? Different crop N contents depending on soil?) -Testing: productionPurposeCode: different result depending on starch or consumption potato (different N contents in tubers; different targetValues? +Testing: in json input data, can certain fields be left out? Which are minimum fields needed? NOT YET +Testing: effect of location (e.g. different targetValue, different N deposition? Different crop N contents depending on soil?) NOT YET +Testing: productionPurposeCode: different result depending on starch or consumption potato (different N contents in tubers; different targetValues? NOT YET diff --git a/Secrets/appsettings.json b/Secrets/appsettings.json index 2aff27f..f224b09 100644 --- a/Secrets/appsettings.json +++ b/Secrets/appsettings.json @@ -2,11 +2,11 @@ //There are three farmmaps environments, uncomment the environemnt you want to use with this sample client //test environment - "Authority": "https://accounts.test.farmmaps.eu/", - "Endpoint": "https://test.farmmaps.eu/", - "BasePath": "api/v1", - "DiscoveryEndpointUrl": "https://accounts.test.farmmaps.eu/.well-known/openid-configuration", - "GrantClientId": "farmmapstesteu", + //"Authority": "https://accounts.test.farmmaps.eu/", + //"Endpoint": "https://test.farmmaps.eu/", + //"BasePath": "api/v1", + //"DiscoveryEndpointUrl": "https://accounts.test.farmmaps.eu/.well-known/openid-configuration", + //"GrantClientId": "farmmapstesteu", ////acceptance environment //"Authority": "https://accounts.acc.farmmaps.eu/", @@ -16,11 +16,11 @@ //"GrantClientId": "farmmapsacceu", ////production environment - //"authority": "https://accounts.farmmaps.eu/", - //"endpoint": "https://farmmaps.eu/", - //"basepath": "api/v1", - //"discoveryendpointurl": "https://accounts.farmmaps.eu/.well-known/openid-configuration", - //"GrantClientId": "farmmaps", + "authority": "https://accounts.farmmaps.eu/", + "endpoint": "https://farmmaps.eu/", + "basepath": "api/v1", + "discoveryendpointurl": "https://accounts.farmmaps.eu/.well-known/openid-configuration", + "GrantClientId": "farmmaps", //overige info "RedirectUri": "http://example.nl/api", From da808362c30ff16c44418a56fe73f64b83949172 Mon Sep 17 00:00:00 2001 From: Pepijn van Oort Date: Wed, 1 Nov 2023 12:32:05 +0100 Subject: [PATCH 30/66] Added codelist CL251: Codering teeltdoelen (CropProductionPurposeCode) --- FarmmapsDownloadCL/DownloadCLInput.json | 2 +- FarmmapsDownloadCL/Models/CodelistsClasses.cs | 22 +++++++++++++++++++ FarmmapsKPI/Models/CropRecordings.cs | 10 --------- 3 files changed, 23 insertions(+), 11 deletions(-) delete mode 100644 FarmmapsKPI/Models/CropRecordings.cs diff --git a/FarmmapsDownloadCL/DownloadCLInput.json b/FarmmapsDownloadCL/DownloadCLInput.json index 0cfa53f..be43c9c 100644 --- a/FarmmapsDownloadCL/DownloadCLInput.json +++ b/FarmmapsDownloadCL/DownloadCLInput.json @@ -1,4 +1,4 @@ { - "codeLists": [ "CL022", "CL127", "CL263", "CL405", "CL042" ] + "codeLists": [ "CL022", "CL127", "CL263", "CL405", "CL251", "CL042" ] //"DownloadFolder": "Downloads", //"C:\\hugoschrererdir\\kpidir\\", // "Downloads", -> if you just put "Downloads" the program will download to somewhere in ..\FarmMapsApiClient_WURtest\FarmmapsDataDownload\bin\Debug\netcoreapp3.1\Downloads\ } \ No newline at end of file diff --git a/FarmmapsDownloadCL/Models/CodelistsClasses.cs b/FarmmapsDownloadCL/Models/CodelistsClasses.cs index b53615a..66f2cbe 100644 --- a/FarmmapsDownloadCL/Models/CodelistsClasses.cs +++ b/FarmmapsDownloadCL/Models/CodelistsClasses.cs @@ -15,6 +15,7 @@ namespace FarmmapsDownloadCL.Models //then below create a new CLxxxitem with fieldnames as you see them in the just downloaded new codelist this.dictCl.Add("CL022", new List { "vnd.farmmaps.itemtype.codelist.cl022", "CL022item" }); this.dictCl.Add("CL127", new List { "vnd.farmmaps.itemtype.codelist.cl127", "CL127item" }); + this.dictCl.Add("CL251", new List { "vnd.farmmaps.itemtype.codelist.cl251", "CL251item" }); this.dictCl.Add("CL263", new List { "vnd.farmmaps.itemtype.codelist.cl263", "CL263item" }); this.dictCl.Add("CL405", new List { "vnd.farmmaps.itemtype.codelist.cl405", "CL405item" }); } @@ -60,6 +61,27 @@ namespace FarmmapsDownloadCL.Models public string culturalPracticeLevel1 { get; set; } public string culturalPracticeLevel2 { get; set; } } + public class CL251item + { + public CL251item() + { + this.headers = new string[] { "codelist", "code", "rvo", "active", "primary", "description", "description_fr", "description_nl" }; + this.codelist = "CL251"; + this.about = new string[] { "Codering teeltdoelen (CropProductionPurposeCode)", "primary: 1 = primary; 2 = secondary", "rvo: 1 = yes; 0 = no" }; + } + + public string[] headers { get; } + public string codelist { get; } + public string[] about { get; } + public string code { get; set; } + public string rvo { get; set; } + public string active { get; set; } + public string primary { get; set; } + public string description { get; set; } + public string description_nl { get; set; } + public string description_fr { get; set; } + + } public class CL263item { public CL263item() diff --git a/FarmmapsKPI/Models/CropRecordings.cs b/FarmmapsKPI/Models/CropRecordings.cs deleted file mode 100644 index 61e667f..0000000 --- a/FarmmapsKPI/Models/CropRecordings.cs +++ /dev/null @@ -1,10 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Text; - -namespace FarmmapsKPI.Models -{ - internal class CropRecordings - { - } -} From cfba86cf751eb9cc36bdf08ba7f5326fc3e4d437 Mon Sep 17 00:00:00 2001 From: Pepijn van Oort Date: Wed, 1 Nov 2023 13:02:54 +0100 Subject: [PATCH 31/66] Get polygon area from geometry and give to operations if operation was on whole field Time tracker for how much time the KPI calculations take --- FarmmapsKPI/KPIApplication.cs | 153 ++++++++++++++++++++++------------ 1 file changed, 102 insertions(+), 51 deletions(-) diff --git a/FarmmapsKPI/KPIApplication.cs b/FarmmapsKPI/KPIApplication.cs index 91f0a9e..a64f442 100644 --- a/FarmmapsKPI/KPIApplication.cs +++ b/FarmmapsKPI/KPIApplication.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.IO; using System.IO.Compression; using System.Linq; @@ -38,13 +39,14 @@ namespace FarmmapsKPI public async Task RunAsync() { - // var fieldsInputJson = File.ReadAllText("KPIInputChemie.json"); // hier gebleven, bad gateway?! + KPIInput input; string fnKPIinput; - Console.WriteLine("Type name of input json file. Example: KPIinput.json (in same directory as FarmmapsKPI.exe) or C:/temp/KPIinputChemieTmp.json"); - fnKPIinput = Console.ReadLine(); - if (string.IsNullOrEmpty(fnKPIinput)) - fnKPIinput = "KPIinput.json"; + //Console.WriteLine("Type name of input json file. Example: KPIinput.json (in same directory as FarmmapsKPI.exe) or C:/temp/KPIinputChemieTmp.json"); + //fnKPIinput = Console.ReadLine(); + //if (string.IsNullOrEmpty(fnKPIinput)) + + fnKPIinput = "KPIinput.json"; var fieldsInputJson = File.ReadAllText(fnKPIinput); @@ -71,11 +73,21 @@ namespace FarmmapsKPI List headerList = new List { "parentName", "area_ha", "cropTypeCode", "cropTypeName", "KPIid", "KPIvariable", "KPIvalue", "KPIunit", "KPItargetvalue", "KPIthresholdValue" }; //Create a new csv file. Means if existing then overwritten !!! sw = new StreamWriter(KPIItemPathCsv); + sw.WriteLine($"FarmmapsKPI backend calculations on input file '{fnKPIinput}' downloaded on {DateTime.Now} with the FarmmapsKPI application in the FarmmapsApSamples.sln"); + sw.WriteLine(); sw.WriteLine(string.Join(",", headerList)); - //Now loop through the list of cropfields in KPIinput.json and get the KPI's for each cropfield - foreach (var input in fieldsInputs) + // For each input download all KPI's. Keep track to time, important when doing bulk calculations + var watch = System.Diagnostics.Stopwatch.StartNew(); + TimeSpan tsSofar = new TimeSpan(); + TimeSpan tsRemaining; + TimeSpan tsTotalEstimated; + + for (int i = 0; i < fieldsInputs.Count; i++) { + watch.Restart(); + input = fieldsInputs[i]; + _logger.LogInformation(string.Format($"// FarmmapsKPI: Downloading KPI's for field {i + 1} out of {fieldsInputs.Count} to single csv file {KPIItemPathCsv}")); try { await Process(roots, input, sw); @@ -84,11 +96,16 @@ namespace FarmmapsKPI { _logger.LogError(ex.Message); } + watch.Stop(); + tsSofar = tsSofar + watch.Elapsed; + tsTotalEstimated = tsSofar / (i + 1) * fieldsInputs.Count; + tsRemaining = tsTotalEstimated - tsSofar; + _logger.LogInformation(string.Format($"// Time (hh:mm:ss): this field: {strTime(watch.Elapsed)}. Sofar: {strTime(tsSofar)}. Estimated total: {strTime(tsTotalEstimated)}. Remaining: {strTime(tsRemaining)}")); } - //Close the csv file, write message to screen sw.Close(); - _logger.LogInformation($"Done! Written all KPI for all fields in 'KPIinput.json' to output file '{KPIItemPathCsv}'"); + _logger.LogInformation(string.Format($"// FarmmapsKPI:")); + _logger.LogInformation($"Done! Written all KPI for all fields in '{fnKPIinput}' to output file '{KPIItemPathCsv}'"); } private async Task Process(List roots, KPIInput input, StreamWriter sw) @@ -134,7 +151,7 @@ namespace FarmmapsKPI //3 useExistingCropfieldWithChildren = true && input.CropfieldItemCode like "deb48a74c5b54299bb852f17288010e9" in KPIinput -> use this one if (useExistingCropfieldWithChildren == false) { - _logger.LogInformation("Creating cropfield, writting to settings file"); + _logger.LogInformation($"Creating cropfield with name '{fieldName}'"); cropfieldItem = await _generalService.CreateCropfieldItemAsync(myDriveRoot.Code, $"{fieldName}", cropYear, input.GeometryJson.ToString(Formatting.None), input.DataCropfield.ToString(Formatting.None)); _settings.CropfieldItemCode = cropfieldItem.Code; @@ -147,13 +164,49 @@ namespace FarmmapsKPI } else { - _logger.LogInformation("reading CropfieldItemCode from KPIinput.json"); + _logger.LogInformation("reading CropfieldItemCode from KPIinput json"); cropfieldItem = await _farmmapsApiService.GetItemAsync(input.CropfieldItemCode); SaveSettings(settingsfile); } + // The cropfieldCharacteristicItem is used to enter crop yields + Item cropfieldCharacteristicItem; + //1 useExistingCropfieldWithChildren = false -> create new + //2 useExistingCropfieldWithChildren = true && input.CropfieldCharacteristicItemCode = "" or absent -> read from settings json + //3 useExistingCropfieldWithChildren = true && input.CropfieldCharacteristicItemCode like "deb48a74c5b54299bb852f17288010e9" in KPIinput -> use this one + if (useExistingCropfieldWithChildren == false) + { + _logger.LogInformation("CreateCropfieldCharacteristicItemAsync ..."); + cropfieldCharacteristicItem = await _generalService.CreateCropfieldCharacteristicItemAsync(cropfieldItem.Code, cropYear, fieldGeom, input.DataCropfieldCharacteristic.ToString(Formatting.None)); + _settings.CropfieldCharacteristicItemCode = cropfieldCharacteristicItem.Code; + SaveSettings(settingsfile); + } + else if (string.IsNullOrEmpty(input.CropfieldItemCode)) + { + _logger.LogInformation("reading OperationItemCode from settings file"); + cropfieldCharacteristicItem = await _farmmapsApiService.GetItemAsync(_settings.CropfieldCharacteristicItemCode); + } + else + { + _logger.LogInformation("reading CropfieldCharacteristicItemCode from KPIinput json"); + cropfieldCharacteristicItem = await _farmmapsApiService.GetItemAsync(input.CropfieldCharacteristicItemCode); + SaveSettings(settingsfile); + } - // Use already created croprecording or create new one, added a Data input, with field specific data for the KPI calculation + // Now we can do a first KPI calculation and get the polygon area from the KPI output (where it is based on geometry) + // We need that because for operations, you need to provide the area on which the operation was applied + // And if we put that to the crop area, then we neatly get everything on a per ha basis + _logger.LogInformation($"Getting polygon area (ha))"); + List KPIItems = await _generalService.GetKpiItemsForCropField(cropfieldItem); + _logger.LogInformation($"Found {KPIItems.Count} KPI items"); + kpio = JsonConvert.DeserializeObject(KPIItems[0].Data.ToString()); + string area_ha = kpio.data.area; + // turn the area into a JObject for later merging with operation data; + string strJarea = JsonConvert.SerializeObject(new { area = area_ha }); + JObject Jarea = JObject.Parse(strJarea); + + // A cropfield has 1 crop recording and the crop recording has 0:many operations + //So first at the crop recording Item crprecItem; //1 useExistingCropfieldWithChildren = false -> create new //2 useExistingCropfieldWithChildren = true && input.CropRecordingItemCode = "" or absent -> read from settings json @@ -172,12 +225,12 @@ namespace FarmmapsKPI } else { - _logger.LogInformation("reading CropRecordingItemCode from KPIinput.json"); + _logger.LogInformation("reading CropRecordingItemCode from KPIinput json"); crprecItem = await _farmmapsApiService.GetItemAsync(input.CropRecordingItemCode); SaveSettings(settingsfile); } - // Use already created operations or create new one,s added a Data input, with field specific data for the KPI calculation + // Now add the operations List crpOperationItems = new List { }; List crpOperationItemCodes = new List { }; Item crpOperationItem; @@ -192,7 +245,30 @@ namespace FarmmapsKPI { dataOperation = input.DataOperations[i].ToString(Formatting.None); dynamic data = JObject.Parse(dataOperation); - _logger.LogInformation($"CreateOperationItemAsync ... for operation {i}: '{data.name}', {data.n} kg N/ha, on date '{data.from}'"); + _logger.LogInformation($"CreateOperationItemAsync ... for operation {i}: '{data.name}', on date '{data.from}'"); + // Now check if the operation has a field called area + string? opArea = data["area"]; + if (string.IsNullOrEmpty(opArea)) + { + // if not having field area: add it to dataOperation ... + input.DataOperations[i].Merge(Jarea); + // ... and update the string + dataOperation = input.DataOperations[i].ToString(Formatting.None); + } + else + { + // if yes having field area: compare with polygon area. If not the same, throw warning + if (data.area != area_ha) + { + double differencePercent = 100.0*(Convert.ToDouble(area_ha) / Convert.ToDouble(data.area) - 1.0); + _logger.LogWarning($"cropfield has area {area_ha}, but operation has area {data.area}" + + $" Difference is {area_ha} / {data.area} - 100% = {differencePercent}%." + + $" Is that correct? Example if operation was applied in part of field, e.g. in case of variable rate (VRA) application." + + $" Or did you accidentally fill in area in the KPIinput json? To use cropfield area, omit field 'area' from json" + + $" then the KPI applicataion will fill in area calculated from geometry"); + } + } + //Now after optionally adding the area, add the Operation to the crop recording crpOperationItem = await _generalService.CreateOperationItemAsync(crprecItem.Code, cropYear, fieldGeom, dataOperation); crpOperationItems.Add(crpOperationItem); crpOperationItemCodes.Add(crpOperationItem.Code); @@ -213,7 +289,7 @@ namespace FarmmapsKPI } else { - _logger.LogInformation("reading OperationItemCodes from KPIinput.json"); + _logger.LogInformation("reading OperationItemCodes from KPIinput json"); for (int i = 0; i < input.OperationItemCodes.Length; i++) { codeOperation = input.OperationItemCodes[i]; @@ -225,35 +301,8 @@ namespace FarmmapsKPI SaveSettings(settingsfile); } - // The cropfieldCharacteristicItem is used to enter crop yields - // So once we have added an operation for fertilizer application and a crop yield, then KPIapp can calculate - // Nutrient balance. - // Use already created cropfieldCharacteristicItem or create new one, added a Data input, with field specific data for the KPI calculation - Item cropfieldCharacteristicItem; - //1 useExistingCropfieldWithChildren = false -> create new - //2 useExistingCropfieldWithChildren = true && input.CropfieldCharacteristicItemCode = "" or absent -> read from settings json - //3 useExistingCropfieldWithChildren = true && input.CropfieldCharacteristicItemCode like "deb48a74c5b54299bb852f17288010e9" in KPIinput -> use this one - if (useExistingCropfieldWithChildren == false) - { - _logger.LogInformation("CreateCropfieldCharacteristicItemAsync ..."); - cropfieldCharacteristicItem = await _generalService.CreateCropfieldCharacteristicItemAsync(cropfieldItem.Code, cropYear, fieldGeom, input.DataCropfieldCharacteristic.ToString(Formatting.None)); - _settings.CropfieldCharacteristicItemCode = cropfieldCharacteristicItem.Code; - SaveSettings(settingsfile); - } - else if (string.IsNullOrEmpty(input.CropfieldItemCode)) - { - _logger.LogInformation("reading OperationItemCode from settings file"); - cropfieldCharacteristicItem = await _farmmapsApiService.GetItemAsync(_settings.CropfieldCharacteristicItemCode); - } - else - { - _logger.LogInformation("reading CropfieldCharacteristicItemCode from KPIinput.json"); - cropfieldCharacteristicItem = await _farmmapsApiService.GetItemAsync(input.CropfieldCharacteristicItemCode); - SaveSettings(settingsfile); - } - // Inspect the children and grandchildren. If all is well, cropfield will have: - // one crprec, with 0-many operations as children. And the Data of an operation will have specification of how much fertilizer was applied + // one crprec, with 0-many operations as children. And the Data of an operation will have specification of how much fertilizer / cropprotection agent was applied // one edicrop.characteristic (with yield in the data) var cropfieldChildren = await _farmmapsApiService.GetItemChildrenAsync(cropfieldItem.Code); var crprecChildren = await _farmmapsApiService.GetItemChildrenAsync(crprecItem.Code); @@ -261,7 +310,7 @@ namespace FarmmapsKPI //Now get the KPIs for this cropfield, mounted with operations & cropyield // Get KPI data for saving it in a file, here the generalsedrvice is called to get the KPI data _logger.LogInformation($"GetKpiItemsForCropField('{cropfieldItem.Code}')"); - List KPIItems = await _generalService.GetKpiItemsForCropField(cropfieldItem); + KPIItems = await _generalService.GetKpiItemsForCropField(cropfieldItem); _logger.LogInformation($"Found {KPIItems.Count} KPI items"); //Download KPI's into a json output file for this specific cropfield (with unique cropfieldItem.Code) @@ -305,7 +354,6 @@ namespace FarmmapsKPI //Total N applied double totalNapplied = 0.0; double operationNapplied; - JObject opData; for (int i = 0; i < crpOperationItemCodes.Count; i++) { codeOperation = crpOperationItemCodes[i]; @@ -321,13 +369,13 @@ namespace FarmmapsKPI dataList = new List { }; //Seems sometimes duplicate KPI items are returned. So check that here and only write if this kpio is different from previous dataList.Add(kpioPrevious.parentName); - dataList.Add(kpioPrevious.data.area); - dataList.Add(kpioPrevious.data.cropTypeCode); - dataList.Add(kpioPrevious.data.cropTypeName); + try { dataList.Add(kpioPrevious.data.area); } catch { dataList.Add(""); }; + try { dataList.Add(kpioPrevious.data.cropTypeCode); } catch { dataList.Add(""); }; + try { dataList.Add(kpioPrevious.data.cropTypeName); } catch { dataList.Add(""); }; dataList.Add(""); dataList.Add("totalNapplied"); dataList.Add(totalNapplied.ToString()); - dataList.Add(kpioPrevious.unit); + dataList.Add("kg/ha"); dataList.Add(""); dataList.Add(""); sw.WriteLine(string.Join(",", dataList)); @@ -386,6 +434,9 @@ namespace FarmmapsKPI File.WriteAllText(file, json); } - + private string strTime(TimeSpan ts) + { + return String.Format("{0:00}:{1:00}:{2:00}", ts.Hours, ts.Minutes, ts.Seconds); + } } } From f1707c432f9172905a685ffed27f17356e48ca10 Mon Sep 17 00:00:00 2001 From: Pepijn van Oort Date: Wed, 1 Nov 2023 13:05:49 +0100 Subject: [PATCH 32/66] More testing & demo's in KPIInput.json: * pesticide * effect of soiltype: yes, different target & threshold for nitrogen (OK) * effect of productionpurpose: none? * effect of location: different nitrogen balance due to different atmospheric deposition (OK) And update TODO.txt --- FarmmapsKPI/KPIInput.json | 391 ++++++++++++++++++++++++++++++++------ FarmmapsKPI/TODO.txt | 37 +++- 2 files changed, 359 insertions(+), 69 deletions(-) diff --git a/FarmmapsKPI/KPIInput.json b/FarmmapsKPI/KPIInput.json index 36922f0..d37b87b 100644 --- a/FarmmapsKPI/KPIInput.json +++ b/FarmmapsKPI/KPIInput.json @@ -8,31 +8,154 @@ //2 useExistingCropfieldWithChildren = true && CropfieldItemCode = "" or absent -> read fieldName (see below) -> read all codes from settings json file //3 useExistingCropfieldWithChildren = true && CropfieldItemCode like "deb48a74c5b54299bb852f17288010e9" in KPIinput -> if you have in your account already an existing cropfield then use that "CropfieldItemCode": "", // could contain for example this: "abae97f89f3c4ac08953b1b8bea9f076" if this is an exisiting CropfieldItemCode in your account. - //And then if useExistingCropfieldWithChildren = true, this would be used. + //And then if useExistingCropfieldWithChildren = true, this would be used. "dataCropfield": { - //"area": 4.22, //not needed for KPI calculation, but shown here to know this is a possible property + //"area": 4.22, //Leave empty, KPI app will calculate it from geometry "final": true, //always true - //"soilCode": "5", //not needed for KPI calculation, but shown here to know this is a possible property - //"soilName": "Loam", //not needed for KPI calculation, but shown here to know this is a possible property - "cropTypeCode": "1010101", - "cropTypeName": "Potato", - //"rootDepthMax": 45, //not needed for KPI calculation, but shown here to know this is a possible property - //"emergenceDate": "2022-05-16T00:00:00", //not needed for KPI calculation, but shown here to know this is a possible property - "productionPurposeCode": "003" - //"productionPurposeName": "consumption" //not needed for KPI calculation, but shown here to know this is a possible property + "soilCode": "1", //From codelist CL405. Can be omitted if unknown + "soilName": "Sand", //From codelist CL405 + "cropTypeCode": "1010101", //From codelist CL263 + "cropTypeName": "Potato", //From codelist CL263 + "productionPurposeCode": "003", //From codelist CL251. For testing, see case with consumption & starch potato + "productionPurposeName": "consumption" //From codelist CL251 }, "CropRecordingItemCode": "", // could contain for example this: "314aa1b6480b493ba28a5f55039a01d4" if this is the CropRecordingItemCode of the provided CropfieldItemCode. And then if useExistingCropfieldWithChildren = true, this would be used "OperationItemCodes": [], // could contain for example this: ["4d1a6f28ea3f414180663baa4b08c60f"] or ["4d1a6f28ea3f414180663baa4b08c60f","21e08386f0454ad79acfebf78649d59b"] if these are the operations of the provided CropRecordingItemCode. And then if useExistingCropfieldWithChildren = true, these would be used "dataOperations": [], //leaving the operations empty means no fertilizer is applied "CropfieldCharacteristicItemCode": "", "DataCropfieldCharacteristic": { + //PO20231029: not different yields from different parts of fields? "code": "860619", //PO20231004: so what does this code mean? Can we see the code list somewhere? "label": "cropyield", "value": "48.01" }, //"DownloadFolder": "Downloads", //"C:\\hugoschrererdir\\kpidir\\", // "Downloads", -> if you just put "Downloads" the program will download to somewhere in ..\FarmMapsApiClient_KB34_MAST\FarmmapsKPI\bin\Debug\netcoreapp3.1\Downloads\ "CropYear": 2022, - "fieldName": "aardappelveld_test_Potato_ZeroFertilizer", + "fieldName": "test_Potato_ZeroFertilizer_Consumption_Sand", + //geometryJson = polygon of location of the cropfield, coordinates in LON,LAT (LON: decimal degrees East, LAT: decimal degrees North) + "geometryJson": { + "type": "Polygon", + "coordinates": [ + [ + [ + 5.5945257993548765, + 52.57080744107003 + ], + [ + 5.598645994070678, + 52.571540800206236 + ], + [ + 5.599381743127071, + 52.57012773140724 + ], + [ + 5.595408698222548, + 52.56968054825188 + ], + [ + 5.5945257993548765, + 52.57080744107003 + ] + ] + ] + } + }, + { + "useExistingCropfieldWithChildren": false, + "deleteNewlyCreatedAfterCalc": true, + //Note when you run the FarmmapsKPI project, somewhere to your C:\git\FarmMapsApiClient_KB34_MAST\FarmmapsKPI\bin\Debug\netcoreapp3.1\ directory, files named like this are written: + //'Settings_aardappelveld_test_Potato_ZeroFertilizer.json', where 'aardappelveld_test_Potato_ZeroFertilizer.json' is your fieldName (see below) + //1 useExistingCropfieldWithChildren = false -> create new + //2 useExistingCropfieldWithChildren = true && CropfieldItemCode = "" or absent -> read fieldName (see below) -> read all codes from settings json file + //3 useExistingCropfieldWithChildren = true && CropfieldItemCode like "deb48a74c5b54299bb852f17288010e9" in KPIinput -> if you have in your account already an existing cropfield then use that + "CropfieldItemCode": "", // could contain for example this: "abae97f89f3c4ac08953b1b8bea9f076" if this is an exisiting CropfieldItemCode in your account. + //And then if useExistingCropfieldWithChildren = true, this would be used. + "dataCropfield": { + //"area": 4.22, //Leave empty, KPI app will calculate it from geometry + "final": true, //always true + "soilCode": "7", //From codelist CL405. Can be omitted if unknown + "soilName": "Clay", //From codelist CL405 + "cropTypeCode": "1010101", //From codelist CL263 + "cropTypeName": "Potato", //From codelist CL263 + "productionPurposeCode": "003", //From codelist CL251. For testing, see case with consumption & starch potato + "productionPurposeName": "consumption" //From codelist CL251 + }, + "CropRecordingItemCode": "", // could contain for example this: "314aa1b6480b493ba28a5f55039a01d4" if this is the CropRecordingItemCode of the provided CropfieldItemCode. And then if useExistingCropfieldWithChildren = true, this would be used + "OperationItemCodes": [], // could contain for example this: ["4d1a6f28ea3f414180663baa4b08c60f"] or ["4d1a6f28ea3f414180663baa4b08c60f","21e08386f0454ad79acfebf78649d59b"] if these are the operations of the provided CropRecordingItemCode. And then if useExistingCropfieldWithChildren = true, these would be used + "dataOperations": [], //leaving the operations empty means no fertilizer is applied + "CropfieldCharacteristicItemCode": "", + "DataCropfieldCharacteristic": { + //PO20231029: not different yields from different parts of fields? + "code": "860619", //PO20231004: so what does this code mean? Can we see the code list somewhere? + "label": "cropyield", + "value": "48.01" + }, + //"DownloadFolder": "Downloads", //"C:\\hugoschrererdir\\kpidir\\", // "Downloads", -> if you just put "Downloads" the program will download to somewhere in ..\FarmMapsApiClient_KB34_MAST\FarmmapsKPI\bin\Debug\netcoreapp3.1\Downloads\ + "CropYear": 2022, + "fieldName": "test_Potato_ZeroFertilizer_Consumption_Clay", + //geometryJson = polygon of location of the cropfield, coordinates in LON,LAT (LON: decimal degrees East, LAT: decimal degrees North) + "geometryJson": { + "type": "Polygon", + "coordinates": [ + [ + [ + 5.5945257993548765, + 52.57080744107003 + ], + [ + 5.598645994070678, + 52.571540800206236 + ], + [ + 5.599381743127071, + 52.57012773140724 + ], + [ + 5.595408698222548, + 52.56968054825188 + ], + [ + 5.5945257993548765, + 52.57080744107003 + ] + ] + ] + } + }, + { + "useExistingCropfieldWithChildren": false, + "deleteNewlyCreatedAfterCalc": true, + //Note when you run the FarmmapsKPI project, somewhere to your C:\git\FarmMapsApiClient_KB34_MAST\FarmmapsKPI\bin\Debug\netcoreapp3.1\ directory, files named like this are written: + //'Settings_aardappelveld_test_Potato_ZeroFertilizer.json', where 'aardappelveld_test_Potato_ZeroFertilizer.json' is your fieldName (see below) + //1 useExistingCropfieldWithChildren = false -> create new + //2 useExistingCropfieldWithChildren = true && CropfieldItemCode = "" or absent -> read fieldName (see below) -> read all codes from settings json file + //3 useExistingCropfieldWithChildren = true && CropfieldItemCode like "deb48a74c5b54299bb852f17288010e9" in KPIinput -> if you have in your account already an existing cropfield then use that + "CropfieldItemCode": "", // could contain for example this: "abae97f89f3c4ac08953b1b8bea9f076" if this is an exisiting CropfieldItemCode in your account. + //And then if useExistingCropfieldWithChildren = true, this would be used. + "dataCropfield": { + //"area": 4.22, //Leave empty, KPI app will calculate it from geometry + "final": true, //always true + "soilCode": "1", //From codelist CL405. Can be omitted if unknown + "soilName": "Sand", + "cropTypeCode": "1010101", //From codelist CL263 + "cropTypeName": "Potato", //From codelist CL263 + "productionPurposeCode": "003", //From codelist CL251. For testing, see case with consumption & starch potato + "productionPurposeName": "consumption" //From codelist CL251 + }, + "CropRecordingItemCode": "", // could contain for example this: "314aa1b6480b493ba28a5f55039a01d4" if this is the CropRecordingItemCode of the provided CropfieldItemCode. And then if useExistingCropfieldWithChildren = true, this would be used + "OperationItemCodes": [], // could contain for example this: ["4d1a6f28ea3f414180663baa4b08c60f"] or ["4d1a6f28ea3f414180663baa4b08c60f","21e08386f0454ad79acfebf78649d59b"] if these are the operations of the provided CropRecordingItemCode. And then if useExistingCropfieldWithChildren = true, these would be used + "dataOperations": [], //leaving the operations empty means no fertilizer is applied + "CropfieldCharacteristicItemCode": "", + "DataCropfieldCharacteristic": { + //PO20231029: not different yields from different parts of fields? + "code": "860619", //PO20231004: so what does this code mean? Can we see the code list somewhere? + "label": "cropyield", + "value": "48.01" + }, + //"DownloadFolder": "Downloads", //"C:\\hugoschrererdir\\kpidir\\", // "Downloads", -> if you just put "Downloads" the program will download to somewhere in ..\FarmMapsApiClient_KB34_MAST\FarmmapsKPI\bin\Debug\netcoreapp3.1\Downloads\ + "CropYear": 2022, + "fieldName": "test_Potato_ZeroFertilizer_Starch_Sand", //geometryJson = polygon of location of the cropfield, coordinates in LON,LAT (LON: decimal degrees East, LAT: decimal degrees North) "geometryJson": { "type": "Polygon", @@ -67,32 +190,32 @@ "deleteNewlyCreatedAfterCalc": true, "CropfieldItemCode": "", "dataCropfield": { - //"area": 4.22, //not needed for KPI calculation, but shown here to know this is a possible property + //"area": 4.22, //Leave empty, KPI app will calculate it from geometry "final": true, //always true - //"soilCode": "5", //not needed for KPI calculation, but shown here to know this is a possible property - //"soilName": "Loam", //not needed for KPI calculation, but shown here to know this is a possible property - "cropTypeCode": "1010101", - "cropTypeName": "Potato", + //"soilCode": "5", //From codelist CL405. Can be omitted if unknown + //"soilName": "Loam", //From codelist CL405. + "cropTypeCode": "1010101", //From codelist CL263 + "cropTypeName": "Potato", //From codelist CL263 //"rootDepthMax": 45, //not needed for KPI calculation, but shown here to know this is a possible property //"emergenceDate": "2022-05-16T00:00:00", //not needed for KPI calculation, but shown here to know this is a possible property - "productionPurposeCode": "003" - //"productionPurposeName": "consumption" //not needed for KPI calculation, but shown here to know this is a possible property + "productionPurposeCode": "003", //From codelist CL251. Can be omitted if unknown + "productionPurposeName": "consumption" //From codelist CL251. Can be omitted if unknown }, "CropRecordingItemCode": "", "OperationItemCodes": [], "dataOperations": [ { - "area": "0.08", //?! + //"area": "0.08", //if leave empty then write code to fill in value from geometry (operation applied to whole field). Fill in different value if operation applied to part of the field (e.g. in case of variable rate application, VRA) "contractor": false, - "designator": "Kunstmest strooien", + "designator": "Kunstmest strooien", //refers to codelist CL127 with operation methods "from": "2022-05-23T11:34:00", - "method": "70400", //refers to codelist 127 with operation types - "n": "92", - "name": "Kunstmest strooien", - "operationCode": "7", - "product": "7360", //refers to codelist 022 with fertilizer types + "method": "70400", //refers to codelist CL127 with operation methods + "n": "92", //refers to codelist CL022 with fertilizer types & npk contents + "name": "Kunstmest strooien", //refers to codelist CL127 with operation methods + "product": "7360", //refers to codelist CL022 with fertilizer types "quantity": "200", - "status": "3", + "status": "3", // Which code list? What does it mean? Is it needed? In any case works if set to 3 + "operationCode": "7", // Which code list? What does it mean? Is it needed? In any case works if set to 7 "to": "2022-05-23T12:34:00", "unit": "kg/ha", "unitCode": "KGMHAR" @@ -102,18 +225,18 @@ "n": "144", "p": "60.8", "to": "2022-04-19T09:19:00", - "area": "0.08", //?!, + //"area": "0.08", //if leave empty then write code to fill in value from geometry (operation applied to whole field). Fill in different value if operation applied to part of the field (e.g. in case of variable rate application, VRA) "from": "2022-04-19T08:19:00", - "name": "Injecteren", + "name": "Injecteren", //refers to codelist CL127 with operation methods "unit": "kg/ha", - "method": "70700", - "status": "3", - "product": "2329", + "method": "70700", //refers to codelist CL127 with operation methods + "status": "3", // Which code list? What does it mean? Is it needed? In any case works if set to 3 + "operationCode": "7", // Which code list? What does it mean? Is it needed? In any case works if set to 7 + "product": "2329", //refers to codelist CL022 with fertilizer types "quantity": "32000", "unitCode": "KGMHAR", "contractor": false, - "designator": "Injecteren", - "operationCode": "7" + "designator": "Injecteren" } ], "CropfieldCharacteristicItemCode": "", @@ -158,32 +281,32 @@ "deleteNewlyCreatedAfterCalc": true, "CropfieldItemCode": "", "dataCropfield": { - //"area": 4.22, //not needed for KPI calculation, but shown here to know this is a possible property + //"area": 4.22, //Leave empty, KPI app will calculate it from geometry "final": true, //always true - //"soilCode": "5", //not needed for KPI calculation, but shown here to know this is a possible property - //"soilName": "Loam", //not needed for KPI calculation, but shown here to know this is a possible property - "cropTypeCode": "1010101", - "cropTypeName": "Potato", + //"soilCode": "5", //From codelist CL405. Can be omitted if unknown + //"soilName": "Loam", //From codelist CL405. + "cropTypeCode": "1010101", //From codelist CL263 + "cropTypeName": "Potato", //From codelist CL263 //"rootDepthMax": 45, //not needed for KPI calculation, but shown here to know this is a possible property //"emergenceDate": "2022-05-16T00:00:00", //not needed for KPI calculation, but shown here to know this is a possible property - "productionPurposeCode": "003" - //"productionPurposeName": "consumption" //not needed for KPI calculation, but shown here to know this is a possible property + "productionPurposeCode": "003", //From codelist CL251. Can be omitted if unknown + "productionPurposeName": "consumption" //From codelist CL251. Can be omitted if unknown }, "CropRecordingItemCode": "", "OperationItemCodes": [], "dataOperations": [ { - "area": "0.08", //?! + //"area": "0.08", //if leave empty then write code to fill in value from geometry (operation applied to whole field). Fill in different value if operation applied to part of the field (e.g. in case of variable rate application, VRA) "contractor": false, - "designator": "Kunstmest strooien", + "designator": "Kunstmest strooien", //refers to codelist CL127 with operation methods "from": "2022-05-23T11:34:00", - "method": "70400", + "method": "70400", //refers to codelist CL127 with operation methods "n": "92", - "name": "Kunstmest strooien", - "operationCode": "7", - "product": "7360", + "name": "Kunstmest strooien", //refers to codelist CL127 with operation methods + "product": "7360", //refers to codelist CL022 with fertilizer types "quantity": "200", - "status": "3", + "status": "3", // Which code list? What does it mean? Is it needed? In any case works if set to 3 + "operationCode": "7", // Which code list? What does it mean? Is it needed? In any case works if set to 7 "to": "2022-05-23T12:34:00", "unit": "kg/ha", "unitCode": "KGMHAR" @@ -231,32 +354,32 @@ "deleteNewlyCreatedAfterCalc": true, "CropfieldItemCode": "", "dataCropfield": { - //"area": 4.22, //not needed for KPI calculation, but shown here to know this is a possible property + //"area": 4.22, //Leave empty, KPI app will calculate it from geometry "final": true, //always true - //"soilCode": "5", //not needed for KPI calculation, but shown here to know this is a possible property - //"soilName": "Loam", //not needed for KPI calculation, but shown here to know this is a possible property - "cropTypeCode": "1010101", - "cropTypeName": "Potato", + //"soilCode": "5", //From codelist CL405. Can be omitted if unknown + //"soilName": "Loam", //From codelist CL405. + "cropTypeCode": "1010101", //From codelist CL263 + "cropTypeName": "Potato", //From codelist CL263 //"rootDepthMax": 45, //not needed for KPI calculation, but shown here to know this is a possible property //"emergenceDate": "2022-05-16T00:00:00", //not needed for KPI calculation, but shown here to know this is a possible property - "productionPurposeCode": "003" - //"productionPurposeName": "consumption" //not needed for KPI calculation, but shown here to know this is a possible property + "productionPurposeCode": "003", //From codelist CL251. Can be omitted if unknown + "productionPurposeName": "consumption" //From codelist CL251. Can be omitted if unknown }, "CropRecordingItemCode": "", "OperationItemCodes": [], "dataOperations": [ { - "area": "0.08", //?! + //"area": "0.08", //if leave empty then write code to fill in value from geometry (operation applied to whole field). Fill in different value if operation applied to part of the field (e.g. in case of variable rate application, VRA) "contractor": false, - "designator": "Kunstmest strooien", + "designator": "Kunstmest strooien", //refers to codelist CL127 with operation methods "from": "2022-05-23T11:34:00", - "method": "70400", - //"n": "92", - "name": "Kunstmest strooien", - "operationCode": "7", - "product": "7360", + "method": "70400", //refers to codelist CL127 with operation methods + "n": "92", + "name": "Kunstmest strooien", //refers to codelist CL127 with operation methods + "status": "3", // Which code list? What does it mean? Is it needed? In any case works if set to 3 + "operationCode": "7", // Which code list? What does it mean? Is it needed? In any case works if set to 7 + "product": "7360", //refers to codelist CL022 with fertilizer types "quantity": "200", - "status": "3", "to": "2022-05-23T12:34:00", "unit": "kg/ha", "unitCode": "KGMHAR" @@ -270,7 +393,149 @@ }, //"DownloadFolder": "Downloads", //"C:\\hugoschrererdir\\kpidir\\", // "Downloads", -> if you just put "Downloads" the program will download to somewhere in ..\FarmMapsApiClient_WURtest\FarmmapsDataDownload\bin\Debug\netcoreapp3.1\Downloads\ "CropYear": 2022, - "fieldName": "aardappelveld_test_Potato_Urea200kgha", + "fieldName": "aardappelveld_test_Potato_Urea92kgNha_ZuidLimburg", + "geometryJson": { + "type": "Polygon", + "coordinates": [ + [ + [ + 5.900, + 50.800 + ], + [ + 5.901, + 50.800 + ], + [ + 5.901, + 50.801 + ], + [ + 5.900, + 50.801 + ], + [ + 5.900, + 50.800 + ] + ] + ] + } + }, + { + "useExistingCropfieldWithChildren": false, + "deleteNewlyCreatedAfterCalc": true, + "CropfieldItemCode": "", + "dataCropfield": { + //"area": 4.22, //Leave empty, KPI app will calculate it from geometry + "final": true, //always true + //"soilCode": "5", //From codelist CL405. Can be omitted if unknown + //"soilName": "Loam", //From codelist CL405. + "cropTypeCode": "1010101", //From codelist CL263 + "cropTypeName": "Potato", //From codelist CL263 + //"rootDepthMax": 45, //not needed for KPI calculation, but shown here to know this is a possible property + //"emergenceDate": "2022-05-16T00:00:00", //not needed for KPI calculation, but shown here to know this is a possible property + "productionPurposeCode": "003", //From codelist CL251. Can be omitted if unknown + "productionPurposeName": "consumption" //From codelist CL251. Can be omitted if unknown + }, + "CropRecordingItemCode": "", + "OperationItemCodes": [], + "dataOperations": [ + { + //"area": "0.08", //if leave empty then write code to fill in value from geometry (operation applied to whole field). Fill in different value if operation applied to part of the field (e.g. in case of variable rate application, VRA) + "contractor": false, + "designator": "Kunstmest strooien", //refers to codelist CL127 with operation methods + "from": "2022-05-23T11:34:00", + "method": "70400", //refers to codelist CL127 with operation methods + //"n": "92", + "name": "Kunstmest strooien", //refers to codelist CL127 with operation methods + "status": "3", // Which code list? What does it mean? Is it needed? In any case works if set to 3 + "operationCode": "7", // Which code list? What does it mean? Is it needed? In any case works if set to 7 + "product": "7360", //refers to codelist CL022 with fertilizer types + "quantity": "200", + "to": "2022-05-23T12:34:00", + "unit": "kg/ha", + "unitCode": "KGMHAR" + } + ], + "CropfieldCharacteristicItemCode": "", + "DataCropfieldCharacteristic": { + "code": "860619", //PO20231004: so what does this code mean? Can we see the code list somewhere? + "label": "cropyield", + "value": "48.01" + }, + //"DownloadFolder": "Downloads", //"C:\\hugoschrererdir\\kpidir\\", // "Downloads", -> if you just put "Downloads" the program will download to somewhere in ..\FarmMapsApiClient_WURtest\FarmmapsDataDownload\bin\Debug\netcoreapp3.1\Downloads\ + "CropYear": 2022, + "fieldName": "caseQuoteNnotProvidedInDataOperations", + "geometryJson": { + "type": "Polygon", + "coordinates": [ + [ + [ + 5.5945257993548765, + 52.57080744107003 + ], + [ + 5.598645994070678, + 52.571540800206236 + ], + [ + 5.599381743127071, + 52.57012773140724 + ], + [ + 5.595408698222548, + 52.56968054825188 + ], + [ + 5.5945257993548765, + 52.57080744107003 + ] + ] + ] + } + }, + { + "useExistingCropfieldWithChildren": false, + "deleteNewlyCreatedAfterCalc": true, + "CropfieldItemCode": "", // could contain for example this: "abae97f89f3c4ac08953b1b8bea9f076" if this is an exisiting CropfieldItemCode in your account. + "dataCropfield": { + "final": true, //always true + "cropTypeCode": "1010101", //From codelist CL263 + "cropTypeName": "Potato", //From codelist CL263 + "productionPurposeCode": "003", //From codelist CL251. Can be omitted if unknown + "productionPurposeName": "consumption" //From codelist CL251. Can be omitted if unknown + }, + "CropRecordingItemCode": "", // could contain for example this: "314aa1b6480b493ba28a5f55039a01d4" if this is the CropRecordingItemCode of the provided CropfieldItemCode. And then if useExistingCropfieldWithChildren = true, this would be used + "OperationItemCodes": [], // could contain for example this: ["4d1a6f28ea3f414180663baa4b08c60f"] or ["4d1a6f28ea3f414180663baa4b08c60f","21e08386f0454ad79acfebf78649d59b"] if these are the operations of the provided CropRecordingItemCode. And then if useExistingCropfieldWithChildren = true, these would be used + "dataOperations": [ + { + "to": "2022-07-13T17:27:00", + "from": "2022-07-13T16:27:00", + "name": "Volveld spuiten (3/3)", //refers to codelist CL127 with operation methods + "method": "80200", //refers to codelist CL127 with operation methods + "status": "3", // Which code list? What does it mean? Is it needed? In any case works if set to 3 + "operationCode": "8", // Which code list? What does it mean? Is it needed? In any case works if set to 8 + "products": [ + { + "product": "11767 N", //CTBG code, lookup via CTBG API. Important: with space N", + "quantity": "1.25", + "unitCode": "LTRHAR" + //"designator": "Amistar" + } + ] + } + ], + "CropfieldCharacteristicItemCode": "", + "DataCropfieldCharacteristic": { + "code": "860619", + "label": "cropyield", + "value": "48.01" + }, + //"DownloadFolder": "Downloads", //"C:\\hugoschrererdir\\kpidir\\", // "Downloads", -> if you just put "Downloads" the program will download to somewhere in ..\FarmMapsApiClient_KB34_MAST\FarmmapsKPI\bin\Debug\netcoreapp3.1\Downloads\ + "CropYear": 2022, + "fieldName": "examplePesticide_11767N", + //geometryJson = polygon of location of the cropfield, coordinates in LON,LAT (LON: decimal degrees East, LAT: decimal degrees North) "geometryJson": { "type": "Polygon", "coordinates": [ diff --git a/FarmmapsKPI/TODO.txt b/FarmmapsKPI/TODO.txt index e51c71f..6232adb 100644 --- a/FarmmapsKPI/TODO.txt +++ b/FarmmapsKPI/TODO.txt @@ -1,13 +1,19 @@ TODO.txt +################################################################################################### +# 1. DONE +################################################################################################### Laat de farmmapsKPI zelf maar uitzoeken wat de n-p gehaltes zijn: a. We willen als input opgeven we hebben 400 kg/ha urea gegeven, of 32000 kg/ha runderdrijfmest. Daarvoor wel de juiste gewas code opgeven -> zie onder, FarmmapsEditeeltList 263 gewassen Daarvoor wel de juiste meststof code opgeven -> zie onder, FarmmapsEditeeltList 022 meststoffen b. Check of de operation data een element “n” bevat. Zoniet of als waarde 0, dan - i. Zoek onder de motorkap op: de omreken factor (bijv 46% -> 400*0.46 = 184 kg N/ha). In editeelt list - ii. Voeg die 184 toe aan de operation data + i. Zoek onder de motorkap op: de omreken factor (bijv 200 kg/ha urea met 46% n -> 200*0.46 = 92 kg N/ha) + ii. Voeg die 92 toe aan de operation data ("n": 92) c. Reken N surplus uit (KPI item id b1) +GEPARKEERD: + Want voor veel meststoffen is geen content bekend -> automatisch opzoeken zal in veel gevallen niet goed werken + Laat de gebruiker zelf maar de codelist CL022 raadplegen en omrekenen. Nieuw project: FarmmapsEditeeltLists DONE a. Why? Because @@ -16,8 +22,27 @@ a. Why? Because b. Download full list of items from vnd.farmmaps.itemtype.codelist.cl263 DONE c. Write them to a csv file DONE d. In any case following lists: 127 operations DONE; 022 meststoffen (fertilizer products) DONE; 263 gewassen DONE; 405 soilcode DONE - in which list the productionPurposeCode NOT YET? + in which list the productionPurposeCode 251 DONE + +Testing: +Effect of soiltype (e.g. different N concents in crop depending on soiltype? (CL405)) + DONE (using KPIinput.json): same N balance, different target & threshold +Effect of location (e.g. different atmospheric N deposition). + DONE (using KPIinput.json): slightly different N balance + +################################################################################################### +# 2. Pending +################################################################################################### +Testing: +productionPurposeCode: different result depending on starch or consumption potato (different N contents in tubers)? + DONE (using KPIinput.json): surprisingly no effect? + +dataOperations: "operationCode": "7" & "8": + a. which codelist? What does it mean? NOT YET TESTED + b. using KPIinput.json: does it have an effect on KPI outcomes? NOT YET TESTED + +dataOperations: "status": "3": + a. which codelist? What does it mean? NOT YET TESTED + b. using KPIinput.json: does it have an effect on KPI outcomes? NOT YET TESTED + -Testing: in json input data, can certain fields be left out? Which are minimum fields needed? NOT YET -Testing: effect of location (e.g. different targetValue, different N deposition? Different crop N contents depending on soil?) NOT YET -Testing: productionPurposeCode: different result depending on starch or consumption potato (different N contents in tubers; different targetValues? NOT YET From aef8c7c44668576428046a6396196c65370c2378 Mon Sep 17 00:00:00 2001 From: Pepijn van Oort Date: Wed, 8 Nov 2023 13:48:01 +0100 Subject: [PATCH 33/66] update KPIdefinitions.csv & TODO.txt --- FarmmapsKPI/KPIdefinitions.csv | 12 ++++++------ FarmmapsKPI/TODO.txt | 2 ++ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/FarmmapsKPI/KPIdefinitions.csv b/FarmmapsKPI/KPIdefinitions.csv index b82cbc5..92aa3d5 100644 --- a/FarmmapsKPI/KPIdefinitions.csv +++ b/FarmmapsKPI/KPIdefinitions.csv @@ -1,9 +1,9 @@ KPIid,KPIvariable,Description -A1,yield,observed yield (user input) -B1,nitrogen,Nitrogen surplus = N fertilizer + N atmospheric deposition + N fixation by crop - N removal through harvested product. Calculated KPI internal model & parameters and from user input: yield and fertilizer applications -B2,phosphate,Phosphate surplus = P fertilizer - P removal through harvested product. Calculated KPI internal model & parameters and from user input: yield and fertilizer applications -C1,organic matter supply,Organic matter surplus = Organic matter from manure + Crop residues to soil - organic matter removal through harvested product. Calculated KPI internal model & parameters and from user input: yield and manure applications -D1,pesticides,? -,KPItargetvalue,target value as in benchmark value for same crop in same region +A1,Yield per crop,observed yield (user input) +B1,Nitrogen surplus,Nitrogen surplus = N artifical fertilizer + N manure+ N atmospheric deposition + N fixation by crop - N removal through harvested product. Calculated KPI internal model & parameters and from user input: yield and fertilizer/manure applications. N fixation (legumes only): internal equations. N depostition: RIVM maps +B2,Phosphate surplus,Phosphate surplus = P fertilizer - P removal through harvested product. Calculated KPI internal model & parameters and from user input: yield and fertilizer applications +C1,Effective OM supply,"Organic matter surplus = Organic matter from manure + Crop residues to soil - organic matter removal through harvested product. Calculated KPI internal model & parameters and from user input: yield and manure applications. ""Effective"" means netto, after correction for carbon respiration by soil microbial activity" +D1,pesticides,"kg/ha of active ingredient. Sum over kg a.i. of the different crop protection agents applied, with concentration (kg a.i. / kg or litre) of crop protection agents retrieved from CTBG" +,KPItargetvalue,target value as in benchmark value for same crop in same region. E.g. cropyield target is from Central Bureau of Statistics (CBS) for same crop in region in which the cropfield (geometry) is located ,KPIthresholdValue,threshold from ??? Ask farmmaps. Surplus nitrogen / phosphate / pesticides must not be above threshold. Surplus organic matter supply must be above threshold ,totalNapplied,"Not a KPI output, simply calculated from the input operations with their respective ""n"" in their ""data""" diff --git a/FarmmapsKPI/TODO.txt b/FarmmapsKPI/TODO.txt index 6232adb..a23ff9d 100644 --- a/FarmmapsKPI/TODO.txt +++ b/FarmmapsKPI/TODO.txt @@ -45,4 +45,6 @@ dataOperations: "status": "3": a. which codelist? What does it mean? NOT YET TESTED b. using KPIinput.json: does it have an effect on KPI outcomes? NOT YET TESTED +"DataCropfieldCharacteristic": { + "code": "860619", //PO20231004: so what does this code mean? Can we see the code list somewhere? NOT YET TESTED From 0b0ec8f767795c619d6b09f3dbe1caaff5c881ff Mon Sep 17 00:00:00 2001 From: Pepijn van Oort Date: Wed, 8 Nov 2023 13:50:37 +0100 Subject: [PATCH 34/66] StartDate & EndDate of cropfield. Is relevant for N fixation by leguminose crops where currently N fixation depends on day between these two dates. If not provided a whole year is assumed which leads to high N fixation --- FarmmapsApi/Services/GeneralService.cs | 41 ++++++++++++++++++-------- FarmmapsKPI/KPIApplication.cs | 13 ++++---- FarmmapsKPI/Models/KPIInput.cs | 4 ++- 3 files changed, 38 insertions(+), 20 deletions(-) diff --git a/FarmmapsApi/Services/GeneralService.cs b/FarmmapsApi/Services/GeneralService.cs index 6b452a0..ce0a792 100644 --- a/FarmmapsApi/Services/GeneralService.cs +++ b/FarmmapsApi/Services/GeneralService.cs @@ -22,14 +22,20 @@ namespace FarmmapsApi.Services } public async Task CreateCropfieldItemAsync(string parentItemCode, string name, int year, - string fieldGeomJson, string data = "{}") { - var currentYear = new DateTime(year, 1, 1); + string fieldGeomJson, string data = "{}", DateTime startDate = new DateTime(), DateTime endDate = new DateTime() ) + { + //If user provides no startDate or endDate, then set startDate and endDate to full year provided through 'year' + if (startDate == new DateTime() || endDate == new DateTime()) + { + startDate = new DateTime(year, 1, 1); + endDate = new DateTime(year, 12, 31); + } var cropfieldItemRequest = new ItemRequest() { ParentCode = parentItemCode, ItemType = CROPFIELD_ITEMTYPE, Name = name, - DataDate = currentYear, - DataEndDate = currentYear.AddYears(1).AddDays(-1), + DataDate = startDate, + DataEndDate = endDate, Data = JObject.Parse(data), Geometry = JObject.Parse(fieldGeomJson) }; @@ -38,10 +44,14 @@ namespace FarmmapsApi.Services } public async Task CreateOperationItemAsync(string cropRecordingItemCode, int year, - string fieldGeomJson, string data = "{}") + string fieldGeomJson, string data = "{}", DateTime startDate = new DateTime(), DateTime endDate = new DateTime()) { - // not sure if here we also need to specify DataDate, DataEndDate & Geometry. Do it just in case - var currentYear = new DateTime(year, 1, 1); + //If user provides no startDate or endDate, then set startDate and endDate to full year provided through 'year' + if (startDate == new DateTime() || endDate == new DateTime()) + { + startDate = new DateTime(year, 1, 1); + endDate = new DateTime(year, 12, 31); + } JObject jdata = JObject.Parse(data); string name = string.Format($"CrpRec Operation, {jdata.GetValue("name")}"); @@ -50,8 +60,8 @@ namespace FarmmapsApi.Services ParentCode = cropRecordingItemCode, ItemType = CROPOP_ITEMTYPE, Name = name, - DataDate = currentYear, - DataEndDate = currentYear.AddYears(1).AddDays(-1), + DataDate = startDate, + DataEndDate = endDate, Data = jdata, Geometry = JObject.Parse(fieldGeomJson) }; @@ -59,18 +69,23 @@ namespace FarmmapsApi.Services return await _farmmapsApiService.CreateItemAsync(operationItemRequest); } public async Task CreateCropfieldCharacteristicItemAsync(string cropfieldItemCode, int year, - string fieldGeomJson, string data = "{}") + string fieldGeomJson, string data = "{}", DateTime startDate = new DateTime(), DateTime endDate = new DateTime()) { + //If user provides no startDate or endDate, then set startDate and endDate to full year provided through 'year' + if (startDate == new DateTime() || endDate == new DateTime()) + { + startDate = new DateTime(year, 1, 1); + endDate = new DateTime(year, 12, 31); + } // not sure if here we also need to specify DataDate, DataEndDate & Geometry. Do it just in case string name = "Cropfield characteristic"; - var currentYear = new DateTime(year, 1, 1); ItemRequest cropfieldCharactericsticItemRequest = new ItemRequest() { ParentCode = cropfieldItemCode, ItemType = CROPCHAR_ITEMTYPE, Name = name, - DataDate = currentYear, - DataEndDate = currentYear.AddYears(1).AddDays(-1), + DataDate = startDate, + DataEndDate = endDate, Data = JObject.Parse(data), Geometry = JObject.Parse(fieldGeomJson) }; diff --git a/FarmmapsKPI/KPIApplication.cs b/FarmmapsKPI/KPIApplication.cs index a64f442..1fa53f6 100644 --- a/FarmmapsKPI/KPIApplication.cs +++ b/FarmmapsKPI/KPIApplication.cs @@ -42,11 +42,12 @@ namespace FarmmapsKPI KPIInput input; string fnKPIinput; - //Console.WriteLine("Type name of input json file. Example: KPIinput.json (in same directory as FarmmapsKPI.exe) or C:/temp/KPIinputChemieTmp.json"); - //fnKPIinput = Console.ReadLine(); - //if (string.IsNullOrEmpty(fnKPIinput)) - - fnKPIinput = "KPIinput.json"; + Console.WriteLine("Type name of input json file. Example: KPIinput.json (in same directory as FarmmapsKPI.exe) or C:/temp/KPIinputChemieTmp.json"); + fnKPIinput = Console.ReadLine(); + if (string.IsNullOrEmpty(fnKPIinput)) + { + fnKPIinput = "KPIinput.json"; + } var fieldsInputJson = File.ReadAllText(fnKPIinput); @@ -153,7 +154,7 @@ namespace FarmmapsKPI { _logger.LogInformation($"Creating cropfield with name '{fieldName}'"); cropfieldItem = await _generalService.CreateCropfieldItemAsync(myDriveRoot.Code, - $"{fieldName}", cropYear, input.GeometryJson.ToString(Formatting.None), input.DataCropfield.ToString(Formatting.None)); + $"{fieldName}", cropYear, input.GeometryJson.ToString(Formatting.None), input.DataCropfield.ToString(Formatting.None), input.StartDate, input.EndDate); _settings.CropfieldItemCode = cropfieldItem.Code; SaveSettings(settingsfile); } diff --git a/FarmmapsKPI/Models/KPIInput.cs b/FarmmapsKPI/Models/KPIInput.cs index 2fef5d9..8d2df12 100644 --- a/FarmmapsKPI/Models/KPIInput.cs +++ b/FarmmapsKPI/Models/KPIInput.cs @@ -7,6 +7,7 @@ namespace FarmmapsKPI.Models { public bool UseExistingCropfieldWithChildren { get; set; } public bool DeleteNewlyCreatedAfterCalc { get; set; } + public string fieldName { get; set; } public string CropfieldItemCode { get; set; } public JObject DataCropfield { get; set; } public string CropRecordingItemCode { get; set; } @@ -17,8 +18,9 @@ namespace FarmmapsKPI.Models public JObject DataCropfieldCharacteristic { get; set; } public string DownloadFolder { get; set; } public int CropYear { get; set; } + public DateTime StartDate { get; set; } + public DateTime EndDate { get; set; } public JObject GeometryJson { get; set; } - public string fieldName { get; set; } } } \ No newline at end of file From aa18393102789e9a497a4d88eefd0e7619129375 Mon Sep 17 00:00:00 2001 From: Pepijn van Oort Date: Wed, 8 Nov 2023 13:51:30 +0100 Subject: [PATCH 35/66] two example inputs for testing garden pea (doperwt) for whole year of part of growing period --- FarmmapsKPI/KPIInput.json | 198 +++++++++++++++++++++++++++++--------- 1 file changed, 150 insertions(+), 48 deletions(-) diff --git a/FarmmapsKPI/KPIInput.json b/FarmmapsKPI/KPIInput.json index d37b87b..a891c5a 100644 --- a/FarmmapsKPI/KPIInput.json +++ b/FarmmapsKPI/KPIInput.json @@ -2,6 +2,7 @@ { "useExistingCropfieldWithChildren": false, "deleteNewlyCreatedAfterCalc": true, + "fieldName": "test_Potato_ZeroFertilizer_Consumption_Sand", //Note when you run the FarmmapsKPI project, somewhere to your C:\git\FarmMapsApiClient_KB34_MAST\FarmmapsKPI\bin\Debug\netcoreapp3.1\ directory, files named like this are written: //'Settings_aardappelveld_test_Potato_ZeroFertilizer.json', where 'aardappelveld_test_Potato_ZeroFertilizer.json' is your fieldName (see below) //1 useExistingCropfieldWithChildren = false -> create new @@ -31,7 +32,6 @@ }, //"DownloadFolder": "Downloads", //"C:\\hugoschrererdir\\kpidir\\", // "Downloads", -> if you just put "Downloads" the program will download to somewhere in ..\FarmMapsApiClient_KB34_MAST\FarmmapsKPI\bin\Debug\netcoreapp3.1\Downloads\ "CropYear": 2022, - "fieldName": "test_Potato_ZeroFertilizer_Consumption_Sand", //geometryJson = polygon of location of the cropfield, coordinates in LON,LAT (LON: decimal degrees East, LAT: decimal degrees North) "geometryJson": { "type": "Polygon", @@ -64,6 +64,7 @@ { "useExistingCropfieldWithChildren": false, "deleteNewlyCreatedAfterCalc": true, + "fieldName": "test_Potato_ZeroFertilizer_Consumption_Clay", //Note when you run the FarmmapsKPI project, somewhere to your C:\git\FarmMapsApiClient_KB34_MAST\FarmmapsKPI\bin\Debug\netcoreapp3.1\ directory, files named like this are written: //'Settings_aardappelveld_test_Potato_ZeroFertilizer.json', where 'aardappelveld_test_Potato_ZeroFertilizer.json' is your fieldName (see below) //1 useExistingCropfieldWithChildren = false -> create new @@ -93,7 +94,6 @@ }, //"DownloadFolder": "Downloads", //"C:\\hugoschrererdir\\kpidir\\", // "Downloads", -> if you just put "Downloads" the program will download to somewhere in ..\FarmMapsApiClient_KB34_MAST\FarmmapsKPI\bin\Debug\netcoreapp3.1\Downloads\ "CropYear": 2022, - "fieldName": "test_Potato_ZeroFertilizer_Consumption_Clay", //geometryJson = polygon of location of the cropfield, coordinates in LON,LAT (LON: decimal degrees East, LAT: decimal degrees North) "geometryJson": { "type": "Polygon", @@ -126,6 +126,7 @@ { "useExistingCropfieldWithChildren": false, "deleteNewlyCreatedAfterCalc": true, + "fieldName": "test_Potato_ZeroFertilizer_Starch_Sand", //Note when you run the FarmmapsKPI project, somewhere to your C:\git\FarmMapsApiClient_KB34_MAST\FarmmapsKPI\bin\Debug\netcoreapp3.1\ directory, files named like this are written: //'Settings_aardappelveld_test_Potato_ZeroFertilizer.json', where 'aardappelveld_test_Potato_ZeroFertilizer.json' is your fieldName (see below) //1 useExistingCropfieldWithChildren = false -> create new @@ -155,7 +156,6 @@ }, //"DownloadFolder": "Downloads", //"C:\\hugoschrererdir\\kpidir\\", // "Downloads", -> if you just put "Downloads" the program will download to somewhere in ..\FarmMapsApiClient_KB34_MAST\FarmmapsKPI\bin\Debug\netcoreapp3.1\Downloads\ "CropYear": 2022, - "fieldName": "test_Potato_ZeroFertilizer_Starch_Sand", //geometryJson = polygon of location of the cropfield, coordinates in LON,LAT (LON: decimal degrees East, LAT: decimal degrees North) "geometryJson": { "type": "Polygon", @@ -188,6 +188,7 @@ { "useExistingCropfieldWithChildren": false, "deleteNewlyCreatedAfterCalc": true, + "fieldName": "aardappelveld_test_Potato_cattlemanure144kgnha_plusUrea92kgNha", "CropfieldItemCode": "", "dataCropfield": { //"area": 4.22, //Leave empty, KPI app will calculate it from geometry @@ -247,7 +248,6 @@ }, //"DownloadFolder": "Downloads", //"C:\\hugoschrererdir\\kpidir\\", // "Downloads", -> if you just put "Downloads" the program will download to somewhere in ..\FarmMapsApiClient_WURtest\FarmmapsDataDownload\bin\Debug\netcoreapp3.1\Downloads\ "CropYear": 2022, - "fieldName": "aardappelveld_test_Potato_cattlemanure144kgnha_plusUrea92kgNha", "geometryJson": { "type": "Polygon", "coordinates": [ @@ -279,48 +279,48 @@ { "useExistingCropfieldWithChildren": false, "deleteNewlyCreatedAfterCalc": true, - "CropfieldItemCode": "", - "dataCropfield": { - //"area": 4.22, //Leave empty, KPI app will calculate it from geometry - "final": true, //always true - //"soilCode": "5", //From codelist CL405. Can be omitted if unknown - //"soilName": "Loam", //From codelist CL405. - "cropTypeCode": "1010101", //From codelist CL263 - "cropTypeName": "Potato", //From codelist CL263 - //"rootDepthMax": 45, //not needed for KPI calculation, but shown here to know this is a possible property - //"emergenceDate": "2022-05-16T00:00:00", //not needed for KPI calculation, but shown here to know this is a possible property - "productionPurposeCode": "003", //From codelist CL251. Can be omitted if unknown - "productionPurposeName": "consumption" //From codelist CL251. Can be omitted if unknown - }, - "CropRecordingItemCode": "", - "OperationItemCodes": [], - "dataOperations": [ - { - //"area": "0.08", //if leave empty then write code to fill in value from geometry (operation applied to whole field). Fill in different value if operation applied to part of the field (e.g. in case of variable rate application, VRA) - "contractor": false, - "designator": "Kunstmest strooien", //refers to codelist CL127 with operation methods - "from": "2022-05-23T11:34:00", - "method": "70400", //refers to codelist CL127 with operation methods - "n": "92", - "name": "Kunstmest strooien", //refers to codelist CL127 with operation methods - "product": "7360", //refers to codelist CL022 with fertilizer types - "quantity": "200", - "status": "3", // Which code list? What does it mean? Is it needed? In any case works if set to 3 - "operationCode": "7", // Which code list? What does it mean? Is it needed? In any case works if set to 7 - "to": "2022-05-23T12:34:00", - "unit": "kg/ha", - "unitCode": "KGMHAR" - } - ], - "CropfieldCharacteristicItemCode": "", - "DataCropfieldCharacteristic": { - "code": "860619", //PO20231004: so what does this code mean? Can we see the code list somewhere? - "label": "cropyield", - "value": "48.01" - }, - //"DownloadFolder": "Downloads", //"C:\\hugoschrererdir\\kpidir\\", // "Downloads", -> if you just put "Downloads" the program will download to somewhere in ..\FarmMapsApiClient_WURtest\FarmmapsDataDownload\bin\Debug\netcoreapp3.1\Downloads\ - "CropYear": 2022, "fieldName": "aardappelveld_test_Potato_Urea92kgNha", + "CropfieldItemCode": "", + "dataCropfield": { + //"area": 4.22, //Leave empty, KPI app will calculate it from geometry + "final": true, //always true + //"soilCode": "5", //From codelist CL405. Can be omitted if unknown + //"soilName": "Loam", //From codelist CL405. + "cropTypeCode": "1010101", //From codelist CL263 + "cropTypeName": "Potato", //From codelist CL263 + //"rootDepthMax": 45, //not needed for KPI calculation, but shown here to know this is a possible property + //"emergenceDate": "2022-05-16T00:00:00", //not needed for KPI calculation, but shown here to know this is a possible property + "productionPurposeCode": "003", //From codelist CL251. Can be omitted if unknown + "productionPurposeName": "consumption" //From codelist CL251. Can be omitted if unknown + }, + "CropRecordingItemCode": "", + "OperationItemCodes": [], + "dataOperations": [ + { + //"area": "0.08", //if leave empty then write code to fill in value from geometry (operation applied to whole field). Fill in different value if operation applied to part of the field (e.g. in case of variable rate application, VRA) + "contractor": false, + "designator": "Kunstmest strooien", //refers to codelist CL127 with operation methods + "from": "2022-05-23T11:34:00", + "method": "70400", //refers to codelist CL127 with operation methods + "n": "92", + "name": "Kunstmest strooien", //refers to codelist CL127 with operation methods + "product": "7360", //refers to codelist CL022 with fertilizer types + "quantity": "200", + "status": "3", // Which code list? What does it mean? Is it needed? In any case works if set to 3 + "operationCode": "7", // Which code list? What does it mean? Is it needed? In any case works if set to 7 + "to": "2022-05-23T12:34:00", + "unit": "kg/ha", + "unitCode": "KGMHAR" + } + ], + "CropfieldCharacteristicItemCode": "", + "DataCropfieldCharacteristic": { + "code": "860619", //PO20231004: so what does this code mean? Can we see the code list somewhere? + "label": "cropyield", + "value": "48.01" + }, + //"DownloadFolder": "Downloads", //"C:\\hugoschrererdir\\kpidir\\", // "Downloads", -> if you just put "Downloads" the program will download to somewhere in ..\FarmMapsApiClient_WURtest\FarmmapsDataDownload\bin\Debug\netcoreapp3.1\Downloads\ + "CropYear": 2022, "geometryJson": { "type": "Polygon", "coordinates": [ @@ -352,6 +352,7 @@ { "useExistingCropfieldWithChildren": false, "deleteNewlyCreatedAfterCalc": true, + "fieldName": "aardappelveld_test_Potato_Urea92kgNha_ZuidLimburg", "CropfieldItemCode": "", "dataCropfield": { //"area": 4.22, //Leave empty, KPI app will calculate it from geometry @@ -393,7 +394,6 @@ }, //"DownloadFolder": "Downloads", //"C:\\hugoschrererdir\\kpidir\\", // "Downloads", -> if you just put "Downloads" the program will download to somewhere in ..\FarmMapsApiClient_WURtest\FarmmapsDataDownload\bin\Debug\netcoreapp3.1\Downloads\ "CropYear": 2022, - "fieldName": "aardappelveld_test_Potato_Urea92kgNha_ZuidLimburg", "geometryJson": { "type": "Polygon", "coordinates": [ @@ -425,6 +425,7 @@ { "useExistingCropfieldWithChildren": false, "deleteNewlyCreatedAfterCalc": true, + "fieldName": "caseQuoteNnotProvidedInDataOperations", "CropfieldItemCode": "", "dataCropfield": { //"area": 4.22, //Leave empty, KPI app will calculate it from geometry @@ -466,7 +467,6 @@ }, //"DownloadFolder": "Downloads", //"C:\\hugoschrererdir\\kpidir\\", // "Downloads", -> if you just put "Downloads" the program will download to somewhere in ..\FarmMapsApiClient_WURtest\FarmmapsDataDownload\bin\Debug\netcoreapp3.1\Downloads\ "CropYear": 2022, - "fieldName": "caseQuoteNnotProvidedInDataOperations", "geometryJson": { "type": "Polygon", "coordinates": [ @@ -498,6 +498,7 @@ { "useExistingCropfieldWithChildren": false, "deleteNewlyCreatedAfterCalc": true, + "fieldName": "examplePesticide_11767N", "CropfieldItemCode": "", // could contain for example this: "abae97f89f3c4ac08953b1b8bea9f076" if this is an exisiting CropfieldItemCode in your account. "dataCropfield": { "final": true, //always true @@ -534,7 +535,6 @@ }, //"DownloadFolder": "Downloads", //"C:\\hugoschrererdir\\kpidir\\", // "Downloads", -> if you just put "Downloads" the program will download to somewhere in ..\FarmMapsApiClient_KB34_MAST\FarmmapsKPI\bin\Debug\netcoreapp3.1\Downloads\ "CropYear": 2022, - "fieldName": "examplePesticide_11767N", //geometryJson = polygon of location of the cropfield, coordinates in LON,LAT (LON: decimal degrees East, LAT: decimal degrees North) "geometryJson": { "type": "Polygon", @@ -563,5 +563,107 @@ ] ] } + }, + { + "useExistingCropfieldWithChildren": false, + "deleteNewlyCreatedAfterCalc": true, + "fieldName": "doperwt_heleJaar", + "CropfieldItemCode": "", + "dataCropfield": { + "final": true, //always true + "cropTypeCode": "1030101", //From codelist CL263 + "cropTypeName": "Garden pea" //From codelist CL263 + }, + "CropRecordingItemCode": "", + "OperationItemCodes": [], + "dataOperations": [], + "CropfieldCharacteristicItemCode": "", + "DataCropfieldCharacteristic": { + "code": "860619", //PO20231004: so what does this code mean? Can we see the code list somewhere? + "label": "cropyield", + "value": "9.58" //CBS: https://www.cbs.nl/nl-nl/cijfers/detail/37738, 2022: 40.4 Mkg / 4216 ha = 40400 / 4216 + }, + //"DownloadFolder": "Downloads", //"C:\\hugoschrererdir\\kpidir\\", // "Downloads", -> if you just put "Downloads" the program will download to somewhere in ..\FarmMapsApiClient_WURtest\FarmmapsDataDownload\bin\Debug\netcoreapp3.1\Downloads\ + "CropYear": 2022, + //"StartDate": "2022-04-15", //PO20231108: StartDate & EndDate important for legumes, where Nitrogen fixation depends on number of days from start to end. More days = more N fixation + //"EndDate": "2022-07-30", // if StartDate & EndDate left empty then whole year from cropYear + "geometryJson": { + "type": "Polygon", + "coordinates": [ + [ + [ + 5.5945257993548765, + 52.57080744107003 + ], + [ + 5.598645994070678, + 52.571540800206236 + ], + [ + 5.599381743127071, + 52.57012773140724 + ], + [ + 5.595408698222548, + 52.56968054825188 + ], + [ + 5.5945257993548765, + 52.57080744107003 + ] + ] + ] + } + }, + { + "useExistingCropfieldWithChildren": false, + "deleteNewlyCreatedAfterCalc": true, + "fieldName": "doperwt_groeiperiode", + "CropfieldItemCode": "", + "dataCropfield": { + "final": true, //always true + "cropTypeCode": "1030101", //From codelist CL263 + "cropTypeName": "Garden pea" //From codelist CL263 + }, + "CropRecordingItemCode": "", + "OperationItemCodes": [], + "dataOperations": [], + "CropfieldCharacteristicItemCode": "", + "DataCropfieldCharacteristic": { + "code": "860619", //PO20231004: so what does this code mean? Can we see the code list somewhere? + "label": "cropyield", + "value": "9.58" //CBS: https://www.cbs.nl/nl-nl/cijfers/detail/37738, 2022: 40.4 Mkg / 4216 ha = 40400 / 4216 + }, + //"DownloadFolder": "Downloads", //"C:\\hugoschrererdir\\kpidir\\", // "Downloads", -> if you just put "Downloads" the program will download to somewhere in ..\FarmMapsApiClient_WURtest\FarmmapsDataDownload\bin\Debug\netcoreapp3.1\Downloads\ + "CropYear": 2022, + "StartDate": "2022-04-15", //PO20231108: StartDate & EndDate important for legumes, where Nitrogen fixation depends on number of days from start to end. More days = more N fixation + "EndDate": "2022-07-30", // if StartDate & EndDate left empty then whole year from cropYear + "geometryJson": { + "type": "Polygon", + "coordinates": [ + [ + [ + 5.5945257993548765, + 52.57080744107003 + ], + [ + 5.598645994070678, + 52.571540800206236 + ], + [ + 5.599381743127071, + 52.57012773140724 + ], + [ + 5.595408698222548, + 52.56968054825188 + ], + [ + 5.5945257993548765, + 52.57080744107003 + ] + ] + ] + } } ] \ No newline at end of file From 3f6667691489b52744af2cd32bd7d2aa6cc684e0 Mon Sep 17 00:00:00 2001 From: Pepijn van Oort Date: Wed, 15 Nov 2023 16:49:46 +0100 Subject: [PATCH 36/66] specify how many seconds to wait for the KPItask to be completed --- FarmmapsApi/Services/GeneralService.cs | 7 +++++-- FarmmapsKPI/KPIApplication.cs | 9 +++++---- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/FarmmapsApi/Services/GeneralService.cs b/FarmmapsApi/Services/GeneralService.cs index ce0a792..977e8df 100644 --- a/FarmmapsApi/Services/GeneralService.cs +++ b/FarmmapsApi/Services/GeneralService.cs @@ -365,12 +365,13 @@ namespace FarmmapsApi.Services return bofekItem; } - public async Task> GetKpiItemsForCropField(Item cropfieldItem) + public async Task> GetKpiItemsForCropField(Item cropfieldItem, int waitsecs = 6) { TaskRequest kpiRequest = new TaskRequest { TaskType = KPI_TASK }; kpiRequest.attributes["processAggregateKpi"] = "false"; int year = cropfieldItem.DataDate.Value.Year; kpiRequest.attributes["year"] = year.ToString(); + int ms = waitsecs * 1000; string itemTaskCode = await _farmmapsApiService.QueueTaskAsync(cropfieldItem.Code, kpiRequest); @@ -387,7 +388,9 @@ namespace FarmmapsApi.Services _logger.LogError($"Something went wrong with task execution: {itemTask.Message}"); return null; } - await Task.Delay(6000); // wait 6000 secs for task to be completed + _logger.LogInformation($"Taking {waitsecs} seconds to look up all KPI items. The longer we wait the more KPI's we get ..." + + $""); + await Task.Delay(ms); // wait 60000ms = 60 secs for task to be completed //After the task is completed we have 1 kpiContainerItem with a code and with no data //Because the data are in the children of this kpiContainerItem. The container will have a list of children each with an "id" aand data, diff --git a/FarmmapsKPI/KPIApplication.cs b/FarmmapsKPI/KPIApplication.cs index 1fa53f6..d1a989c 100644 --- a/FarmmapsKPI/KPIApplication.cs +++ b/FarmmapsKPI/KPIApplication.cs @@ -85,8 +85,9 @@ namespace FarmmapsKPI TimeSpan tsTotalEstimated; for (int i = 0; i < fieldsInputs.Count; i++) - { - watch.Restart(); + //for (int i = 7; i < 8; i++) + { + watch.Restart(); input = fieldsInputs[i]; _logger.LogInformation(string.Format($"// FarmmapsKPI: Downloading KPI's for field {i + 1} out of {fieldsInputs.Count} to single csv file {KPIItemPathCsv}")); try @@ -262,7 +263,7 @@ namespace FarmmapsKPI if (data.area != area_ha) { double differencePercent = 100.0*(Convert.ToDouble(area_ha) / Convert.ToDouble(data.area) - 1.0); - _logger.LogWarning($"cropfield has area {area_ha}, but operation has area {data.area}" + + _logger.LogWarning($"cropfield has area {area_ha}, but in your KPIinput.json or file like that, you have an operation with area {data.area}" + $" Difference is {area_ha} / {data.area} - 100% = {differencePercent}%." + $" Is that correct? Example if operation was applied in part of field, e.g. in case of variable rate (VRA) application." + $" Or did you accidentally fill in area in the KPIinput json? To use cropfield area, omit field 'area' from json" + @@ -311,7 +312,7 @@ namespace FarmmapsKPI //Now get the KPIs for this cropfield, mounted with operations & cropyield // Get KPI data for saving it in a file, here the generalsedrvice is called to get the KPI data _logger.LogInformation($"GetKpiItemsForCropField('{cropfieldItem.Code}')"); - KPIItems = await _generalService.GetKpiItemsForCropField(cropfieldItem); + KPIItems = await _generalService.GetKpiItemsForCropField(cropfieldItem, 60); _logger.LogInformation($"Found {KPIItems.Count} KPI items"); //Download KPI's into a json output file for this specific cropfield (with unique cropfieldItem.Code) From d616fd3dfcacfd0055d5054cbc3de58d31a8f565 Mon Sep 17 00:00:00 2001 From: Pepijn van Oort Date: Tue, 21 Nov 2023 14:03:14 +0100 Subject: [PATCH 37/66] in KPIApplication add bofek to cropfield, because KPI E1 (milieubelastingspunten) only calculated if soil is known --- FarmmapsKPI/KPIApplication.cs | 36 ++++++++++++++++++++++++++++++----- 1 file changed, 31 insertions(+), 5 deletions(-) diff --git a/FarmmapsKPI/KPIApplication.cs b/FarmmapsKPI/KPIApplication.cs index d1a989c..dc3299d 100644 --- a/FarmmapsKPI/KPIApplication.cs +++ b/FarmmapsKPI/KPIApplication.cs @@ -42,7 +42,7 @@ namespace FarmmapsKPI KPIInput input; string fnKPIinput; - Console.WriteLine("Type name of input json file. Example: KPIinput.json (in same directory as FarmmapsKPI.exe) or C:/temp/KPIinputChemieTmp.json"); + Console.WriteLine("Type name of input json file. Example: KPIinput.json (in same directory as FarmmapsKPI.exe) or for example like this: C:/temp/KPIinputChemieTmp.json"); fnKPIinput = Console.ReadLine(); if (string.IsNullOrEmpty(fnKPIinput)) { @@ -112,6 +112,8 @@ namespace FarmmapsKPI private async Task Process(List roots, KPIInput input, StreamWriter sw) { + List cropfieldChildren; + List crprecChildren; KPIOutput kpio; KPIOutput kpioPrevious = new KPIOutput(); //creates a new empty @@ -207,6 +209,19 @@ namespace FarmmapsKPI string strJarea = JsonConvert.SerializeObject(new { area = area_ha }); JObject Jarea = JObject.Parse(strJarea); + //Retreiving BOFEK. A cropfield has 1 soil + //Have a look at the cropfieldChildren before and after running this task, see one child (i.e. soil) has been added) + cropfieldChildren = await _farmmapsApiService.GetItemChildrenAsync(cropfieldItem.Code); + _logger.LogInformation("Get BOFEK for field"); + Item bofekItem = await _generalService.RunBofekTask(cropfieldItem); + if (bofekItem == null) + { + _logger.LogError("Something went wrong while obtaining the BOFEK data"); + return; + } + cropfieldChildren = await _farmmapsApiService.GetItemChildrenAsync(cropfieldItem.Code); + + // A cropfield has 1 crop recording and the crop recording has 0:many operations //So first at the crop recording Item crprecItem; @@ -306,14 +321,25 @@ namespace FarmmapsKPI // Inspect the children and grandchildren. If all is well, cropfield will have: // one crprec, with 0-many operations as children. And the Data of an operation will have specification of how much fertilizer / cropprotection agent was applied // one edicrop.characteristic (with yield in the data) - var cropfieldChildren = await _farmmapsApiService.GetItemChildrenAsync(cropfieldItem.Code); - var crprecChildren = await _farmmapsApiService.GetItemChildrenAsync(crprecItem.Code); + cropfieldChildren = await _farmmapsApiService.GetItemChildrenAsync(cropfieldItem.Code); + crprecChildren = await _farmmapsApiService.GetItemChildrenAsync(crprecItem.Code); //Now get the KPIs for this cropfield, mounted with operations & cropyield // Get KPI data for saving it in a file, here the generalsedrvice is called to get the KPI data _logger.LogInformation($"GetKpiItemsForCropField('{cropfieldItem.Code}')"); - KPIItems = await _generalService.GetKpiItemsForCropField(cropfieldItem, 60); - _logger.LogInformation($"Found {KPIItems.Count} KPI items"); + //Pesticide KPI's D1 and E1 are retreived from API's of CTBG and CLM and may take a bit longer to retrieve + int targetKPIitemsCount = 8; //if we know we should be getting 8 + int maxtries = 5; // but don't keep on trying forever; there is a maximum number of tries + int trycnt = 0; + _logger.LogInformation($"Firing calls GetKpiItemsForCropField() until we have {targetKPIitemsCount}, but don't keep firing forever, stop after {maxtries} calls"); + while (KPIItems.Count < targetKPIitemsCount & trycnt < maxtries) { + KPIItems = await _generalService.GetKpiItemsForCropField(cropfieldItem, 30); + _logger.LogInformation($"Found {KPIItems.Count} KPI items"); + trycnt ++; + } + if (KPIItems.Count < targetKPIitemsCount) { + _logger.LogWarning($"Found {KPIItems.Count} KPIItems while you were aiming for {targetKPIitemsCount} KPIItems"); + } //Download KPI's into a json output file for this specific cropfield (with unique cropfieldItem.Code) string KPIItemPathJson = Path.Combine(downloadFolder, $"KPIItems_{cropfieldItem.Code}.json"); From 9d691dfed11946a7a4210009d4acc777b550a670 Mon Sep 17 00:00:00 2001 From: Pepijn van Oort Date: Wed, 22 Nov 2023 16:16:59 +0100 Subject: [PATCH 38/66] read & write KPI "E1" = milieu belastingspunten ("mbp") --- FarmmapsKPI/KPIApplication.cs | 103 +++++++++++++++++++++++--------- FarmmapsKPI/Models/KPIoutput.cs | 13 ++++ 2 files changed, 89 insertions(+), 27 deletions(-) diff --git a/FarmmapsKPI/KPIApplication.cs b/FarmmapsKPI/KPIApplication.cs index dc3299d..61960ed 100644 --- a/FarmmapsKPI/KPIApplication.cs +++ b/FarmmapsKPI/KPIApplication.cs @@ -71,7 +71,8 @@ namespace FarmmapsKPI StreamWriter sw; string KPIItemCsv = Path.GetFileNameWithoutExtension(fnKPIinput) + "_Items.csv"; string KPIItemPathCsv = Path.Combine(downloadFolder, KPIItemCsv); - List headerList = new List { "parentName", "area_ha", "cropTypeCode", "cropTypeName", "KPIid", "KPIvariable", "KPIvalue", "KPIunit", "KPItargetvalue", "KPIthresholdValue" }; + List headerList = new List { "parentName", "area_ha", "cropTypeCode", "cropTypeName", "KPIid", "KPIvariable", "KPIvalue", "KPIunit", "KPItargetvalue", "KPIthresholdValue", + "mbp_productCode","mbp_productName","mbp_quantity","mbp_unitCode","mbp_date","mbp_KPIvariable","mbp_KPIvalue"}; //Create a new csv file. Means if existing then overwritten !!! sw = new StreamWriter(KPIItemPathCsv); sw.WriteLine($"FarmmapsKPI backend calculations on input file '{fnKPIinput}' downloaded on {DateTime.Now} with the FarmmapsKPI application in the FarmmapsApSamples.sln"); @@ -86,8 +87,8 @@ namespace FarmmapsKPI for (int i = 0; i < fieldsInputs.Count; i++) //for (int i = 7; i < 8; i++) - { - watch.Restart(); + { + watch.Restart(); input = fieldsInputs[i]; _logger.LogInformation(string.Format($"// FarmmapsKPI: Downloading KPI's for field {i + 1} out of {fieldsInputs.Count} to single csv file {KPIItemPathCsv}")); try @@ -116,6 +117,9 @@ namespace FarmmapsKPI List crprecChildren; KPIOutput kpio; KPIOutput kpioPrevious = new KPIOutput(); //creates a new empty + //If KPI E1 is calculated, write these sub kpi's to output + string[] mbp_KPIvariables = new string[] { "aquaticLife", "groundWater", "soilLife" }; + string mbp_KPIvalue; string downloadFolder = input.DownloadFolder; if (string.IsNullOrEmpty(downloadFolder)) { @@ -209,18 +213,24 @@ namespace FarmmapsKPI string strJarea = JsonConvert.SerializeObject(new { area = area_ha }); JObject Jarea = JObject.Parse(strJarea); - //Retreiving BOFEK. A cropfield has 1 soil - //Have a look at the cropfieldChildren before and after running this task, see one child (i.e. soil) has been added) - cropfieldChildren = await _farmmapsApiService.GetItemChildrenAsync(cropfieldItem.Code); - _logger.LogInformation("Get BOFEK for field"); - Item bofekItem = await _generalService.RunBofekTask(cropfieldItem); - if (bofekItem == null) + if (useExistingCropfieldWithChildren == false) { - _logger.LogError("Something went wrong while obtaining the BOFEK data"); - return; + //Retreiving BOFEK. A cropfield has 1 soil + //Have a look at the cropfieldChildren before and after running this task, see one child (i.e. soil) has been added) + //cropfieldChildren = await _farmmapsApiService.GetItemChildrenAsync(cropfieldItem.Code); + _logger.LogInformation("Get BOFEK for field"); + Item bofekItem = await _generalService.RunBofekTask(cropfieldItem); + if (bofekItem == null) + { + _logger.LogError("Something went wrong while obtaining the BOFEK data"); + return; + } + //cropfieldChildren = await _farmmapsApiService.GetItemChildrenAsync(cropfieldItem.Code); + } + else + { + _logger.LogInformation("For existing cropfield we assume it already has BOFEK soil data"); } - cropfieldChildren = await _farmmapsApiService.GetItemChildrenAsync(cropfieldItem.Code); - // A cropfield has 1 crop recording and the crop recording has 0:many operations //So first at the crop recording @@ -331,9 +341,10 @@ namespace FarmmapsKPI int targetKPIitemsCount = 8; //if we know we should be getting 8 int maxtries = 5; // but don't keep on trying forever; there is a maximum number of tries int trycnt = 0; + _logger.LogInformation($"Firing calls GetKpiItemsForCropField() until we have {targetKPIitemsCount}, but don't keep firing forever, stop after {maxtries} calls"); while (KPIItems.Count < targetKPIitemsCount & trycnt < maxtries) { - KPIItems = await _generalService.GetKpiItemsForCropField(cropfieldItem, 30); + KPIItems = await _generalService.GetKpiItemsForCropField(cropfieldItem, 5); _logger.LogInformation($"Found {KPIItems.Count} KPI items"); trycnt ++; } @@ -356,24 +367,62 @@ namespace FarmmapsKPI List dataList; foreach (Item item in KPIItems) { - dataList = new List { }; kpio = JsonConvert.DeserializeObject(item.Data.ToString()); - //Seems sometimes duplicate KPI items are returned. So check that here and only write if this kpio is different from previous if (kpio.id != null) { if (kpio.id != kpioPrevious.id) { - dataList.Add(kpio.parentName); - dataList.Add(kpio.data.area); - dataList.Add(kpio.data.cropTypeCode); - dataList.Add(kpio.data.cropTypeName); - dataList.Add(kpio.id); - dataList.Add(kpio.quantity); - dataList.Add(kpio.value); - dataList.Add(kpio.unit); - dataList.Add(kpio.targetValue); - dataList.Add(kpio.thresholdValue); - sw.WriteLine(string.Join(",", dataList)); + if (kpio.id != "E1") + { + //Make a new dataList = new line to be written + dataList = new List { }; + //Fill the datalist with this kpi + dataList.Add(kpio.parentName); + dataList.Add(kpio.data.area); + dataList.Add(kpio.data.cropTypeCode); + dataList.Add(kpio.data.cropTypeName); + dataList.Add(kpio.id); + dataList.Add(kpio.quantity); // in KPI output quantity is what we call KPIvariable in headerlist of csv file + dataList.Add(kpio.value); + dataList.Add(kpio.unit); + dataList.Add(kpio.targetValue); + dataList.Add(kpio.thresholdValue); + //Write the datalist to a line to the streamwrieter sw for the output csv file + sw.WriteLine(string.Join(",", dataList)); + } + else + { + //for E1, environmentMeasureData is an array of elements with 1 per pesticide. Inside each element there are multiple mbp_KPIvariables to be written + foreach (KPIenvironmentMeasureData e in kpio.data.environmentMeasureData) + { + foreach (string mbp_KPIvariable in mbp_KPIvariables) + { + if (mbp_KPIvariable == "aquaticLife") { mbp_KPIvalue = e.aquaticLife; } else if (mbp_KPIvariable == "groundWater") { mbp_KPIvalue = e.groundWater; } else if (mbp_KPIvariable == "soilLife") { mbp_KPIvalue = e.soilLife; } else { mbp_KPIvalue = ""; } + //Make a new dataList = new line to be written + dataList = new List { }; + //Fill the datalist with this kpi + dataList.Add(kpio.parentName); + dataList.Add(kpio.data.area); + dataList.Add(kpio.data.cropTypeCode); + dataList.Add(kpio.data.cropTypeName); + dataList.Add(kpio.id); //"E1" + dataList.Add(kpio.quantity); // "mbp" + dataList.Add(""); // not here the value + dataList.Add(""); // not here KPIunit for this indicator + dataList.Add(""); // not here KPItargetvalue for this indicator + dataList.Add(""); // not here KPIthresholdValue for this indicator + dataList.Add(e.productCode); + dataList.Add(e.productName); + dataList.Add(e.quantity); + dataList.Add(e.unitCode); + dataList.Add(e.date); + dataList.Add(mbp_KPIvariable); + dataList.Add(mbp_KPIvalue); + //Write the datalist to a line to the streamwrieter sw for the output csv file + sw.WriteLine(string.Join(",", dataList)); + } + } + } } } kpioPrevious = kpio; diff --git a/FarmmapsKPI/Models/KPIoutput.cs b/FarmmapsKPI/Models/KPIoutput.cs index 5cf2acd..2a64b40 100644 --- a/FarmmapsKPI/Models/KPIoutput.cs +++ b/FarmmapsKPI/Models/KPIoutput.cs @@ -21,5 +21,18 @@ namespace FarmmapsKPI.Models public string area { get; set; } public string cropTypeCode { get; set; } public string cropTypeName { get; set; } + public KPIenvironmentMeasureData[] environmentMeasureData { get; set; } + + } + public class KPIenvironmentMeasureData + { + public string date { get; set; } + public string productCode { get; set; } + public string productName { get; set; } + public string quantity { get; set; } + public string unitCode { get; set; } + public string soilLife { get; set; } + public string aquaticLife { get; set; } + public string groundWater { get; set; } } } \ No newline at end of file From 476116647250860a431309f07405c84a3c2fad21 Mon Sep 17 00:00:00 2001 From: Pepijn van Oort Date: Tue, 28 Nov 2023 15:12:51 +0100 Subject: [PATCH 39/66] new Task GetConfiguration --- FarmmapsApi/Services/FarmmapsApiService.cs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/FarmmapsApi/Services/FarmmapsApiService.cs b/FarmmapsApi/Services/FarmmapsApiService.cs index dec584c..b0d1719 100644 --- a/FarmmapsApi/Services/FarmmapsApiService.cs +++ b/FarmmapsApi/Services/FarmmapsApiService.cs @@ -46,6 +46,10 @@ namespace FarmmapsApi.Services } } + public async Task GetConfiguration() + { + return _configuration; + } public async Task AuthenticateAsync() { if (_httpClient.DefaultRequestHeaders.Authorization != null && From e4087adc77812898217d3ef2a4ef4b5d7f895f93 Mon Sep 17 00:00:00 2001 From: Pepijn van Oort Date: Tue, 28 Nov 2023 15:16:44 +0100 Subject: [PATCH 40/66] add miliebelastingspunten (mbp) and checks on whether KPI task is finished or not yet (if not yet then wait and call again) --- FarmmapsKPI/KPIApplication.cs | 93 +++++++++++++++++++++++++++------- FarmmapsKPI/KPIdefinitions.csv | 3 ++ 2 files changed, 78 insertions(+), 18 deletions(-) diff --git a/FarmmapsKPI/KPIApplication.cs b/FarmmapsKPI/KPIApplication.cs index 61960ed..cbd4991 100644 --- a/FarmmapsKPI/KPIApplication.cs +++ b/FarmmapsKPI/KPIApplication.cs @@ -42,12 +42,12 @@ namespace FarmmapsKPI KPIInput input; string fnKPIinput; - Console.WriteLine("Type name of input json file. Example: KPIinput.json (in same directory as FarmmapsKPI.exe) or for example like this: C:/temp/KPIinputChemieTmp.json"); - fnKPIinput = Console.ReadLine(); - if (string.IsNullOrEmpty(fnKPIinput)) - { + //Console.WriteLine("Type name of input json file. Example: KPIinput.json (in same directory as FarmmapsKPI.exe) or for example like this: C:/temp/KPIinputChemieTmp.json"); + //fnKPIinput = Console.ReadLine(); + //if (string.IsNullOrEmpty(fnKPIinput)) + //{ fnKPIinput = "KPIinput.json"; - } + //} var fieldsInputJson = File.ReadAllText(fnKPIinput); @@ -71,11 +71,13 @@ namespace FarmmapsKPI StreamWriter sw; string KPIItemCsv = Path.GetFileNameWithoutExtension(fnKPIinput) + "_Items.csv"; string KPIItemPathCsv = Path.Combine(downloadFolder, KPIItemCsv); - List headerList = new List { "parentName", "area_ha", "cropTypeCode", "cropTypeName", "KPIid", "KPIvariable", "KPIvalue", "KPIunit", "KPItargetvalue", "KPIthresholdValue", + List headerList = new List { "parentName", "cropfieldcode","area_ha", "cropTypeCode", "cropTypeName", "KPIid", "KPIvariable", "KPIvalue", "KPIunit", "KPItargetvalue", "KPIthresholdValue", "mbp_productCode","mbp_productName","mbp_quantity","mbp_unitCode","mbp_date","mbp_KPIvariable","mbp_KPIvalue"}; //Create a new csv file. Means if existing then overwritten !!! sw = new StreamWriter(KPIItemPathCsv); - sw.WriteLine($"FarmmapsKPI backend calculations on input file '{fnKPIinput}' downloaded on {DateTime.Now} with the FarmmapsKPI application in the FarmmapsApSamples.sln"); + Configuration cf = await _farmmapsApiService.GetConfiguration(); + string endPoint = cf.Endpoint; + sw.WriteLine($"Using FarmmapsKPI application in FarmmapsApSamples.sln. Input file: '{fnKPIinput}'. Download DateTime: '{DateTime.Now}'. Calculations on: '{endPoint}'"); sw.WriteLine(); sw.WriteLine(string.Join(",", headerList)); @@ -120,6 +122,9 @@ namespace FarmmapsKPI //If KPI E1 is calculated, write these sub kpi's to output string[] mbp_KPIvariables = new string[] { "aquaticLife", "groundWater", "soilLife" }; string mbp_KPIvalue; + int targetKPIitemsCount; //if we know we should be getting 8 KPI items (A1,B1,B2,C1,D1,E1,F1,F2) + int maxtries = 5; // but don't keep on trying forever; there is a maximum number of tries + int trycnt; string downloadFolder = input.DownloadFolder; if (string.IsNullOrEmpty(downloadFolder)) { @@ -205,8 +210,15 @@ namespace FarmmapsKPI // We need that because for operations, you need to provide the area on which the operation was applied // And if we put that to the crop area, then we neatly get everything on a per ha basis _logger.LogInformation($"Getting polygon area (ha))"); - List KPIItems = await _generalService.GetKpiItemsForCropField(cropfieldItem); - _logger.LogInformation($"Found {KPIItems.Count} KPI items"); + List KPIItems = await _generalService.GetKpiItemsForCropField(cropfieldItem, 3); + trycnt = 1; + targetKPIitemsCount = 3; // here for the area we need at least 3, but not more than that + while (KPIItems.Count < targetKPIitemsCount & trycnt < maxtries) + { + KPIItems = await _generalService.GetKpiItemsForCropField(cropfieldItem,3); + _logger.LogInformation($"Found {KPIItems.Count} KPI items"); + trycnt++; + } kpio = JsonConvert.DeserializeObject(KPIItems[0].Data.ToString()); string area_ha = kpio.data.area; // turn the area into a JObject for later merging with operation data; @@ -335,17 +347,28 @@ namespace FarmmapsKPI crprecChildren = await _farmmapsApiService.GetItemChildrenAsync(crprecItem.Code); //Now get the KPIs for this cropfield, mounted with operations & cropyield - // Get KPI data for saving it in a file, here the generalsedrvice is called to get the KPI data + //Note sometimes the KPIItems.Count is already for some crazy reason greater than or equal to targetKPIitemsCount + //But that would have strange results, since that was from above before adding the crop recordings. We want to do at least one new call -> 'while (trycnt == 0 || ' _logger.LogInformation($"GetKpiItemsForCropField('{cropfieldItem.Code}')"); //Pesticide KPI's D1 and E1 are retreived from API's of CTBG and CLM and may take a bit longer to retrieve - int targetKPIitemsCount = 8; //if we know we should be getting 8 - int maxtries = 5; // but don't keep on trying forever; there is a maximum number of tries - int trycnt = 0; - - _logger.LogInformation($"Firing calls GetKpiItemsForCropField() until we have {targetKPIitemsCount}, but don't keep firing forever, stop after {maxtries} calls"); - while (KPIItems.Count < targetKPIitemsCount & trycnt < maxtries) { - KPIItems = await _generalService.GetKpiItemsForCropField(cropfieldItem, 5); + targetKPIitemsCount = 8; //if we know we should be getting 8 KPI items (A1,B1,B2,C1,D1,E1,F1,F2) + trycnt = 0; + bool boolAquaticLife = false; + _logger.LogInformation($"Firing calls GetKpiItemsForCropField() until we have {targetKPIitemsCount} KPIitems, but don't keep firing forever, stop after {maxtries} calls"); + _logger.LogInformation($"Before we start:"); + _logger.LogInformation($"* KPIItems.Count = {KPIItems.Count}"); + _logger.LogInformation($"* trycnt = {trycnt}"); + _logger.LogInformation($"* boolAquaticLife = {boolAquaticLife}"); + //additional criterion for while loop: check if it really contains the E1 mbp elements. + //while (trycnt == 0 || ((KPIItems.Count < targetKPIitemsCount || boolAquaticLife == false) & trycnt < maxtries)) + //normal while loop + targetKPIitemsCount = 7; //if we know we should be getting 7 KPI items (A1,B1,B2,C1,D1,F1,F2) + while (trycnt == 0 || (KPIItems.Count < targetKPIitemsCount & trycnt < maxtries)) + { + _logger.LogInformation($"Call nr {trycnt + 1}"); + KPIItems = await _generalService.GetKpiItemsForCropField(cropfieldItem, 3); //number after comma is how many seconds per try _logger.LogInformation($"Found {KPIItems.Count} KPI items"); + //boolAquaticLife = GetBoolAquaticLife(KPIItems); trycnt ++; } if (KPIItems.Count < targetKPIitemsCount) { @@ -362,6 +385,7 @@ namespace FarmmapsKPI count++; } _logger.LogInformation($"Downloaded file {KPIItemPathJson}"); + _logger.LogInformation($""); //Write to the csv file that collects all KPI's for all the crop fields List dataList; @@ -378,6 +402,7 @@ namespace FarmmapsKPI dataList = new List { }; //Fill the datalist with this kpi dataList.Add(kpio.parentName); + dataList.Add(cropfieldItem.Code); dataList.Add(kpio.data.area); dataList.Add(kpio.data.cropTypeCode); dataList.Add(kpio.data.cropTypeName); @@ -402,6 +427,7 @@ namespace FarmmapsKPI dataList = new List { }; //Fill the datalist with this kpi dataList.Add(kpio.parentName); + dataList.Add(cropfieldItem.Code); dataList.Add(kpio.data.area); dataList.Add(kpio.data.cropTypeCode); dataList.Add(kpio.data.cropTypeName); @@ -444,8 +470,8 @@ namespace FarmmapsKPI } //Also add totalNapplied to the csv dataList = new List { }; - //Seems sometimes duplicate KPI items are returned. So check that here and only write if this kpio is different from previous dataList.Add(kpioPrevious.parentName); + dataList.Add(cropfieldItem.Code); try { dataList.Add(kpioPrevious.data.area); } catch { dataList.Add(""); }; try { dataList.Add(kpioPrevious.data.cropTypeCode); } catch { dataList.Add(""); }; try { dataList.Add(kpioPrevious.data.cropTypeName); } catch { dataList.Add(""); }; @@ -515,5 +541,36 @@ namespace FarmmapsKPI { return String.Format("{0:00}:{1:00}:{2:00}", ts.Hours, ts.Minutes, ts.Seconds); } + + private bool GetBoolAquaticLife(List KPIItems) + { + bool found = false; + foreach (Item item in KPIItems) + { + KPIOutput kpio = JsonConvert.DeserializeObject(item.Data.ToString()); + if (kpio.id == "E1") + { + try + { + foreach (KPIenvironmentMeasureData e in kpio.data.environmentMeasureData) + { + if (e.aquaticLife != null) + { + found = true; + _logger.LogInformation($"Found e.aquaticLife = {e.aquaticLife}"); + _logger.LogInformation(""); + } + } + } + catch (Exception ex) + { + _logger.LogWarning($"NOT Found e.aquaticLife"); + _logger.LogError(ex.Message); + } + } + } + + return found; + } } } diff --git a/FarmmapsKPI/KPIdefinitions.csv b/FarmmapsKPI/KPIdefinitions.csv index 92aa3d5..556520a 100644 --- a/FarmmapsKPI/KPIdefinitions.csv +++ b/FarmmapsKPI/KPIdefinitions.csv @@ -4,6 +4,9 @@ B1,Nitrogen surplus,Nitrogen surplus = N artifical fertilizer + N manure+ N atmo B2,Phosphate surplus,Phosphate surplus = P fertilizer - P removal through harvested product. Calculated KPI internal model & parameters and from user input: yield and fertilizer applications C1,Effective OM supply,"Organic matter surplus = Organic matter from manure + Crop residues to soil - organic matter removal through harvested product. Calculated KPI internal model & parameters and from user input: yield and manure applications. ""Effective"" means netto, after correction for carbon respiration by soil microbial activity" D1,pesticides,"kg/ha of active ingredient. Sum over kg a.i. of the different crop protection agents applied, with concentration (kg a.i. / kg or litre) of crop protection agents retrieved from CTBG" +E1,mbp,"milieubelastingspunten. 3 Quantitative sub items & 2 Qualitative (A, B, C)" +F1,greenness,? +F2,rotationindex,farmlevel index of diversity of crops ,KPItargetvalue,target value as in benchmark value for same crop in same region. E.g. cropyield target is from Central Bureau of Statistics (CBS) for same crop in region in which the cropfield (geometry) is located ,KPIthresholdValue,threshold from ??? Ask farmmaps. Surplus nitrogen / phosphate / pesticides must not be above threshold. Surplus organic matter supply must be above threshold ,totalNapplied,"Not a KPI output, simply calculated from the input operations with their respective ""n"" in their ""data""" From 5d02370f560a47c8073ccde5f9fb0dd4b3f34901 Mon Sep 17 00:00:00 2001 From: tamara Date: Mon, 12 Feb 2024 15:43:00 +0100 Subject: [PATCH 41/66] details about N & P balance written to output file --- FarmmapsKPI/KPIApplication.cs | 147 +++++++++++++++++++++++++------- FarmmapsKPI/Models/KPIoutput.cs | 21 +++++ FarmmapsKPI/TODO.txt | 11 +++ 3 files changed, 146 insertions(+), 33 deletions(-) diff --git a/FarmmapsKPI/KPIApplication.cs b/FarmmapsKPI/KPIApplication.cs index cbd4991..0d6e514 100644 --- a/FarmmapsKPI/KPIApplication.cs +++ b/FarmmapsKPI/KPIApplication.cs @@ -71,7 +71,7 @@ namespace FarmmapsKPI StreamWriter sw; string KPIItemCsv = Path.GetFileNameWithoutExtension(fnKPIinput) + "_Items.csv"; string KPIItemPathCsv = Path.Combine(downloadFolder, KPIItemCsv); - List headerList = new List { "parentName", "cropfieldcode","area_ha", "cropTypeCode", "cropTypeName", "KPIid", "KPIvariable", "KPIvalue", "KPIunit", "KPItargetvalue", "KPIthresholdValue", + List headerList = new List { "parentName", "cropfieldcode", "area_ha", "cropTypeCode", "cropTypeName", "KPIid", "KPIvariable", "KPIvalue", "KPIunit", "KPItargetvalue", "KPIthresholdValue", "mbp_productCode","mbp_productName","mbp_quantity","mbp_unitCode","mbp_date","mbp_KPIvariable","mbp_KPIvalue"}; //Create a new csv file. Means if existing then overwritten !!! sw = new StreamWriter(KPIItemPathCsv); @@ -88,7 +88,7 @@ namespace FarmmapsKPI TimeSpan tsTotalEstimated; for (int i = 0; i < fieldsInputs.Count; i++) - //for (int i = 7; i < 8; i++) + //for (int i = 3; i < 4; i++) { watch.Restart(); input = fieldsInputs[i]; @@ -362,7 +362,7 @@ namespace FarmmapsKPI //additional criterion for while loop: check if it really contains the E1 mbp elements. //while (trycnt == 0 || ((KPIItems.Count < targetKPIitemsCount || boolAquaticLife == false) & trycnt < maxtries)) //normal while loop - targetKPIitemsCount = 7; //if we know we should be getting 7 KPI items (A1,B1,B2,C1,D1,F1,F2) + targetKPIitemsCount = 7; //if we know we should be getting 7 KPI items (A1,B1,B2,C1,D1,F1,F2) //8 if also E1 while (trycnt == 0 || (KPIItems.Count < targetKPIitemsCount & trycnt < maxtries)) { _logger.LogInformation($"Call nr {trycnt + 1}"); @@ -396,7 +396,7 @@ namespace FarmmapsKPI { if (kpio.id != kpioPrevious.id) { - if (kpio.id != "E1") + if (kpio.id == "B1") { //Make a new dataList = new line to be written dataList = new List { }; @@ -414,8 +414,69 @@ namespace FarmmapsKPI dataList.Add(kpio.thresholdValue); //Write the datalist to a line to the streamwrieter sw for the output csv file sw.WriteLine(string.Join(",", dataList)); + KPIelementsOfBalance kPIelementsOfBalance = kpio.data.values; + + foreach (string elementName in kpio.B1elements) + { + // get B1element from the element called values + dataList = new List { }; + string elementValue = (string)kPIelementsOfBalance.GetType().GetProperty(elementName).GetValue(kPIelementsOfBalance, null); + dataList.Add(kpio.parentName); + dataList.Add(cropfieldItem.Code); + dataList.Add(kpio.data.area); + dataList.Add(kpio.data.cropTypeCode); + dataList.Add(kpio.data.cropTypeName); + dataList.Add(kpio.id); + dataList.Add(elementName); // specific output variable name for B1 element + dataList.Add(elementValue); // specific output value name for B1 element + dataList.Add(kpio.unit); + dataList.Add(""); + dataList.Add(""); + //Write the datalist to a line to the streamwrieter sw for the output csv file + sw.WriteLine(string.Join(",", dataList)); + } } - else + else if (kpio.id == "B2") + { + //Make a new dataList = new line to be written + dataList = new List { }; + //Fill the datalist with this kpi + dataList.Add(kpio.parentName); + dataList.Add(cropfieldItem.Code); + dataList.Add(kpio.data.area); + dataList.Add(kpio.data.cropTypeCode); + dataList.Add(kpio.data.cropTypeName); + dataList.Add(kpio.id); + dataList.Add(kpio.quantity); // in KPI output quantity is what we call KPIvariable in headerlist of csv file + dataList.Add(kpio.value); + dataList.Add(kpio.unit); + dataList.Add(kpio.targetValue); + dataList.Add(kpio.thresholdValue); + //Write the datalist to a line to the streamwrieter sw for the output csv file + sw.WriteLine(string.Join(",", dataList)); + KPIelementsOfBalance kPIelementsOfBalance = kpio.data.values; + + foreach (string elementName in kpio.B2elements) + { + // get B1element from the element called values + dataList = new List { }; + string elementValue = (string)kPIelementsOfBalance.GetType().GetProperty(elementName).GetValue(kPIelementsOfBalance, null); + dataList.Add(kpio.parentName); + dataList.Add(cropfieldItem.Code); + dataList.Add(kpio.data.area); + dataList.Add(kpio.data.cropTypeCode); + dataList.Add(kpio.data.cropTypeName); + dataList.Add(kpio.id); + dataList.Add(elementName); // specific output variable name for B2 element + dataList.Add(elementValue); // specific output value name for B2 element + dataList.Add(kpio.unit); + dataList.Add(""); + dataList.Add(""); + //Write the datalist to a line to the streamwrieter sw for the output csv file + sw.WriteLine(string.Join(",", dataList)); + } + } + else if (kpio.id == "E1") { //for E1, environmentMeasureData is an array of elements with 1 per pesticide. Inside each element there are multiple mbp_KPIvariables to be written foreach (KPIenvironmentMeasureData e in kpio.data.environmentMeasureData) @@ -449,39 +510,59 @@ namespace FarmmapsKPI } } } + else + { + //Make a new dataList = new line to be written + dataList = new List { }; + //Fill the datalist with this kpi + dataList.Add(kpio.parentName); + dataList.Add(cropfieldItem.Code); + dataList.Add(kpio.data.area); + dataList.Add(kpio.data.cropTypeCode); + dataList.Add(kpio.data.cropTypeName); + dataList.Add(kpio.id); + dataList.Add(kpio.quantity); // in KPI output quantity is what we call KPIvariable in headerlist of csv file + dataList.Add(kpio.value); + dataList.Add(kpio.unit); + dataList.Add(kpio.targetValue); + dataList.Add(kpio.thresholdValue); + //Write the datalist to a line to the streamwrieter sw for the output csv file + sw.WriteLine(string.Join(",", dataList)); + } + } } kpioPrevious = kpio; } - //Total N applied - double totalNapplied = 0.0; - double operationNapplied; - for (int i = 0; i < crpOperationItemCodes.Count; i++) - { - codeOperation = crpOperationItemCodes[i]; - crpOperationItem = await _farmmapsApiService.GetItemAsync(codeOperation); - dynamic data = JObject.Parse(crpOperationItem.Data.ToString(Formatting.None)); - if (data.n != null) - { - operationNapplied = crpOperationItem.Data.GetValue("n").ToObject(); - totalNapplied = totalNapplied + operationNapplied; - } - } - //Also add totalNapplied to the csv - dataList = new List { }; - dataList.Add(kpioPrevious.parentName); - dataList.Add(cropfieldItem.Code); - try { dataList.Add(kpioPrevious.data.area); } catch { dataList.Add(""); }; - try { dataList.Add(kpioPrevious.data.cropTypeCode); } catch { dataList.Add(""); }; - try { dataList.Add(kpioPrevious.data.cropTypeName); } catch { dataList.Add(""); }; - dataList.Add(""); - dataList.Add("totalNapplied"); - dataList.Add(totalNapplied.ToString()); - dataList.Add("kg/ha"); - dataList.Add(""); - dataList.Add(""); - sw.WriteLine(string.Join(",", dataList)); + ////Total N applied from input + //double totalNapplied = 0.0; + //double operationNapplied; + //for (int i = 0; i < crpOperationItemCodes.Count; i++) + //{ + // codeOperation = crpOperationItemCodes[i]; + // crpOperationItem = await _farmmapsApiService.GetItemAsync(codeOperation); + // dynamic data = JObject.Parse(crpOperationItem.Data.ToString(Formatting.None)); + // if (data.n != null) + // { + // operationNapplied = crpOperationItem.Data.GetValue("n").ToObject(); + // totalNapplied = totalNapplied + operationNapplied; + // } + //} + ////Also add totalNapplied to the csv + //dataList = new List { }; + //dataList.Add(kpioPrevious.parentName); + //dataList.Add(cropfieldItem.Code); + //try { dataList.Add(kpioPrevious.data.area); } catch { dataList.Add(""); }; + //try { dataList.Add(kpioPrevious.data.cropTypeCode); } catch { dataList.Add(""); }; + //try { dataList.Add(kpioPrevious.data.cropTypeName); } catch { dataList.Add(""); }; + //dataList.Add(""); + //dataList.Add("totalNapplied"); + //dataList.Add(totalNapplied.ToString()); + //dataList.Add("kg/ha"); + //dataList.Add(""); + //dataList.Add(""); + //sw.WriteLine(string.Join(",", dataList)); //Clean up. Only newly created fields //Look up instruction in Swagger / api / v1 / items /{ code} diff --git a/FarmmapsKPI/Models/KPIoutput.cs b/FarmmapsKPI/Models/KPIoutput.cs index 2a64b40..5073eda 100644 --- a/FarmmapsKPI/Models/KPIoutput.cs +++ b/FarmmapsKPI/Models/KPIoutput.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using Newtonsoft.Json.Linq; namespace FarmmapsKPI.Models @@ -15,12 +16,21 @@ namespace FarmmapsKPI.Models public string parentName { get; set; } public string targetValue { get; set; } public string thresholdValue { get; set; } + + public List B1elements; + public List B2elements; + public KPIOutput() + { + this.B1elements = new List() { "disposal", "nFixationNKgHa", "nDepositionNKgHa","nFertilizerNKgHa","sowPlantingNKgHa" }; //TODO Tamara: rename disposal to nHarvestedNKgHa + this.B2elements = new List() { "disposal", "pFertillizerNKgHa", "sowPlantingNKgHa" }; //TODO Tamara: rename from pFertilizerNKgHa to pFertilizerPKgHa; rename disposal to pHarvestedPKgHa + } } public class KPIOutputData { public string area { get; set; } public string cropTypeCode { get; set; } public string cropTypeName { get; set; } + public KPIelementsOfBalance values { get; set; } public KPIenvironmentMeasureData[] environmentMeasureData { get; set; } } @@ -35,4 +45,15 @@ namespace FarmmapsKPI.Models public string aquaticLife { get; set; } public string groundWater { get; set; } } + public class KPIelementsOfBalance + { + public string disposal { get; set; } + public string nFixationNKgHa { get; set; } + public string nDepositionNKgHa { get; set; } + public string nFertilizerNKgHa { get; set; } + public string sowPlantingNKgHa { get; set; } + public string pFertillizerNKgHa { get; set; } //TODO Tamara: delete this one after renaming + public string pFertilizerPKgHa { get; set; } + public string sowPlantingPKgHa { get; set; } + } } \ No newline at end of file diff --git a/FarmmapsKPI/TODO.txt b/FarmmapsKPI/TODO.txt index a23ff9d..9419da2 100644 --- a/FarmmapsKPI/TODO.txt +++ b/FarmmapsKPI/TODO.txt @@ -48,3 +48,14 @@ dataOperations: "status": "3": "DataCropfieldCharacteristic": { "code": "860619", //PO20231004: so what does this code mean? Can we see the code list somewhere? NOT YET TESTED +KPIelementsOfBalance: Tamara + a. define elements for C1 organic matter supply + b. for nitrogen B1 rename disposal to nHarvestedNKgHa + c. for phosporous B2 rename disposal to pHarvestedPKgHa + d. for phosporous B2 rename from pFertilizerNKgHa to pFertilizerPKgHa + e. idem sowPlantingNKgHa + f. add straw for N & P in output + g. add 2 examples in KPIinput.json: straw yes/no removed. Bijv. nStrawLeftInField EN nStrawRemoved + h. update KPIdefinitions.csv + +MBP (E1): Pepijn: in same way as for B1 & B2 From 5399fd66df6147582854bf3e87943b096929fc59 Mon Sep 17 00:00:00 2001 From: tamara Date: Fri, 16 Feb 2024 12:58:00 +0100 Subject: [PATCH 42/66] added detailed output for KPI C1 effective organic matter supply --- FarmmapsKPI/KPIApplication.cs | 46 +++++++++++++++++++++++++++++++-- FarmmapsKPI/Models/KPIoutput.cs | 10 ++++++- FarmmapsKPI/TODO.txt | 10 +++---- 3 files changed, 58 insertions(+), 8 deletions(-) diff --git a/FarmmapsKPI/KPIApplication.cs b/FarmmapsKPI/KPIApplication.cs index 0d6e514..0416615 100644 --- a/FarmmapsKPI/KPIApplication.cs +++ b/FarmmapsKPI/KPIApplication.cs @@ -87,8 +87,8 @@ namespace FarmmapsKPI TimeSpan tsRemaining; TimeSpan tsTotalEstimated; - for (int i = 0; i < fieldsInputs.Count; i++) - //for (int i = 3; i < 4; i++) + //for (int i = 0; i < fieldsInputs.Count; i++) + for (int i = 3; i < 4; i++) { watch.Restart(); input = fieldsInputs[i]; @@ -476,6 +476,48 @@ namespace FarmmapsKPI sw.WriteLine(string.Join(",", dataList)); } } + + else if (kpio.id == "C1") //TtD + { + //Make a new dataList = new line to be written + dataList = new List { }; + //Fill the datalist with this kpi + dataList.Add(kpio.parentName); + dataList.Add(cropfieldItem.Code); + dataList.Add(kpio.data.area); + dataList.Add(kpio.data.cropTypeCode); + dataList.Add(kpio.data.cropTypeName); + dataList.Add(kpio.id); + dataList.Add(kpio.quantity); // in KPI output quantity is what we call KPIvariable in headerlist of csv file + dataList.Add(kpio.value); + dataList.Add(kpio.unit); + dataList.Add(kpio.targetValue); + dataList.Add(kpio.thresholdValue); + //Write the datalist to a line to the streamwrieter sw for the output csv file + sw.WriteLine(string.Join(",", dataList)); + KPIelementsOfBalance kPIelementsOfBalance = kpio.data.values; + + foreach (string elementName in kpio.C1elements) + { + // get C1element from the element called values + dataList = new List { }; + string elementValue = (string)kPIelementsOfBalance.GetType().GetProperty(elementName).GetValue(kPIelementsOfBalance, null); + dataList.Add(kpio.parentName); + dataList.Add(cropfieldItem.Code); + dataList.Add(kpio.data.area); + dataList.Add(kpio.data.cropTypeCode); + dataList.Add(kpio.data.cropTypeName); + dataList.Add(kpio.id); + dataList.Add(elementName); // specific output variable name for C1 element + dataList.Add(elementValue); // specific output value name for C1 element + dataList.Add(kpio.unit); + dataList.Add(""); + dataList.Add(""); + //Write the datalist to a line to the streamwrieter sw for the output csv file + sw.WriteLine(string.Join(",", dataList)); + } + } + else if (kpio.id == "E1") { //for E1, environmentMeasureData is an array of elements with 1 per pesticide. Inside each element there are multiple mbp_KPIvariables to be written diff --git a/FarmmapsKPI/Models/KPIoutput.cs b/FarmmapsKPI/Models/KPIoutput.cs index 5073eda..b7dd992 100644 --- a/FarmmapsKPI/Models/KPIoutput.cs +++ b/FarmmapsKPI/Models/KPIoutput.cs @@ -19,10 +19,13 @@ namespace FarmmapsKPI.Models public List B1elements; public List B2elements; + public List C1elements; //TtD public KPIOutput() { this.B1elements = new List() { "disposal", "nFixationNKgHa", "nDepositionNKgHa","nFertilizerNKgHa","sowPlantingNKgHa" }; //TODO Tamara: rename disposal to nHarvestedNKgHa this.B2elements = new List() { "disposal", "pFertillizerNKgHa", "sowPlantingNKgHa" }; //TODO Tamara: rename from pFertilizerNKgHa to pFertilizerPKgHa; rename disposal to pHarvestedPKgHa + this.C1elements = new List() { "fertilizerEom", "greenManureEom", "cropLeftoversEom" }; //TtD + } } public class KPIOutputData @@ -47,7 +50,7 @@ namespace FarmmapsKPI.Models } public class KPIelementsOfBalance { - public string disposal { get; set; } + public string disposal { get; set; } //TODO Tamara: rename this one public string nFixationNKgHa { get; set; } public string nDepositionNKgHa { get; set; } public string nFertilizerNKgHa { get; set; } @@ -55,5 +58,10 @@ namespace FarmmapsKPI.Models public string pFertillizerNKgHa { get; set; } //TODO Tamara: delete this one after renaming public string pFertilizerPKgHa { get; set; } public string sowPlantingPKgHa { get; set; } + public string fertilizerEom { get; set; } //TtD + public string greenManureEom { get; set; } //TtD + public string cropLeftoversEom { get; set; } //TtD + + } } \ No newline at end of file diff --git a/FarmmapsKPI/TODO.txt b/FarmmapsKPI/TODO.txt index 9419da2..dfbbb93 100644 --- a/FarmmapsKPI/TODO.txt +++ b/FarmmapsKPI/TODO.txt @@ -49,11 +49,11 @@ dataOperations: "status": "3": "code": "860619", //PO20231004: so what does this code mean? Can we see the code list somewhere? NOT YET TESTED KPIelementsOfBalance: Tamara - a. define elements for C1 organic matter supply - b. for nitrogen B1 rename disposal to nHarvestedNKgHa - c. for phosporous B2 rename disposal to pHarvestedPKgHa - d. for phosporous B2 rename from pFertilizerNKgHa to pFertilizerPKgHa - e. idem sowPlantingNKgHa + a. define elements for C1 organic matter supply DONE + b. for nitrogen B1 rename disposal to nHarvestedNKgHa PARTLY + c. for phosporous B2 rename disposal to pHarvestedPKgHa PARTLY + d. for phosporous B2 rename from pFertilizerNKgHa to pFertilizerPKgHa PARTLY + e. idem sowPlantingNKgHa PARTLY f. add straw for N & P in output g. add 2 examples in KPIinput.json: straw yes/no removed. Bijv. nStrawLeftInField EN nStrawRemoved h. update KPIdefinitions.csv From 619cb5e92cfe72bbf0ff96206cf183a9f542905e Mon Sep 17 00:00:00 2001 From: Pepijn van Oort Date: Fri, 16 Feb 2024 14:36:10 +0100 Subject: [PATCH 43/66] small updates on KPIdefinitions.csv & TODO.txt --- FarmmapsKPI/KPIApplication.cs | 4 ++-- FarmmapsKPI/KPIdefinitions.csv | 28 ++++++++++++++++++++-------- FarmmapsKPI/TODO.txt | 8 ++++---- 3 files changed, 26 insertions(+), 14 deletions(-) diff --git a/FarmmapsKPI/KPIApplication.cs b/FarmmapsKPI/KPIApplication.cs index 0416615..92be0ab 100644 --- a/FarmmapsKPI/KPIApplication.cs +++ b/FarmmapsKPI/KPIApplication.cs @@ -87,8 +87,8 @@ namespace FarmmapsKPI TimeSpan tsRemaining; TimeSpan tsTotalEstimated; - //for (int i = 0; i < fieldsInputs.Count; i++) - for (int i = 3; i < 4; i++) + for (int i = 0; i < fieldsInputs.Count; i++) + //for (int i = 3; i < 4; i++) { watch.Restart(); input = fieldsInputs[i]; diff --git a/FarmmapsKPI/KPIdefinitions.csv b/FarmmapsKPI/KPIdefinitions.csv index 556520a..b26da88 100644 --- a/FarmmapsKPI/KPIdefinitions.csv +++ b/FarmmapsKPI/KPIdefinitions.csv @@ -1,12 +1,24 @@ KPIid,KPIvariable,Description -A1,Yield per crop,observed yield (user input) -B1,Nitrogen surplus,Nitrogen surplus = N artifical fertilizer + N manure+ N atmospheric deposition + N fixation by crop - N removal through harvested product. Calculated KPI internal model & parameters and from user input: yield and fertilizer/manure applications. N fixation (legumes only): internal equations. N depostition: RIVM maps -B2,Phosphate surplus,Phosphate surplus = P fertilizer - P removal through harvested product. Calculated KPI internal model & parameters and from user input: yield and fertilizer applications -C1,Effective OM supply,"Organic matter surplus = Organic matter from manure + Crop residues to soil - organic matter removal through harvested product. Calculated KPI internal model & parameters and from user input: yield and manure applications. ""Effective"" means netto, after correction for carbon respiration by soil microbial activity" +A1,yield,observed yield (user input) +B1,nitrogen,Nitrogen surplus = N artifical fertilizer + N manure+ N atmospheric deposition + N fixation by crop - N removal through harvested product. Calculated KPI internal model & parameters and from user input: yield and fertilizer/manure applications. N fixation (legumes only): internal equations. N depostition: RIVM maps +B1,disposal, +B1,nFixationNKgHa, +B1,nDepositionNKgHa, +B1,nFertilizerNKgHa, +B1,sowPlantingNKgHa, +B2,phosphate,Phosphate surplus = P fertilizer - P removal through harvested product. Calculated KPI internal model & parameters and from user input: yield and fertilizer applications +B2,disposal, +B2,pFertillizerNKgHa, +B2,sowPlantingNKgHa, +C1,organic matter supply,"Organic matter surplus = Organic matter from manure + Crop residues to soil - organic matter removal through harvested product. Calculated KPI internal model & parameters and from user input: yield and manure applications. ""Effective"" means netto, after correction for carbon respiration by soil microbial activity" +C1,fertilizerEom, +C1,greenManureEom, +C1,cropLeftoversEom, D1,pesticides,"kg/ha of active ingredient. Sum over kg a.i. of the different crop protection agents applied, with concentration (kg a.i. / kg or litre) of crop protection agents retrieved from CTBG" -E1,mbp,"milieubelastingspunten. 3 Quantitative sub items & 2 Qualitative (A, B, C)" -F1,greenness,? -F2,rotationindex,farmlevel index of diversity of crops +E1,mpb,"milieubelastingspunten. For each pesticide applied show 3 sub-mbp's: aquaticLife, groundWater, soilLife" +E1,mpb, +E1,mpb,? +F1,greenness,farmlevel index of diversity of crops +F2,rotationindex, ,KPItargetvalue,target value as in benchmark value for same crop in same region. E.g. cropyield target is from Central Bureau of Statistics (CBS) for same crop in region in which the cropfield (geometry) is located ,KPIthresholdValue,threshold from ??? Ask farmmaps. Surplus nitrogen / phosphate / pesticides must not be above threshold. Surplus organic matter supply must be above threshold -,totalNapplied,"Not a KPI output, simply calculated from the input operations with their respective ""n"" in their ""data""" diff --git a/FarmmapsKPI/TODO.txt b/FarmmapsKPI/TODO.txt index dfbbb93..fda88ee 100644 --- a/FarmmapsKPI/TODO.txt +++ b/FarmmapsKPI/TODO.txt @@ -30,6 +30,9 @@ Effect of soiltype (e.g. different N concents in crop depending on soiltype? (CL Effect of location (e.g. different atmospheric N deposition). DONE (using KPIinput.json): slightly different N balance +KPIelementsOfBalance: Tamara + a. define elements for C1 organic matter supply + ################################################################################################### # 2. Pending ################################################################################################### @@ -48,8 +51,6 @@ dataOperations: "status": "3": "DataCropfieldCharacteristic": { "code": "860619", //PO20231004: so what does this code mean? Can we see the code list somewhere? NOT YET TESTED -KPIelementsOfBalance: Tamara - a. define elements for C1 organic matter supply DONE b. for nitrogen B1 rename disposal to nHarvestedNKgHa PARTLY c. for phosporous B2 rename disposal to pHarvestedPKgHa PARTLY d. for phosporous B2 rename from pFertilizerNKgHa to pFertilizerPKgHa PARTLY @@ -57,5 +58,4 @@ KPIelementsOfBalance: Tamara f. add straw for N & P in output g. add 2 examples in KPIinput.json: straw yes/no removed. Bijv. nStrawLeftInField EN nStrawRemoved h. update KPIdefinitions.csv - -MBP (E1): Pepijn: in same way as for B1 & B2 + i. for phosporous B2 rename 'cropLeftoversEom' to 'cropResiduesEom' \ No newline at end of file From 39ee3263a716e17d02b867570b49f2dd117f9b21 Mon Sep 17 00:00:00 2001 From: Pepijn van Oort Date: Fri, 16 Feb 2024 14:52:48 +0100 Subject: [PATCH 44/66] bit of code simplification --- FarmmapsKPI/KPIApplication.cs | 233 ++++++++++++++++++---------------- 1 file changed, 122 insertions(+), 111 deletions(-) diff --git a/FarmmapsKPI/KPIApplication.cs b/FarmmapsKPI/KPIApplication.cs index 92be0ab..312c002 100644 --- a/FarmmapsKPI/KPIApplication.cs +++ b/FarmmapsKPI/KPIApplication.cs @@ -396,42 +396,46 @@ namespace FarmmapsKPI { if (kpio.id != kpioPrevious.id) { + KPIelementsOfBalance kPIelementsOfBalance = kpio.data.values; if (kpio.id == "B1") { //Make a new dataList = new line to be written - dataList = new List { }; //Fill the datalist with this kpi - dataList.Add(kpio.parentName); - dataList.Add(cropfieldItem.Code); - dataList.Add(kpio.data.area); - dataList.Add(kpio.data.cropTypeCode); - dataList.Add(kpio.data.cropTypeName); - dataList.Add(kpio.id); - dataList.Add(kpio.quantity); // in KPI output quantity is what we call KPIvariable in headerlist of csv file - dataList.Add(kpio.value); - dataList.Add(kpio.unit); - dataList.Add(kpio.targetValue); - dataList.Add(kpio.thresholdValue); + dataList = new List + { + kpio.parentName, + cropfieldItem.Code, + kpio.data.area, + kpio.data.cropTypeCode, + kpio.data.cropTypeName, + kpio.id, + kpio.quantity, // in KPI output quantity is what we call KPIvariable in headerlist of csv file + kpio.value, + kpio.unit, + kpio.targetValue, + kpio.thresholdValue + }; //Write the datalist to a line to the streamwrieter sw for the output csv file sw.WriteLine(string.Join(",", dataList)); - KPIelementsOfBalance kPIelementsOfBalance = kpio.data.values; foreach (string elementName in kpio.B1elements) { // get B1element from the element called values - dataList = new List { }; string elementValue = (string)kPIelementsOfBalance.GetType().GetProperty(elementName).GetValue(kPIelementsOfBalance, null); - dataList.Add(kpio.parentName); - dataList.Add(cropfieldItem.Code); - dataList.Add(kpio.data.area); - dataList.Add(kpio.data.cropTypeCode); - dataList.Add(kpio.data.cropTypeName); - dataList.Add(kpio.id); - dataList.Add(elementName); // specific output variable name for B1 element - dataList.Add(elementValue); // specific output value name for B1 element - dataList.Add(kpio.unit); - dataList.Add(""); - dataList.Add(""); + dataList = new List + { + kpio.parentName, + cropfieldItem.Code, + kpio.data.area, + kpio.data.cropTypeCode, + kpio.data.cropTypeName, + kpio.id, + elementName, // specific output variable name for B1 element + elementValue, // specific output value name for B1 element + kpio.unit, + "", + "" + }; //Write the datalist to a line to the streamwrieter sw for the output csv file sw.WriteLine(string.Join(",", dataList)); } @@ -439,39 +443,42 @@ namespace FarmmapsKPI else if (kpio.id == "B2") { //Make a new dataList = new line to be written - dataList = new List { }; - //Fill the datalist with this kpi - dataList.Add(kpio.parentName); - dataList.Add(cropfieldItem.Code); - dataList.Add(kpio.data.area); - dataList.Add(kpio.data.cropTypeCode); - dataList.Add(kpio.data.cropTypeName); - dataList.Add(kpio.id); - dataList.Add(kpio.quantity); // in KPI output quantity is what we call KPIvariable in headerlist of csv file - dataList.Add(kpio.value); - dataList.Add(kpio.unit); - dataList.Add(kpio.targetValue); - dataList.Add(kpio.thresholdValue); + dataList = new List + { + //Fill the datalist with this kpi + kpio.parentName, + cropfieldItem.Code, + kpio.data.area, + kpio.data.cropTypeCode, + kpio.data.cropTypeName, + kpio.id, + kpio.quantity, // in KPI output quantity is what we call KPIvariable in headerlist of csv file + kpio.value, + kpio.unit, + kpio.targetValue, + kpio.thresholdValue + }; //Write the datalist to a line to the streamwrieter sw for the output csv file sw.WriteLine(string.Join(",", dataList)); - KPIelementsOfBalance kPIelementsOfBalance = kpio.data.values; foreach (string elementName in kpio.B2elements) { // get B1element from the element called values - dataList = new List { }; string elementValue = (string)kPIelementsOfBalance.GetType().GetProperty(elementName).GetValue(kPIelementsOfBalance, null); - dataList.Add(kpio.parentName); - dataList.Add(cropfieldItem.Code); - dataList.Add(kpio.data.area); - dataList.Add(kpio.data.cropTypeCode); - dataList.Add(kpio.data.cropTypeName); - dataList.Add(kpio.id); - dataList.Add(elementName); // specific output variable name for B2 element - dataList.Add(elementValue); // specific output value name for B2 element - dataList.Add(kpio.unit); - dataList.Add(""); - dataList.Add(""); + dataList = new List + { + kpio.parentName, + cropfieldItem.Code, + kpio.data.area, + kpio.data.cropTypeCode, + kpio.data.cropTypeName, + kpio.id, + elementName, // specific output variable name for B2 element + elementValue, // specific output value name for B2 element + kpio.unit, + "", + "" + }; //Write the datalist to a line to the streamwrieter sw for the output csv file sw.WriteLine(string.Join(",", dataList)); } @@ -480,39 +487,41 @@ namespace FarmmapsKPI else if (kpio.id == "C1") //TtD { //Make a new dataList = new line to be written - dataList = new List { }; - //Fill the datalist with this kpi - dataList.Add(kpio.parentName); - dataList.Add(cropfieldItem.Code); - dataList.Add(kpio.data.area); - dataList.Add(kpio.data.cropTypeCode); - dataList.Add(kpio.data.cropTypeName); - dataList.Add(kpio.id); - dataList.Add(kpio.quantity); // in KPI output quantity is what we call KPIvariable in headerlist of csv file - dataList.Add(kpio.value); - dataList.Add(kpio.unit); - dataList.Add(kpio.targetValue); - dataList.Add(kpio.thresholdValue); + dataList = new List + { + kpio.parentName, + cropfieldItem.Code, + kpio.data.area, + kpio.data.cropTypeCode, + kpio.data.cropTypeName, + kpio.id, + kpio.quantity, // in KPI output quantity is what we call KPIvariable in headerlist of csv file + kpio.value, + kpio.unit, + kpio.targetValue, + kpio.thresholdValue + }; //Write the datalist to a line to the streamwrieter sw for the output csv file sw.WriteLine(string.Join(",", dataList)); - KPIelementsOfBalance kPIelementsOfBalance = kpio.data.values; foreach (string elementName in kpio.C1elements) { // get C1element from the element called values - dataList = new List { }; string elementValue = (string)kPIelementsOfBalance.GetType().GetProperty(elementName).GetValue(kPIelementsOfBalance, null); - dataList.Add(kpio.parentName); - dataList.Add(cropfieldItem.Code); - dataList.Add(kpio.data.area); - dataList.Add(kpio.data.cropTypeCode); - dataList.Add(kpio.data.cropTypeName); - dataList.Add(kpio.id); - dataList.Add(elementName); // specific output variable name for C1 element - dataList.Add(elementValue); // specific output value name for C1 element - dataList.Add(kpio.unit); - dataList.Add(""); - dataList.Add(""); + dataList = new List + { + kpio.parentName, + cropfieldItem.Code, + kpio.data.area, + kpio.data.cropTypeCode, + kpio.data.cropTypeName, + kpio.id, + elementName, // specific output variable name for C1 element + elementValue, // specific output value name for C1 element + kpio.unit, + "", + "" + }; //Write the datalist to a line to the streamwrieter sw for the output csv file sw.WriteLine(string.Join(",", dataList)); } @@ -527,26 +536,27 @@ namespace FarmmapsKPI { if (mbp_KPIvariable == "aquaticLife") { mbp_KPIvalue = e.aquaticLife; } else if (mbp_KPIvariable == "groundWater") { mbp_KPIvalue = e.groundWater; } else if (mbp_KPIvariable == "soilLife") { mbp_KPIvalue = e.soilLife; } else { mbp_KPIvalue = ""; } //Make a new dataList = new line to be written - dataList = new List { }; - //Fill the datalist with this kpi - dataList.Add(kpio.parentName); - dataList.Add(cropfieldItem.Code); - dataList.Add(kpio.data.area); - dataList.Add(kpio.data.cropTypeCode); - dataList.Add(kpio.data.cropTypeName); - dataList.Add(kpio.id); //"E1" - dataList.Add(kpio.quantity); // "mbp" - dataList.Add(""); // not here the value - dataList.Add(""); // not here KPIunit for this indicator - dataList.Add(""); // not here KPItargetvalue for this indicator - dataList.Add(""); // not here KPIthresholdValue for this indicator - dataList.Add(e.productCode); - dataList.Add(e.productName); - dataList.Add(e.quantity); - dataList.Add(e.unitCode); - dataList.Add(e.date); - dataList.Add(mbp_KPIvariable); - dataList.Add(mbp_KPIvalue); + dataList = new List + { + kpio.parentName, + cropfieldItem.Code, + kpio.data.area, + kpio.data.cropTypeCode, + kpio.data.cropTypeName, + kpio.id, //"E1" + kpio.quantity, // "mbp" + "", // not here the value + "", // not here KPIunit for this indicator + "", // not here KPItargetvalue for this indicator + "", // not here KPIthresholdValue for this indicator + e.productCode, + e.productName, + e.quantity, + e.unitCode, + e.date, + mbp_KPIvariable, + mbp_KPIvalue + }; //Write the datalist to a line to the streamwrieter sw for the output csv file sw.WriteLine(string.Join(",", dataList)); } @@ -554,20 +564,21 @@ namespace FarmmapsKPI } else { - //Make a new dataList = new line to be written - dataList = new List { }; - //Fill the datalist with this kpi - dataList.Add(kpio.parentName); - dataList.Add(cropfieldItem.Code); - dataList.Add(kpio.data.area); - dataList.Add(kpio.data.cropTypeCode); - dataList.Add(kpio.data.cropTypeName); - dataList.Add(kpio.id); - dataList.Add(kpio.quantity); // in KPI output quantity is what we call KPIvariable in headerlist of csv file - dataList.Add(kpio.value); - dataList.Add(kpio.unit); - dataList.Add(kpio.targetValue); - dataList.Add(kpio.thresholdValue); + //Any other KPI, example A1 or D1, with just 1 record to write + dataList = new List + { + kpio.parentName, + cropfieldItem.Code, + kpio.data.area, + kpio.data.cropTypeCode, + kpio.data.cropTypeName, + kpio.id, + kpio.quantity, + kpio.value, + kpio.unit, + kpio.targetValue, + kpio.thresholdValue + }; //Write the datalist to a line to the streamwrieter sw for the output csv file sw.WriteLine(string.Join(",", dataList)); } From dea4f102f56370db8ed8fb242e43e2203fd32663 Mon Sep 17 00:00:00 2001 From: Pepijn van Oort Date: Fri, 16 Feb 2024 15:28:15 +0100 Subject: [PATCH 45/66] C1 defs --- FarmmapsKPI/KPIdefinitions.csv | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/FarmmapsKPI/KPIdefinitions.csv b/FarmmapsKPI/KPIdefinitions.csv index b26da88..5460086 100644 --- a/FarmmapsKPI/KPIdefinitions.csv +++ b/FarmmapsKPI/KPIdefinitions.csv @@ -11,9 +11,9 @@ B2,disposal, B2,pFertillizerNKgHa, B2,sowPlantingNKgHa, C1,organic matter supply,"Organic matter surplus = Organic matter from manure + Crop residues to soil - organic matter removal through harvested product. Calculated KPI internal model & parameters and from user input: yield and manure applications. ""Effective"" means netto, after correction for carbon respiration by soil microbial activity" -C1,fertilizerEom, -C1,greenManureEom, -C1,cropLeftoversEom, +C1,fertilizerEom,"Effective organic mater added through fertilizer (animal manure, compost, etc)" +C1,greenManureEom,Effective organic mater added through green manure (also called cover crop) +C1,cropLeftoversEom,Effective organic mater added through crop residues D1,pesticides,"kg/ha of active ingredient. Sum over kg a.i. of the different crop protection agents applied, with concentration (kg a.i. / kg or litre) of crop protection agents retrieved from CTBG" E1,mpb,"milieubelastingspunten. For each pesticide applied show 3 sub-mbp's: aquaticLife, groundWater, soilLife" E1,mpb, From 5d2789a806ba44fa2c4b2d6c211732e7a6b8442a Mon Sep 17 00:00:00 2001 From: tamara Date: Mon, 19 Feb 2024 15:07:26 +0100 Subject: [PATCH 46/66] updated KPI definitions --- FarmmapsKPI/KPIdefinitions.csv | 32 ++++++++++++++++---------------- FarmmapsKPI/TODO.txt | 5 +++-- 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/FarmmapsKPI/KPIdefinitions.csv b/FarmmapsKPI/KPIdefinitions.csv index 5460086..f6f2ac6 100644 --- a/FarmmapsKPI/KPIdefinitions.csv +++ b/FarmmapsKPI/KPIdefinitions.csv @@ -1,24 +1,24 @@ KPIid,KPIvariable,Description A1,yield,observed yield (user input) -B1,nitrogen,Nitrogen surplus = N artifical fertilizer + N manure+ N atmospheric deposition + N fixation by crop - N removal through harvested product. Calculated KPI internal model & parameters and from user input: yield and fertilizer/manure applications. N fixation (legumes only): internal equations. N depostition: RIVM maps -B1,disposal, -B1,nFixationNKgHa, -B1,nDepositionNKgHa, -B1,nFertilizerNKgHa, -B1,sowPlantingNKgHa, -B2,phosphate,Phosphate surplus = P fertilizer - P removal through harvested product. Calculated KPI internal model & parameters and from user input: yield and fertilizer applications -B2,disposal, -B2,pFertillizerNKgHa, -B2,sowPlantingNKgHa, -C1,organic matter supply,"Organic matter surplus = Organic matter from manure + Crop residues to soil - organic matter removal through harvested product. Calculated KPI internal model & parameters and from user input: yield and manure applications. ""Effective"" means netto, after correction for carbon respiration by soil microbial activity" -C1,fertilizerEom,"Effective organic mater added through fertilizer (animal manure, compost, etc)" -C1,greenManureEom,Effective organic mater added through green manure (also called cover crop) -C1,cropLeftoversEom,Effective organic mater added through crop residues +B1,nitrogen,Nitrogen surplus = N artifical fertilizer + N manure+ N atmospheric deposition + N fixation by crop + N from seed potatoes - N removal through harvested product. Calculated KPI internal model & parameters and from user input: yield and fertilizer/manure applications. +B1,disposal, N removal through harvested product. Calculated KPI internal model & parameters and from user input: yield. +B1,nFixationNKgHa, N fixated by crop (legumes only). Calculated KPI internal model & parameters and from user input: planting date +B1,nDepositionNKgHa, N deposition based on RIVM maps. +B1,nFertilizerNKgHa, N from artifical fertilizers and manure.Calculated KPI internal model & parameters and from user input: fertilizer/manure applications. +B1,sowPlantingNKgHa, N from seed potatoes (potatoes only). Calculated KPI internal model & parameters and from user input: amount of planted seed potatoes +B2,phosphate,Phosphate surplus = P fertilizer + P from seed potatoes - P removal through harvested product. Calculated KPI internal model & parameters and from user input: yield and fertilizer applications +B2,disposal, P2O5 removal through harvested product. Calculated KPI internal model & parameters and from user input: yield +B2,pFertillizerNKgHa, P2O5 from artifical fertilizers and manure.Calculated KPI internal model & parameters and from user input: fertilizer/manure applications. +B2,sowPlantingNKgHa, P2O5 from seeds potatoes (potatoes only). Calculated KPI internal model & parameters and from user input: amount of planted seed potatoes +C1,effective organic matter supply,"Organic matter supply = Organic matter from manure + Crop residues to soil + organic matter from green manure. Calculated KPI internal model & parameters and from user input: yield and manure applications. ""Effective"" means netto, after correction for carbon respiration by soil microbial activity" +C1,fertilizerEom,Effective organic matter added through fertilizer (animal manure, compost, etc). Calculated KPI internal model & parameters and from user input: manure applications. +C1,greenManureEom,Effective organic matter added through green manure (also called cover crop). Calculated KPI internal model & parameters and from user input: cover crop. +C1,cropLeftoversEom,Effective organic matter added through crop residues of main product and (when relevant) straw. Calculated KPI internal model & parameters. D1,pesticides,"kg/ha of active ingredient. Sum over kg a.i. of the different crop protection agents applied, with concentration (kg a.i. / kg or litre) of crop protection agents retrieved from CTBG" E1,mpb,"milieubelastingspunten. For each pesticide applied show 3 sub-mbp's: aquaticLife, groundWater, soilLife" E1,mpb, E1,mpb,? -F1,greenness,farmlevel index of diversity of crops -F2,rotationindex, +F1,greenness, time of year that the field was covered. Based on NDVI data from AgroDataCube. +F2,rotationindex, index of diversity of crops over time. Calculated in AgroDataCube ,KPItargetvalue,target value as in benchmark value for same crop in same region. E.g. cropyield target is from Central Bureau of Statistics (CBS) for same crop in region in which the cropfield (geometry) is located ,KPIthresholdValue,threshold from ??? Ask farmmaps. Surplus nitrogen / phosphate / pesticides must not be above threshold. Surplus organic matter supply must be above threshold diff --git a/FarmmapsKPI/TODO.txt b/FarmmapsKPI/TODO.txt index fda88ee..dbf7b7a 100644 --- a/FarmmapsKPI/TODO.txt +++ b/FarmmapsKPI/TODO.txt @@ -51,11 +51,12 @@ dataOperations: "status": "3": "DataCropfieldCharacteristic": { "code": "860619", //PO20231004: so what does this code mean? Can we see the code list somewhere? NOT YET TESTED +KPIelementsOfBalance: Tamara b. for nitrogen B1 rename disposal to nHarvestedNKgHa PARTLY c. for phosporous B2 rename disposal to pHarvestedPKgHa PARTLY d. for phosporous B2 rename from pFertilizerNKgHa to pFertilizerPKgHa PARTLY e. idem sowPlantingNKgHa PARTLY f. add straw for N & P in output g. add 2 examples in KPIinput.json: straw yes/no removed. Bijv. nStrawLeftInField EN nStrawRemoved - h. update KPIdefinitions.csv - i. for phosporous B2 rename 'cropLeftoversEom' to 'cropResiduesEom' \ No newline at end of file + h. update KPIdefinitions.csv PARTLY + i. for C1 rename 'cropLeftoversEom' to 'cropResiduesEom' \ No newline at end of file From 77532fee9f95a7a18a4d6935d5aee2131e1d4ce3 Mon Sep 17 00:00:00 2001 From: tamara Date: Tue, 27 Feb 2024 13:55:51 +0100 Subject: [PATCH 47/66] add codelist 265. Empty result? --- FarmmapsDownloadCL/DownloadCLInput.json | 3 +- FarmmapsDownloadCL/Models/CodelistsClasses.cs | 33 ++++++++++++++++--- 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/FarmmapsDownloadCL/DownloadCLInput.json b/FarmmapsDownloadCL/DownloadCLInput.json index be43c9c..42340f6 100644 --- a/FarmmapsDownloadCL/DownloadCLInput.json +++ b/FarmmapsDownloadCL/DownloadCLInput.json @@ -1,4 +1,5 @@ { - "codeLists": [ "CL022", "CL127", "CL263", "CL405", "CL251", "CL042" ] + //"codeLists": [ "CL022", "CL127", "CL263", "CL405", "CL251", "CL042" ], + "codeLists": [ "CL263", "CL265"] //"DownloadFolder": "Downloads", //"C:\\hugoschrererdir\\kpidir\\", // "Downloads", -> if you just put "Downloads" the program will download to somewhere in ..\FarmMapsApiClient_WURtest\FarmmapsDataDownload\bin\Debug\netcoreapp3.1\Downloads\ } \ No newline at end of file diff --git a/FarmmapsDownloadCL/Models/CodelistsClasses.cs b/FarmmapsDownloadCL/Models/CodelistsClasses.cs index 66f2cbe..1264e34 100644 --- a/FarmmapsDownloadCL/Models/CodelistsClasses.cs +++ b/FarmmapsDownloadCL/Models/CodelistsClasses.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Reflection; using System.Text; namespace FarmmapsDownloadCL.Models @@ -17,6 +18,7 @@ namespace FarmmapsDownloadCL.Models this.dictCl.Add("CL127", new List { "vnd.farmmaps.itemtype.codelist.cl127", "CL127item" }); this.dictCl.Add("CL251", new List { "vnd.farmmaps.itemtype.codelist.cl251", "CL251item" }); this.dictCl.Add("CL263", new List { "vnd.farmmaps.itemtype.codelist.cl263", "CL263item" }); + this.dictCl.Add("CL265", new List { "vnd.farmmaps.itemtype.codelist.cl265", "CL265item" }); this.dictCl.Add("CL405", new List { "vnd.farmmaps.itemtype.codelist.cl405", "CL405item" }); } public string itemcode; @@ -86,7 +88,7 @@ namespace FarmmapsDownloadCL.Models { public CL263item() { - this.headers = new string[] { "codelist", "code", "eppoCode", "botanicName", "description", "description_fr", "description_nl", "cropGroupCode", "cropGroupName", "cultivationGroupCode", "cultivationGroupName" }; + this.headers = new string[] { "codelist", "code", "eppoCode", "botanicName", "description", "description_fr", "description_nl", "cropGroupCode", "cropGroupName", "subCropGroupCode", "subCropGroupName", "cultivationGroupCode", "cultivationGroupName" }; this.codelist = "CL263"; this.about = new string[] { "EDI-Crop, coderingslijst gewassoorten" }; } @@ -95,18 +97,39 @@ namespace FarmmapsDownloadCL.Models public string codelist { get; } public string[] about { get; } public string code { get; set; } - public string eppoCode { get; set; } - public string botanicName { get; set; } - public string description { get; set; } + public string eppoCode { get; set; } //? + public string botanicName { get; set; } //? + public string description { get; set; } //? public string description_nl { get; set; } public string description_fr { get; set; } public string cropGroupCode { get; set; } public string cropGroupName { get; set; } + public string subCropGroupCode { get; set; } + public string subCropGroupName { get; set; } public string cultivationGroupCode { get; set; } public string cultivationGroupName { get; set; } - public string composition { get; set; } + public string composition { get; set; } //? } + public class CL265item + { + public CL265item() + { + this.headers = new string[] { "Code", "Formaat", "Eenheid", "Waarde" }; + this.codelist = "CL265"; + this.about = new string[] { "EDI-Crop CL265: FarmCharacterictics, FieldCharacteristics, CropFieldCharacteristics, TreatmentCharacteristics", + "Betreft codelijst eigenschappen teelt en teelthandelingen CL265.Daar waar in kolom 'Waarde' wordt verwezen naar CL.., wordt de codelijst bedoeld die onder het betreffende nummer bekend is.", + "In het EDI-Crop bericht wordt de code gebruikt onder resp. CropFieldCharacteristic c.q.TreatmentCharacteristic middels de dataelementen PropertyVariableCode (invullen vanuit CL265) en PropertyVariableValue(voor de waarde)"}; + } + + public string[] headers { get; } + public string codelist { get; } + public string[] about { get; } + public string Code { get; set; } + public string Formaat { get; set; } + public string Eenheid { get; set; } + public string Waarde { get; set; } + } public class CL405item { public CL405item() From d756f08b4cf3e1db87d7bc6a4bb72fe7e775c270 Mon Sep 17 00:00:00 2001 From: tamara Date: Tue, 27 Feb 2024 13:58:09 +0100 Subject: [PATCH 48/66] typos --- FarmmapsDownloadCL/DownloadCLInput.json | 4 ++-- FarmmapsDownloadCL/Models/CodelistsClasses.cs | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/FarmmapsDownloadCL/DownloadCLInput.json b/FarmmapsDownloadCL/DownloadCLInput.json index 42340f6..ad850fc 100644 --- a/FarmmapsDownloadCL/DownloadCLInput.json +++ b/FarmmapsDownloadCL/DownloadCLInput.json @@ -1,5 +1,5 @@ { - //"codeLists": [ "CL022", "CL127", "CL263", "CL405", "CL251", "CL042" ], - "codeLists": [ "CL263", "CL265"] + "codeLists": [ "CL022", "CL127", "CL263", "CL265", "CL405", "CL251", "CL042" ] + //"codeLists": [ "CL263", "CL265"] //"DownloadFolder": "Downloads", //"C:\\hugoschrererdir\\kpidir\\", // "Downloads", -> if you just put "Downloads" the program will download to somewhere in ..\FarmMapsApiClient_WURtest\FarmmapsDataDownload\bin\Debug\netcoreapp3.1\Downloads\ } \ No newline at end of file diff --git a/FarmmapsDownloadCL/Models/CodelistsClasses.cs b/FarmmapsDownloadCL/Models/CodelistsClasses.cs index 1264e34..b3d4edc 100644 --- a/FarmmapsDownloadCL/Models/CodelistsClasses.cs +++ b/FarmmapsDownloadCL/Models/CodelistsClasses.cs @@ -97,9 +97,9 @@ namespace FarmmapsDownloadCL.Models public string codelist { get; } public string[] about { get; } public string code { get; set; } - public string eppoCode { get; set; } //? - public string botanicName { get; set; } //? - public string description { get; set; } //? + public string eppoCode { get; set; } + public string botanicName { get; set; } + public string description { get; set; } public string description_nl { get; set; } public string description_fr { get; set; } public string cropGroupCode { get; set; } From 84f27db16e828d20b1511de9f0a44ff613eae8de Mon Sep 17 00:00:00 2001 From: tamara Date: Tue, 27 Feb 2024 15:44:07 +0100 Subject: [PATCH 49/66] DataCropfieldCharacteristics as an array containing cropyield and possibly strawyield. If no strawyield is filled in then straw remains in field. --- FarmmapsKPI/KPIApplication.cs | 22 ++- FarmmapsKPI/KPIInput.json | 258 ++++++++++++++++++++++++++------- FarmmapsKPI/Models/KPIInput.cs | 2 +- 3 files changed, 218 insertions(+), 64 deletions(-) diff --git a/FarmmapsKPI/KPIApplication.cs b/FarmmapsKPI/KPIApplication.cs index 312c002..e76b07f 100644 --- a/FarmmapsKPI/KPIApplication.cs +++ b/FarmmapsKPI/KPIApplication.cs @@ -88,7 +88,7 @@ namespace FarmmapsKPI TimeSpan tsTotalEstimated; for (int i = 0; i < fieldsInputs.Count; i++) - //for (int i = 3; i < 4; i++) + //for (int i = 10; i < 12; i++) { watch.Restart(); input = fieldsInputs[i]; @@ -182,21 +182,27 @@ namespace FarmmapsKPI SaveSettings(settingsfile); } - // The cropfieldCharacteristicItem is used to enter crop yields + // The cropfieldCharacteristicItem is used to enter crop yields and/or straw yields Item cropfieldCharacteristicItem; + string dataCropfieldCharacteristic; //1 useExistingCropfieldWithChildren = false -> create new //2 useExistingCropfieldWithChildren = true && input.CropfieldCharacteristicItemCode = "" or absent -> read from settings json //3 useExistingCropfieldWithChildren = true && input.CropfieldCharacteristicItemCode like "deb48a74c5b54299bb852f17288010e9" in KPIinput -> use this one if (useExistingCropfieldWithChildren == false) { - _logger.LogInformation("CreateCropfieldCharacteristicItemAsync ..."); - cropfieldCharacteristicItem = await _generalService.CreateCropfieldCharacteristicItemAsync(cropfieldItem.Code, cropYear, fieldGeom, input.DataCropfieldCharacteristic.ToString(Formatting.None)); - _settings.CropfieldCharacteristicItemCode = cropfieldCharacteristicItem.Code; - SaveSettings(settingsfile); + for (int i = 0; i < input.DataCropfieldCharacteristics.Length; i++) + { + dataCropfieldCharacteristic = input.DataCropfieldCharacteristics[i].ToString(Formatting.None); + dynamic data = JObject.Parse(dataCropfieldCharacteristic); + _logger.LogInformation($"CreateCropfieldCharacteristicItemAsync ... for cropfieldCharacteristic {i}: '{data.label}', value '{data.value}'"); + cropfieldCharacteristicItem = await _generalService.CreateCropfieldCharacteristicItemAsync(cropfieldItem.Code, cropYear, fieldGeom, dataCropfieldCharacteristic); + _settings.CropfieldCharacteristicItemCode = cropfieldCharacteristicItem.Code; + SaveSettings(settingsfile); + } } else if (string.IsNullOrEmpty(input.CropfieldItemCode)) { - _logger.LogInformation("reading OperationItemCode from settings file"); + _logger.LogInformation("reading CropfieldCharacteristicItemCode from settings file"); cropfieldCharacteristicItem = await _farmmapsApiService.GetItemAsync(_settings.CropfieldCharacteristicItemCode); } else @@ -362,7 +368,7 @@ namespace FarmmapsKPI //additional criterion for while loop: check if it really contains the E1 mbp elements. //while (trycnt == 0 || ((KPIItems.Count < targetKPIitemsCount || boolAquaticLife == false) & trycnt < maxtries)) //normal while loop - targetKPIitemsCount = 7; //if we know we should be getting 7 KPI items (A1,B1,B2,C1,D1,F1,F2) //8 if also E1 + targetKPIitemsCount = 8; //if we know we should be getting 8 KPI items (A1,B1,B2,C1,D1,E1,F1,F2) while (trycnt == 0 || (KPIItems.Count < targetKPIitemsCount & trycnt < maxtries)) { _logger.LogInformation($"Call nr {trycnt + 1}"); diff --git a/FarmmapsKPI/KPIInput.json b/FarmmapsKPI/KPIInput.json index a891c5a..c43fc1e 100644 --- a/FarmmapsKPI/KPIInput.json +++ b/FarmmapsKPI/KPIInput.json @@ -24,12 +24,14 @@ "OperationItemCodes": [], // could contain for example this: ["4d1a6f28ea3f414180663baa4b08c60f"] or ["4d1a6f28ea3f414180663baa4b08c60f","21e08386f0454ad79acfebf78649d59b"] if these are the operations of the provided CropRecordingItemCode. And then if useExistingCropfieldWithChildren = true, these would be used "dataOperations": [], //leaving the operations empty means no fertilizer is applied "CropfieldCharacteristicItemCode": "", - "DataCropfieldCharacteristic": { - //PO20231029: not different yields from different parts of fields? - "code": "860619", //PO20231004: so what does this code mean? Can we see the code list somewhere? - "label": "cropyield", - "value": "48.01" - }, + "DataCropfieldCharacteristics": [ + { + //PO20231029: not different yields from different parts of fields? + "code": "860619", //From codelist CL265. 860619 refers to cropyield in ton/ha + "label": "cropyield", + "value": "48.01" + } + ], //"DownloadFolder": "Downloads", //"C:\\hugoschrererdir\\kpidir\\", // "Downloads", -> if you just put "Downloads" the program will download to somewhere in ..\FarmMapsApiClient_KB34_MAST\FarmmapsKPI\bin\Debug\netcoreapp3.1\Downloads\ "CropYear": 2022, //geometryJson = polygon of location of the cropfield, coordinates in LON,LAT (LON: decimal degrees East, LAT: decimal degrees North) @@ -86,12 +88,14 @@ "OperationItemCodes": [], // could contain for example this: ["4d1a6f28ea3f414180663baa4b08c60f"] or ["4d1a6f28ea3f414180663baa4b08c60f","21e08386f0454ad79acfebf78649d59b"] if these are the operations of the provided CropRecordingItemCode. And then if useExistingCropfieldWithChildren = true, these would be used "dataOperations": [], //leaving the operations empty means no fertilizer is applied "CropfieldCharacteristicItemCode": "", - "DataCropfieldCharacteristic": { - //PO20231029: not different yields from different parts of fields? - "code": "860619", //PO20231004: so what does this code mean? Can we see the code list somewhere? - "label": "cropyield", - "value": "48.01" - }, + "DataCropfieldCharacteristics": [ + { + //PO20231029: not different yields from different parts of fields? + "code": "860619", //From codelist CL265. 860619 refers to cropyield in ton/ha + "label": "cropyield", + "value": "48.01" + } + ], //"DownloadFolder": "Downloads", //"C:\\hugoschrererdir\\kpidir\\", // "Downloads", -> if you just put "Downloads" the program will download to somewhere in ..\FarmMapsApiClient_KB34_MAST\FarmmapsKPI\bin\Debug\netcoreapp3.1\Downloads\ "CropYear": 2022, //geometryJson = polygon of location of the cropfield, coordinates in LON,LAT (LON: decimal degrees East, LAT: decimal degrees North) @@ -148,12 +152,14 @@ "OperationItemCodes": [], // could contain for example this: ["4d1a6f28ea3f414180663baa4b08c60f"] or ["4d1a6f28ea3f414180663baa4b08c60f","21e08386f0454ad79acfebf78649d59b"] if these are the operations of the provided CropRecordingItemCode. And then if useExistingCropfieldWithChildren = true, these would be used "dataOperations": [], //leaving the operations empty means no fertilizer is applied "CropfieldCharacteristicItemCode": "", - "DataCropfieldCharacteristic": { - //PO20231029: not different yields from different parts of fields? - "code": "860619", //PO20231004: so what does this code mean? Can we see the code list somewhere? - "label": "cropyield", - "value": "48.01" - }, + "DataCropfieldCharacteristics": [ + { + //PO20231029: not different yields from different parts of fields? + "code": "860619", //From codelist CL265. 860619 refers to cropyield in ton/ha + "label": "cropyield", + "value": "48.01" + } + ], //"DownloadFolder": "Downloads", //"C:\\hugoschrererdir\\kpidir\\", // "Downloads", -> if you just put "Downloads" the program will download to somewhere in ..\FarmMapsApiClient_KB34_MAST\FarmmapsKPI\bin\Debug\netcoreapp3.1\Downloads\ "CropYear": 2022, //geometryJson = polygon of location of the cropfield, coordinates in LON,LAT (LON: decimal degrees East, LAT: decimal degrees North) @@ -241,11 +247,13 @@ } ], "CropfieldCharacteristicItemCode": "", - "DataCropfieldCharacteristic": { - "code": "860619", //PO20231004: so what does this code mean? Can we see the code list somewhere? - "label": "cropyield", - "value": "48.01" - }, + "DataCropfieldCharacteristics": [ + { + "code": "860619", //From codelist CL265. 860619 refers to cropyield in ton/ha + "label": "cropyield", + "value": "48.01" + } + ], //"DownloadFolder": "Downloads", //"C:\\hugoschrererdir\\kpidir\\", // "Downloads", -> if you just put "Downloads" the program will download to somewhere in ..\FarmMapsApiClient_WURtest\FarmmapsDataDownload\bin\Debug\netcoreapp3.1\Downloads\ "CropYear": 2022, "geometryJson": { @@ -314,11 +322,13 @@ } ], "CropfieldCharacteristicItemCode": "", - "DataCropfieldCharacteristic": { - "code": "860619", //PO20231004: so what does this code mean? Can we see the code list somewhere? - "label": "cropyield", - "value": "48.01" - }, + "DataCropfieldCharacteristics": [ + { + "code": "860619", //From codelist CL265. 860619 refers to cropyield in ton/ha + "label": "cropyield", + "value": "48.01" + } + ], //"DownloadFolder": "Downloads", //"C:\\hugoschrererdir\\kpidir\\", // "Downloads", -> if you just put "Downloads" the program will download to somewhere in ..\FarmMapsApiClient_WURtest\FarmmapsDataDownload\bin\Debug\netcoreapp3.1\Downloads\ "CropYear": 2022, "geometryJson": { @@ -387,11 +397,13 @@ } ], "CropfieldCharacteristicItemCode": "", - "DataCropfieldCharacteristic": { - "code": "860619", //PO20231004: so what does this code mean? Can we see the code list somewhere? - "label": "cropyield", - "value": "48.01" - }, + "DataCropfieldCharacteristics": [ + { + "code": "860619", //From codelist CL265. 860619 refers to cropyield in ton/ha + "label": "cropyield", + "value": "48.01" + } + ], //"DownloadFolder": "Downloads", //"C:\\hugoschrererdir\\kpidir\\", // "Downloads", -> if you just put "Downloads" the program will download to somewhere in ..\FarmMapsApiClient_WURtest\FarmmapsDataDownload\bin\Debug\netcoreapp3.1\Downloads\ "CropYear": 2022, "geometryJson": { @@ -460,11 +472,13 @@ } ], "CropfieldCharacteristicItemCode": "", - "DataCropfieldCharacteristic": { - "code": "860619", //PO20231004: so what does this code mean? Can we see the code list somewhere? - "label": "cropyield", - "value": "48.01" - }, + "DataCropfieldCharacteristics": [ + { + "code": "860619", //From codelist CL265. 860619 refers to cropyield in ton/ha + "label": "cropyield", + "value": "48.01" + } + ], //"DownloadFolder": "Downloads", //"C:\\hugoschrererdir\\kpidir\\", // "Downloads", -> if you just put "Downloads" the program will download to somewhere in ..\FarmMapsApiClient_WURtest\FarmmapsDataDownload\bin\Debug\netcoreapp3.1\Downloads\ "CropYear": 2022, "geometryJson": { @@ -528,11 +542,13 @@ } ], "CropfieldCharacteristicItemCode": "", - "DataCropfieldCharacteristic": { - "code": "860619", - "label": "cropyield", - "value": "48.01" - }, + "DataCropfieldCharacteristics": [ + { + "code": "860619", + "label": "cropyield", + "value": "48.01" + } + ], //"DownloadFolder": "Downloads", //"C:\\hugoschrererdir\\kpidir\\", // "Downloads", -> if you just put "Downloads" the program will download to somewhere in ..\FarmMapsApiClient_KB34_MAST\FarmmapsKPI\bin\Debug\netcoreapp3.1\Downloads\ "CropYear": 2022, //geometryJson = polygon of location of the cropfield, coordinates in LON,LAT (LON: decimal degrees East, LAT: decimal degrees North) @@ -578,11 +594,13 @@ "OperationItemCodes": [], "dataOperations": [], "CropfieldCharacteristicItemCode": "", - "DataCropfieldCharacteristic": { - "code": "860619", //PO20231004: so what does this code mean? Can we see the code list somewhere? - "label": "cropyield", - "value": "9.58" //CBS: https://www.cbs.nl/nl-nl/cijfers/detail/37738, 2022: 40.4 Mkg / 4216 ha = 40400 / 4216 - }, + "DataCropfieldCharacteristics": [ + { + "code": "860619", //From codelist CL265. 860619 refers to cropyield in ton/ha + "label": "cropyield", + "value": "9.58" //CBS: https://www.cbs.nl/nl-nl/cijfers/detail/37738, 2022: 40.4 Mkg / 4216 ha = 40400 / 4216 + } + ], //"DownloadFolder": "Downloads", //"C:\\hugoschrererdir\\kpidir\\", // "Downloads", -> if you just put "Downloads" the program will download to somewhere in ..\FarmMapsApiClient_WURtest\FarmmapsDataDownload\bin\Debug\netcoreapp3.1\Downloads\ "CropYear": 2022, //"StartDate": "2022-04-15", //PO20231108: StartDate & EndDate important for legumes, where Nitrogen fixation depends on number of days from start to end. More days = more N fixation @@ -629,15 +647,145 @@ "OperationItemCodes": [], "dataOperations": [], "CropfieldCharacteristicItemCode": "", - "DataCropfieldCharacteristic": { - "code": "860619", //PO20231004: so what does this code mean? Can we see the code list somewhere? - "label": "cropyield", - "value": "9.58" //CBS: https://www.cbs.nl/nl-nl/cijfers/detail/37738, 2022: 40.4 Mkg / 4216 ha = 40400 / 4216 - }, + "DataCropfieldCharacteristics": [ + { + "code": "860619", //From codelist CL265. 860619 refers to cropyield in ton/ha + "label": "cropyield", + "value": "9.58" //CBS: https://www.cbs.nl/nl-nl/cijfers/detail/37738, 2022: 40.4 Mkg / 4216 ha = 40400 / 4216 + } + ], //"DownloadFolder": "Downloads", //"C:\\hugoschrererdir\\kpidir\\", // "Downloads", -> if you just put "Downloads" the program will download to somewhere in ..\FarmMapsApiClient_WURtest\FarmmapsDataDownload\bin\Debug\netcoreapp3.1\Downloads\ "CropYear": 2022, - "StartDate": "2022-04-15", //PO20231108: StartDate & EndDate important for legumes, where Nitrogen fixation depends on number of days from start to end. More days = more N fixation - "EndDate": "2022-07-30", // if StartDate & EndDate left empty then whole year from cropYear + "StartDate": "2022-04-15", //PO20231108: StartDate & EndDate important for legumes, where Nitrogen fixation depends on number of days from start to end. More days = more N fixation + "EndDate": "2022-07-30", // if StartDate & EndDate left empty then whole year from cropYear + "geometryJson": { + "type": "Polygon", + "coordinates": [ + [ + [ + 5.5945257993548765, + 52.57080744107003 + ], + [ + 5.598645994070678, + 52.571540800206236 + ], + [ + 5.599381743127071, + 52.57012773140724 + ], + [ + 5.595408698222548, + 52.56968054825188 + ], + [ + 5.5945257993548765, + 52.57080744107003 + ] + ] + ] + } + }, + { + "useExistingCropfieldWithChildren": false, + "deleteNewlyCreatedAfterCalc": true, + "fieldName": "test_WinterWheat_DefaultStrawRemainsInField", + //Note when you run the FarmmapsKPI project, somewhere to your C:\git\FarmMapsApiClient_KB34_MAST\FarmmapsKPI\bin\Debug\netcoreapp3.1\ directory, files named like this are written: + //'Settings_aardappelveld_test_Potato_ZeroFertilizer.json', where 'aardappelveld_test_Potato_ZeroFertilizer.json' is your fieldName (see below) + //1 useExistingCropfieldWithChildren = false -> create new + //2 useExistingCropfieldWithChildren = true && CropfieldItemCode = "" or absent -> read fieldName (see below) -> read all codes from settings json file + //3 useExistingCropfieldWithChildren = true && CropfieldItemCode like "deb48a74c5b54299bb852f17288010e9" in KPIinput -> if you have in your account already an existing cropfield then use that + "CropfieldItemCode": "", // could contain for example this: "abae97f89f3c4ac08953b1b8bea9f076" if this is an exisiting CropfieldItemCode in your account. + //And then if useExistingCropfieldWithChildren = true, this would be used. + "dataCropfield": { + //"area": 4.22, //Leave empty, KPI app will calculate it from geometry + "final": true, //always true + "soilCode": "1", //From codelist CL405. Can be omitted if unknown + "soilName": "Sand", //From codelist CL405 + "cropTypeCode": "1020101", //From codelist CL263 + "cropTypeName": "Winter wheat" //From codelist CL263 + }, + "CropRecordingItemCode": "", // could contain for example this: "314aa1b6480b493ba28a5f55039a01d4" if this is the CropRecordingItemCode of the provided CropfieldItemCode. And then if useExistingCropfieldWithChildren = true, this would be used + "OperationItemCodes": [], // could contain for example this: ["4d1a6f28ea3f414180663baa4b08c60f"] or ["4d1a6f28ea3f414180663baa4b08c60f","21e08386f0454ad79acfebf78649d59b"] if these are the operations of the provided CropRecordingItemCode. And then if useExistingCropfieldWithChildren = true, these would be used + "dataOperations": [], //leaving the operations empty means no fertilizer is applied + "CropfieldCharacteristicItemCode": "", + "DataCropfieldCharacteristics": [ + { + //PO20231029: not different yields from different parts of fields? + "code": "860619", //From codelist CL265. 860619 refers to cropyield in ton/ha + "label": "cropyield", + "value": "10.0" + } + ], + //"DownloadFolder": "Downloads", //"C:\\hugoschrererdir\\kpidir\\", // "Downloads", -> if you just put "Downloads" the program will download to somewhere in ..\FarmMapsApiClient_KB34_MAST\FarmmapsKPI\bin\Debug\netcoreapp3.1\Downloads\ + "CropYear": 2022, + //geometryJson = polygon of location of the cropfield, coordinates in LON,LAT (LON: decimal degrees East, LAT: decimal degrees North) + "geometryJson": { + "type": "Polygon", + "coordinates": [ + [ + [ + 5.5945257993548765, + 52.57080744107003 + ], + [ + 5.598645994070678, + 52.571540800206236 + ], + [ + 5.599381743127071, + 52.57012773140724 + ], + [ + 5.595408698222548, + 52.56968054825188 + ], + [ + 5.5945257993548765, + 52.57080744107003 + ] + ] + ] + } + }, + { + "useExistingCropfieldWithChildren": false, + "deleteNewlyCreatedAfterCalc": true, + "fieldName": "test_WinterWheat_StrawRemovedFromField", + //Note when you run the FarmmapsKPI project, somewhere to your C:\git\FarmMapsApiClient_KB34_MAST\FarmmapsKPI\bin\Debug\netcoreapp3.1\ directory, files named like this are written: + //'Settings_aardappelveld_test_Potato_ZeroFertilizer.json', where 'aardappelveld_test_Potato_ZeroFertilizer.json' is your fieldName (see below) + //1 useExistingCropfieldWithChildren = false -> create new + //2 useExistingCropfieldWithChildren = true && CropfieldItemCode = "" or absent -> read fieldName (see below) -> read all codes from settings json file + //3 useExistingCropfieldWithChildren = true && CropfieldItemCode like "deb48a74c5b54299bb852f17288010e9" in KPIinput -> if you have in your account already an existing cropfield then use that + "CropfieldItemCode": "", // could contain for example this: "abae97f89f3c4ac08953b1b8bea9f076" if this is an exisiting CropfieldItemCode in your account. + //And then if useExistingCropfieldWithChildren = true, this would be used. + "dataCropfield": { + //"area": 4.22, //Leave empty, KPI app will calculate it from geometry + "final": true, //always true + "soilCode": "1", //From codelist CL405. Can be omitted if unknown + "soilName": "Sand", //From codelist CL405 + "cropTypeCode": "1020101", //From codelist CL263 + "cropTypeName": "Winter wheat" //From codelist CL263 + }, + "CropRecordingItemCode": "", // could contain for example this: "314aa1b6480b493ba28a5f55039a01d4" if this is the CropRecordingItemCode of the provided CropfieldItemCode. And then if useExistingCropfieldWithChildren = true, this would be used + "OperationItemCodes": [], // could contain for example this: ["4d1a6f28ea3f414180663baa4b08c60f"] or ["4d1a6f28ea3f414180663baa4b08c60f","21e08386f0454ad79acfebf78649d59b"] if these are the operations of the provided CropRecordingItemCode. And then if useExistingCropfieldWithChildren = true, these would be used + "dataOperations": [], //leaving the operations empty means no fertilizer is applied + "CropfieldCharacteristicItemCode": "", + "DataCropfieldCharacteristics": [ + { + //PO20231029: not different yields from different parts of fields? + "code": "860619", //From codelist CL265. 860619 refers to cropyield in ton/ha + "label": "cropyield", + "value": "10.0" + }, + { + "code": "fm009", //TODO: which codelist? + "label": "strawyield", + "value": "5.9" + } + ], + "CropYear": 2022, + //geometryJson = polygon of location of the cropfield, coordinates in LON,LAT (LON: decimal degrees East, LAT: decimal degrees North) "geometryJson": { "type": "Polygon", "coordinates": [ diff --git a/FarmmapsKPI/Models/KPIInput.cs b/FarmmapsKPI/Models/KPIInput.cs index 8d2df12..1a89084 100644 --- a/FarmmapsKPI/Models/KPIInput.cs +++ b/FarmmapsKPI/Models/KPIInput.cs @@ -15,7 +15,7 @@ namespace FarmmapsKPI.Models public string[] OperationItemCodes { get; set; } public JObject[] DataOperations { get; set; } public string CropfieldCharacteristicItemCode { get; set; } - public JObject DataCropfieldCharacteristic { get; set; } + public JObject[] DataCropfieldCharacteristics { get; set; } public string DownloadFolder { get; set; } public int CropYear { get; set; } public DateTime StartDate { get; set; } From 67014e20f51e06e7a376a45911b76ccfd35f8d60 Mon Sep 17 00:00:00 2001 From: tamara Date: Thu, 7 Mar 2024 10:37:27 +0100 Subject: [PATCH 50/66] updated names of output KPI B1 and B2 + added yieldStraw to output (non functional) --- FarmmapsKPI/KPIApplication.cs | 45 ++++++++++++++++++++++++++++++++- FarmmapsKPI/KPIdefinitions.csv | 12 +++++---- FarmmapsKPI/Models/KPIoutput.cs | 29 +++++++++++---------- FarmmapsKPI/TODO.txt | 23 +++++++++-------- 4 files changed, 79 insertions(+), 30 deletions(-) diff --git a/FarmmapsKPI/KPIApplication.cs b/FarmmapsKPI/KPIApplication.cs index e76b07f..c25acc2 100644 --- a/FarmmapsKPI/KPIApplication.cs +++ b/FarmmapsKPI/KPIApplication.cs @@ -401,8 +401,51 @@ namespace FarmmapsKPI if (kpio.id != null) { if (kpio.id != kpioPrevious.id) - { + { KPIelementsOfBalance kPIelementsOfBalance = kpio.data.values; + //if (kpio.id == "A1") //TtD not functional + //{ + // //Make a new dataList = new line to be written + // //Fill the datalist with this kpi + // dataList = new List + // { + // kpio.parentName, + // cropfieldItem.Code, + // kpio.data.area, + // kpio.data.cropTypeCode, + // kpio.data.cropTypeName, + // kpio.id, + // kpio.quantity, // in KPI output quantity is what we call KPIvariable in headerlist of csv file + // kpio.value, + // kpio.unit, + // kpio.targetValue, + // kpio.thresholdValue + // }; + // //Write the datalist to a line to the streamwrieter sw for the output csv file + // sw.WriteLine(string.Join(",", dataList)); + // foreach (string elementName in kpio.A1elements) + // { + // // get A1element from the element called values + // string elementValue = (string)kPIelementsOfBalance.GetType().GetProperty(elementName).GetValue(kPIelementsOfBalance, null); + // dataList = new List + // { + // kpio.parentName, + // cropfieldItem.Code, + // kpio.data.area, + // kpio.data.cropTypeCode, + // kpio.data.cropTypeName, + // kpio.id, + // elementName, // specific output variable name for A1 element + // elementValue, // specific output value name for A1 element + // kpio.unit, + // "", + // "" + // }; + // //Write the datalist to a line to the streamwrieter sw for the output csv file + // sw.WriteLine(string.Join(",", dataList)); + // } + //} + //else if (kpio.id == "B1") if (kpio.id == "B1") { //Make a new dataList = new line to be written diff --git a/FarmmapsKPI/KPIdefinitions.csv b/FarmmapsKPI/KPIdefinitions.csv index f6f2ac6..b832ccd 100644 --- a/FarmmapsKPI/KPIdefinitions.csv +++ b/FarmmapsKPI/KPIdefinitions.csv @@ -1,15 +1,17 @@ KPIid,KPIvariable,Description A1,yield,observed yield (user input) B1,nitrogen,Nitrogen surplus = N artifical fertilizer + N manure+ N atmospheric deposition + N fixation by crop + N from seed potatoes - N removal through harvested product. Calculated KPI internal model & parameters and from user input: yield and fertilizer/manure applications. -B1,disposal, N removal through harvested product. Calculated KPI internal model & parameters and from user input: yield. +B1,nHarvestedKgHa, N removal through harvested product. Calculated KPI internal model & parameters and from user input: yield. +B1,nHarvestedStrawKgHa, N removal through harvested product. Calculated KPI internal model & parameters and from user input: yieldStraw. B1,nFixationNKgHa, N fixated by crop (legumes only). Calculated KPI internal model & parameters and from user input: planting date B1,nDepositionNKgHa, N deposition based on RIVM maps. B1,nFertilizerNKgHa, N from artifical fertilizers and manure.Calculated KPI internal model & parameters and from user input: fertilizer/manure applications. B1,sowPlantingNKgHa, N from seed potatoes (potatoes only). Calculated KPI internal model & parameters and from user input: amount of planted seed potatoes B2,phosphate,Phosphate surplus = P fertilizer + P from seed potatoes - P removal through harvested product. Calculated KPI internal model & parameters and from user input: yield and fertilizer applications -B2,disposal, P2O5 removal through harvested product. Calculated KPI internal model & parameters and from user input: yield -B2,pFertillizerNKgHa, P2O5 from artifical fertilizers and manure.Calculated KPI internal model & parameters and from user input: fertilizer/manure applications. -B2,sowPlantingNKgHa, P2O5 from seeds potatoes (potatoes only). Calculated KPI internal model & parameters and from user input: amount of planted seed potatoes +B2,pHarvestedKgHa, P2O5 removal through harvested product. Calculated KPI internal model & parameters and from user input: yield +B2,pHarvestedStrawKgHa, P2O5 removal through harvested product. Calculated KPI internal model & parameters and from user input: yieldStraw +B2,pFertillizerPKgHa, P2O5 from artifical fertilizers and manure.Calculated KPI internal model & parameters and from user input: fertilizer/manure applications. +B2,sowPlantingPKgHa, P2O5 from seeds potatoes (potatoes only). Calculated KPI internal model & parameters and from user input: amount of planted seed potatoes C1,effective organic matter supply,"Organic matter supply = Organic matter from manure + Crop residues to soil + organic matter from green manure. Calculated KPI internal model & parameters and from user input: yield and manure applications. ""Effective"" means netto, after correction for carbon respiration by soil microbial activity" C1,fertilizerEom,Effective organic matter added through fertilizer (animal manure, compost, etc). Calculated KPI internal model & parameters and from user input: manure applications. C1,greenManureEom,Effective organic matter added through green manure (also called cover crop). Calculated KPI internal model & parameters and from user input: cover crop. @@ -21,4 +23,4 @@ E1,mpb,? F1,greenness, time of year that the field was covered. Based on NDVI data from AgroDataCube. F2,rotationindex, index of diversity of crops over time. Calculated in AgroDataCube ,KPItargetvalue,target value as in benchmark value for same crop in same region. E.g. cropyield target is from Central Bureau of Statistics (CBS) for same crop in region in which the cropfield (geometry) is located -,KPIthresholdValue,threshold from ??? Ask farmmaps. Surplus nitrogen / phosphate / pesticides must not be above threshold. Surplus organic matter supply must be above threshold +,KPIthresholdValue,threshold from ??? Ask farmmaps. Surplus nitrogen / phosphate / pesticides must not be above threshold. Surplus organic matter supply must be above threshold \ No newline at end of file diff --git a/FarmmapsKPI/Models/KPIoutput.cs b/FarmmapsKPI/Models/KPIoutput.cs index b7dd992..6a8255a 100644 --- a/FarmmapsKPI/Models/KPIoutput.cs +++ b/FarmmapsKPI/Models/KPIoutput.cs @@ -16,16 +16,17 @@ namespace FarmmapsKPI.Models public string parentName { get; set; } public string targetValue { get; set; } public string thresholdValue { get; set; } - + + public List A1elements; //TtD public List B1elements; public List B2elements; - public List C1elements; //TtD + public List C1elements; public KPIOutput() { - this.B1elements = new List() { "disposal", "nFixationNKgHa", "nDepositionNKgHa","nFertilizerNKgHa","sowPlantingNKgHa" }; //TODO Tamara: rename disposal to nHarvestedNKgHa - this.B2elements = new List() { "disposal", "pFertillizerNKgHa", "sowPlantingNKgHa" }; //TODO Tamara: rename from pFertilizerNKgHa to pFertilizerPKgHa; rename disposal to pHarvestedPKgHa - this.C1elements = new List() { "fertilizerEom", "greenManureEom", "cropLeftoversEom" }; //TtD - + this.A1elements = new List() { "yield", "strawYield" }; //TtD + this.B1elements = new List() { "nHarvestedKgHa", "nHarvestedStrawKgHa", "nFixationNKgHa", "nDepositionNKgHa","nFertilizerNKgHa","sowPlantingNKgHa" }; //TtD + this.B2elements = new List() { "pHarvestedKgHa", "pHarvestedStrawKgHa", "pFertillizerPKgHa", "sowPlantingPKgHa" }; //TtD + this.C1elements = new List() { "fertilizerEom", "greenManureEom", "cropLeftoversEom" }; //TtD to do rename } } public class KPIOutputData @@ -50,18 +51,20 @@ namespace FarmmapsKPI.Models } public class KPIelementsOfBalance { - public string disposal { get; set; } //TODO Tamara: rename this one + public string yield { get; set; } //TtD + public string strawYield { get; set; } //TtD + public string nHarvestedKgHa { get; set; } //TtD + public string nHarvestedStrawKgHa { get; set; } //TtD public string nFixationNKgHa { get; set; } public string nDepositionNKgHa { get; set; } public string nFertilizerNKgHa { get; set; } public string sowPlantingNKgHa { get; set; } - public string pFertillizerNKgHa { get; set; } //TODO Tamara: delete this one after renaming - public string pFertilizerPKgHa { get; set; } - public string sowPlantingPKgHa { get; set; } + public string pHarvestedKgHa { get; set; } //TtD + public string pHarvestedStrawKgHa { get; set; } //TtD + public string pFertillizerPKgHa { get; set; } //TtD + public string sowPlantingPKgHa { get; set; } //TtD public string fertilizerEom { get; set; } //TtD public string greenManureEom { get; set; } //TtD - public string cropLeftoversEom { get; set; } //TtD - - + public string cropLeftoversEom { get; set; } //TtD tamara: rename } } \ No newline at end of file diff --git a/FarmmapsKPI/TODO.txt b/FarmmapsKPI/TODO.txt index dbf7b7a..f08ace5 100644 --- a/FarmmapsKPI/TODO.txt +++ b/FarmmapsKPI/TODO.txt @@ -30,8 +30,16 @@ Effect of soiltype (e.g. different N concents in crop depending on soiltype? (CL Effect of location (e.g. different atmospheric N deposition). DONE (using KPIinput.json): slightly different N balance + "DataCropfieldCharacteristic": { + "code": "860619", //PO20231004: so what does this code mean? Can we see the code list somewhere? COMPLETED code comes from codelist CL265. 860619 refers to cropyield in ton/ha + KPIelementsOfBalance: Tamara a. define elements for C1 organic matter supply + b. for nitrogen B1 rename disposal to nHarvestedNKgHa COMPLETED + c. for phosporous B2 rename disposal to pHarvestedPKgHa COMPLETED + d. for phosporous B2 rename from pFertilizerNKgHa to pFertilizerPKgHa COMPLETED + e. idem sowPlantingNKgHa COMPLETED + f. add straw for N & P in output COMPLETED ################################################################################################### # 2. Pending @@ -48,15 +56,8 @@ dataOperations: "status": "3": a. which codelist? What does it mean? NOT YET TESTED b. using KPIinput.json: does it have an effect on KPI outcomes? NOT YET TESTED -"DataCropfieldCharacteristic": { - "code": "860619", //PO20231004: so what does this code mean? Can we see the code list somewhere? NOT YET TESTED - KPIelementsOfBalance: Tamara - b. for nitrogen B1 rename disposal to nHarvestedNKgHa PARTLY - c. for phosporous B2 rename disposal to pHarvestedPKgHa PARTLY - d. for phosporous B2 rename from pFertilizerNKgHa to pFertilizerPKgHa PARTLY - e. idem sowPlantingNKgHa PARTLY - f. add straw for N & P in output - g. add 2 examples in KPIinput.json: straw yes/no removed. Bijv. nStrawLeftInField EN nStrawRemoved - h. update KPIdefinitions.csv PARTLY - i. for C1 rename 'cropLeftoversEom' to 'cropResiduesEom' \ No newline at end of file + g. add 2 examples in KPIinput.json: straw yes/no removed. Bijv. nStrawLeftInField EN nStrawRemoved COMPLETED but not yet fully functional + h. add amount of removed straw in A1 + i. update KPIdefinitions.csv PARTLY + j. for C1 rename 'cropLeftoversEom' to 'cropResiduesEom' \ No newline at end of file From ffbb45894ad20f151c169cb92a7f78568d6e958f Mon Sep 17 00:00:00 2001 From: Pepijn van Oort Date: Mon, 11 Mar 2024 14:35:03 +0100 Subject: [PATCH 51/66] write A1 yield & strawYield to output update definitions update TODO.txt --- FarmmapsKPI/KPIApplication.cs | 88 +++++++++---------- FarmmapsKPI/KPIInput.json | 4 +- FarmmapsKPI/KPIdefinitions.csv | 49 +++++------ .../KPIdefinitionsTargetsThresholds.csv | 9 ++ FarmmapsKPI/Models/KPIoutput.cs | 3 + FarmmapsKPI/TODO.txt | 11 ++- 6 files changed, 89 insertions(+), 75 deletions(-) create mode 100644 FarmmapsKPI/KPIdefinitionsTargetsThresholds.csv diff --git a/FarmmapsKPI/KPIApplication.cs b/FarmmapsKPI/KPIApplication.cs index c25acc2..a81248b 100644 --- a/FarmmapsKPI/KPIApplication.cs +++ b/FarmmapsKPI/KPIApplication.cs @@ -403,50 +403,50 @@ namespace FarmmapsKPI if (kpio.id != kpioPrevious.id) { KPIelementsOfBalance kPIelementsOfBalance = kpio.data.values; - //if (kpio.id == "A1") //TtD not functional - //{ - // //Make a new dataList = new line to be written - // //Fill the datalist with this kpi - // dataList = new List - // { - // kpio.parentName, - // cropfieldItem.Code, - // kpio.data.area, - // kpio.data.cropTypeCode, - // kpio.data.cropTypeName, - // kpio.id, - // kpio.quantity, // in KPI output quantity is what we call KPIvariable in headerlist of csv file - // kpio.value, - // kpio.unit, - // kpio.targetValue, - // kpio.thresholdValue - // }; - // //Write the datalist to a line to the streamwrieter sw for the output csv file - // sw.WriteLine(string.Join(",", dataList)); - // foreach (string elementName in kpio.A1elements) - // { - // // get A1element from the element called values - // string elementValue = (string)kPIelementsOfBalance.GetType().GetProperty(elementName).GetValue(kPIelementsOfBalance, null); - // dataList = new List - // { - // kpio.parentName, - // cropfieldItem.Code, - // kpio.data.area, - // kpio.data.cropTypeCode, - // kpio.data.cropTypeName, - // kpio.id, - // elementName, // specific output variable name for A1 element - // elementValue, // specific output value name for A1 element - // kpio.unit, - // "", - // "" - // }; - // //Write the datalist to a line to the streamwrieter sw for the output csv file - // sw.WriteLine(string.Join(",", dataList)); - // } - //} - //else if (kpio.id == "B1") - if (kpio.id == "B1") + if (kpio.id == "A1") //TtD not functional + { + //Make a new dataList = new line to be written + //Fill the datalist with this kpi + dataList = new List + { + kpio.parentName, + cropfieldItem.Code, + kpio.data.area, + kpio.data.cropTypeCode, + kpio.data.cropTypeName, + kpio.id, + kpio.quantity, // in KPI output quantity is what we call KPIvariable in headerlist of csv file + kpio.value, + kpio.unit, + kpio.targetValue, + kpio.thresholdValue + }; + //Write the datalist to a line to the streamwrieter sw for the output csv file + sw.WriteLine(string.Join(",", dataList)); + foreach (string elementName in kpio.A1elements) + { + // get A1element from the element called values + // TtD 20240311: note elements of A1 are in the data structure elements of kpio.data and not as below for B1, B2, etc as kpio.data.values + string elementValue = (string)kpio.data.GetType().GetProperty(elementName).GetValue(kpio.data, null); + dataList = new List + { + kpio.parentName, + cropfieldItem.Code, + kpio.data.area, + kpio.data.cropTypeCode, + kpio.data.cropTypeName, + kpio.id, + elementName, // specific output variable name for A1 element + elementValue, // specific output value name for A1 element + kpio.unit, + "", + "" + }; + //Write the datalist to a line to the streamwrieter sw for the output csv file + sw.WriteLine(string.Join(",", dataList)); + } + } + else if (kpio.id == "B1") { //Make a new dataList = new line to be written //Fill the datalist with this kpi diff --git a/FarmmapsKPI/KPIInput.json b/FarmmapsKPI/KPIInput.json index c43fc1e..9484ba4 100644 --- a/FarmmapsKPI/KPIInput.json +++ b/FarmmapsKPI/KPIInput.json @@ -145,8 +145,8 @@ "soilName": "Sand", "cropTypeCode": "1010101", //From codelist CL263 "cropTypeName": "Potato", //From codelist CL263 - "productionPurposeCode": "003", //From codelist CL251. For testing, see case with consumption & starch potato - "productionPurposeName": "consumption" //From codelist CL251 + "productionPurposeCode": "002", //From codelist CL251. For testing, see case with consumption & starch potato + "productionPurposeName": "starch" //From codelist CL251 }, "CropRecordingItemCode": "", // could contain for example this: "314aa1b6480b493ba28a5f55039a01d4" if this is the CropRecordingItemCode of the provided CropfieldItemCode. And then if useExistingCropfieldWithChildren = true, this would be used "OperationItemCodes": [], // could contain for example this: ["4d1a6f28ea3f414180663baa4b08c60f"] or ["4d1a6f28ea3f414180663baa4b08c60f","21e08386f0454ad79acfebf78649d59b"] if these are the operations of the provided CropRecordingItemCode. And then if useExistingCropfieldWithChildren = true, these would be used diff --git a/FarmmapsKPI/KPIdefinitions.csv b/FarmmapsKPI/KPIdefinitions.csv index b832ccd..fb5fc11 100644 --- a/FarmmapsKPI/KPIdefinitions.csv +++ b/FarmmapsKPI/KPIdefinitions.csv @@ -1,26 +1,25 @@ -KPIid,KPIvariable,Description -A1,yield,observed yield (user input) -B1,nitrogen,Nitrogen surplus = N artifical fertilizer + N manure+ N atmospheric deposition + N fixation by crop + N from seed potatoes - N removal through harvested product. Calculated KPI internal model & parameters and from user input: yield and fertilizer/manure applications. -B1,nHarvestedKgHa, N removal through harvested product. Calculated KPI internal model & parameters and from user input: yield. -B1,nHarvestedStrawKgHa, N removal through harvested product. Calculated KPI internal model & parameters and from user input: yieldStraw. -B1,nFixationNKgHa, N fixated by crop (legumes only). Calculated KPI internal model & parameters and from user input: planting date -B1,nDepositionNKgHa, N deposition based on RIVM maps. -B1,nFertilizerNKgHa, N from artifical fertilizers and manure.Calculated KPI internal model & parameters and from user input: fertilizer/manure applications. -B1,sowPlantingNKgHa, N from seed potatoes (potatoes only). Calculated KPI internal model & parameters and from user input: amount of planted seed potatoes -B2,phosphate,Phosphate surplus = P fertilizer + P from seed potatoes - P removal through harvested product. Calculated KPI internal model & parameters and from user input: yield and fertilizer applications -B2,pHarvestedKgHa, P2O5 removal through harvested product. Calculated KPI internal model & parameters and from user input: yield -B2,pHarvestedStrawKgHa, P2O5 removal through harvested product. Calculated KPI internal model & parameters and from user input: yieldStraw -B2,pFertillizerPKgHa, P2O5 from artifical fertilizers and manure.Calculated KPI internal model & parameters and from user input: fertilizer/manure applications. -B2,sowPlantingPKgHa, P2O5 from seeds potatoes (potatoes only). Calculated KPI internal model & parameters and from user input: amount of planted seed potatoes -C1,effective organic matter supply,"Organic matter supply = Organic matter from manure + Crop residues to soil + organic matter from green manure. Calculated KPI internal model & parameters and from user input: yield and manure applications. ""Effective"" means netto, after correction for carbon respiration by soil microbial activity" +KPIid,KPIvariable,Description,, +A1,yield,observed yield (user input),, +A1,strawYield,straw yield (user input; value 0 or not filled in = straw remains in field; value > 0 means straw was exported),, +B1,nitrogen,Nitrogen surplus = N artifical fertilizer + N manure+ N atmospheric deposition + N fixation by crop + N from seed potatoes - N removal through harvested product. Calculated KPI internal model & parameters and from user input: yield and fertilizer/manure applications.,, +B1,nHarvestedKgHa, N removal through harvested product. Calculated KPI internal model & parameters and from user input: yield.,, +B1,nHarvestedStrawKgHa, N removal through harvested product. Calculated KPI internal model & parameters and from user input: yieldStraw.,, +B1,nFixationNKgHa, N fixated by crop (legumes only). Calculated KPI internal model & parameters and from user input: planting date,, +B1,nDepositionNKgHa, N deposition based on RIVM maps. ,, +B1,nFertilizerNKgHa, N from artifical fertilizers and manure.Calculated KPI internal model & parameters and from user input: fertilizer/manure applications.,, +B1,sowPlantingNKgHa, N from seed potatoes (potatoes only). Calculated KPI internal model & parameters and from user input: amount of planted seed potatoes,, +B2,phosphate,Phosphate surplus = P fertilizer + P from seed potatoes - P removal through harvested product. Calculated KPI internal model & parameters and from user input: yield and fertilizer applications ,, +B2,pHarvestedKgHa, P2O5 removal through harvested product. Calculated KPI internal model & parameters and from user input: yield,, +B2,pHarvestedStrawKgHa, P2O5 removal through harvested product. Calculated KPI internal model & parameters and from user input: yieldStraw,, +B2,pFertillizerPKgHa, P2O5 from artifical fertilizers and manure.Calculated KPI internal model & parameters and from user input: fertilizer/manure applications.,, +B2,sowPlantingPKgHa, P2O5 from seeds potatoes (potatoes only). Calculated KPI internal model & parameters and from user input: amount of planted seed potatoes,, +C1,effective organic matter supply,"Organic matter supply = Organic matter from manure + Crop residues to soil + organic matter from green manure. Calculated KPI internal model & parameters and from user input: yield and manure applications. ""Effective"" means netto, after correction for carbon respiration by soil microbial activity",, C1,fertilizerEom,Effective organic matter added through fertilizer (animal manure, compost, etc). Calculated KPI internal model & parameters and from user input: manure applications. -C1,greenManureEom,Effective organic matter added through green manure (also called cover crop). Calculated KPI internal model & parameters and from user input: cover crop. -C1,cropLeftoversEom,Effective organic matter added through crop residues of main product and (when relevant) straw. Calculated KPI internal model & parameters. -D1,pesticides,"kg/ha of active ingredient. Sum over kg a.i. of the different crop protection agents applied, with concentration (kg a.i. / kg or litre) of crop protection agents retrieved from CTBG" -E1,mpb,"milieubelastingspunten. For each pesticide applied show 3 sub-mbp's: aquaticLife, groundWater, soilLife" -E1,mpb, -E1,mpb,? -F1,greenness, time of year that the field was covered. Based on NDVI data from AgroDataCube. -F2,rotationindex, index of diversity of crops over time. Calculated in AgroDataCube -,KPItargetvalue,target value as in benchmark value for same crop in same region. E.g. cropyield target is from Central Bureau of Statistics (CBS) for same crop in region in which the cropfield (geometry) is located -,KPIthresholdValue,threshold from ??? Ask farmmaps. Surplus nitrogen / phosphate / pesticides must not be above threshold. Surplus organic matter supply must be above threshold \ No newline at end of file +C1,greenManureEom,Effective organic matter added through green manure (also called cover crop). Calculated KPI internal model & parameters and from user input: cover crop.,, +C1,cropLeftoversEom,Effective organic matter added through crop residues of main product and (when relevant) straw. Calculated KPI internal model & parameters.,, +D1,pesticides,"kg/ha of active ingredient. Sum over kg a.i. of the different crop protection agents applied, with concentration (kg a.i. / kg or litre) of crop protection agents retrieved from CTBG",, +E1,mpb,"milieubelastingspunten. For each pesticide applied show 3 sub-mbp's: aquaticLife, groundWater, soilLife",, +E1,mpb,,, +E1,mpb,?,, +F1,greenness,time of year that the field was covered. Based on NDVI data from AgroDataCube.,, +F2,rotationindex,index of diversity of crops over time. Calculated in AgroDataCube,, diff --git a/FarmmapsKPI/KPIdefinitionsTargetsThresholds.csv b/FarmmapsKPI/KPIdefinitionsTargetsThresholds.csv new file mode 100644 index 0000000..4d80804 --- /dev/null +++ b/FarmmapsKPI/KPIdefinitionsTargetsThresholds.csv @@ -0,0 +1,9 @@ +KPIid,KPIvariable,KPItargetvalue,KPIthresholdValue +A1,yield,target value as in benchmark value for same crop in same region. E.g. cropyield target is from Central Bureau of Statistics (CBS) for same crop in region in which the cropfield (geometry) is located,no threshold for this KPI +B1,nitrogen,etc, +B2,phosphate,, +C1,effective organic matter supply,, +D1,pesticides,, +E1,mpb,, +F1,greenness,, +F2,rotationindex,, diff --git a/FarmmapsKPI/Models/KPIoutput.cs b/FarmmapsKPI/Models/KPIoutput.cs index 6a8255a..ba22106 100644 --- a/FarmmapsKPI/Models/KPIoutput.cs +++ b/FarmmapsKPI/Models/KPIoutput.cs @@ -34,6 +34,9 @@ namespace FarmmapsKPI.Models public string area { get; set; } public string cropTypeCode { get; set; } public string cropTypeName { get; set; } + public string yield { get; set; } + public string strawYield { get; set; } + public KPIelementsOfBalance values { get; set; } public KPIenvironmentMeasureData[] environmentMeasureData { get; set; } diff --git a/FarmmapsKPI/TODO.txt b/FarmmapsKPI/TODO.txt index f08ace5..18528b2 100644 --- a/FarmmapsKPI/TODO.txt +++ b/FarmmapsKPI/TODO.txt @@ -40,12 +40,17 @@ KPIelementsOfBalance: Tamara d. for phosporous B2 rename from pFertilizerNKgHa to pFertilizerPKgHa COMPLETED e. idem sowPlantingNKgHa COMPLETED f. add straw for N & P in output COMPLETED + g. add 2 examples in KPIinput.json: straw yes/no removed. Bijv. nStrawLeftInField EN nStrawRemoved COMPLETED but not yet fully functional + h. add amount of removed straw in A1 + i. update KPIdefinitions.csv PARTLY ################################################################################################### # 2. Pending ################################################################################################### Testing: productionPurposeCode: different result depending on starch or consumption potato (different N contents in tubers)? + nContentFactor seems not to be adjusted by productionPurposeCode. Tamara check with Francisco + idem for pContentFactor DONE (using KPIinput.json): surprisingly no effect? dataOperations: "operationCode": "7" & "8": @@ -57,7 +62,5 @@ dataOperations: "status": "3": b. using KPIinput.json: does it have an effect on KPI outcomes? NOT YET TESTED KPIelementsOfBalance: Tamara - g. add 2 examples in KPIinput.json: straw yes/no removed. Bijv. nStrawLeftInField EN nStrawRemoved COMPLETED but not yet fully functional - h. add amount of removed straw in A1 - i. update KPIdefinitions.csv PARTLY - j. for C1 rename 'cropLeftoversEom' to 'cropResiduesEom' \ No newline at end of file + j. for C1 rename 'cropLeftoversEom' to 'cropResiduesEom' GEPARKEERD + k. fill in new file KPIdefinitionsTargetsThresholds.csv with definitions of targets & thresholds per KPI element (A1, B1, etc) \ No newline at end of file From fe2bd98589faf41669edf2d79fcb85b4b4700b65 Mon Sep 17 00:00:00 2001 From: tamara Date: Tue, 12 Mar 2024 11:42:16 +0100 Subject: [PATCH 52/66] added target and threshold definitions + changed productionPurposeCode to be functional --- FarmmapsKPI/KPIApplication.cs | 2 +- FarmmapsKPI/KPIInput.json | 18 +++++++++--------- .../KPIdefinitionsTargetsThresholds.csv | 18 +++++++++--------- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/FarmmapsKPI/KPIApplication.cs b/FarmmapsKPI/KPIApplication.cs index a81248b..149e0fd 100644 --- a/FarmmapsKPI/KPIApplication.cs +++ b/FarmmapsKPI/KPIApplication.cs @@ -403,7 +403,7 @@ namespace FarmmapsKPI if (kpio.id != kpioPrevious.id) { KPIelementsOfBalance kPIelementsOfBalance = kpio.data.values; - if (kpio.id == "A1") //TtD not functional + if (kpio.id == "A1") //TtD { //Make a new dataList = new line to be written //Fill the datalist with this kpi diff --git a/FarmmapsKPI/KPIInput.json b/FarmmapsKPI/KPIInput.json index 9484ba4..99f33e8 100644 --- a/FarmmapsKPI/KPIInput.json +++ b/FarmmapsKPI/KPIInput.json @@ -17,7 +17,7 @@ "soilName": "Sand", //From codelist CL405 "cropTypeCode": "1010101", //From codelist CL263 "cropTypeName": "Potato", //From codelist CL263 - "productionPurposeCode": "003", //From codelist CL251. For testing, see case with consumption & starch potato + "productionPurposeCode": "3", //From codelist CL251. For testing, see case with consumption & starch potato "productionPurposeName": "consumption" //From codelist CL251 }, "CropRecordingItemCode": "", // could contain for example this: "314aa1b6480b493ba28a5f55039a01d4" if this is the CropRecordingItemCode of the provided CropfieldItemCode. And then if useExistingCropfieldWithChildren = true, this would be used @@ -81,7 +81,7 @@ "soilName": "Clay", //From codelist CL405 "cropTypeCode": "1010101", //From codelist CL263 "cropTypeName": "Potato", //From codelist CL263 - "productionPurposeCode": "003", //From codelist CL251. For testing, see case with consumption & starch potato + "productionPurposeCode": "3", //From codelist CL251. For testing, see case with consumption & starch potato "productionPurposeName": "consumption" //From codelist CL251 }, "CropRecordingItemCode": "", // could contain for example this: "314aa1b6480b493ba28a5f55039a01d4" if this is the CropRecordingItemCode of the provided CropfieldItemCode. And then if useExistingCropfieldWithChildren = true, this would be used @@ -145,7 +145,7 @@ "soilName": "Sand", "cropTypeCode": "1010101", //From codelist CL263 "cropTypeName": "Potato", //From codelist CL263 - "productionPurposeCode": "002", //From codelist CL251. For testing, see case with consumption & starch potato + "productionPurposeCode": "2", //From codelist CL251. For testing, see case with consumption & starch potato "productionPurposeName": "starch" //From codelist CL251 }, "CropRecordingItemCode": "", // could contain for example this: "314aa1b6480b493ba28a5f55039a01d4" if this is the CropRecordingItemCode of the provided CropfieldItemCode. And then if useExistingCropfieldWithChildren = true, this would be used @@ -205,7 +205,7 @@ "cropTypeName": "Potato", //From codelist CL263 //"rootDepthMax": 45, //not needed for KPI calculation, but shown here to know this is a possible property //"emergenceDate": "2022-05-16T00:00:00", //not needed for KPI calculation, but shown here to know this is a possible property - "productionPurposeCode": "003", //From codelist CL251. Can be omitted if unknown + "productionPurposeCode": "3", //From codelist CL251. Can be omitted if unknown "productionPurposeName": "consumption" //From codelist CL251. Can be omitted if unknown }, "CropRecordingItemCode": "", @@ -298,7 +298,7 @@ "cropTypeName": "Potato", //From codelist CL263 //"rootDepthMax": 45, //not needed for KPI calculation, but shown here to know this is a possible property //"emergenceDate": "2022-05-16T00:00:00", //not needed for KPI calculation, but shown here to know this is a possible property - "productionPurposeCode": "003", //From codelist CL251. Can be omitted if unknown + "productionPurposeCode": "3", //From codelist CL251. Can be omitted if unknown "productionPurposeName": "consumption" //From codelist CL251. Can be omitted if unknown }, "CropRecordingItemCode": "", @@ -373,7 +373,7 @@ "cropTypeName": "Potato", //From codelist CL263 //"rootDepthMax": 45, //not needed for KPI calculation, but shown here to know this is a possible property //"emergenceDate": "2022-05-16T00:00:00", //not needed for KPI calculation, but shown here to know this is a possible property - "productionPurposeCode": "003", //From codelist CL251. Can be omitted if unknown + "productionPurposeCode": "3", //From codelist CL251. Can be omitted if unknown "productionPurposeName": "consumption" //From codelist CL251. Can be omitted if unknown }, "CropRecordingItemCode": "", @@ -448,7 +448,7 @@ "cropTypeName": "Potato", //From codelist CL263 //"rootDepthMax": 45, //not needed for KPI calculation, but shown here to know this is a possible property //"emergenceDate": "2022-05-16T00:00:00", //not needed for KPI calculation, but shown here to know this is a possible property - "productionPurposeCode": "003", //From codelist CL251. Can be omitted if unknown + "productionPurposeCode": "3", //From codelist CL251. Can be omitted if unknown "productionPurposeName": "consumption" //From codelist CL251. Can be omitted if unknown }, "CropRecordingItemCode": "", @@ -518,7 +518,7 @@ "final": true, //always true "cropTypeCode": "1010101", //From codelist CL263 "cropTypeName": "Potato", //From codelist CL263 - "productionPurposeCode": "003", //From codelist CL251. Can be omitted if unknown + "productionPurposeCode": "3", //From codelist CL251. Can be omitted if unknown "productionPurposeName": "consumption" //From codelist CL251. Can be omitted if unknown }, "CropRecordingItemCode": "", // could contain for example this: "314aa1b6480b493ba28a5f55039a01d4" if this is the CropRecordingItemCode of the provided CropfieldItemCode. And then if useExistingCropfieldWithChildren = true, this would be used @@ -781,7 +781,7 @@ { "code": "fm009", //TODO: which codelist? "label": "strawyield", - "value": "5.9" + "value": "6.8" //was 5.9 } ], "CropYear": 2022, diff --git a/FarmmapsKPI/KPIdefinitionsTargetsThresholds.csv b/FarmmapsKPI/KPIdefinitionsTargetsThresholds.csv index 4d80804..665266c 100644 --- a/FarmmapsKPI/KPIdefinitionsTargetsThresholds.csv +++ b/FarmmapsKPI/KPIdefinitionsTargetsThresholds.csv @@ -1,9 +1,9 @@ -KPIid,KPIvariable,KPItargetvalue,KPIthresholdValue -A1,yield,target value as in benchmark value for same crop in same region. E.g. cropyield target is from Central Bureau of Statistics (CBS) for same crop in region in which the cropfield (geometry) is located,no threshold for this KPI -B1,nitrogen,etc, -B2,phosphate,, -C1,effective organic matter supply,, -D1,pesticides,, -E1,mpb,, -F1,greenness,, -F2,rotationindex,, +KPIid,KPIvariable,KPItargetvalue,KPIthresholdValue, +A1,yield,target value as in benchmark value for same crop in same region. E.g. cropyield target is from Central Bureau of Statistics (CBS) for same crop in region in which the cropfield (geometry) is located.,No threshold for this KPI., +B1,nitrogen,The target value depends on the soil type. To calculate the threshold and target values data from CBS and RIVM are used. The same calculation method is also used in biodiversiteitsmonitoring akkerbouw (https://www.bo-akkerbouw.nl/kennis-en-innovatie/pps-biodiversiteitsmonitor-akkerbouw).,The threshold value depends on soil type and groundwater levels. To calculate the threshold and target values data from CBS and RIVM are used. The same calculation method is also used in biodiversiteitsmonitoring akkerbouw (https://www.bo-akkerbouw.nl/kennis-en-innovatie/pps-biodiversiteitsmonitor-akkerbouw)., +B2,phosphate,The phosphate target value is set at 20 kg/ha ,No threshold for this KPI., +C1,effective organic matter supply,The target value for this KPI is the amount of EOM that companies with ‘ Goede Landbouwpraktijk’ supply (2500 kg EOM/ha/year). ,The threshold value is 2000 kg EOM/ha/year. This value comes from van Doorn et al. (2022) which states that on average 2000 kg of EOM/ha degrades on Dutch agricultural land. This is the also the threshold value from biodiversiteitsmonitoring akkerbouw (https://www.bo-akkerbouw.nl/kennis-en-innovatie/pps-biodiversiteitsmonitor-akkerbouw), +D1,pesticides,The target value fort his KPI is to reduce pesticide use to 50% of the amount used in the Netherlands during 2020. Data for 2020 is obtained from CBS statline ,No threshold for this KPI., +E1,mpb,"0 applications with exceedances of either the target MBP or risks. MBP's consist of three categories (groundwater, soil life, and aquatic life). An value of <100 in either or these categories counts as exceedance. Each application of pesticides cannot have more then one exceedance for MBP. Risks consist of two categories (natural enemies and pollinators), a value of B or C in either category counts as exceedance. Each application of pesticides cannot have more then one exceedance for risks. Values and methodology come from Environmental Yardstick for Pesticides (https://www.milieumeetlat.nl/). The same calculation method is also used in biodiversiteitsmonitoring akkerbouw (https://www.bo-akkerbouw.nl/kennis-en-innovatie/pps-biodiversiteitsmonitor-akkerbouw).","0 applications with exceedances of either the target MBP or risks. MBP's consist of three categories (groundwater, soil life, and aquatic life). An value of <10 in either or these categories counts as exceedance. Each application of pesticides cannot have more then one exceedance for MBP. Risks consist of two categories (natural enemies and pollinators), a value of B or C in either category counts as exceedance. Each application of pesticides cannot have more then one exceedance for risks. Values and methodology come from Environmental Yardstick for Pesticides (https://www.milieumeetlat.nl/). The same calculation method is also used in biodiversiteitsmonitoring akkerbouw (https://www.bo-akkerbouw.nl/kennis-en-innovatie/pps-biodiversiteitsmonitor-akkerbouw).", +F1,greenness,Target value is 75%.,Threshold value is 25%., +F2,rotationindex,Target value is 75%.,Threshold value is 25%., From 0a9a83a5a6ee46f1fecf7bd8642bab621697f0f2 Mon Sep 17 00:00:00 2001 From: tamara Date: Wed, 10 Apr 2024 10:29:30 +0200 Subject: [PATCH 53/66] updated KPIdefinitions and todolist --- FarmmapsKPI/KPIdefinitions.csv | 48 +++++++++---------- .../KPIdefinitionsTargetsThresholds.csv | 9 ---- FarmmapsKPI/TODO.txt | 17 +++---- 3 files changed, 32 insertions(+), 42 deletions(-) delete mode 100644 FarmmapsKPI/KPIdefinitionsTargetsThresholds.csv diff --git a/FarmmapsKPI/KPIdefinitions.csv b/FarmmapsKPI/KPIdefinitions.csv index fb5fc11..317f999 100644 --- a/FarmmapsKPI/KPIdefinitions.csv +++ b/FarmmapsKPI/KPIdefinitions.csv @@ -1,25 +1,23 @@ -KPIid,KPIvariable,Description,, -A1,yield,observed yield (user input),, -A1,strawYield,straw yield (user input; value 0 or not filled in = straw remains in field; value > 0 means straw was exported),, -B1,nitrogen,Nitrogen surplus = N artifical fertilizer + N manure+ N atmospheric deposition + N fixation by crop + N from seed potatoes - N removal through harvested product. Calculated KPI internal model & parameters and from user input: yield and fertilizer/manure applications.,, -B1,nHarvestedKgHa, N removal through harvested product. Calculated KPI internal model & parameters and from user input: yield.,, -B1,nHarvestedStrawKgHa, N removal through harvested product. Calculated KPI internal model & parameters and from user input: yieldStraw.,, -B1,nFixationNKgHa, N fixated by crop (legumes only). Calculated KPI internal model & parameters and from user input: planting date,, -B1,nDepositionNKgHa, N deposition based on RIVM maps. ,, -B1,nFertilizerNKgHa, N from artifical fertilizers and manure.Calculated KPI internal model & parameters and from user input: fertilizer/manure applications.,, -B1,sowPlantingNKgHa, N from seed potatoes (potatoes only). Calculated KPI internal model & parameters and from user input: amount of planted seed potatoes,, -B2,phosphate,Phosphate surplus = P fertilizer + P from seed potatoes - P removal through harvested product. Calculated KPI internal model & parameters and from user input: yield and fertilizer applications ,, -B2,pHarvestedKgHa, P2O5 removal through harvested product. Calculated KPI internal model & parameters and from user input: yield,, -B2,pHarvestedStrawKgHa, P2O5 removal through harvested product. Calculated KPI internal model & parameters and from user input: yieldStraw,, -B2,pFertillizerPKgHa, P2O5 from artifical fertilizers and manure.Calculated KPI internal model & parameters and from user input: fertilizer/manure applications.,, -B2,sowPlantingPKgHa, P2O5 from seeds potatoes (potatoes only). Calculated KPI internal model & parameters and from user input: amount of planted seed potatoes,, -C1,effective organic matter supply,"Organic matter supply = Organic matter from manure + Crop residues to soil + organic matter from green manure. Calculated KPI internal model & parameters and from user input: yield and manure applications. ""Effective"" means netto, after correction for carbon respiration by soil microbial activity",, -C1,fertilizerEom,Effective organic matter added through fertilizer (animal manure, compost, etc). Calculated KPI internal model & parameters and from user input: manure applications. -C1,greenManureEom,Effective organic matter added through green manure (also called cover crop). Calculated KPI internal model & parameters and from user input: cover crop.,, -C1,cropLeftoversEom,Effective organic matter added through crop residues of main product and (when relevant) straw. Calculated KPI internal model & parameters.,, -D1,pesticides,"kg/ha of active ingredient. Sum over kg a.i. of the different crop protection agents applied, with concentration (kg a.i. / kg or litre) of crop protection agents retrieved from CTBG",, -E1,mpb,"milieubelastingspunten. For each pesticide applied show 3 sub-mbp's: aquaticLife, groundWater, soilLife",, -E1,mpb,,, -E1,mpb,?,, -F1,greenness,time of year that the field was covered. Based on NDVI data from AgroDataCube.,, -F2,rotationindex,index of diversity of crops over time. Calculated in AgroDataCube,, +KPIid,KPIvariable,Description,KPItargetvalue,KPIthresholdValue +A1,yield,observed yield (user input),target value as in benchmark value for same crop in same region. E.g. cropyield target is from Central Bureau of Statistics (CBS) for same crop in region in which the cropfield (geometry) is located.,No threshold for this KPI. +A1,strawYield,straw yield (user input; value 0 or not filled in = straw remains in field; value > 0 means straw was exported),there are no target values for individual compenets of KPI's,there are no threshold values for individual compenets of KPI's +B1,nitrogen,Nitrogen surplus = N artifical fertilizer + N manure+ N atmospheric deposition + N fixation by crop + N from seed potatoes - N removal through harvested product. Calculated KPI internal model & parameters and from user input: yield and fertilizer/manure applications.,The target value depends on the soil type. To calculate the threshold and target values data from CBS and RIVM are used. The same calculation method is also used in biodiversiteitsmonitoring akkerbouw (https://www.bo-akkerbouw.nl/kennis-en-innovatie/pps-biodiversiteitsmonitor-akkerbouw).,The threshold value depends on soil type and groundwater levels. To calculate the threshold and target values data from CBS and RIVM are used. The same calculation method is also used in biodiversiteitsmonitoring akkerbouw (https://www.bo-akkerbouw.nl/kennis-en-innovatie/pps-biodiversiteitsmonitor-akkerbouw). +B1,nHarvestedKgHa, N removal through harvested product. Calculated KPI internal model & parameters and from user input: yield.,there are no target values for individual compenets of KPI's,there are no threshold values for individual compenets of KPI's +B1,nHarvestedStrawKgHa, N removal through harvested product. Calculated KPI internal model & parameters and from user input: yieldStraw.,there are no target values for individual compenets of KPI's,there are no threshold values for individual compenets of KPI's +B1,nFixationNKgHa, N fixated by crop (legumes only). Calculated KPI internal model & parameters and from user input: planting date,there are no target values for individual compenets of KPI's,there are no threshold values for individual compenets of KPI's +B1,nDepositionNKgHa, N deposition based on RIVM maps. ,there are no target values for individual compenets of KPI's,there are no threshold values for individual compenets of KPI's +B1,nFertilizerNKgHa, N from artifical fertilizers and manure.Calculated KPI internal model & parameters and from user input: fertilizer/manure applications.,there are no target values for individual compenets of KPI's,there are no threshold values for individual compenets of KPI's +B1,sowPlantingNKgHa, N from seed potatoes (potatoes only). Calculated KPI internal model & parameters and from user input: amount of planted seed potatoes,there are no target values for individual compenets of KPI's,there are no threshold values for individual compenets of KPI's +B2,phosphate,Phosphate surplus = P fertilizer + P from seed potatoes - P removal through harvested product. Calculated KPI internal model & parameters and from user input: yield and fertilizer applications ,The phosphate target value is set at 20 kg/ha ,No threshold for this KPI. +B2,pHarvestedKgHa, P2O5 removal through harvested product. Calculated KPI internal model & parameters and from user input: yield,there are no target values for individual compenets of KPI's,there are no threshold values for individual compenets of KPI's +B2,pHarvestedStrawKgHa, P2O5 removal through harvested product. Calculated KPI internal model & parameters and from user input: yieldStraw,there are no target values for individual compenets of KPI's,there are no threshold values for individual compenets of KPI's +B2,pFertillizerPKgHa, P2O5 from artifical fertilizers and manure.Calculated KPI internal model & parameters and from user input: fertilizer/manure applications.,there are no target values for individual compenets of KPI's,there are no threshold values for individual compenets of KPI's +B2,sowPlantingPKgHa, P2O5 from seeds potatoes (potatoes only). Calculated KPI internal model & parameters and from user input: amount of planted seed potatoes,there are no target values for individual compenets of KPI's,there are no threshold values for individual compenets of KPI's +C1,effective organic matter supply,"Organic matter supply = Organic matter from manure + Crop residues to soil + organic matter from green manure. Calculated KPI internal model & parameters and from user input: yield and manure applications. ""Effective"" means netto, after correction for carbon respiration by soil microbial activity",The target value for this KPI is the amount of EOM that companies with ‘ Goede Landbouwpraktijk’ supply (2500 kg EOM/ha/year). ,The threshold value is 2000 kg EOM/ha/year. This value comes from van Doorn et al. (2022) which states that on average 2000 kg of EOM/ha degrades on Dutch agricultural land. This is the also the threshold value from biodiversiteitsmonitoring akkerbouw (https://www.bo-akkerbouw.nl/kennis-en-innovatie/pps-biodiversiteitsmonitor-akkerbouw) +C1,fertilizerEom,"Effective organic matter added through fertilizer (animal manure, compost, etc). Calculated KPI internal model & parameters and from user input: manure applications.",there are no target values for individual compenets of KPI's,there are no threshold values for individual compenets of KPI's +C1,greenManureEom,Effective organic matter added through green manure (also called cover crop). Calculated KPI internal model & parameters and from user input: cover crop.,there are no target values for individual compenets of KPI's,there are no threshold values for individual compenets of KPI's +C1,cropLeftoversEom,Effective organic matter added through crop residues of main product and (when relevant) straw. Calculated KPI internal model & parameters.,there are no target values for individual compenets of KPI's,there are no threshold values for individual compenets of KPI's +D1,pesticides,"kg/ha of active ingredient. Sum over kg a.i. of the different crop protection agents applied, with concentration (kg a.i. / kg or litre) of crop protection agents retrieved from CTBG",The target value fort his KPI is to reduce pesticide use to 50% of the amount used in the Netherlands during 2020. Data for 2020 is obtained from CBS statline ,No threshold for this KPI. +E1,mpb,"milieubelastingspunten. For each pesticide applied show 3 sub-mbp's: aquaticLife, groundWater, soilLife","0 applications with exceedances of either the target MBP or risks. MBP's consist of three categories (groundwater, soil life, and aquatic life). An value of <100 in either or these categories counts as exceedance. Each application of pesticides cannot have more then one exceedance for MBP. Risks consist of two categories (natural enemies and pollinators), a value of B or C in either category counts as exceedance. Each application of pesticides cannot have more then one exceedance for risks. Values and methodology come from Environmental Yardstick for Pesticides (https://www.milieumeetlat.nl/). The same calculation method is also used in biodiversiteitsmonitoring akkerbouw (https://www.bo-akkerbouw.nl/kennis-en-innovatie/pps-biodiversiteitsmonitor-akkerbouw).","0 applications with exceedances of either the target MBP or risks. MBP's consist of three categories (groundwater, soil life, and aquatic life). An value of <10 in either or these categories counts as exceedance. Each application of pesticides cannot have more then one exceedance for MBP. Risks consist of two categories (natural enemies and pollinators), a value of B or C in either category counts as exceedance. Each application of pesticides cannot have more then one exceedance for risks. Values and methodology come from Environmental Yardstick for Pesticides (https://www.milieumeetlat.nl/). The same calculation method is also used in biodiversiteitsmonitoring akkerbouw (https://www.bo-akkerbouw.nl/kennis-en-innovatie/pps-biodiversiteitsmonitor-akkerbouw)." +F1,greenness,time of year that the field was covered. Based on NDVI data from AgroDataCube.,Target value is 75%.,Threshold value is 25%. +F2,rotationindex,index of diversity of crops over time. Calculated in AgroDataCube,Target value is 75%.,Threshold value is 25%. diff --git a/FarmmapsKPI/KPIdefinitionsTargetsThresholds.csv b/FarmmapsKPI/KPIdefinitionsTargetsThresholds.csv deleted file mode 100644 index 665266c..0000000 --- a/FarmmapsKPI/KPIdefinitionsTargetsThresholds.csv +++ /dev/null @@ -1,9 +0,0 @@ -KPIid,KPIvariable,KPItargetvalue,KPIthresholdValue, -A1,yield,target value as in benchmark value for same crop in same region. E.g. cropyield target is from Central Bureau of Statistics (CBS) for same crop in region in which the cropfield (geometry) is located.,No threshold for this KPI., -B1,nitrogen,The target value depends on the soil type. To calculate the threshold and target values data from CBS and RIVM are used. The same calculation method is also used in biodiversiteitsmonitoring akkerbouw (https://www.bo-akkerbouw.nl/kennis-en-innovatie/pps-biodiversiteitsmonitor-akkerbouw).,The threshold value depends on soil type and groundwater levels. To calculate the threshold and target values data from CBS and RIVM are used. The same calculation method is also used in biodiversiteitsmonitoring akkerbouw (https://www.bo-akkerbouw.nl/kennis-en-innovatie/pps-biodiversiteitsmonitor-akkerbouw)., -B2,phosphate,The phosphate target value is set at 20 kg/ha ,No threshold for this KPI., -C1,effective organic matter supply,The target value for this KPI is the amount of EOM that companies with ‘ Goede Landbouwpraktijk’ supply (2500 kg EOM/ha/year). ,The threshold value is 2000 kg EOM/ha/year. This value comes from van Doorn et al. (2022) which states that on average 2000 kg of EOM/ha degrades on Dutch agricultural land. This is the also the threshold value from biodiversiteitsmonitoring akkerbouw (https://www.bo-akkerbouw.nl/kennis-en-innovatie/pps-biodiversiteitsmonitor-akkerbouw), -D1,pesticides,The target value fort his KPI is to reduce pesticide use to 50% of the amount used in the Netherlands during 2020. Data for 2020 is obtained from CBS statline ,No threshold for this KPI., -E1,mpb,"0 applications with exceedances of either the target MBP or risks. MBP's consist of three categories (groundwater, soil life, and aquatic life). An value of <100 in either or these categories counts as exceedance. Each application of pesticides cannot have more then one exceedance for MBP. Risks consist of two categories (natural enemies and pollinators), a value of B or C in either category counts as exceedance. Each application of pesticides cannot have more then one exceedance for risks. Values and methodology come from Environmental Yardstick for Pesticides (https://www.milieumeetlat.nl/). The same calculation method is also used in biodiversiteitsmonitoring akkerbouw (https://www.bo-akkerbouw.nl/kennis-en-innovatie/pps-biodiversiteitsmonitor-akkerbouw).","0 applications with exceedances of either the target MBP or risks. MBP's consist of three categories (groundwater, soil life, and aquatic life). An value of <10 in either or these categories counts as exceedance. Each application of pesticides cannot have more then one exceedance for MBP. Risks consist of two categories (natural enemies and pollinators), a value of B or C in either category counts as exceedance. Each application of pesticides cannot have more then one exceedance for risks. Values and methodology come from Environmental Yardstick for Pesticides (https://www.milieumeetlat.nl/). The same calculation method is also used in biodiversiteitsmonitoring akkerbouw (https://www.bo-akkerbouw.nl/kennis-en-innovatie/pps-biodiversiteitsmonitor-akkerbouw).", -F1,greenness,Target value is 75%.,Threshold value is 25%., -F2,rotationindex,Target value is 75%.,Threshold value is 25%., diff --git a/FarmmapsKPI/TODO.txt b/FarmmapsKPI/TODO.txt index 18528b2..7e0e705 100644 --- a/FarmmapsKPI/TODO.txt +++ b/FarmmapsKPI/TODO.txt @@ -32,6 +32,11 @@ Effect of location (e.g. different atmospheric N deposition). "DataCropfieldCharacteristic": { "code": "860619", //PO20231004: so what does this code mean? Can we see the code list somewhere? COMPLETED code comes from codelist CL265. 860619 refers to cropyield in ton/ha +productionPurposeCode: different result depending on starch or consumption potato (different N contents in tubers)? + nContentFactor seems not to be adjusted by productionPurposeCode. Tamara check with Francisco + idem for pContentFactor + DONE (using KPIinput.json): surprisingly no effect?, format of the productionPurposeCode was wrong i.e was '2' instead of '002'. Format has been adjusted. + KPIelementsOfBalance: Tamara a. define elements for C1 organic matter supply @@ -40,18 +45,14 @@ KPIelementsOfBalance: Tamara d. for phosporous B2 rename from pFertilizerNKgHa to pFertilizerPKgHa COMPLETED e. idem sowPlantingNKgHa COMPLETED f. add straw for N & P in output COMPLETED - g. add 2 examples in KPIinput.json: straw yes/no removed. Bijv. nStrawLeftInField EN nStrawRemoved COMPLETED but not yet fully functional - h. add amount of removed straw in A1 - i. update KPIdefinitions.csv PARTLY + g. add 2 examples in KPIinput.json: straw yes/no removed. Bijv. nStrawLeftInField EN nStrawRemoved COMPLETED + h. add amount of removed straw in A1 COMPLETED + i. update KPIdefinitions.csv COMPLETED ################################################################################################### # 2. Pending ################################################################################################### Testing: -productionPurposeCode: different result depending on starch or consumption potato (different N contents in tubers)? - nContentFactor seems not to be adjusted by productionPurposeCode. Tamara check with Francisco - idem for pContentFactor - DONE (using KPIinput.json): surprisingly no effect? dataOperations: "operationCode": "7" & "8": a. which codelist? What does it mean? NOT YET TESTED @@ -63,4 +64,4 @@ dataOperations: "status": "3": KPIelementsOfBalance: Tamara j. for C1 rename 'cropLeftoversEom' to 'cropResiduesEom' GEPARKEERD - k. fill in new file KPIdefinitionsTargetsThresholds.csv with definitions of targets & thresholds per KPI element (A1, B1, etc) \ No newline at end of file + \ No newline at end of file From efb4d6833fe71bd9db55e39bd8ec6d7b07902019 Mon Sep 17 00:00:00 2001 From: tamara Date: Thu, 11 Apr 2024 11:31:56 +0200 Subject: [PATCH 54/66] Included relevant codelist for dataOperations operationCode and status. --- FarmmapsKPI/KPIInput.json | 24 ++++++++++++------------ FarmmapsKPI/TODO.txt | 19 +++++++++---------- 2 files changed, 21 insertions(+), 22 deletions(-) diff --git a/FarmmapsKPI/KPIInput.json b/FarmmapsKPI/KPIInput.json index 99f33e8..11aa2b4 100644 --- a/FarmmapsKPI/KPIInput.json +++ b/FarmmapsKPI/KPIInput.json @@ -221,8 +221,8 @@ "name": "Kunstmest strooien", //refers to codelist CL127 with operation methods "product": "7360", //refers to codelist CL022 with fertilizer types "quantity": "200", - "status": "3", // Which code list? What does it mean? Is it needed? In any case works if set to 3 - "operationCode": "7", // Which code list? What does it mean? Is it needed? In any case works if set to 7 + //"status": "3", // refers to codelist CL256. where 3 stand for completed. The status does not matter and is not used in the KPI calculations + "operationCode": "7", // refers to codelist CL018 consisting of main catergorie of operations. Code 7 stands for fertilization "to": "2022-05-23T12:34:00", "unit": "kg/ha", "unitCode": "KGMHAR" @@ -237,8 +237,8 @@ "name": "Injecteren", //refers to codelist CL127 with operation methods "unit": "kg/ha", "method": "70700", //refers to codelist CL127 with operation methods - "status": "3", // Which code list? What does it mean? Is it needed? In any case works if set to 3 - "operationCode": "7", // Which code list? What does it mean? Is it needed? In any case works if set to 7 + //"status": "3", // refers to codelist CL256. where 3 stand for completed. The status does not matter and is not used in the KPI calculations + "operationCode": "7", // refers to codelist CL018 consisting of main catergorie of operations. Code 7 stands for fertilization "product": "2329", //refers to codelist CL022 with fertilizer types "quantity": "32000", "unitCode": "KGMHAR", @@ -314,8 +314,8 @@ "name": "Kunstmest strooien", //refers to codelist CL127 with operation methods "product": "7360", //refers to codelist CL022 with fertilizer types "quantity": "200", - "status": "3", // Which code list? What does it mean? Is it needed? In any case works if set to 3 - "operationCode": "7", // Which code list? What does it mean? Is it needed? In any case works if set to 7 + //"status": "3", // refers to codelist CL256. where 3 stand for completed. The status does not matter and is not used in the KPI calculations + "operationCode": "7", // refers to codelist CL018 consisting of main catergorie of operations. Code 7 stands for fertilization "to": "2022-05-23T12:34:00", "unit": "kg/ha", "unitCode": "KGMHAR" @@ -387,8 +387,8 @@ "method": "70400", //refers to codelist CL127 with operation methods "n": "92", "name": "Kunstmest strooien", //refers to codelist CL127 with operation methods - "status": "3", // Which code list? What does it mean? Is it needed? In any case works if set to 3 - "operationCode": "7", // Which code list? What does it mean? Is it needed? In any case works if set to 7 + //"status": "3", // refers to codelist CL256. where 3 stand for completed. The status does not matter and is not used in the KPI calculations + "operationCode": "7", // refers to codelist CL018 consisting of main catergorie of operations. Code 7 stands for fertilization "product": "7360", //refers to codelist CL022 with fertilizer types "quantity": "200", "to": "2022-05-23T12:34:00", @@ -462,8 +462,8 @@ "method": "70400", //refers to codelist CL127 with operation methods //"n": "92", "name": "Kunstmest strooien", //refers to codelist CL127 with operation methods - "status": "3", // Which code list? What does it mean? Is it needed? In any case works if set to 3 - "operationCode": "7", // Which code list? What does it mean? Is it needed? In any case works if set to 7 + //"status": "3", // refers to codelist CL256. where 3 stand for completed. The status does not matter and is not used in the KPI calculations + "operationCode": "7", // refers to codelist CL018 consisting of main catergorie of operations. Code 7 stands for fertilization "product": "7360", //refers to codelist CL022 with fertilizer types "quantity": "200", "to": "2022-05-23T12:34:00", @@ -529,8 +529,8 @@ "from": "2022-07-13T16:27:00", "name": "Volveld spuiten (3/3)", //refers to codelist CL127 with operation methods "method": "80200", //refers to codelist CL127 with operation methods - "status": "3", // Which code list? What does it mean? Is it needed? In any case works if set to 3 - "operationCode": "8", // Which code list? What does it mean? Is it needed? In any case works if set to 8 + //"status": "3", // refers to codelist CL256. where 3 stand for completed. The status does not matter and is not used in the KPI calculations + "operationCode": "8", // refers to codelist CL018 consisting of main catergorie of operations. Code 8 stands for crop protection "products": [ { "product": "11767 N", //CTBG code, lookup via CTBG API. Important: with space N", diff --git a/FarmmapsKPI/TODO.txt b/FarmmapsKPI/TODO.txt index 7e0e705..32c15f3 100644 --- a/FarmmapsKPI/TODO.txt +++ b/FarmmapsKPI/TODO.txt @@ -35,8 +35,15 @@ Effect of location (e.g. different atmospheric N deposition). productionPurposeCode: different result depending on starch or consumption potato (different N contents in tubers)? nContentFactor seems not to be adjusted by productionPurposeCode. Tamara check with Francisco idem for pContentFactor - DONE (using KPIinput.json): surprisingly no effect?, format of the productionPurposeCode was wrong i.e was '2' instead of '002'. Format has been adjusted. - + DONE (using KPIinput.json): surprisingly no effect?, format of the productionPurposeCode was wrong i.e was '002' instead of '2'. Format has been adjusted. +dataOperations: "operationCode": "7" & "8": DONE + a. which codelist? What does it mean? this refers to CL018 (bewerkinghoofdcaterogie), which contains information about the caterogie of the crop operations. + 7 stands for fertilization while 8 refers to crop protection + b. using KPIinput.json: does it have an effect on KPI outcomes? YES +dataOperations: "status": "3": DONE + a. which codelist? What does it mean? this refers to CL256 (statusBewerking), which contains information about a crop operation has been planned, completed, cancelled etc. 3 refers to completed + b. using KPIinput.json: does it have an effect on KPI outcomes? NO, even without the status the crop operations is included in the KPI calculation. + As the status has no effect on the KPI calculations I have changed the lines about the status in the KPIinput.json file to comment KPIelementsOfBalance: Tamara a. define elements for C1 organic matter supply @@ -54,14 +61,6 @@ KPIelementsOfBalance: Tamara ################################################################################################### Testing: -dataOperations: "operationCode": "7" & "8": - a. which codelist? What does it mean? NOT YET TESTED - b. using KPIinput.json: does it have an effect on KPI outcomes? NOT YET TESTED - -dataOperations: "status": "3": - a. which codelist? What does it mean? NOT YET TESTED - b. using KPIinput.json: does it have an effect on KPI outcomes? NOT YET TESTED - KPIelementsOfBalance: Tamara j. for C1 rename 'cropLeftoversEom' to 'cropResiduesEom' GEPARKEERD \ No newline at end of file From 6faa5104c8e5ddd11e6d1f1ccbfbb4ed7bbbaca2 Mon Sep 17 00:00:00 2001 From: tamara Date: Thu, 2 May 2024 15:57:32 +0200 Subject: [PATCH 55/66] added example with greenmanure --- FarmmapsKPI/KPIInput.json | 68 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 67 insertions(+), 1 deletion(-) diff --git a/FarmmapsKPI/KPIInput.json b/FarmmapsKPI/KPIInput.json index 11aa2b4..1e4bc24 100644 --- a/FarmmapsKPI/KPIInput.json +++ b/FarmmapsKPI/KPIInput.json @@ -781,7 +781,7 @@ { "code": "fm009", //TODO: which codelist? "label": "strawyield", - "value": "6.8" //was 5.9 + "value": "6.8" } ], "CropYear": 2022, @@ -813,5 +813,71 @@ ] ] } + }, + { + "useExistingCropfieldWithChildren": false, + "deleteNewlyCreatedAfterCalc": true, + "fieldName": "test_WinterWheat_greenmanure", + //Note when you run the FarmmapsKPI project, somewhere to your C:\git\FarmMapsApiClient_KB34_MAST\FarmmapsKPI\bin\Debug\netcoreapp3.1\ directory, files named like this are written: + //'Settings_aardappelveld_test_Potato_ZeroFertilizer.json', where 'aardappelveld_test_Potato_ZeroFertilizer.json' is your fieldName (see below) + //1 useExistingCropfieldWithChildren = false -> create new + //2 useExistingCropfieldWithChildren = true && CropfieldItemCode = "" or absent -> read fieldName (see below) -> read all codes from settings json file + //3 useExistingCropfieldWithChildren = true && CropfieldItemCode like "deb48a74c5b54299bb852f17288010e9" in KPIinput -> if you have in your account already an existing cropfield then use that + "CropfieldItemCode": "", // could contain for example this: "abae97f89f3c4ac08953b1b8bea9f076" if this is an exisiting CropfieldItemCode in your account. + //And then if useExistingCropfieldWithChildren = true, this would be used. + "dataCropfield": { + //"area": 4.22, //Leave empty, KPI app will calculate it from geometry + "final": true, //always true + "soilCode": "1", //From codelist CL405. Can be omitted if unknown + "soilName": "Sand", //From codelist CL405 + "cropTypeCode": "1020101", //From codelist CL263 + "cropTypeName": "Winter wheat" //From codelist CL263 + }, + "CropRecordingItemCode": "", // could contain for example this: "314aa1b6480b493ba28a5f55039a01d4" if this is the CropRecordingItemCode of the provided CropfieldItemCode. And then if useExistingCropfieldWithChildren = true, this would be used + "OperationItemCodes": [], // could contain for example this: ["4d1a6f28ea3f414180663baa4b08c60f"] or ["4d1a6f28ea3f414180663baa4b08c60f","21e08386f0454ad79acfebf78649d59b"] if these are the operations of the provided CropRecordingItemCode. And then if useExistingCropfieldWithChildren = true, these would be used + "dataOperations": [], + "CropfieldCharacteristicItemCode": "", + "DataCropfieldCharacteristics": [ + { + "code": "860619", //From codelist CL265. 860619 refers to cropyield in ton/ha + "label": "cropyield", + "value": "10" + }, + { + "code": "501505", //From codelist CL265. 501505 refers to green manure after main crop. + "label": "cropgreenmanureaftercultivation", + "value": "1040102", // Code is from CL263. You can put any CL263 crop here. However, the KPI app only works with a limited selection of greenmanure crops, see CL104. + "cropHeight": 100, //optional input. Height of greemmanure in cm. If this value is 0 then it is assumed that the crop height is unknown. + "plantingDate": "2022-08-01T12:34:00" // optional input. Plantingsdate of greenmanure. required layout "2022-07-23T12:34:00". If this value is null then it is assumed that the plantingsdate is unknown. + } + ], + "CropYear": 2022, + "geometryJson": { + "type": "Polygon", + "coordinates": [ + [ + [ + 5.5945257993548765, + 52.57080744107003 + ], + [ + 5.598645994070678, + 52.571540800206236 + ], + [ + 5.599381743127071, + 52.57012773140724 + ], + [ + 5.595408698222548, + 52.56968054825188 + ], + [ + 5.5945257993548765, + 52.57080744107003 + ] + ] + ] + } } ] \ No newline at end of file From e05bb5a8f7502afb2b8c3050ebcb99612eea3615 Mon Sep 17 00:00:00 2001 From: Pepijn van Oort Date: Wed, 15 May 2024 15:45:53 +0200 Subject: [PATCH 56/66] added code for checking if multiple runs give same KPI output (should be but is not, so where is it going wrong?) --- FarmmapsKPI/KPIApplication.cs | 83 ++++++++++++++++++++++++++--------- 1 file changed, 62 insertions(+), 21 deletions(-) diff --git a/FarmmapsKPI/KPIApplication.cs b/FarmmapsKPI/KPIApplication.cs index 149e0fd..17512ed 100644 --- a/FarmmapsKPI/KPIApplication.cs +++ b/FarmmapsKPI/KPIApplication.cs @@ -5,6 +5,7 @@ using System.IO; using System.IO.Compression; using System.Linq; using System.Threading.Tasks; +using System.Transactions; using FarmmapsApi; using FarmmapsApi.Models; using FarmmapsApi.Services; @@ -46,7 +47,7 @@ namespace FarmmapsKPI //fnKPIinput = Console.ReadLine(); //if (string.IsNullOrEmpty(fnKPIinput)) //{ - fnKPIinput = "KPIinput.json"; + fnKPIinput = "KPIinput.json"; //} var fieldsInputJson = File.ReadAllText(fnKPIinput); @@ -71,7 +72,7 @@ namespace FarmmapsKPI StreamWriter sw; string KPIItemCsv = Path.GetFileNameWithoutExtension(fnKPIinput) + "_Items.csv"; string KPIItemPathCsv = Path.Combine(downloadFolder, KPIItemCsv); - List headerList = new List { "parentName", "cropfieldcode", "area_ha", "cropTypeCode", "cropTypeName", "KPIid", "KPIvariable", "KPIvalue", "KPIunit", "KPItargetvalue", "KPIthresholdValue", + List headerList = new List { "run","parentName", "cropfieldcode", "area_ha", "cropTypeCode", "cropTypeName", "KPIid", "KPIvariable", "KPIvalue", "KPIunit", "KPItargetvalue", "KPIthresholdValue", "mbp_productCode","mbp_productName","mbp_quantity","mbp_unitCode","mbp_date","mbp_KPIvariable","mbp_KPIvalue"}; //Create a new csv file. Means if existing then overwritten !!! sw = new StreamWriter(KPIItemPathCsv); @@ -87,25 +88,29 @@ namespace FarmmapsKPI TimeSpan tsRemaining; TimeSpan tsTotalEstimated; - for (int i = 0; i < fieldsInputs.Count; i++) - //for (int i = 10; i < 12; i++) - { - watch.Restart(); - input = fieldsInputs[i]; - _logger.LogInformation(string.Format($"// FarmmapsKPI: Downloading KPI's for field {i + 1} out of {fieldsInputs.Count} to single csv file {KPIItemPathCsv}")); - try + //Per default just 1 run per field. For debugging check if when we run multiple times do we get (should be) always the same output? + int nrun = 1; + for (int run = 1; run <= nrun; run++) { + //for (int i = 0; i < fieldsInputs.Count; i++) + for (int i = 3; i < 4; i++) // for testing { - await Process(roots, input, sw); - } - catch (Exception ex) - { - _logger.LogError(ex.Message); - } - watch.Stop(); - tsSofar = tsSofar + watch.Elapsed; - tsTotalEstimated = tsSofar / (i + 1) * fieldsInputs.Count; - tsRemaining = tsTotalEstimated - tsSofar; - _logger.LogInformation(string.Format($"// Time (hh:mm:ss): this field: {strTime(watch.Elapsed)}. Sofar: {strTime(tsSofar)}. Estimated total: {strTime(tsTotalEstimated)}. Remaining: {strTime(tsRemaining)}")); + watch.Restart(); + input = fieldsInputs[i]; + _logger.LogInformation(string.Format($"// FarmmapsKPI: Run {run}: Downloading KPI's for field {i + 1} out of {fieldsInputs.Count} to single csv file {KPIItemPathCsv}")); + try + { + await Process(roots, input, sw, run); + } + catch (Exception ex) + { + _logger.LogError(ex.Message); + } + watch.Stop(); + tsSofar = tsSofar + watch.Elapsed; + tsTotalEstimated = tsSofar / (i + 1) * fieldsInputs.Count; + tsRemaining = tsTotalEstimated - tsSofar; + _logger.LogInformation(string.Format($"// Time (hh:mm:ss): this field: {strTime(watch.Elapsed)}. Sofar: {strTime(tsSofar)}. Estimated total: {strTime(tsTotalEstimated)}. Remaining: {strTime(tsRemaining)}")); + } } //Close the csv file, write message to screen sw.Close(); @@ -113,7 +118,7 @@ namespace FarmmapsKPI _logger.LogInformation($"Done! Written all KPI for all fields in '{fnKPIinput}' to output file '{KPIItemPathCsv}'"); } - private async Task Process(List roots, KPIInput input, StreamWriter sw) + private async Task Process(List roots, KPIInput input, StreamWriter sw, int run) { List cropfieldChildren; List crprecChildren; @@ -125,6 +130,8 @@ namespace FarmmapsKPI int targetKPIitemsCount; //if we know we should be getting 8 KPI items (A1,B1,B2,C1,D1,E1,F1,F2) int maxtries = 5; // but don't keep on trying forever; there is a maximum number of tries int trycnt; + double totalNferiliserInput = 0; + double totalNferiliserCropfield = 0; string downloadFolder = input.DownloadFolder; if (string.IsNullOrEmpty(downloadFolder)) { @@ -317,6 +324,8 @@ namespace FarmmapsKPI crpOperationItem = await _generalService.CreateOperationItemAsync(crprecItem.Code, cropYear, fieldGeom, dataOperation); crpOperationItems.Add(crpOperationItem); crpOperationItemCodes.Add(crpOperationItem.Code); + //Keep track of totalNferiliserInput + totalNferiliserInput = totalNferiliserInput + (double)data.n; } _settings.OperationItemCodes = crpOperationItemCodes.ToArray(); SaveSettings(settingsfile); @@ -352,6 +361,22 @@ namespace FarmmapsKPI cropfieldChildren = await _farmmapsApiService.GetItemChildrenAsync(cropfieldItem.Code); crprecChildren = await _farmmapsApiService.GetItemChildrenAsync(crprecItem.Code); + //Check total operations + int nOperationsInput = input.DataOperations.Length; + int nOperationsCropfield = crprecChildren.Count; + if (nOperationsInput != nOperationsCropfield) + throw new Exception(String.Format($"run {run}: nOperationsInput != nOperationsCropfield?!")); + + //Check totalNfertiliser input and Cropfield + for (int o = 0; o < crprecChildren.Count; o++) + { + dynamic data = JObject.Parse(crprecChildren[o].Data.ToString()); + totalNferiliserCropfield = totalNferiliserCropfield + (double)data.n; + } + if (totalNferiliserInput != totalNferiliserCropfield) + throw new Exception(String.Format($"run {run}: totalNferiliserInput != totalNferiliserCropfield?!")); + + //Now get the KPIs for this cropfield, mounted with operations & cropyield //Note sometimes the KPIItems.Count is already for some crazy reason greater than or equal to targetKPIitemsCount //But that would have strange results, since that was from above before adding the crop recordings. We want to do at least one new call -> 'while (trycnt == 0 || ' @@ -409,6 +434,7 @@ namespace FarmmapsKPI //Fill the datalist with this kpi dataList = new List { + run.ToString(), kpio.parentName, cropfieldItem.Code, kpio.data.area, @@ -430,6 +456,7 @@ namespace FarmmapsKPI string elementValue = (string)kpio.data.GetType().GetProperty(elementName).GetValue(kpio.data, null); dataList = new List { + run.ToString(), kpio.parentName, cropfieldItem.Code, kpio.data.area, @@ -452,6 +479,7 @@ namespace FarmmapsKPI //Fill the datalist with this kpi dataList = new List { + run.ToString(), kpio.parentName, cropfieldItem.Code, kpio.data.area, @@ -473,6 +501,7 @@ namespace FarmmapsKPI string elementValue = (string)kPIelementsOfBalance.GetType().GetProperty(elementName).GetValue(kPIelementsOfBalance, null); dataList = new List { + run.ToString(), kpio.parentName, cropfieldItem.Code, kpio.data.area, @@ -487,6 +516,12 @@ namespace FarmmapsKPI }; //Write the datalist to a line to the streamwrieter sw for the output csv file sw.WriteLine(string.Join(",", dataList)); + + //Check if totalNferiliserInput is equal to what comes out of the KPI calculation + //PO20240515: Hier gaat het mis!!! + if (elementName == "nFertilizerNKgHa") + if (totalNferiliserInput != Convert.ToDouble(elementValue)) + throw new Exception(String.Format($"run {run}: totalNferiliserInput != nFertilizerNKgHa?!")); } } else if (kpio.id == "B2") @@ -495,6 +530,7 @@ namespace FarmmapsKPI dataList = new List { //Fill the datalist with this kpi + run.ToString(), kpio.parentName, cropfieldItem.Code, kpio.data.area, @@ -516,6 +552,7 @@ namespace FarmmapsKPI string elementValue = (string)kPIelementsOfBalance.GetType().GetProperty(elementName).GetValue(kPIelementsOfBalance, null); dataList = new List { + run.ToString(), kpio.parentName, cropfieldItem.Code, kpio.data.area, @@ -538,6 +575,7 @@ namespace FarmmapsKPI //Make a new dataList = new line to be written dataList = new List { + run.ToString(), kpio.parentName, cropfieldItem.Code, kpio.data.area, @@ -559,6 +597,7 @@ namespace FarmmapsKPI string elementValue = (string)kPIelementsOfBalance.GetType().GetProperty(elementName).GetValue(kPIelementsOfBalance, null); dataList = new List { + run.ToString(), kpio.parentName, cropfieldItem.Code, kpio.data.area, @@ -587,6 +626,7 @@ namespace FarmmapsKPI //Make a new dataList = new line to be written dataList = new List { + run.ToString(), kpio.parentName, cropfieldItem.Code, kpio.data.area, @@ -616,6 +656,7 @@ namespace FarmmapsKPI //Any other KPI, example A1 or D1, with just 1 record to write dataList = new List { + run.ToString(), kpio.parentName, cropfieldItem.Code, kpio.data.area, From a98a81776db316781fb72cce59df4dcddf748ad4 Mon Sep 17 00:00:00 2001 From: ttenden Date: Wed, 15 May 2024 16:06:19 +0200 Subject: [PATCH 57/66] calculate KPI's for all example fields instead of one field --- FarmmapsKPI/KPIApplication.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/FarmmapsKPI/KPIApplication.cs b/FarmmapsKPI/KPIApplication.cs index 17512ed..1ab4d22 100644 --- a/FarmmapsKPI/KPIApplication.cs +++ b/FarmmapsKPI/KPIApplication.cs @@ -91,8 +91,8 @@ namespace FarmmapsKPI //Per default just 1 run per field. For debugging check if when we run multiple times do we get (should be) always the same output? int nrun = 1; for (int run = 1; run <= nrun; run++) { - //for (int i = 0; i < fieldsInputs.Count; i++) - for (int i = 3; i < 4; i++) // for testing + for (int i = 0; i < fieldsInputs.Count; i++) + //for (int i = 3; i < 4; i++) // for testing { watch.Restart(); input = fieldsInputs[i]; From 24feeba58eb8d033ab33f72cb3a4af326608fed2 Mon Sep 17 00:00:00 2001 From: Pepijn van Oort Date: Thu, 16 May 2024 10:04:48 +0200 Subject: [PATCH 58/66] throw warning instead of error --- FarmmapsKPI/KPIApplication.cs | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/FarmmapsKPI/KPIApplication.cs b/FarmmapsKPI/KPIApplication.cs index 17512ed..b486f89 100644 --- a/FarmmapsKPI/KPIApplication.cs +++ b/FarmmapsKPI/KPIApplication.cs @@ -89,7 +89,7 @@ namespace FarmmapsKPI TimeSpan tsTotalEstimated; //Per default just 1 run per field. For debugging check if when we run multiple times do we get (should be) always the same output? - int nrun = 1; + int nrun = 5; for (int run = 1; run <= nrun; run++) { //for (int i = 0; i < fieldsInputs.Count; i++) for (int i = 3; i < 4; i++) // for testing @@ -365,7 +365,11 @@ namespace FarmmapsKPI int nOperationsInput = input.DataOperations.Length; int nOperationsCropfield = crprecChildren.Count; if (nOperationsInput != nOperationsCropfield) - throw new Exception(String.Format($"run {run}: nOperationsInput != nOperationsCropfield?!")); + { + _logger.LogWarning(String.Format($"1. run {run}: nOperationsInput != nOperationsCropfield?!")); + //throw new Exception(String.Format($"run {run}: nOperationsInput != nOperationsCropfield?!")); + } + //Check totalNfertiliser input and Cropfield for (int o = 0; o < crprecChildren.Count; o++) @@ -374,7 +378,10 @@ namespace FarmmapsKPI totalNferiliserCropfield = totalNferiliserCropfield + (double)data.n; } if (totalNferiliserInput != totalNferiliserCropfield) - throw new Exception(String.Format($"run {run}: totalNferiliserInput != totalNferiliserCropfield?!")); + { + _logger.LogWarning(String.Format($"2. run {run}: totalNferiliserInput != totalNferiliserCropfield?!")); + //throw new Exception(String.Format($"run {run}: totalNferiliserInput != totalNferiliserCropfield?!")); + } //Now get the KPIs for this cropfield, mounted with operations & cropyield @@ -520,8 +527,13 @@ namespace FarmmapsKPI //Check if totalNferiliserInput is equal to what comes out of the KPI calculation //PO20240515: Hier gaat het mis!!! if (elementName == "nFertilizerNKgHa") + { if (totalNferiliserInput != Convert.ToDouble(elementValue)) - throw new Exception(String.Format($"run {run}: totalNferiliserInput != nFertilizerNKgHa?!")); + { + _logger.LogWarning(String.Format($"3. run {run}: totalNferiliserInput != nFertilizerNKgHa")); + //throw new Exception(String.Format($"run {run}: totalNferiliserInput != nFertilizerNKgHa")); + } + } } } else if (kpio.id == "B2") From 6d6b5b1b7a84fc9ec84dbd0af847c528c8c1062a Mon Sep 17 00:00:00 2001 From: ttenden Date: Mon, 27 May 2024 13:52:15 +0200 Subject: [PATCH 59/66] removed incorrect duplicate KPIs --- FarmmapsKPI/KPIApplication.cs | 248 +++++++++++++++++++--------------- 1 file changed, 142 insertions(+), 106 deletions(-) diff --git a/FarmmapsKPI/KPIApplication.cs b/FarmmapsKPI/KPIApplication.cs index 18e33a2..d55fc98 100644 --- a/FarmmapsKPI/KPIApplication.cs +++ b/FarmmapsKPI/KPIApplication.cs @@ -89,7 +89,7 @@ namespace FarmmapsKPI TimeSpan tsTotalEstimated; //Per default just 1 run per field. For debugging check if when we run multiple times do we get (should be) always the same output? - int nrun = 5; + int nrun = 1; for (int run = 1; run <= nrun; run++) { for (int i = 0; i < fieldsInputs.Count; i++) //for (int i = 3; i < 4; i++) // for testing @@ -223,16 +223,16 @@ namespace FarmmapsKPI // We need that because for operations, you need to provide the area on which the operation was applied // And if we put that to the crop area, then we neatly get everything on a per ha basis _logger.LogInformation($"Getting polygon area (ha))"); - List KPIItems = await _generalService.GetKpiItemsForCropField(cropfieldItem, 3); + List KPIItemsArea = await _generalService.GetKpiItemsForCropField(cropfieldItem, 3); trycnt = 1; - targetKPIitemsCount = 3; // here for the area we need at least 3, but not more than that - while (KPIItems.Count < targetKPIitemsCount & trycnt < maxtries) + targetKPIitemsCount = 3; // here for the area we need at least 3, but not more than that + while (KPIItemsArea.Count < targetKPIitemsCount & trycnt < maxtries) { - KPIItems = await _generalService.GetKpiItemsForCropField(cropfieldItem,3); - _logger.LogInformation($"Found {KPIItems.Count} KPI items"); + KPIItemsArea = await _generalService.GetKpiItemsForCropField(cropfieldItem,3); + _logger.LogInformation($"Found {KPIItemsArea.Count} KPI items"); trycnt++; } - kpio = JsonConvert.DeserializeObject(KPIItems[0].Data.ToString()); + kpio = JsonConvert.DeserializeObject(KPIItemsArea[0].Data.ToString()); string area_ha = kpio.data.area; // turn the area into a JObject for later merging with operation data; string strJarea = JsonConvert.SerializeObject(new { area = area_ha }); @@ -383,10 +383,8 @@ namespace FarmmapsKPI //throw new Exception(String.Format($"run {run}: totalNferiliserInput != totalNferiliserCropfield?!")); } - //Now get the KPIs for this cropfield, mounted with operations & cropyield - //Note sometimes the KPIItems.Count is already for some crazy reason greater than or equal to targetKPIitemsCount - //But that would have strange results, since that was from above before adding the crop recordings. We want to do at least one new call -> 'while (trycnt == 0 || ' + List KPIItems = await _generalService.GetKpiItemsForCropField(cropfieldItem, 3); _logger.LogInformation($"GetKpiItemsForCropField('{cropfieldItem.Code}')"); //Pesticide KPI's D1 and E1 are retreived from API's of CTBG and CLM and may take a bit longer to retrieve targetKPIitemsCount = 8; //if we know we should be getting 8 KPI items (A1,B1,B2,C1,D1,E1,F1,F2) @@ -394,17 +392,16 @@ namespace FarmmapsKPI bool boolAquaticLife = false; _logger.LogInformation($"Firing calls GetKpiItemsForCropField() until we have {targetKPIitemsCount} KPIitems, but don't keep firing forever, stop after {maxtries} calls"); _logger.LogInformation($"Before we start:"); - _logger.LogInformation($"* KPIItems.Count = {KPIItems.Count}"); + //_logger.LogInformation($"* KPIItems.Count = {KPIItems.Count}"); _logger.LogInformation($"* trycnt = {trycnt}"); _logger.LogInformation($"* boolAquaticLife = {boolAquaticLife}"); //additional criterion for while loop: check if it really contains the E1 mbp elements. //while (trycnt == 0 || ((KPIItems.Count < targetKPIitemsCount || boolAquaticLife == false) & trycnt < maxtries)) - //normal while loop targetKPIitemsCount = 8; //if we know we should be getting 8 KPI items (A1,B1,B2,C1,D1,E1,F1,F2) while (trycnt == 0 || (KPIItems.Count < targetKPIitemsCount & trycnt < maxtries)) { _logger.LogInformation($"Call nr {trycnt + 1}"); - KPIItems = await _generalService.GetKpiItemsForCropField(cropfieldItem, 3); //number after comma is how many seconds per try + KPIItems = await _generalService.GetKpiItemsForCropField(cropfieldItem, 30); //number after comma is how many seconds per try, should be between 30 seconds and 5 minutes _logger.LogInformation($"Found {KPIItems.Count} KPI items"); //boolAquaticLife = GetBoolAquaticLife(KPIItems); trycnt ++; @@ -425,23 +422,65 @@ namespace FarmmapsKPI _logger.LogInformation($"Downloaded file {KPIItemPathJson}"); _logger.LogInformation($""); + + //Remove duplicate KPI items + List KPIItemsClean = new List(); + Item itemi; + string id, idNext; + double value, valueNext; + value = 0; + id = ""; + for (int i = 0; i < KPIItems.Count; i++) + { + itemi = KPIItems[i]; + idNext = JsonConvert.DeserializeObject(itemi.Data.ToString()).id; + valueNext = Convert.ToDouble(JsonConvert.DeserializeObject(itemi.Data.ToString()).value); //hoe doe ik dit voor een getal? + if (idNext != null) + { + if (id != idNext) + { + KPIItemsClean.Add(itemi); + } + else + { + if (valueNext >= value) + { + //Remove the previous element from this list with same id but wrong value (valueNext > value) or duplicate value (valueNext == value). Presumes list is always sorted by kpiid, e.g. we may havce " + KPIItemsClean.RemoveAt(i - 1); + KPIItemsClean.Add(itemi); + } + else + { + //Previous element was correct and already added so do nothing here + } + } + id = idNext; + value = valueNext; + } + } + + ////Order again from A zo Z. + //List KPIItemsCleanSorted = new List(); + //for (int i = KPIItemsClean.Count - 1; i >= 0; i--) + //{ + // KPIItemsCleanSorted.Add(KPIItemsClean[i]); + //} + //Write to the csv file that collects all KPI's for all the crop fields List dataList; - foreach (Item item in KPIItems) - { + foreach (Item item in KPIItemsClean) + { kpio = JsonConvert.DeserializeObject(item.Data.ToString()); if (kpio.id != null) { - if (kpio.id != kpioPrevious.id) - { - KPIelementsOfBalance kPIelementsOfBalance = kpio.data.values; - if (kpio.id == "A1") //TtD - { - //Make a new dataList = new line to be written - //Fill the datalist with this kpi - dataList = new List + KPIelementsOfBalance kPIelementsOfBalance = kpio.data.values; + if (kpio.id == "A1") //TtD + { + //Make a new dataList = new line to be written + //Fill the datalist with this kpi + dataList = new List { - run.ToString(), + run.ToString(), kpio.parentName, cropfieldItem.Code, kpio.data.area, @@ -454,14 +493,14 @@ namespace FarmmapsKPI kpio.targetValue, kpio.thresholdValue }; - //Write the datalist to a line to the streamwrieter sw for the output csv file - sw.WriteLine(string.Join(",", dataList)); - foreach (string elementName in kpio.A1elements) - { - // get A1element from the element called values - // TtD 20240311: note elements of A1 are in the data structure elements of kpio.data and not as below for B1, B2, etc as kpio.data.values - string elementValue = (string)kpio.data.GetType().GetProperty(elementName).GetValue(kpio.data, null); - dataList = new List + //Write the datalist to a line to the streamwrieter sw for the output csv file + sw.WriteLine(string.Join(",", dataList)); + foreach (string elementName in kpio.A1elements) + { + // get A1element from the element called values + // TtD 20240311: note elements of A1 are in the data structure elements of kpio.data and not as below for B1, B2, etc as kpio.data.values + string elementValue = (string)kpio.data.GetType().GetProperty(elementName).GetValue(kpio.data, null); + dataList = new List { run.ToString(), kpio.parentName, @@ -476,15 +515,15 @@ namespace FarmmapsKPI "", "" }; - //Write the datalist to a line to the streamwrieter sw for the output csv file - sw.WriteLine(string.Join(",", dataList)); - } + //Write the datalist to a line to the streamwrieter sw for the output csv file + sw.WriteLine(string.Join(",", dataList)); } - else if (kpio.id == "B1") - { - //Make a new dataList = new line to be written - //Fill the datalist with this kpi - dataList = new List + } + else if (kpio.id == "B1") + { + //Make a new dataList = new line to be written + //Fill the datalist with this kpi + dataList = new List { run.ToString(), kpio.parentName, @@ -498,15 +537,15 @@ namespace FarmmapsKPI kpio.unit, kpio.targetValue, kpio.thresholdValue - }; - //Write the datalist to a line to the streamwrieter sw for the output csv file - sw.WriteLine(string.Join(",", dataList)); + }; + //Write the datalist to a line to the streamwrieter sw for the output csv file + sw.WriteLine(string.Join(",", dataList)); - foreach (string elementName in kpio.B1elements) - { - // get B1element from the element called values - string elementValue = (string)kPIelementsOfBalance.GetType().GetProperty(elementName).GetValue(kPIelementsOfBalance, null); - dataList = new List + foreach (string elementName in kpio.B1elements) + { + // get B1element from the element called values + string elementValue = (string)kPIelementsOfBalance.GetType().GetProperty(elementName).GetValue(kPIelementsOfBalance, null); + dataList = new List { run.ToString(), kpio.parentName, @@ -521,25 +560,25 @@ namespace FarmmapsKPI "", "" }; - //Write the datalist to a line to the streamwrieter sw for the output csv file - sw.WriteLine(string.Join(",", dataList)); + //Write the datalist to a line to the streamwrieter sw for the output csv file + sw.WriteLine(string.Join(",", dataList)); - //Check if totalNferiliserInput is equal to what comes out of the KPI calculation - //PO20240515: Hier gaat het mis!!! - if (elementName == "nFertilizerNKgHa") + //Check if totalNferiliserInput is equal to what comes out of the KPI calculation + //PO20240515: Hier gaat het mis!!! + if (elementName == "nFertilizerNKgHa") + { + if (totalNferiliserInput != Convert.ToDouble(elementValue)) { - if (totalNferiliserInput != Convert.ToDouble(elementValue)) - { - _logger.LogWarning(String.Format($"3. run {run}: totalNferiliserInput != nFertilizerNKgHa")); - //throw new Exception(String.Format($"run {run}: totalNferiliserInput != nFertilizerNKgHa")); - } + _logger.LogWarning(String.Format($"3. run {run}: totalNferiliserInput != nFertilizerNKgHa")); + //throw new Exception(String.Format($"run {run}: totalNferiliserInput != nFertilizerNKgHa")); } } } - else if (kpio.id == "B2") - { - //Make a new dataList = new line to be written - dataList = new List + } + else if (kpio.id == "B2") + { + //Make a new dataList = new line to be written + dataList = new List { //Fill the datalist with this kpi run.ToString(), @@ -555,14 +594,14 @@ namespace FarmmapsKPI kpio.targetValue, kpio.thresholdValue }; - //Write the datalist to a line to the streamwrieter sw for the output csv file - sw.WriteLine(string.Join(",", dataList)); + //Write the datalist to a line to the streamwrieter sw for the output csv file + sw.WriteLine(string.Join(",", dataList)); - foreach (string elementName in kpio.B2elements) - { - // get B1element from the element called values - string elementValue = (string)kPIelementsOfBalance.GetType().GetProperty(elementName).GetValue(kPIelementsOfBalance, null); - dataList = new List + foreach (string elementName in kpio.B2elements) + { + // get B1element from the element called values + string elementValue = (string)kPIelementsOfBalance.GetType().GetProperty(elementName).GetValue(kPIelementsOfBalance, null); + dataList = new List { run.ToString(), kpio.parentName, @@ -577,15 +616,15 @@ namespace FarmmapsKPI "", "" }; - //Write the datalist to a line to the streamwrieter sw for the output csv file - sw.WriteLine(string.Join(",", dataList)); - } + //Write the datalist to a line to the streamwrieter sw for the output csv file + sw.WriteLine(string.Join(",", dataList)); } + } - else if (kpio.id == "C1") //TtD - { - //Make a new dataList = new line to be written - dataList = new List + else if (kpio.id == "C1") //TtD + { + //Make a new dataList = new line to be written + dataList = new List { run.ToString(), kpio.parentName, @@ -600,14 +639,14 @@ namespace FarmmapsKPI kpio.targetValue, kpio.thresholdValue }; - //Write the datalist to a line to the streamwrieter sw for the output csv file - sw.WriteLine(string.Join(",", dataList)); + //Write the datalist to a line to the streamwrieter sw for the output csv file + sw.WriteLine(string.Join(",", dataList)); - foreach (string elementName in kpio.C1elements) - { - // get C1element from the element called values - string elementValue = (string)kPIelementsOfBalance.GetType().GetProperty(elementName).GetValue(kPIelementsOfBalance, null); - dataList = new List + foreach (string elementName in kpio.C1elements) + { + // get C1element from the element called values + string elementValue = (string)kPIelementsOfBalance.GetType().GetProperty(elementName).GetValue(kPIelementsOfBalance, null); + dataList = new List { run.ToString(), kpio.parentName, @@ -622,21 +661,21 @@ namespace FarmmapsKPI "", "" }; - //Write the datalist to a line to the streamwrieter sw for the output csv file - sw.WriteLine(string.Join(",", dataList)); - } + //Write the datalist to a line to the streamwrieter sw for the output csv file + sw.WriteLine(string.Join(",", dataList)); } + } - else if (kpio.id == "E1") + else if (kpio.id == "E1") + { + //for E1, environmentMeasureData is an array of elements with 1 per pesticide. Inside each element there are multiple mbp_KPIvariables to be written + foreach (KPIenvironmentMeasureData e in kpio.data.environmentMeasureData) { - //for E1, environmentMeasureData is an array of elements with 1 per pesticide. Inside each element there are multiple mbp_KPIvariables to be written - foreach (KPIenvironmentMeasureData e in kpio.data.environmentMeasureData) + foreach (string mbp_KPIvariable in mbp_KPIvariables) { - foreach (string mbp_KPIvariable in mbp_KPIvariables) - { - if (mbp_KPIvariable == "aquaticLife") { mbp_KPIvalue = e.aquaticLife; } else if (mbp_KPIvariable == "groundWater") { mbp_KPIvalue = e.groundWater; } else if (mbp_KPIvariable == "soilLife") { mbp_KPIvalue = e.soilLife; } else { mbp_KPIvalue = ""; } - //Make a new dataList = new line to be written - dataList = new List + if (mbp_KPIvariable == "aquaticLife") { mbp_KPIvalue = e.aquaticLife; } else if (mbp_KPIvariable == "groundWater") { mbp_KPIvalue = e.groundWater; } else if (mbp_KPIvariable == "soilLife") { mbp_KPIvalue = e.soilLife; } else { mbp_KPIvalue = ""; } + //Make a new dataList = new line to be written + dataList = new List { run.ToString(), kpio.parentName, @@ -658,15 +697,15 @@ namespace FarmmapsKPI mbp_KPIvariable, mbp_KPIvalue }; - //Write the datalist to a line to the streamwrieter sw for the output csv file - sw.WriteLine(string.Join(",", dataList)); - } + //Write the datalist to a line to the streamwrieter sw for the output csv file + sw.WriteLine(string.Join(",", dataList)); } } - else - { - //Any other KPI, example A1 or D1, with just 1 record to write - dataList = new List + } + else + { + //Any other KPI, example A1 or D1, with just 1 record to write + dataList = new List { run.ToString(), kpio.parentName, @@ -681,13 +720,10 @@ namespace FarmmapsKPI kpio.targetValue, kpio.thresholdValue }; - //Write the datalist to a line to the streamwrieter sw for the output csv file - sw.WriteLine(string.Join(",", dataList)); - } - + //Write the datalist to a line to the streamwrieter sw for the output csv file + sw.WriteLine(string.Join(",", dataList)); } } - kpioPrevious = kpio; } ////Total N applied from input From c2e4f9d861a6d162690fe3fce3e1cdd36e8f5517 Mon Sep 17 00:00:00 2001 From: ttenden Date: Mon, 27 May 2024 15:35:25 +0200 Subject: [PATCH 60/66] commented out the totalNfertiliserInput = totalNferiliserCropfield check --- FarmmapsKPI/KPIApplication.cs | 47 +++++++++++++++++------------------ 1 file changed, 23 insertions(+), 24 deletions(-) diff --git a/FarmmapsKPI/KPIApplication.cs b/FarmmapsKPI/KPIApplication.cs index d55fc98..aedef28 100644 --- a/FarmmapsKPI/KPIApplication.cs +++ b/FarmmapsKPI/KPIApplication.cs @@ -92,7 +92,7 @@ namespace FarmmapsKPI int nrun = 1; for (int run = 1; run <= nrun; run++) { for (int i = 0; i < fieldsInputs.Count; i++) - //for (int i = 3; i < 4; i++) // for testing + //for (int i = 6; i < 7; i++) // for testing { watch.Restart(); input = fieldsInputs[i]; @@ -130,8 +130,8 @@ namespace FarmmapsKPI int targetKPIitemsCount; //if we know we should be getting 8 KPI items (A1,B1,B2,C1,D1,E1,F1,F2) int maxtries = 5; // but don't keep on trying forever; there is a maximum number of tries int trycnt; - double totalNferiliserInput = 0; - double totalNferiliserCropfield = 0; + //double totalNferiliserInput = 0; + //double totalNferiliserCropfield = 0; string downloadFolder = input.DownloadFolder; if (string.IsNullOrEmpty(downloadFolder)) { @@ -325,7 +325,7 @@ namespace FarmmapsKPI crpOperationItems.Add(crpOperationItem); crpOperationItemCodes.Add(crpOperationItem.Code); //Keep track of totalNferiliserInput - totalNferiliserInput = totalNferiliserInput + (double)data.n; + //totalNferiliserInput = totalNferiliserInput + (double)data.n; / this causes a problem if N content of the fertilizer is missing. As this check is no longer needed I just commented it out } _settings.OperationItemCodes = crpOperationItemCodes.ToArray(); SaveSettings(settingsfile); @@ -371,17 +371,17 @@ namespace FarmmapsKPI } - //Check totalNfertiliser input and Cropfield - for (int o = 0; o < crprecChildren.Count; o++) - { - dynamic data = JObject.Parse(crprecChildren[o].Data.ToString()); - totalNferiliserCropfield = totalNferiliserCropfield + (double)data.n; - } - if (totalNferiliserInput != totalNferiliserCropfield) - { - _logger.LogWarning(String.Format($"2. run {run}: totalNferiliserInput != totalNferiliserCropfield?!")); - //throw new Exception(String.Format($"run {run}: totalNferiliserInput != totalNferiliserCropfield?!")); - } + //Check totalNfertiliser input and Cropfield, + //for (int o = 0; o < crprecChildren.Count; o++) + //{ + // dynamic data = JObject.Parse(crprecChildren[o].Data.ToString()); + // totalNferiliserCropfield = totalNferiliserCropfield + (double)data.n; + //} + //if (totalNferiliserInput != totalNferiliserCropfield) + //{ + // _logger.LogWarning(String.Format($"2. run {run}: totalNferiliserInput != totalNferiliserCropfield?!")); + // //throw new Exception(String.Format($"run {run}: totalNferiliserInput != totalNferiliserCropfield?!")); + //} //Now get the KPIs for this cropfield, mounted with operations & cropyield List KPIItems = await _generalService.GetKpiItemsForCropField(cropfieldItem, 3); @@ -564,15 +564,14 @@ namespace FarmmapsKPI sw.WriteLine(string.Join(",", dataList)); //Check if totalNferiliserInput is equal to what comes out of the KPI calculation - //PO20240515: Hier gaat het mis!!! - if (elementName == "nFertilizerNKgHa") - { - if (totalNferiliserInput != Convert.ToDouble(elementValue)) - { - _logger.LogWarning(String.Format($"3. run {run}: totalNferiliserInput != nFertilizerNKgHa")); - //throw new Exception(String.Format($"run {run}: totalNferiliserInput != nFertilizerNKgHa")); - } - } + //if (elementName == "nFertilizerNKgHa") + //{ + // if (totalNferiliserInput != Convert.ToDouble(elementValue)) + // { + // _logger.LogWarning(String.Format($"3. run {run}: totalNferiliserInput != nFertilizerNKgHa")); + // //throw new Exception(String.Format($"run {run}: totalNferiliserInput != nFertilizerNKgHa")); + // } + //} } } else if (kpio.id == "B2") From 178ba1ea5ba7c26149abb7e8a115e4148fa21c65 Mon Sep 17 00:00:00 2001 From: ttenden Date: Tue, 28 May 2024 10:06:48 +0200 Subject: [PATCH 61/66] //added a duplicate counter to deal with calculations where there is more 1 duplicate. --- FarmmapsKPI/KPIApplication.cs | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/FarmmapsKPI/KPIApplication.cs b/FarmmapsKPI/KPIApplication.cs index aedef28..97d4994 100644 --- a/FarmmapsKPI/KPIApplication.cs +++ b/FarmmapsKPI/KPIApplication.cs @@ -4,6 +4,7 @@ using System.Diagnostics; using System.IO; using System.IO.Compression; using System.Linq; +using System.Net.NetworkInformation; using System.Threading.Tasks; using System.Transactions; using FarmmapsApi; @@ -92,7 +93,7 @@ namespace FarmmapsKPI int nrun = 1; for (int run = 1; run <= nrun; run++) { for (int i = 0; i < fieldsInputs.Count; i++) - //for (int i = 6; i < 7; i++) // for testing + //for (int i = 11; i < 13; i++) // for testing { watch.Restart(); input = fieldsInputs[i]; @@ -426,15 +427,17 @@ namespace FarmmapsKPI //Remove duplicate KPI items List KPIItemsClean = new List(); Item itemi; - string id, idNext; - double value, valueNext; + string id, idNext; + double value, valueNext; + int DuplicateCounter; value = 0; id = ""; + DuplicateCounter = 1; for (int i = 0; i < KPIItems.Count; i++) { itemi = KPIItems[i]; idNext = JsonConvert.DeserializeObject(itemi.Data.ToString()).id; - valueNext = Convert.ToDouble(JsonConvert.DeserializeObject(itemi.Data.ToString()).value); //hoe doe ik dit voor een getal? + valueNext = Convert.ToDouble(JsonConvert.DeserializeObject(itemi.Data.ToString()).value); if (idNext != null) { if (id != idNext) @@ -446,8 +449,9 @@ namespace FarmmapsKPI if (valueNext >= value) { //Remove the previous element from this list with same id but wrong value (valueNext > value) or duplicate value (valueNext == value). Presumes list is always sorted by kpiid, e.g. we may havce " - KPIItemsClean.RemoveAt(i - 1); + KPIItemsClean.RemoveAt(i - DuplicateCounter); //added a duplicate counter to deal with calculations where there is more 1 duplicate. KPIItemsClean.Add(itemi); + DuplicateCounter = DuplicateCounter + 1; } else { From 752b538780b4b75541280f78313b3983c3631a3c Mon Sep 17 00:00:00 2001 From: ttenden Date: Wed, 5 Jun 2024 14:06:13 +0200 Subject: [PATCH 62/66] changed explanation of optional inputs for green manure. --- FarmmapsKPI/KPIInput.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/FarmmapsKPI/KPIInput.json b/FarmmapsKPI/KPIInput.json index 1e4bc24..394e108 100644 --- a/FarmmapsKPI/KPIInput.json +++ b/FarmmapsKPI/KPIInput.json @@ -846,9 +846,9 @@ { "code": "501505", //From codelist CL265. 501505 refers to green manure after main crop. "label": "cropgreenmanureaftercultivation", - "value": "1040102", // Code is from CL263. You can put any CL263 crop here. However, the KPI app only works with a limited selection of greenmanure crops, see CL104. - "cropHeight": 100, //optional input. Height of greemmanure in cm. If this value is 0 then it is assumed that the crop height is unknown. - "plantingDate": "2022-08-01T12:34:00" // optional input. Plantingsdate of greenmanure. required layout "2022-07-23T12:34:00". If this value is null then it is assumed that the plantingsdate is unknown. + "value": "1040102", // Code is from CL263. You can put any CL263 crop here. However, the KPI app only works with a limited selection of greenmanure crops, see CL104 for the names. + //"cropHeight": 100, //optional input. Height of greemmanure in cm. + //"plantingDate": "2022-08-01T12:34:00" // optional input. Plantingsdate of greenmanure. required layout "2022-07-23T12:34:00". } ], "CropYear": 2022, From da82c232e30b931a331ec7320a60394f1e6102a2 Mon Sep 17 00:00:00 2001 From: ttenden Date: Fri, 28 Jun 2024 11:27:24 +0200 Subject: [PATCH 63/66] Added CL104 --- FarmmapsDownloadCL/DownloadCLInput.json | 2 +- FarmmapsDownloadCL/Models/CodelistsClasses.cs | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/FarmmapsDownloadCL/DownloadCLInput.json b/FarmmapsDownloadCL/DownloadCLInput.json index ad850fc..77fdea7 100644 --- a/FarmmapsDownloadCL/DownloadCLInput.json +++ b/FarmmapsDownloadCL/DownloadCLInput.json @@ -1,5 +1,5 @@ { - "codeLists": [ "CL022", "CL127", "CL263", "CL265", "CL405", "CL251", "CL042" ] + "codeLists": [ "CL022", "CL127", "CL263", "CL265", "CL405", "CL251", "CL104" ] //"codeLists": [ "CL263", "CL265"] //"DownloadFolder": "Downloads", //"C:\\hugoschrererdir\\kpidir\\", // "Downloads", -> if you just put "Downloads" the program will download to somewhere in ..\FarmMapsApiClient_WURtest\FarmmapsDataDownload\bin\Debug\netcoreapp3.1\Downloads\ } \ No newline at end of file diff --git a/FarmmapsDownloadCL/Models/CodelistsClasses.cs b/FarmmapsDownloadCL/Models/CodelistsClasses.cs index b3d4edc..d10ae8f 100644 --- a/FarmmapsDownloadCL/Models/CodelistsClasses.cs +++ b/FarmmapsDownloadCL/Models/CodelistsClasses.cs @@ -13,8 +13,10 @@ namespace FarmmapsDownloadCL.Models this.dictCl = new Dictionary>(); //Add here once you have defined a new CLxxxitem below //To add a new item, look it up in https://test.farmmaps.eu/swagger/index.html. Download the item you are interested in (like "vnd.farmmaps.itemtype.codelist.cl263") + //do this under Items <- get/api/v1/items, use the it field. Depending on the size of the list retrieving the list can take a few minutes. //then below create a new CLxxxitem with fieldnames as you see them in the just downloaded new codelist this.dictCl.Add("CL022", new List { "vnd.farmmaps.itemtype.codelist.cl022", "CL022item" }); + this.dictCl.Add("CL104", new List { "vnd.farmmaps.itemtype.codelist.cl104", "CL104item" }); this.dictCl.Add("CL127", new List { "vnd.farmmaps.itemtype.codelist.cl127", "CL127item" }); this.dictCl.Add("CL251", new List { "vnd.farmmaps.itemtype.codelist.cl251", "CL251item" }); this.dictCl.Add("CL263", new List { "vnd.farmmaps.itemtype.codelist.cl263", "CL263item" }); @@ -47,6 +49,21 @@ namespace FarmmapsDownloadCL.Models public string description { get; set; } } + public class CL104item + { + public CL104item() + { + this.headers = new string[] { "codelist", "code", "description" }; + this.codelist = "CL104"; + this.about = new string[] { "EDI-crop, coderingslijst groenbemesters" }; + } + + public string[] headers { get; } + public string codelist { get; } + public string[] about { get; } + public string code { get; set; } + public string description { get; set; } + } public class CL127item { public CL127item() From 67e5cb6bb486af5197dba8fd8c029a19f6e8a3de Mon Sep 17 00:00:00 2001 From: ttenden Date: Thu, 4 Jul 2024 13:35:33 +0200 Subject: [PATCH 64/66] Added KPI G1 ammonia emissions (only on test servers) - Ammonia emissions from fertilizers not yet fully functional --- FarmmapsKPI/KPIApplication.cs | 54 ++++++++++++++++++++++++++++++--- FarmmapsKPI/KPIInput.json | 18 ++++++----- FarmmapsKPI/KPIdefinitions.csv | 51 +++++++++++++++++-------------- FarmmapsKPI/Models/KPIoutput.cs | 8 ++++- 4 files changed, 94 insertions(+), 37 deletions(-) diff --git a/FarmmapsKPI/KPIApplication.cs b/FarmmapsKPI/KPIApplication.cs index 97d4994..e3d8af9 100644 --- a/FarmmapsKPI/KPIApplication.cs +++ b/FarmmapsKPI/KPIApplication.cs @@ -48,7 +48,8 @@ namespace FarmmapsKPI //fnKPIinput = Console.ReadLine(); //if (string.IsNullOrEmpty(fnKPIinput)) //{ - fnKPIinput = "KPIinput.json"; + //fnKPIinput = "KPIinput.json"; + fnKPIinput = "C:\\git\\FarmMapsApiClient_KB34_MAST\\FarmmapsKPI\\KPIinput.json"; //} var fieldsInputJson = File.ReadAllText(fnKPIinput); @@ -93,7 +94,7 @@ namespace FarmmapsKPI int nrun = 1; for (int run = 1; run <= nrun; run++) { for (int i = 0; i < fieldsInputs.Count; i++) - //for (int i = 11; i < 13; i++) // for testing + //for (int i = 2; i < 3; i++) // for testing { watch.Restart(); input = fieldsInputs[i]; @@ -432,7 +433,7 @@ namespace FarmmapsKPI int DuplicateCounter; value = 0; id = ""; - DuplicateCounter = 1; + DuplicateCounter = 1; for (int i = 0; i < KPIItems.Count; i++) { itemi = KPIItems[i]; @@ -668,7 +669,6 @@ namespace FarmmapsKPI sw.WriteLine(string.Join(",", dataList)); } } - else if (kpio.id == "E1") { //for E1, environmentMeasureData is an array of elements with 1 per pesticide. Inside each element there are multiple mbp_KPIvariables to be written @@ -705,9 +705,53 @@ namespace FarmmapsKPI } } } + else if (kpio.id == "G1") //TtD + { + //Make a new dataList = new line to be written + dataList = new List + { + run.ToString(), + kpio.parentName, + cropfieldItem.Code, + kpio.data.area, + kpio.data.cropTypeCode, + kpio.data.cropTypeName, + kpio.id, + kpio.quantity, // in KPI output quantity is what we call KPIvariable in headerlist of csv file + kpio.value, + kpio.unit, + kpio.targetValue, + kpio.thresholdValue + }; + //Write the datalist to a line to the streamwrieter sw for the output csv file + sw.WriteLine(string.Join(",", dataList)); + + foreach (string elementName in kpio.G1elements) + { + // get G1element from the element called values + string elementValue = (string)kPIelementsOfBalance.GetType().GetProperty(elementName).GetValue(kPIelementsOfBalance, null); + dataList = new List + { + run.ToString(), + kpio.parentName, + cropfieldItem.Code, + kpio.data.area, + kpio.data.cropTypeCode, + kpio.data.cropTypeName, + kpio.id, + elementName, // specific output variable name for G1 element + elementValue, // specific output value name for G1 element + kpio.unit, + "", + "" + }; + //Write the datalist to a line to the streamwrieter sw for the output csv file + sw.WriteLine(string.Join(",", dataList)); + } + } else { - //Any other KPI, example A1 or D1, with just 1 record to write + //Any other KPI, (D1, F1, and F2), with just 1 record to write dataList = new List { run.ToString(), diff --git a/FarmmapsKPI/KPIInput.json b/FarmmapsKPI/KPIInput.json index 394e108..6fdf94d 100644 --- a/FarmmapsKPI/KPIInput.json +++ b/FarmmapsKPI/KPIInput.json @@ -17,7 +17,7 @@ "soilName": "Sand", //From codelist CL405 "cropTypeCode": "1010101", //From codelist CL263 "cropTypeName": "Potato", //From codelist CL263 - "productionPurposeCode": "3", //From codelist CL251. For testing, see case with consumption & starch potato + "productionPurposeCode": "003", //From codelist CL251. For testing, see case with consumption & starch potato "productionPurposeName": "consumption" //From codelist CL251 }, "CropRecordingItemCode": "", // could contain for example this: "314aa1b6480b493ba28a5f55039a01d4" if this is the CropRecordingItemCode of the provided CropfieldItemCode. And then if useExistingCropfieldWithChildren = true, this would be used @@ -81,7 +81,7 @@ "soilName": "Clay", //From codelist CL405 "cropTypeCode": "1010101", //From codelist CL263 "cropTypeName": "Potato", //From codelist CL263 - "productionPurposeCode": "3", //From codelist CL251. For testing, see case with consumption & starch potato + "productionPurposeCode": "003", //From codelist CL251. For testing, see case with consumption & starch potato "productionPurposeName": "consumption" //From codelist CL251 }, "CropRecordingItemCode": "", // could contain for example this: "314aa1b6480b493ba28a5f55039a01d4" if this is the CropRecordingItemCode of the provided CropfieldItemCode. And then if useExistingCropfieldWithChildren = true, this would be used @@ -145,7 +145,7 @@ "soilName": "Sand", "cropTypeCode": "1010101", //From codelist CL263 "cropTypeName": "Potato", //From codelist CL263 - "productionPurposeCode": "2", //From codelist CL251. For testing, see case with consumption & starch potato + "productionPurposeCode": "002", //From codelist CL251. For testing, see case with consumption & starch potato "productionPurposeName": "starch" //From codelist CL251 }, "CropRecordingItemCode": "", // could contain for example this: "314aa1b6480b493ba28a5f55039a01d4" if this is the CropRecordingItemCode of the provided CropfieldItemCode. And then if useExistingCropfieldWithChildren = true, this would be used @@ -205,7 +205,7 @@ "cropTypeName": "Potato", //From codelist CL263 //"rootDepthMax": 45, //not needed for KPI calculation, but shown here to know this is a possible property //"emergenceDate": "2022-05-16T00:00:00", //not needed for KPI calculation, but shown here to know this is a possible property - "productionPurposeCode": "3", //From codelist CL251. Can be omitted if unknown + "productionPurposeCode": "003", //From codelist CL251. Can be omitted if unknown "productionPurposeName": "consumption" //From codelist CL251. Can be omitted if unknown }, "CropRecordingItemCode": "", @@ -220,6 +220,7 @@ "n": "92", //refers to codelist CL022 with fertilizer types & npk contents "name": "Kunstmest strooien", //refers to codelist CL127 with operation methods "product": "7360", //refers to codelist CL022 with fertilizer types + "type": "MOR", //indicates wether the fertilization is anorganic (MAN) or organic (MOR) "quantity": "200", //"status": "3", // refers to codelist CL256. where 3 stand for completed. The status does not matter and is not used in the KPI calculations "operationCode": "7", // refers to codelist CL018 consisting of main catergorie of operations. Code 7 stands for fertilization @@ -240,6 +241,7 @@ //"status": "3", // refers to codelist CL256. where 3 stand for completed. The status does not matter and is not used in the KPI calculations "operationCode": "7", // refers to codelist CL018 consisting of main catergorie of operations. Code 7 stands for fertilization "product": "2329", //refers to codelist CL022 with fertilizer types + "type": "MAN",//indicates wether the fertilization is anorganic (MAN) or organic (MOR) "quantity": "32000", "unitCode": "KGMHAR", "contractor": false, @@ -298,7 +300,7 @@ "cropTypeName": "Potato", //From codelist CL263 //"rootDepthMax": 45, //not needed for KPI calculation, but shown here to know this is a possible property //"emergenceDate": "2022-05-16T00:00:00", //not needed for KPI calculation, but shown here to know this is a possible property - "productionPurposeCode": "3", //From codelist CL251. Can be omitted if unknown + "productionPurposeCode": "003", //From codelist CL251. Can be omitted if unknown "productionPurposeName": "consumption" //From codelist CL251. Can be omitted if unknown }, "CropRecordingItemCode": "", @@ -373,7 +375,7 @@ "cropTypeName": "Potato", //From codelist CL263 //"rootDepthMax": 45, //not needed for KPI calculation, but shown here to know this is a possible property //"emergenceDate": "2022-05-16T00:00:00", //not needed for KPI calculation, but shown here to know this is a possible property - "productionPurposeCode": "3", //From codelist CL251. Can be omitted if unknown + "productionPurposeCode": "003", //From codelist CL251. Can be omitted if unknown "productionPurposeName": "consumption" //From codelist CL251. Can be omitted if unknown }, "CropRecordingItemCode": "", @@ -448,7 +450,7 @@ "cropTypeName": "Potato", //From codelist CL263 //"rootDepthMax": 45, //not needed for KPI calculation, but shown here to know this is a possible property //"emergenceDate": "2022-05-16T00:00:00", //not needed for KPI calculation, but shown here to know this is a possible property - "productionPurposeCode": "3", //From codelist CL251. Can be omitted if unknown + "productionPurposeCode": "003", //From codelist CL251. Can be omitted if unknown "productionPurposeName": "consumption" //From codelist CL251. Can be omitted if unknown }, "CropRecordingItemCode": "", @@ -518,7 +520,7 @@ "final": true, //always true "cropTypeCode": "1010101", //From codelist CL263 "cropTypeName": "Potato", //From codelist CL263 - "productionPurposeCode": "3", //From codelist CL251. Can be omitted if unknown + "productionPurposeCode": "003", //From codelist CL251. Can be omitted if unknown "productionPurposeName": "consumption" //From codelist CL251. Can be omitted if unknown }, "CropRecordingItemCode": "", // could contain for example this: "314aa1b6480b493ba28a5f55039a01d4" if this is the CropRecordingItemCode of the provided CropfieldItemCode. And then if useExistingCropfieldWithChildren = true, this would be used diff --git a/FarmmapsKPI/KPIdefinitions.csv b/FarmmapsKPI/KPIdefinitions.csv index 317f999..c140e48 100644 --- a/FarmmapsKPI/KPIdefinitions.csv +++ b/FarmmapsKPI/KPIdefinitions.csv @@ -1,23 +1,28 @@ -KPIid,KPIvariable,Description,KPItargetvalue,KPIthresholdValue -A1,yield,observed yield (user input),target value as in benchmark value for same crop in same region. E.g. cropyield target is from Central Bureau of Statistics (CBS) for same crop in region in which the cropfield (geometry) is located.,No threshold for this KPI. -A1,strawYield,straw yield (user input; value 0 or not filled in = straw remains in field; value > 0 means straw was exported),there are no target values for individual compenets of KPI's,there are no threshold values for individual compenets of KPI's -B1,nitrogen,Nitrogen surplus = N artifical fertilizer + N manure+ N atmospheric deposition + N fixation by crop + N from seed potatoes - N removal through harvested product. Calculated KPI internal model & parameters and from user input: yield and fertilizer/manure applications.,The target value depends on the soil type. To calculate the threshold and target values data from CBS and RIVM are used. The same calculation method is also used in biodiversiteitsmonitoring akkerbouw (https://www.bo-akkerbouw.nl/kennis-en-innovatie/pps-biodiversiteitsmonitor-akkerbouw).,The threshold value depends on soil type and groundwater levels. To calculate the threshold and target values data from CBS and RIVM are used. The same calculation method is also used in biodiversiteitsmonitoring akkerbouw (https://www.bo-akkerbouw.nl/kennis-en-innovatie/pps-biodiversiteitsmonitor-akkerbouw). -B1,nHarvestedKgHa, N removal through harvested product. Calculated KPI internal model & parameters and from user input: yield.,there are no target values for individual compenets of KPI's,there are no threshold values for individual compenets of KPI's -B1,nHarvestedStrawKgHa, N removal through harvested product. Calculated KPI internal model & parameters and from user input: yieldStraw.,there are no target values for individual compenets of KPI's,there are no threshold values for individual compenets of KPI's -B1,nFixationNKgHa, N fixated by crop (legumes only). Calculated KPI internal model & parameters and from user input: planting date,there are no target values for individual compenets of KPI's,there are no threshold values for individual compenets of KPI's -B1,nDepositionNKgHa, N deposition based on RIVM maps. ,there are no target values for individual compenets of KPI's,there are no threshold values for individual compenets of KPI's -B1,nFertilizerNKgHa, N from artifical fertilizers and manure.Calculated KPI internal model & parameters and from user input: fertilizer/manure applications.,there are no target values for individual compenets of KPI's,there are no threshold values for individual compenets of KPI's -B1,sowPlantingNKgHa, N from seed potatoes (potatoes only). Calculated KPI internal model & parameters and from user input: amount of planted seed potatoes,there are no target values for individual compenets of KPI's,there are no threshold values for individual compenets of KPI's -B2,phosphate,Phosphate surplus = P fertilizer + P from seed potatoes - P removal through harvested product. Calculated KPI internal model & parameters and from user input: yield and fertilizer applications ,The phosphate target value is set at 20 kg/ha ,No threshold for this KPI. -B2,pHarvestedKgHa, P2O5 removal through harvested product. Calculated KPI internal model & parameters and from user input: yield,there are no target values for individual compenets of KPI's,there are no threshold values for individual compenets of KPI's -B2,pHarvestedStrawKgHa, P2O5 removal through harvested product. Calculated KPI internal model & parameters and from user input: yieldStraw,there are no target values for individual compenets of KPI's,there are no threshold values for individual compenets of KPI's -B2,pFertillizerPKgHa, P2O5 from artifical fertilizers and manure.Calculated KPI internal model & parameters and from user input: fertilizer/manure applications.,there are no target values for individual compenets of KPI's,there are no threshold values for individual compenets of KPI's -B2,sowPlantingPKgHa, P2O5 from seeds potatoes (potatoes only). Calculated KPI internal model & parameters and from user input: amount of planted seed potatoes,there are no target values for individual compenets of KPI's,there are no threshold values for individual compenets of KPI's -C1,effective organic matter supply,"Organic matter supply = Organic matter from manure + Crop residues to soil + organic matter from green manure. Calculated KPI internal model & parameters and from user input: yield and manure applications. ""Effective"" means netto, after correction for carbon respiration by soil microbial activity",The target value for this KPI is the amount of EOM that companies with ‘ Goede Landbouwpraktijk’ supply (2500 kg EOM/ha/year). ,The threshold value is 2000 kg EOM/ha/year. This value comes from van Doorn et al. (2022) which states that on average 2000 kg of EOM/ha degrades on Dutch agricultural land. This is the also the threshold value from biodiversiteitsmonitoring akkerbouw (https://www.bo-akkerbouw.nl/kennis-en-innovatie/pps-biodiversiteitsmonitor-akkerbouw) -C1,fertilizerEom,"Effective organic matter added through fertilizer (animal manure, compost, etc). Calculated KPI internal model & parameters and from user input: manure applications.",there are no target values for individual compenets of KPI's,there are no threshold values for individual compenets of KPI's -C1,greenManureEom,Effective organic matter added through green manure (also called cover crop). Calculated KPI internal model & parameters and from user input: cover crop.,there are no target values for individual compenets of KPI's,there are no threshold values for individual compenets of KPI's -C1,cropLeftoversEom,Effective organic matter added through crop residues of main product and (when relevant) straw. Calculated KPI internal model & parameters.,there are no target values for individual compenets of KPI's,there are no threshold values for individual compenets of KPI's -D1,pesticides,"kg/ha of active ingredient. Sum over kg a.i. of the different crop protection agents applied, with concentration (kg a.i. / kg or litre) of crop protection agents retrieved from CTBG",The target value fort his KPI is to reduce pesticide use to 50% of the amount used in the Netherlands during 2020. Data for 2020 is obtained from CBS statline ,No threshold for this KPI. -E1,mpb,"milieubelastingspunten. For each pesticide applied show 3 sub-mbp's: aquaticLife, groundWater, soilLife","0 applications with exceedances of either the target MBP or risks. MBP's consist of three categories (groundwater, soil life, and aquatic life). An value of <100 in either or these categories counts as exceedance. Each application of pesticides cannot have more then one exceedance for MBP. Risks consist of two categories (natural enemies and pollinators), a value of B or C in either category counts as exceedance. Each application of pesticides cannot have more then one exceedance for risks. Values and methodology come from Environmental Yardstick for Pesticides (https://www.milieumeetlat.nl/). The same calculation method is also used in biodiversiteitsmonitoring akkerbouw (https://www.bo-akkerbouw.nl/kennis-en-innovatie/pps-biodiversiteitsmonitor-akkerbouw).","0 applications with exceedances of either the target MBP or risks. MBP's consist of three categories (groundwater, soil life, and aquatic life). An value of <10 in either or these categories counts as exceedance. Each application of pesticides cannot have more then one exceedance for MBP. Risks consist of two categories (natural enemies and pollinators), a value of B or C in either category counts as exceedance. Each application of pesticides cannot have more then one exceedance for risks. Values and methodology come from Environmental Yardstick for Pesticides (https://www.milieumeetlat.nl/). The same calculation method is also used in biodiversiteitsmonitoring akkerbouw (https://www.bo-akkerbouw.nl/kennis-en-innovatie/pps-biodiversiteitsmonitor-akkerbouw)." -F1,greenness,time of year that the field was covered. Based on NDVI data from AgroDataCube.,Target value is 75%.,Threshold value is 25%. -F2,rotationindex,index of diversity of crops over time. Calculated in AgroDataCube,Target value is 75%.,Threshold value is 25%. +KPIid,KPIvariable,Description,Calculated,KPItargetvalue,KPIthresholdValue +A1,yield,observed yield,User input,target value as in benchmark value for same crop in same region. E.g. cropyield target is from Central Bureau of Statistics (CBS) for same crop in region in which the cropfield (geometry) is located.,No threshold for this KPI. +A1,strawYield,straw yield; value 0 or not filled in = straw remains in field; value > 0 means straw was exported,User input,there are no target values for individual compenets of KPI's,there are no threshold values for individual compenets of KPI's +B1,nitrogen,Nitrogen surplus = N artifical fertilizer + N manure+ N atmospheric deposition + N fixation by crop + N from seed potatoes - N removal through harvested product. ,Calculated KPI internal model & parameters and from user input: yield and fertilizer/manure applications.,The target value depends on the soil type. To calculate the threshold and target values data from CBS and RIVM are used. The same calculation method is also used in biodiversiteitsmonitoring akkerbouw (https://www.bo-akkerbouw.nl/kennis-en-innovatie/pps-biodiversiteitsmonitor-akkerbouw).,The threshold value depends on soil type and groundwater levels. To calculate the threshold and target values data from CBS and RIVM are used. The same calculation method is also used in biodiversiteitsmonitoring akkerbouw (https://www.bo-akkerbouw.nl/kennis-en-innovatie/pps-biodiversiteitsmonitor-akkerbouw). +B1,nHarvestedKgHa, N removal through harvested product. ,Calculated KPI internal model & parameters and from user input: yield.,there are no target values for individual compenets of KPI's,there are no threshold values for individual compenets of KPI's +B1,nHarvestedStrawKgHa, N removal through harvested product. ,Calculated KPI internal model & parameters and from user input: yieldStraw.,there are no target values for individual compenets of KPI's,there are no threshold values for individual compenets of KPI's +B1,nFixationNKgHa, N fixated by crop (legumes only). ,Calculated KPI internal model & parameters and from user input: planting date,there are no target values for individual compenets of KPI's,there are no threshold values for individual compenets of KPI's +B1,nDepositionNKgHa, N deposition,Based on RIVM maps. Value depens on location,there are no target values for individual compenets of KPI's,there are no threshold values for individual compenets of KPI's +B1,nFertilizerNKgHa, N from artifical fertilizers and manure.,KPI internal model & parameters and from user input: fertilizer/manure applications.,there are no target values for individual compenets of KPI's,there are no threshold values for individual compenets of KPI's +B1,sowPlantingNKgHa, N from seed potatoes (potatoes only). ,Calculated KPI internal model & parameters and from user input: amount of planted seed potatoes,there are no target values for individual compenets of KPI's,there are no threshold values for individual compenets of KPI's +B2,phosphate,Phosphate surplus = P fertilizer + P from seed potatoes - P removal through harvested product. ,Calculated KPI internal model & parameters and from user input: yield and fertilizer applications ,The phosphate target value is set at 20 kg/ha ,No threshold for this KPI. +B2,pHarvestedKgHa, P2O5 removal through harvested product. ,Calculated KPI internal model & parameters and from user input: yield,there are no target values for individual compenets of KPI's,there are no threshold values for individual compenets of KPI's +B2,pHarvestedStrawKgHa, P2O5 removal through harvested product. ,Calculated KPI internal model & parameters and from user input: yieldStraw,there are no target values for individual compenets of KPI's,there are no threshold values for individual compenets of KPI's +B2,pFertillizerPKgHa, P2O5 from artifical fertilizers and manure.,Calculated KPI internal model & parameters and from user input: fertilizer/manure applications.,there are no target values for individual compenets of KPI's,there are no threshold values for individual compenets of KPI's +B2,sowPlantingPKgHa, P2O5 from seeds potatoes (potatoes only). ,Calculated KPI internal model & parameters and from user input: amount of planted seed potatoes,there are no target values for individual compenets of KPI's,there are no threshold values for individual compenets of KPI's +C1,effective organic matter supply,"Organic matter supply = Organic matter from manure + Crop residues to soil + organic matter from green manure. ""Effective"" means netto, after correction for carbon respiration by soil microbial activity",Calculated KPI internal model & parameters and from user input: yield and manure applications.,The target value for this KPI is the amount of EOM that companies with ‘ Goede Landbouwpraktijk’ supply (2500 kg EOM/ha/year). ,The threshold value is 2000 kg EOM/ha/year. This value comes from van Doorn et al. (2022) which states that on average 2000 kg of EOM/ha degrades on Dutch agricultural land. This is the also the threshold value from biodiversiteitsmonitoring akkerbouw (https://www.bo-akkerbouw.nl/kennis-en-innovatie/pps-biodiversiteitsmonitor-akkerbouw) +C1,fertilizerEom,"Effective organic matter added through fertilizer (animal manure, compost, etc). ",Calculated KPI internal model & parameters and from user input: manure applications.,there are no target values for individual compenets of KPI's,there are no threshold values for individual compenets of KPI's +C1,greenManureEom,Effective organic matter added through green manure (also called cover crop). ,Calculated KPI internal model & parameters and from user input: cover crop.,there are no target values for individual compenets of KPI's,there are no threshold values for individual compenets of KPI's +C1,cropLeftoversEom,Effective organic matter added through crop residues of main product and (when relevant) straw.,Calculated KPI internal model & parameters.,there are no target values for individual compenets of KPI's,there are no threshold values for individual compenets of KPI's +D1,pesticides,"kg/ha of active ingredient. Sum over kg a.i. of the different crop protection agents applied, with concentration (kg a.i. / kg or litre) of crop protection agents ",Retrieved from CTBG,The target value fort his KPI is to reduce pesticide use to 50% of the amount used in the Netherlands during 2020. Data for 2020 is obtained from CBS statline ,No threshold for this KPI. +E1,mpb,"milieubelastingspunten. For each pesticide applied show 3 sub-mbp's: aquaticLife, groundWater, soilLife",Retrieved from CTBG,"0 applications with exceedances of either the target MBP or risks. MBP's consist of three categories (groundwater, soil life, and aquatic life). An value of <100 in either or these categories counts as exceedance. Each application of pesticides cannot have more then one exceedance for MBP. Risks consist of two categories (natural enemies and pollinators), a value of B or C in either category counts as exceedance. Each application of pesticides cannot have more then one exceedance for risks. Values and methodology come from Environmental Yardstick for Pesticides (https://www.milieumeetlat.nl/). The same calculation method is also used in biodiversiteitsmonitoring akkerbouw (https://www.bo-akkerbouw.nl/kennis-en-innovatie/pps-biodiversiteitsmonitor-akkerbouw).","0 applications with exceedances of either the target MBP or risks. MBP's consist of three categories (groundwater, soil life, and aquatic life). An value of <10 in either or these categories counts as exceedance. Each application of pesticides cannot have more then one exceedance for MBP. Risks consist of two categories (natural enemies and pollinators), a value of B or C in either category counts as exceedance. Each application of pesticides cannot have more then one exceedance for risks. Values and methodology come from Environmental Yardstick for Pesticides (https://www.milieumeetlat.nl/). The same calculation method is also used in biodiversiteitsmonitoring akkerbouw (https://www.bo-akkerbouw.nl/kennis-en-innovatie/pps-biodiversiteitsmonitor-akkerbouw)." +F1,greenness,time of year that the field was covered. ,Based on NDVI data from AgroDataCube.,Target value is 75%.,Threshold value is 25%. +F2,rotationindex,index of diversity of crops over time. ,Calculated in AgroDataCube,Target value is 75%.,Threshold value is 25%. +G1,ammonia emissions,"ammonia emissions from crop residues, green manure, artificial fertilizers and organic fertilizers.",Calculated KPI internal model & parameters and from user input: fertilizer/manure applications.,There is not target value yet for this KPI. It currently has a dummy value of 9999.,There is not target value yet for this KPI. It currently has a dummy value of 9999. +G1,Manure,ammonia emissions from manure.,Calculated KPI internal model & parameters and from user input: manure applications.,there are no target values for individual compenets of KPI's,there are no threshold values for individual compenets of KPI's +G1,GreenManure,ammonia emissions from green manure (also called cover crop). ,Calculated KPI internal model & parameters.,there are no target values for individual compenets of KPI's,there are no threshold values for individual compenets of KPI's +G1,CropResidues,ammonia emissions from crop residues. ,Calculated KPI internal model & parameters.,there are no target values for individual compenets of KPI's,there are no threshold values for individual compenets of KPI's +G1,ArtificialFertilize,ammonia emissions from artificial fertilizers ,Calculated KPI internal model & parameters and from user input: artificial fertilizer applications.,there are no target values for individual compenets of KPI's,there are no threshold values for individual compenets of KPI's diff --git a/FarmmapsKPI/Models/KPIoutput.cs b/FarmmapsKPI/Models/KPIoutput.cs index ba22106..a9db192 100644 --- a/FarmmapsKPI/Models/KPIoutput.cs +++ b/FarmmapsKPI/Models/KPIoutput.cs @@ -20,13 +20,15 @@ namespace FarmmapsKPI.Models public List A1elements; //TtD public List B1elements; public List B2elements; - public List C1elements; + public List C1elements; + public List G1elements; //TtD public KPIOutput() { this.A1elements = new List() { "yield", "strawYield" }; //TtD this.B1elements = new List() { "nHarvestedKgHa", "nHarvestedStrawKgHa", "nFixationNKgHa", "nDepositionNKgHa","nFertilizerNKgHa","sowPlantingNKgHa" }; //TtD this.B2elements = new List() { "pHarvestedKgHa", "pHarvestedStrawKgHa", "pFertillizerPKgHa", "sowPlantingPKgHa" }; //TtD this.C1elements = new List() { "fertilizerEom", "greenManureEom", "cropLeftoversEom" }; //TtD to do rename + this.G1elements = new List() { "Manure", "GreenManure", "CropResidues", "ArtificialFertilize" }; //TtD } } public class KPIOutputData @@ -69,5 +71,9 @@ namespace FarmmapsKPI.Models public string fertilizerEom { get; set; } //TtD public string greenManureEom { get; set; } //TtD public string cropLeftoversEom { get; set; } //TtD tamara: rename + public string Manure { get; set; } //TtD + public string GreenManure { get; set; } //TtD + public string CropResidues { get; set; } //TtD + public string ArtificialFertilize { get; set; } //TtD } } \ No newline at end of file From fb92fa0d1c30b2506fbf4def6c912aa4891b8276 Mon Sep 17 00:00:00 2001 From: ttenden Date: Mon, 26 Aug 2024 10:35:58 +0200 Subject: [PATCH 65/66] KPI ammonia emissions is now fully functional (including in the example input file) --- FarmmapsKPI/KPIApplication.cs | 2 +- FarmmapsKPI/KPIInput.json | 46 +++++++++++++++++++--------------- FarmmapsKPI/KPIdefinitions.csv | 2 +- 3 files changed, 28 insertions(+), 22 deletions(-) diff --git a/FarmmapsKPI/KPIApplication.cs b/FarmmapsKPI/KPIApplication.cs index e3d8af9..66494dd 100644 --- a/FarmmapsKPI/KPIApplication.cs +++ b/FarmmapsKPI/KPIApplication.cs @@ -94,7 +94,7 @@ namespace FarmmapsKPI int nrun = 1; for (int run = 1; run <= nrun; run++) { for (int i = 0; i < fieldsInputs.Count; i++) - //for (int i = 2; i < 3; i++) // for testing + //for (int i = 3; i < 4; i++) // for testing { watch.Restart(); input = fieldsInputs[i]; diff --git a/FarmmapsKPI/KPIInput.json b/FarmmapsKPI/KPIInput.json index 6fdf94d..e24d7aa 100644 --- a/FarmmapsKPI/KPIInput.json +++ b/FarmmapsKPI/KPIInput.json @@ -13,8 +13,8 @@ "dataCropfield": { //"area": 4.22, //Leave empty, KPI app will calculate it from geometry "final": true, //always true - "soilCode": "1", //From codelist CL405. Can be omitted if unknown - "soilName": "Sand", //From codelist CL405 + "soilCode": "1", //From codelist CL405. Can be no longer be omitted + "soilName": "Sand", "cropTypeCode": "1010101", //From codelist CL263 "cropTypeName": "Potato", //From codelist CL263 "productionPurposeCode": "003", //From codelist CL251. For testing, see case with consumption & starch potato @@ -77,7 +77,7 @@ "dataCropfield": { //"area": 4.22, //Leave empty, KPI app will calculate it from geometry "final": true, //always true - "soilCode": "7", //From codelist CL405. Can be omitted if unknown + "soilCode": "7", //From codelist CL405. Can no longer be omitted "soilName": "Clay", //From codelist CL405 "cropTypeCode": "1010101", //From codelist CL263 "cropTypeName": "Potato", //From codelist CL263 @@ -141,7 +141,7 @@ "dataCropfield": { //"area": 4.22, //Leave empty, KPI app will calculate it from geometry "final": true, //always true - "soilCode": "1", //From codelist CL405. Can be omitted if unknown + "soilCode": "1", //From codelist CL405. Can be no longer be omitted "soilName": "Sand", "cropTypeCode": "1010101", //From codelist CL263 "cropTypeName": "Potato", //From codelist CL263 @@ -199,8 +199,8 @@ "dataCropfield": { //"area": 4.22, //Leave empty, KPI app will calculate it from geometry "final": true, //always true - //"soilCode": "5", //From codelist CL405. Can be omitted if unknown - //"soilName": "Loam", //From codelist CL405. + "soilCode": "5", //From codelist CL405. Can be no longer be omitted + "soilName": "Loam", //From codelist CL405. "cropTypeCode": "1010101", //From codelist CL263 "cropTypeName": "Potato", //From codelist CL263 //"rootDepthMax": 45, //not needed for KPI calculation, but shown here to know this is a possible property @@ -220,7 +220,7 @@ "n": "92", //refers to codelist CL022 with fertilizer types & npk contents "name": "Kunstmest strooien", //refers to codelist CL127 with operation methods "product": "7360", //refers to codelist CL022 with fertilizer types - "type": "MOR", //indicates wether the fertilization is anorganic (MAN) or organic (MOR) + //"type": "MAN", //indicates wether the fertilization is anorganic (MAN) or organic (MOR) "quantity": "200", //"status": "3", // refers to codelist CL256. where 3 stand for completed. The status does not matter and is not used in the KPI calculations "operationCode": "7", // refers to codelist CL018 consisting of main catergorie of operations. Code 7 stands for fertilization @@ -241,7 +241,7 @@ //"status": "3", // refers to codelist CL256. where 3 stand for completed. The status does not matter and is not used in the KPI calculations "operationCode": "7", // refers to codelist CL018 consisting of main catergorie of operations. Code 7 stands for fertilization "product": "2329", //refers to codelist CL022 with fertilizer types - "type": "MAN",//indicates wether the fertilization is anorganic (MAN) or organic (MOR) + //"type": "MAN",//indicates wether the fertilization is anorganic (MAN) or organic (MOR) "quantity": "32000", "unitCode": "KGMHAR", "contractor": false, @@ -294,8 +294,8 @@ "dataCropfield": { //"area": 4.22, //Leave empty, KPI app will calculate it from geometry "final": true, //always true - //"soilCode": "5", //From codelist CL405. Can be omitted if unknown - //"soilName": "Loam", //From codelist CL405. + "soilCode": "5", //From codelist CL405. Can be no longer be omitted + "soilName": "Loam", //From codelist CL405. "cropTypeCode": "1010101", //From codelist CL263 "cropTypeName": "Potato", //From codelist CL263 //"rootDepthMax": 45, //not needed for KPI calculation, but shown here to know this is a possible property @@ -369,8 +369,8 @@ "dataCropfield": { //"area": 4.22, //Leave empty, KPI app will calculate it from geometry "final": true, //always true - //"soilCode": "5", //From codelist CL405. Can be omitted if unknown - //"soilName": "Loam", //From codelist CL405. + "soilCode": "5", //From codelist CL405. Can be no longer be omitted + "soilName": "Loam", //From codelist CL405. "cropTypeCode": "1010101", //From codelist CL263 "cropTypeName": "Potato", //From codelist CL263 //"rootDepthMax": 45, //not needed for KPI calculation, but shown here to know this is a possible property @@ -442,10 +442,10 @@ "fieldName": "caseQuoteNnotProvidedInDataOperations", "CropfieldItemCode": "", "dataCropfield": { + "soilCode": "1", //From codelist CL405. Can be no longer be omitted + "soilName": "Sand", //"area": 4.22, //Leave empty, KPI app will calculate it from geometry "final": true, //always true - //"soilCode": "5", //From codelist CL405. Can be omitted if unknown - //"soilName": "Loam", //From codelist CL405. "cropTypeCode": "1010101", //From codelist CL263 "cropTypeName": "Potato", //From codelist CL263 //"rootDepthMax": 45, //not needed for KPI calculation, but shown here to know this is a possible property @@ -517,6 +517,8 @@ "fieldName": "examplePesticide_11767N", "CropfieldItemCode": "", // could contain for example this: "abae97f89f3c4ac08953b1b8bea9f076" if this is an exisiting CropfieldItemCode in your account. "dataCropfield": { + "soilCode": "1", //From codelist CL405. Can be no longer be omitted + "soilName": "Sand", "final": true, //always true "cropTypeCode": "1010101", //From codelist CL263 "cropTypeName": "Potato", //From codelist CL263 @@ -588,6 +590,8 @@ "fieldName": "doperwt_heleJaar", "CropfieldItemCode": "", "dataCropfield": { + "soilCode": "1", //From codelist CL405. Can be no longer be omitted + "soilName": "Sand", "final": true, //always true "cropTypeCode": "1030101", //From codelist CL263 "cropTypeName": "Garden pea" //From codelist CL263 @@ -641,6 +645,8 @@ "fieldName": "doperwt_groeiperiode", "CropfieldItemCode": "", "dataCropfield": { + "soilCode": "1", //From codelist CL405. Can be no longer be omitted + "soilName": "Sand", "final": true, //always true "cropTypeCode": "1030101", //From codelist CL263 "cropTypeName": "Garden pea" //From codelist CL263 @@ -702,8 +708,8 @@ "dataCropfield": { //"area": 4.22, //Leave empty, KPI app will calculate it from geometry "final": true, //always true - "soilCode": "1", //From codelist CL405. Can be omitted if unknown - "soilName": "Sand", //From codelist CL405 + "soilCode": "1", //From codelist CL405. Can be no longer be omitted + "soilName": "Sand", "cropTypeCode": "1020101", //From codelist CL263 "cropTypeName": "Winter wheat" //From codelist CL263 }, @@ -764,8 +770,8 @@ "dataCropfield": { //"area": 4.22, //Leave empty, KPI app will calculate it from geometry "final": true, //always true - "soilCode": "1", //From codelist CL405. Can be omitted if unknown - "soilName": "Sand", //From codelist CL405 + "soilCode": "1", //From codelist CL405. Can be no longer be omitted + "soilName": "Sand", "cropTypeCode": "1020101", //From codelist CL263 "cropTypeName": "Winter wheat" //From codelist CL263 }, @@ -830,8 +836,8 @@ "dataCropfield": { //"area": 4.22, //Leave empty, KPI app will calculate it from geometry "final": true, //always true - "soilCode": "1", //From codelist CL405. Can be omitted if unknown - "soilName": "Sand", //From codelist CL405 + "soilCode": "1", //From codelist CL405. Can be no longer be omitted + "soilName": "Sand", "cropTypeCode": "1020101", //From codelist CL263 "cropTypeName": "Winter wheat" //From codelist CL263 }, diff --git a/FarmmapsKPI/KPIdefinitions.csv b/FarmmapsKPI/KPIdefinitions.csv index c140e48..17b6faf 100644 --- a/FarmmapsKPI/KPIdefinitions.csv +++ b/FarmmapsKPI/KPIdefinitions.csv @@ -21,7 +21,7 @@ D1,pesticides,"kg/ha of active ingredient. Sum over kg a.i. of the different cr E1,mpb,"milieubelastingspunten. For each pesticide applied show 3 sub-mbp's: aquaticLife, groundWater, soilLife",Retrieved from CTBG,"0 applications with exceedances of either the target MBP or risks. MBP's consist of three categories (groundwater, soil life, and aquatic life). An value of <100 in either or these categories counts as exceedance. Each application of pesticides cannot have more then one exceedance for MBP. Risks consist of two categories (natural enemies and pollinators), a value of B or C in either category counts as exceedance. Each application of pesticides cannot have more then one exceedance for risks. Values and methodology come from Environmental Yardstick for Pesticides (https://www.milieumeetlat.nl/). The same calculation method is also used in biodiversiteitsmonitoring akkerbouw (https://www.bo-akkerbouw.nl/kennis-en-innovatie/pps-biodiversiteitsmonitor-akkerbouw).","0 applications with exceedances of either the target MBP or risks. MBP's consist of three categories (groundwater, soil life, and aquatic life). An value of <10 in either or these categories counts as exceedance. Each application of pesticides cannot have more then one exceedance for MBP. Risks consist of two categories (natural enemies and pollinators), a value of B or C in either category counts as exceedance. Each application of pesticides cannot have more then one exceedance for risks. Values and methodology come from Environmental Yardstick for Pesticides (https://www.milieumeetlat.nl/). The same calculation method is also used in biodiversiteitsmonitoring akkerbouw (https://www.bo-akkerbouw.nl/kennis-en-innovatie/pps-biodiversiteitsmonitor-akkerbouw)." F1,greenness,time of year that the field was covered. ,Based on NDVI data from AgroDataCube.,Target value is 75%.,Threshold value is 25%. F2,rotationindex,index of diversity of crops over time. ,Calculated in AgroDataCube,Target value is 75%.,Threshold value is 25%. -G1,ammonia emissions,"ammonia emissions from crop residues, green manure, artificial fertilizers and organic fertilizers.",Calculated KPI internal model & parameters and from user input: fertilizer/manure applications.,There is not target value yet for this KPI. It currently has a dummy value of 9999.,There is not target value yet for this KPI. It currently has a dummy value of 9999. +G1,ammonia emissions,"ammonia emissions from crop residues, green manure, artificial fertilizers and organic fertilizers.",Calculated KPI internal model & parameters and from user input: fertilizer/manure applications.,There is not target value yet for this KPI. Currently has a dummy value of 9999.,There is not target value yet for this KPI. Currently has a dummy value of 9999. G1,Manure,ammonia emissions from manure.,Calculated KPI internal model & parameters and from user input: manure applications.,there are no target values for individual compenets of KPI's,there are no threshold values for individual compenets of KPI's G1,GreenManure,ammonia emissions from green manure (also called cover crop). ,Calculated KPI internal model & parameters.,there are no target values for individual compenets of KPI's,there are no threshold values for individual compenets of KPI's G1,CropResidues,ammonia emissions from crop residues. ,Calculated KPI internal model & parameters.,there are no target values for individual compenets of KPI's,there are no threshold values for individual compenets of KPI's From c6e4f7368ff6e6e6e5389e90bc603aa4d54aa547 Mon Sep 17 00:00:00 2001 From: Pepijn van Oort Date: Fri, 10 Jan 2025 12:50:49 +0100 Subject: [PATCH 66/66] Delete the FarmmapsCleanUpApplication --- FarmmapsApiSamples.sln | 10 -- FarmmapsCleanUp/CleanUpApplication.cs | 143 ------------------------- FarmmapsCleanUp/CleanUpService.cs | 28 ----- FarmmapsCleanUp/FarmmapsCleanUp.csproj | 23 ---- FarmmapsCleanUp/Program.cs | 21 ---- 5 files changed, 225 deletions(-) delete mode 100644 FarmmapsCleanUp/CleanUpApplication.cs delete mode 100644 FarmmapsCleanUp/CleanUpService.cs delete mode 100644 FarmmapsCleanUp/FarmmapsCleanUp.csproj delete mode 100644 FarmmapsCleanUp/Program.cs diff --git a/FarmmapsApiSamples.sln b/FarmmapsApiSamples.sln index 8f35ae1..1183f84 100644 --- a/FarmmapsApiSamples.sln +++ b/FarmmapsApiSamples.sln @@ -29,8 +29,6 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FarmmapsBulkSatDownload", " EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Secrets", "Secrets\Secrets.csproj", "{C4EE5ECA-253A-4B71-9F67-D231AC4517D6}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FarmmapsCleanUp", "FarmmapsCleanUp\FarmmapsCleanUp.csproj", "{5E4387F9-5953-4A9B-BCA5-DF3964EED3CB}" -EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FarmmapsKPI", "FarmmapsKPI\FarmmapsKPI.csproj", "{14575235-9867-4CE5-A22F-3F9FE002FF42}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FarmmapsDownloadCL", "FarmmapsDownloadCL\FarmmapsDownloadCL.csproj", "{63E69101-D804-4DBC-B2E4-A33771CD5C5F}" @@ -123,14 +121,6 @@ Global {C4EE5ECA-253A-4B71-9F67-D231AC4517D6}.Release|Any CPU.Build.0 = Release|Any CPU {C4EE5ECA-253A-4B71-9F67-D231AC4517D6}.Release|x64.ActiveCfg = Release|x64 {C4EE5ECA-253A-4B71-9F67-D231AC4517D6}.Release|x64.Build.0 = Release|x64 - {5E4387F9-5953-4A9B-BCA5-DF3964EED3CB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {5E4387F9-5953-4A9B-BCA5-DF3964EED3CB}.Debug|Any CPU.Build.0 = Debug|Any CPU - {5E4387F9-5953-4A9B-BCA5-DF3964EED3CB}.Debug|x64.ActiveCfg = Debug|Any CPU - {5E4387F9-5953-4A9B-BCA5-DF3964EED3CB}.Debug|x64.Build.0 = Debug|Any CPU - {5E4387F9-5953-4A9B-BCA5-DF3964EED3CB}.Release|Any CPU.ActiveCfg = Release|Any CPU - {5E4387F9-5953-4A9B-BCA5-DF3964EED3CB}.Release|Any CPU.Build.0 = Release|Any CPU - {5E4387F9-5953-4A9B-BCA5-DF3964EED3CB}.Release|x64.ActiveCfg = Release|Any CPU - {5E4387F9-5953-4A9B-BCA5-DF3964EED3CB}.Release|x64.Build.0 = Release|Any CPU {14575235-9867-4CE5-A22F-3F9FE002FF42}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {14575235-9867-4CE5-A22F-3F9FE002FF42}.Debug|Any CPU.Build.0 = Debug|Any CPU {14575235-9867-4CE5-A22F-3F9FE002FF42}.Debug|x64.ActiveCfg = Debug|x64 diff --git a/FarmmapsCleanUp/CleanUpApplication.cs b/FarmmapsCleanUp/CleanUpApplication.cs deleted file mode 100644 index ab95e24..0000000 --- a/FarmmapsCleanUp/CleanUpApplication.cs +++ /dev/null @@ -1,143 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.IO.Compression; -using System.Linq; -using System.Threading.Tasks; -using FarmmapsApi; -using FarmmapsApi.Models; -using FarmmapsApi.Services; -using FarmmapsBulkSatDownload.Models; -using Microsoft.Extensions.Logging; -using Newtonsoft.Json; -using Npgsql; -using Newtonsoft.Json.Linq; -using static FarmmapsApiSamples.Constants; - -namespace FarmmapsCleanup -{ - public class CleanupApplication : IApplication - { - private readonly ILogger _logger; - private readonly FarmmapsApiService _farmmapsApiService; - private readonly CleanupService _cleanupService; - private readonly GeneralService _generalService; - - public CleanupApplication(ILogger logger, FarmmapsApiService farmmapsApiService, - GeneralService generalService, CleanupService cleanupService) - { - _logger = logger; - _farmmapsApiService = farmmapsApiService; - _generalService = generalService; - _cleanupService = cleanupService; - } - - public async Task RunAsync() - { - // !! this call is needed the first time an api is called with a fresh clientid and secret !! - await _farmmapsApiService.GetCurrentUserCodeAsync(); - var roots = await _farmmapsApiService.GetCurrentUserRootsAsync(); - - //Establish a database connection. Expecting you are using same database server as in project FarmmapsBulkSatDownload - DB dbparcels = JsonConvert.DeserializeObject(File.ReadAllText("DBsettings.secrets.json")); - string schemaname = "bigdata"; - string parceltablename = "parcel_bollenrevolutie_tulips2020"; //"parcelsijbrandij" "parcel"; "parcel_flowerbulbs"; "parcel_disac"; ""parcel_bollenrevolutie_tulips2020"" - - //Query to get lists of items to delete from FarmmapsDatabase and from parceltablename - string cropfielditemcode; - string satellitetaskcode; - List cropfieldItemCodes = new List(); - List satellitetaskCodes = new List(); - string connectionString = dbparcels.GetConnectionString(); - string readSql = string.Format( -@" -SELECT pt.cropfielditemcode, pt.satellitetaskcode -FROM {0}.{1} pt -WHERE - pt.arbid IN(1,2) -ORDER BY pt.arbid -;", schemaname, parceltablename); - - string updateCropfieldItemCodesSql = string.Format( -@" -UPDATE {0}.{1} - SET cropfielditemcode=NULL - WHERE arbid IN(1,2) -;", schemaname, parceltablename); //Same WHERE AS above - string updateSatellitetaskCodesSql = string.Format( -@" -UPDATE {0}.{1} - SET satellitetaskcode=NULL - WHERE arbid IN(1,2) -;", schemaname, parceltablename); //Same WHERE AS above - - using (NpgsqlConnection connection = new NpgsqlConnection(connectionString)) - { - connection.Open(); - - // Read data (run query) = build a list of fields for which to download images - NpgsqlCommand command = connection.CreateCommand(); - command.CommandText = readSql; - NpgsqlDataReader dr = command.ExecuteReader(); - while (dr.Read()) - { - cropfielditemcode = dr.GetString(0); - satellitetaskcode = dr.GetString(1); - if(!string.IsNullOrEmpty(cropfielditemcode)) - cropfieldItemCodes.Add(cropfielditemcode); - if (!string.IsNullOrEmpty(satellitetaskcode)) - satellitetaskCodes.Add(satellitetaskcode); - } - connection.Close(); - } - _logger.LogWarning($"// FarmmapsCleanUp: WARNING: you are about to delete {cropfieldItemCodes.Count} cropfieldItemCodes and {satellitetaskCodes.Count} satellitetaskCodes from the FarmMaps database and your own table {schemaname}.{parceltablename}"); - _logger.LogInformation($"// Nice of you to clean up after the work is done. You would typically do this for cropfieldItemCodes used only once."); - _logger.LogInformation($"// You do NOT want to do this if you think you may later on still want to use these items."); - _logger.LogInformation($"// Please carefully check the SQL queries 'readSql' 'updateSql' in CleanUpApplication.cs before proceeding."); - - _logger.LogInformation($"// FarmmapsCleanUp: delete selected cropfieldItemCodes from FarmMaps database and table {schemaname}.{parceltablename}? 0 = no, 1 = yes"); - int i; - i = Int32.Parse(Console.ReadLine()); - if (i == 1) - { - await _farmmapsApiService.DeleteItemsAsync(cropfieldItemCodes); - //TODO _farmmapsApiService.DeleteItemsAsync throws an error: {StatusCode: 415, ReasonPhrase: 'Unsupported Media Type', Version: 1.1, Content: System.Net.Http.HttpConnectionResponseContent, Headers: {...}} - //what is wrong with cropfieldItemCodes? - //and shouldn't we be telling _farmmapsApiService.DeleteItemsAsync what item type to delete? - using (NpgsqlConnection connection = new NpgsqlConnection(connectionString)) - { - connection.Open(); - NpgsqlCommand updateCmd = connection.CreateCommand(); - updateCmd.CommandText = updateCropfieldItemCodesSql; - int r = updateCmd.ExecuteNonQuery(); - if (r == -1) - throw new Exception("// FarmmapsCleanUp: Update cropfielditemcode Failed"); - connection.Close(); - } - } - _logger.LogInformation($"// FarmmapsCleanUp: delete selected satellitetaskCodes from FarmMaps database and table {schemaname}.{parceltablename}? 0 = no, 1 = yes"); - i = Int32.Parse(Console.ReadLine()); - if (i == 1) - { - await _farmmapsApiService.DeleteItemsAsync(satellitetaskCodes); - //TODO _farmmapsApiService.DeleteItemsAsync throws an error: {StatusCode: 415, ReasonPhrase: 'Unsupported Media Type', Version: 1.1, Content: System.Net.Http.HttpConnectionResponseContent, Headers: {...}} - //what is wrong with satellitetaskCodes? - //and shouldn't we be telling _farmmapsApiService.DeleteItemsAsync what item type to delete? - using (NpgsqlConnection connection = new NpgsqlConnection(connectionString)) - { - connection.Open(); - NpgsqlCommand updateCmd = connection.CreateCommand(); - updateCmd.CommandText = updateSatellitetaskCodesSql; - int r = updateCmd.ExecuteNonQuery(); - if (r == -1) - throw new Exception("// FarmmapsCleanUp: Update cropfielditemcode Failed"); - connection.Close(); - } - } - _logger.LogInformation($"// FarmmapsCleanUp: done! Hit any key to exit ..."); - Console.ReadKey(); - - - } - } -} diff --git a/FarmmapsCleanUp/CleanUpService.cs b/FarmmapsCleanUp/CleanUpService.cs deleted file mode 100644 index b569e19..0000000 --- a/FarmmapsCleanUp/CleanUpService.cs +++ /dev/null @@ -1,28 +0,0 @@ -using System; -using System.Globalization; -using System.Threading.Tasks; -using FarmmapsApi.Models; -using FarmmapsApi.Services; -using Microsoft.Extensions.Logging; -using static FarmmapsApi.Extensions; -using static FarmmapsApiSamples.Constants; - -namespace FarmmapsCleanup -{ - public class CleanupService - { - private readonly ILogger _logger; - private readonly FarmmapsApiService _farmmapsApiService; - private readonly GeneralService _generalService; - - public CleanupService(ILogger logger, FarmmapsApiService farmmapsApiService, - GeneralService generalService) - { - _logger = logger; - _farmmapsApiService = farmmapsApiService; - _generalService = generalService; - } - - - } -} \ No newline at end of file diff --git a/FarmmapsCleanUp/FarmmapsCleanUp.csproj b/FarmmapsCleanUp/FarmmapsCleanUp.csproj deleted file mode 100644 index 74cf8ea..0000000 --- a/FarmmapsCleanUp/FarmmapsCleanUp.csproj +++ /dev/null @@ -1,23 +0,0 @@ - - - - Exe - netcoreapp3.1 - - - - - - - - - Always - - - - - - - - - diff --git a/FarmmapsCleanUp/Program.cs b/FarmmapsCleanUp/Program.cs deleted file mode 100644 index 182d69e..0000000 --- a/FarmmapsCleanUp/Program.cs +++ /dev/null @@ -1,21 +0,0 @@ -using System.Threading.Tasks; -using FarmmapsApi; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging; - -namespace FarmmapsCleanup -{ - class Program : FarmmapsProgram - { - private static async Task Main(string[] args) - { - await new Program().Start(args); - } - - protected override void Configure(IServiceCollection serviceCollection) - { - serviceCollection.AddLogging() - .AddTransient(); - } - } -} \ No newline at end of file