From e4087adc77812898217d3ef2a4ef4b5d7f895f93 Mon Sep 17 00:00:00 2001 From: Pepijn van Oort Date: Tue, 28 Nov 2023 15:16:44 +0100 Subject: [PATCH] add miliebelastingspunten (mbp) and checks on whether KPI task is finished or not yet (if not yet then wait and call again) --- FarmmapsKPI/KPIApplication.cs | 93 +++++++++++++++++++++++++++------- FarmmapsKPI/KPIdefinitions.csv | 3 ++ 2 files changed, 78 insertions(+), 18 deletions(-) diff --git a/FarmmapsKPI/KPIApplication.cs b/FarmmapsKPI/KPIApplication.cs index 61960ed..cbd4991 100644 --- a/FarmmapsKPI/KPIApplication.cs +++ b/FarmmapsKPI/KPIApplication.cs @@ -42,12 +42,12 @@ namespace FarmmapsKPI KPIInput input; string fnKPIinput; - Console.WriteLine("Type name of input json file. Example: KPIinput.json (in same directory as FarmmapsKPI.exe) or for example like this: C:/temp/KPIinputChemieTmp.json"); - fnKPIinput = Console.ReadLine(); - if (string.IsNullOrEmpty(fnKPIinput)) - { + //Console.WriteLine("Type name of input json file. Example: KPIinput.json (in same directory as FarmmapsKPI.exe) or for example like this: C:/temp/KPIinputChemieTmp.json"); + //fnKPIinput = Console.ReadLine(); + //if (string.IsNullOrEmpty(fnKPIinput)) + //{ fnKPIinput = "KPIinput.json"; - } + //} var fieldsInputJson = File.ReadAllText(fnKPIinput); @@ -71,11 +71,13 @@ namespace FarmmapsKPI StreamWriter sw; string KPIItemCsv = Path.GetFileNameWithoutExtension(fnKPIinput) + "_Items.csv"; string KPIItemPathCsv = Path.Combine(downloadFolder, KPIItemCsv); - List headerList = new List { "parentName", "area_ha", "cropTypeCode", "cropTypeName", "KPIid", "KPIvariable", "KPIvalue", "KPIunit", "KPItargetvalue", "KPIthresholdValue", + List headerList = new List { "parentName", "cropfieldcode","area_ha", "cropTypeCode", "cropTypeName", "KPIid", "KPIvariable", "KPIvalue", "KPIunit", "KPItargetvalue", "KPIthresholdValue", "mbp_productCode","mbp_productName","mbp_quantity","mbp_unitCode","mbp_date","mbp_KPIvariable","mbp_KPIvalue"}; //Create a new csv file. Means if existing then overwritten !!! sw = new StreamWriter(KPIItemPathCsv); - sw.WriteLine($"FarmmapsKPI backend calculations on input file '{fnKPIinput}' downloaded on {DateTime.Now} with the FarmmapsKPI application in the FarmmapsApSamples.sln"); + Configuration cf = await _farmmapsApiService.GetConfiguration(); + string endPoint = cf.Endpoint; + sw.WriteLine($"Using FarmmapsKPI application in FarmmapsApSamples.sln. Input file: '{fnKPIinput}'. Download DateTime: '{DateTime.Now}'. Calculations on: '{endPoint}'"); sw.WriteLine(); sw.WriteLine(string.Join(",", headerList)); @@ -120,6 +122,9 @@ namespace FarmmapsKPI //If KPI E1 is calculated, write these sub kpi's to output string[] mbp_KPIvariables = new string[] { "aquaticLife", "groundWater", "soilLife" }; string mbp_KPIvalue; + int targetKPIitemsCount; //if we know we should be getting 8 KPI items (A1,B1,B2,C1,D1,E1,F1,F2) + int maxtries = 5; // but don't keep on trying forever; there is a maximum number of tries + int trycnt; string downloadFolder = input.DownloadFolder; if (string.IsNullOrEmpty(downloadFolder)) { @@ -205,8 +210,15 @@ namespace FarmmapsKPI // We need that because for operations, you need to provide the area on which the operation was applied // And if we put that to the crop area, then we neatly get everything on a per ha basis _logger.LogInformation($"Getting polygon area (ha))"); - List KPIItems = await _generalService.GetKpiItemsForCropField(cropfieldItem); - _logger.LogInformation($"Found {KPIItems.Count} KPI items"); + List KPIItems = await _generalService.GetKpiItemsForCropField(cropfieldItem, 3); + trycnt = 1; + targetKPIitemsCount = 3; // here for the area we need at least 3, but not more than that + while (KPIItems.Count < targetKPIitemsCount & trycnt < maxtries) + { + KPIItems = await _generalService.GetKpiItemsForCropField(cropfieldItem,3); + _logger.LogInformation($"Found {KPIItems.Count} KPI items"); + trycnt++; + } kpio = JsonConvert.DeserializeObject(KPIItems[0].Data.ToString()); string area_ha = kpio.data.area; // turn the area into a JObject for later merging with operation data; @@ -335,17 +347,28 @@ namespace FarmmapsKPI crprecChildren = await _farmmapsApiService.GetItemChildrenAsync(crprecItem.Code); //Now get the KPIs for this cropfield, mounted with operations & cropyield - // Get KPI data for saving it in a file, here the generalsedrvice is called to get the KPI data + //Note sometimes the KPIItems.Count is already for some crazy reason greater than or equal to targetKPIitemsCount + //But that would have strange results, since that was from above before adding the crop recordings. We want to do at least one new call -> 'while (trycnt == 0 || ' _logger.LogInformation($"GetKpiItemsForCropField('{cropfieldItem.Code}')"); //Pesticide KPI's D1 and E1 are retreived from API's of CTBG and CLM and may take a bit longer to retrieve - int targetKPIitemsCount = 8; //if we know we should be getting 8 - int maxtries = 5; // but don't keep on trying forever; there is a maximum number of tries - int trycnt = 0; - - _logger.LogInformation($"Firing calls GetKpiItemsForCropField() until we have {targetKPIitemsCount}, but don't keep firing forever, stop after {maxtries} calls"); - while (KPIItems.Count < targetKPIitemsCount & trycnt < maxtries) { - KPIItems = await _generalService.GetKpiItemsForCropField(cropfieldItem, 5); + targetKPIitemsCount = 8; //if we know we should be getting 8 KPI items (A1,B1,B2,C1,D1,E1,F1,F2) + trycnt = 0; + bool boolAquaticLife = false; + _logger.LogInformation($"Firing calls GetKpiItemsForCropField() until we have {targetKPIitemsCount} KPIitems, but don't keep firing forever, stop after {maxtries} calls"); + _logger.LogInformation($"Before we start:"); + _logger.LogInformation($"* KPIItems.Count = {KPIItems.Count}"); + _logger.LogInformation($"* trycnt = {trycnt}"); + _logger.LogInformation($"* boolAquaticLife = {boolAquaticLife}"); + //additional criterion for while loop: check if it really contains the E1 mbp elements. + //while (trycnt == 0 || ((KPIItems.Count < targetKPIitemsCount || boolAquaticLife == false) & trycnt < maxtries)) + //normal while loop + targetKPIitemsCount = 7; //if we know we should be getting 7 KPI items (A1,B1,B2,C1,D1,F1,F2) + while (trycnt == 0 || (KPIItems.Count < targetKPIitemsCount & trycnt < maxtries)) + { + _logger.LogInformation($"Call nr {trycnt + 1}"); + KPIItems = await _generalService.GetKpiItemsForCropField(cropfieldItem, 3); //number after comma is how many seconds per try _logger.LogInformation($"Found {KPIItems.Count} KPI items"); + //boolAquaticLife = GetBoolAquaticLife(KPIItems); trycnt ++; } if (KPIItems.Count < targetKPIitemsCount) { @@ -362,6 +385,7 @@ namespace FarmmapsKPI count++; } _logger.LogInformation($"Downloaded file {KPIItemPathJson}"); + _logger.LogInformation($""); //Write to the csv file that collects all KPI's for all the crop fields List dataList; @@ -378,6 +402,7 @@ namespace FarmmapsKPI dataList = new List { }; //Fill the datalist with this kpi dataList.Add(kpio.parentName); + dataList.Add(cropfieldItem.Code); dataList.Add(kpio.data.area); dataList.Add(kpio.data.cropTypeCode); dataList.Add(kpio.data.cropTypeName); @@ -402,6 +427,7 @@ namespace FarmmapsKPI dataList = new List { }; //Fill the datalist with this kpi dataList.Add(kpio.parentName); + dataList.Add(cropfieldItem.Code); dataList.Add(kpio.data.area); dataList.Add(kpio.data.cropTypeCode); dataList.Add(kpio.data.cropTypeName); @@ -444,8 +470,8 @@ namespace FarmmapsKPI } //Also add totalNapplied to the csv dataList = new List { }; - //Seems sometimes duplicate KPI items are returned. So check that here and only write if this kpio is different from previous dataList.Add(kpioPrevious.parentName); + dataList.Add(cropfieldItem.Code); try { dataList.Add(kpioPrevious.data.area); } catch { dataList.Add(""); }; try { dataList.Add(kpioPrevious.data.cropTypeCode); } catch { dataList.Add(""); }; try { dataList.Add(kpioPrevious.data.cropTypeName); } catch { dataList.Add(""); }; @@ -515,5 +541,36 @@ namespace FarmmapsKPI { return String.Format("{0:00}:{1:00}:{2:00}", ts.Hours, ts.Minutes, ts.Seconds); } + + private bool GetBoolAquaticLife(List KPIItems) + { + bool found = false; + foreach (Item item in KPIItems) + { + KPIOutput kpio = JsonConvert.DeserializeObject(item.Data.ToString()); + if (kpio.id == "E1") + { + try + { + foreach (KPIenvironmentMeasureData e in kpio.data.environmentMeasureData) + { + if (e.aquaticLife != null) + { + found = true; + _logger.LogInformation($"Found e.aquaticLife = {e.aquaticLife}"); + _logger.LogInformation(""); + } + } + } + catch (Exception ex) + { + _logger.LogWarning($"NOT Found e.aquaticLife"); + _logger.LogError(ex.Message); + } + } + } + + return found; + } } } diff --git a/FarmmapsKPI/KPIdefinitions.csv b/FarmmapsKPI/KPIdefinitions.csv index 92aa3d5..556520a 100644 --- a/FarmmapsKPI/KPIdefinitions.csv +++ b/FarmmapsKPI/KPIdefinitions.csv @@ -4,6 +4,9 @@ B1,Nitrogen surplus,Nitrogen surplus = N artifical fertilizer + N manure+ N atmo B2,Phosphate surplus,Phosphate surplus = P fertilizer - P removal through harvested product. Calculated KPI internal model & parameters and from user input: yield and fertilizer applications C1,Effective OM supply,"Organic matter surplus = Organic matter from manure + Crop residues to soil - organic matter removal through harvested product. Calculated KPI internal model & parameters and from user input: yield and manure applications. ""Effective"" means netto, after correction for carbon respiration by soil microbial activity" D1,pesticides,"kg/ha of active ingredient. Sum over kg a.i. of the different crop protection agents applied, with concentration (kg a.i. / kg or litre) of crop protection agents retrieved from CTBG" +E1,mbp,"milieubelastingspunten. 3 Quantitative sub items & 2 Qualitative (A, B, C)" +F1,greenness,? +F2,rotationindex,farmlevel index of diversity of crops ,KPItargetvalue,target value as in benchmark value for same crop in same region. E.g. cropyield target is from Central Bureau of Statistics (CBS) for same crop in region in which the cropfield (geometry) is located ,KPIthresholdValue,threshold from ??? Ask farmmaps. Surplus nitrogen / phosphate / pesticides must not be above threshold. Surplus organic matter supply must be above threshold ,totalNapplied,"Not a KPI output, simply calculated from the input operations with their respective ""n"" in their ""data"""