From e05bb5a8f7502afb2b8c3050ebcb99612eea3615 Mon Sep 17 00:00:00 2001 From: Pepijn van Oort Date: Wed, 15 May 2024 15:45:53 +0200 Subject: [PATCH] added code for checking if multiple runs give same KPI output (should be but is not, so where is it going wrong?) --- FarmmapsKPI/KPIApplication.cs | 83 ++++++++++++++++++++++++++--------- 1 file changed, 62 insertions(+), 21 deletions(-) diff --git a/FarmmapsKPI/KPIApplication.cs b/FarmmapsKPI/KPIApplication.cs index 149e0fd..17512ed 100644 --- a/FarmmapsKPI/KPIApplication.cs +++ b/FarmmapsKPI/KPIApplication.cs @@ -5,6 +5,7 @@ using System.IO; using System.IO.Compression; using System.Linq; using System.Threading.Tasks; +using System.Transactions; using FarmmapsApi; using FarmmapsApi.Models; using FarmmapsApi.Services; @@ -46,7 +47,7 @@ namespace FarmmapsKPI //fnKPIinput = Console.ReadLine(); //if (string.IsNullOrEmpty(fnKPIinput)) //{ - fnKPIinput = "KPIinput.json"; + fnKPIinput = "KPIinput.json"; //} var fieldsInputJson = File.ReadAllText(fnKPIinput); @@ -71,7 +72,7 @@ namespace FarmmapsKPI StreamWriter sw; string KPIItemCsv = Path.GetFileNameWithoutExtension(fnKPIinput) + "_Items.csv"; string KPIItemPathCsv = Path.Combine(downloadFolder, KPIItemCsv); - List headerList = new List { "parentName", "cropfieldcode", "area_ha", "cropTypeCode", "cropTypeName", "KPIid", "KPIvariable", "KPIvalue", "KPIunit", "KPItargetvalue", "KPIthresholdValue", + List headerList = new List { "run","parentName", "cropfieldcode", "area_ha", "cropTypeCode", "cropTypeName", "KPIid", "KPIvariable", "KPIvalue", "KPIunit", "KPItargetvalue", "KPIthresholdValue", "mbp_productCode","mbp_productName","mbp_quantity","mbp_unitCode","mbp_date","mbp_KPIvariable","mbp_KPIvalue"}; //Create a new csv file. Means if existing then overwritten !!! sw = new StreamWriter(KPIItemPathCsv); @@ -87,25 +88,29 @@ namespace FarmmapsKPI TimeSpan tsRemaining; TimeSpan tsTotalEstimated; - for (int i = 0; i < fieldsInputs.Count; i++) - //for (int i = 10; i < 12; i++) - { - watch.Restart(); - input = fieldsInputs[i]; - _logger.LogInformation(string.Format($"// FarmmapsKPI: Downloading KPI's for field {i + 1} out of {fieldsInputs.Count} to single csv file {KPIItemPathCsv}")); - try + //Per default just 1 run per field. For debugging check if when we run multiple times do we get (should be) always the same output? + int nrun = 1; + for (int run = 1; run <= nrun; run++) { + //for (int i = 0; i < fieldsInputs.Count; i++) + for (int i = 3; i < 4; i++) // for testing { - await Process(roots, input, sw); - } - catch (Exception ex) - { - _logger.LogError(ex.Message); - } - watch.Stop(); - tsSofar = tsSofar + watch.Elapsed; - tsTotalEstimated = tsSofar / (i + 1) * fieldsInputs.Count; - tsRemaining = tsTotalEstimated - tsSofar; - _logger.LogInformation(string.Format($"// Time (hh:mm:ss): this field: {strTime(watch.Elapsed)}. Sofar: {strTime(tsSofar)}. Estimated total: {strTime(tsTotalEstimated)}. Remaining: {strTime(tsRemaining)}")); + watch.Restart(); + input = fieldsInputs[i]; + _logger.LogInformation(string.Format($"// FarmmapsKPI: Run {run}: Downloading KPI's for field {i + 1} out of {fieldsInputs.Count} to single csv file {KPIItemPathCsv}")); + try + { + await Process(roots, input, sw, run); + } + catch (Exception ex) + { + _logger.LogError(ex.Message); + } + watch.Stop(); + tsSofar = tsSofar + watch.Elapsed; + tsTotalEstimated = tsSofar / (i + 1) * fieldsInputs.Count; + tsRemaining = tsTotalEstimated - tsSofar; + _logger.LogInformation(string.Format($"// Time (hh:mm:ss): this field: {strTime(watch.Elapsed)}. Sofar: {strTime(tsSofar)}. Estimated total: {strTime(tsTotalEstimated)}. Remaining: {strTime(tsRemaining)}")); + } } //Close the csv file, write message to screen sw.Close(); @@ -113,7 +118,7 @@ namespace FarmmapsKPI _logger.LogInformation($"Done! Written all KPI for all fields in '{fnKPIinput}' to output file '{KPIItemPathCsv}'"); } - private async Task Process(List roots, KPIInput input, StreamWriter sw) + private async Task Process(List roots, KPIInput input, StreamWriter sw, int run) { List cropfieldChildren; List crprecChildren; @@ -125,6 +130,8 @@ namespace FarmmapsKPI int targetKPIitemsCount; //if we know we should be getting 8 KPI items (A1,B1,B2,C1,D1,E1,F1,F2) int maxtries = 5; // but don't keep on trying forever; there is a maximum number of tries int trycnt; + double totalNferiliserInput = 0; + double totalNferiliserCropfield = 0; string downloadFolder = input.DownloadFolder; if (string.IsNullOrEmpty(downloadFolder)) { @@ -317,6 +324,8 @@ namespace FarmmapsKPI crpOperationItem = await _generalService.CreateOperationItemAsync(crprecItem.Code, cropYear, fieldGeom, dataOperation); crpOperationItems.Add(crpOperationItem); crpOperationItemCodes.Add(crpOperationItem.Code); + //Keep track of totalNferiliserInput + totalNferiliserInput = totalNferiliserInput + (double)data.n; } _settings.OperationItemCodes = crpOperationItemCodes.ToArray(); SaveSettings(settingsfile); @@ -352,6 +361,22 @@ namespace FarmmapsKPI cropfieldChildren = await _farmmapsApiService.GetItemChildrenAsync(cropfieldItem.Code); crprecChildren = await _farmmapsApiService.GetItemChildrenAsync(crprecItem.Code); + //Check total operations + int nOperationsInput = input.DataOperations.Length; + int nOperationsCropfield = crprecChildren.Count; + if (nOperationsInput != nOperationsCropfield) + throw new Exception(String.Format($"run {run}: nOperationsInput != nOperationsCropfield?!")); + + //Check totalNfertiliser input and Cropfield + for (int o = 0; o < crprecChildren.Count; o++) + { + dynamic data = JObject.Parse(crprecChildren[o].Data.ToString()); + totalNferiliserCropfield = totalNferiliserCropfield + (double)data.n; + } + if (totalNferiliserInput != totalNferiliserCropfield) + throw new Exception(String.Format($"run {run}: totalNferiliserInput != totalNferiliserCropfield?!")); + + //Now get the KPIs for this cropfield, mounted with operations & cropyield //Note sometimes the KPIItems.Count is already for some crazy reason greater than or equal to targetKPIitemsCount //But that would have strange results, since that was from above before adding the crop recordings. We want to do at least one new call -> 'while (trycnt == 0 || ' @@ -409,6 +434,7 @@ namespace FarmmapsKPI //Fill the datalist with this kpi dataList = new List { + run.ToString(), kpio.parentName, cropfieldItem.Code, kpio.data.area, @@ -430,6 +456,7 @@ namespace FarmmapsKPI string elementValue = (string)kpio.data.GetType().GetProperty(elementName).GetValue(kpio.data, null); dataList = new List { + run.ToString(), kpio.parentName, cropfieldItem.Code, kpio.data.area, @@ -452,6 +479,7 @@ namespace FarmmapsKPI //Fill the datalist with this kpi dataList = new List { + run.ToString(), kpio.parentName, cropfieldItem.Code, kpio.data.area, @@ -473,6 +501,7 @@ namespace FarmmapsKPI string elementValue = (string)kPIelementsOfBalance.GetType().GetProperty(elementName).GetValue(kPIelementsOfBalance, null); dataList = new List { + run.ToString(), kpio.parentName, cropfieldItem.Code, kpio.data.area, @@ -487,6 +516,12 @@ namespace FarmmapsKPI }; //Write the datalist to a line to the streamwrieter sw for the output csv file sw.WriteLine(string.Join(",", dataList)); + + //Check if totalNferiliserInput is equal to what comes out of the KPI calculation + //PO20240515: Hier gaat het mis!!! + if (elementName == "nFertilizerNKgHa") + if (totalNferiliserInput != Convert.ToDouble(elementValue)) + throw new Exception(String.Format($"run {run}: totalNferiliserInput != nFertilizerNKgHa?!")); } } else if (kpio.id == "B2") @@ -495,6 +530,7 @@ namespace FarmmapsKPI dataList = new List { //Fill the datalist with this kpi + run.ToString(), kpio.parentName, cropfieldItem.Code, kpio.data.area, @@ -516,6 +552,7 @@ namespace FarmmapsKPI string elementValue = (string)kPIelementsOfBalance.GetType().GetProperty(elementName).GetValue(kPIelementsOfBalance, null); dataList = new List { + run.ToString(), kpio.parentName, cropfieldItem.Code, kpio.data.area, @@ -538,6 +575,7 @@ namespace FarmmapsKPI //Make a new dataList = new line to be written dataList = new List { + run.ToString(), kpio.parentName, cropfieldItem.Code, kpio.data.area, @@ -559,6 +597,7 @@ namespace FarmmapsKPI string elementValue = (string)kPIelementsOfBalance.GetType().GetProperty(elementName).GetValue(kPIelementsOfBalance, null); dataList = new List { + run.ToString(), kpio.parentName, cropfieldItem.Code, kpio.data.area, @@ -587,6 +626,7 @@ namespace FarmmapsKPI //Make a new dataList = new line to be written dataList = new List { + run.ToString(), kpio.parentName, cropfieldItem.Code, kpio.data.area, @@ -616,6 +656,7 @@ namespace FarmmapsKPI //Any other KPI, example A1 or D1, with just 1 record to write dataList = new List { + run.ToString(), kpio.parentName, cropfieldItem.Code, kpio.data.area,