using System; using System.Collections.Generic; using System.Diagnostics; using System.IO; using System.IO.Compression; using System.Linq; using System.Threading.Tasks; using FarmmapsApi; using FarmmapsApi.Models; using FarmmapsApi.Services; using FarmmapsKPI.Models; using Microsoft.Extensions.Logging; using Newtonsoft.Json; using Newtonsoft.Json.Linq; using static FarmmapsApiSamples.Constants; namespace FarmmapsKPI { public class KPIApplication : IApplication { //private const string DownloadFolder = "Downloads"; private const string SettingsFile = "settings.json"; private readonly ILogger _logger; private readonly FarmmapsApiService _farmmapsApiService; private readonly KPIService _kpiService; private readonly GeneralService _generalService; private Settings _settings; public KPIApplication(ILogger logger, FarmmapsApiService farmmapsApiService, GeneralService generalService, KPIService kpiService) { _logger = logger; _farmmapsApiService = farmmapsApiService; _generalService = generalService; _kpiService = kpiService; } public async Task RunAsync() { KPIInput input; string fnKPIinput; Console.WriteLine("Type name of input json file. Example: KPIinput.json (in same directory as FarmmapsKPI.exe) or for example like this: C:/temp/KPIinputChemieTmp.json"); fnKPIinput = Console.ReadLine(); if (string.IsNullOrEmpty(fnKPIinput)) { fnKPIinput = "KPIinput.json"; } var fieldsInputJson = File.ReadAllText(fnKPIinput); List fieldsInputs = JsonConvert.DeserializeObject>(fieldsInputJson); // !! this call is needed the first time an api is called with a fresh clientid and secret !! await _farmmapsApiService.GetCurrentUserCodeAsync(); var roots = await _farmmapsApiService.GetCurrentUserRootsAsync(); //Where to write the output string downloadFolder = fieldsInputs[0].DownloadFolder; if (string.IsNullOrEmpty(downloadFolder)) { downloadFolder = "Downloads"; } if (!Directory.Exists(downloadFolder)) Directory.CreateDirectory(downloadFolder); //Write the same info to a single csv file. Note this means existing file will be overwritten! StreamWriter sw; string KPIItemCsv = Path.GetFileNameWithoutExtension(fnKPIinput) + "_Items.csv"; string KPIItemPathCsv = Path.Combine(downloadFolder, KPIItemCsv); List headerList = new List { "parentName", "area_ha", "cropTypeCode", "cropTypeName", "KPIid", "KPIvariable", "KPIvalue", "KPIunit", "KPItargetvalue", "KPIthresholdValue", "mbp_productCode","mbp_productName","mbp_quantity","mbp_unitCode","mbp_date","mbp_KPIvariable","mbp_KPIvalue"}; //Create a new csv file. Means if existing then overwritten !!! sw = new StreamWriter(KPIItemPathCsv); sw.WriteLine($"FarmmapsKPI backend calculations on input file '{fnKPIinput}' downloaded on {DateTime.Now} with the FarmmapsKPI application in the FarmmapsApSamples.sln"); sw.WriteLine(); sw.WriteLine(string.Join(",", headerList)); // For each input download all KPI's. Keep track to time, important when doing bulk calculations var watch = System.Diagnostics.Stopwatch.StartNew(); TimeSpan tsSofar = new TimeSpan(); TimeSpan tsRemaining; TimeSpan tsTotalEstimated; for (int i = 0; i < fieldsInputs.Count; i++) //for (int i = 7; i < 8; i++) { watch.Restart(); input = fieldsInputs[i]; _logger.LogInformation(string.Format($"// FarmmapsKPI: Downloading KPI's for field {i + 1} out of {fieldsInputs.Count} to single csv file {KPIItemPathCsv}")); try { await Process(roots, input, sw); } catch (Exception ex) { _logger.LogError(ex.Message); } watch.Stop(); tsSofar = tsSofar + watch.Elapsed; tsTotalEstimated = tsSofar / (i + 1) * fieldsInputs.Count; tsRemaining = tsTotalEstimated - tsSofar; _logger.LogInformation(string.Format($"// Time (hh:mm:ss): this field: {strTime(watch.Elapsed)}. Sofar: {strTime(tsSofar)}. Estimated total: {strTime(tsTotalEstimated)}. Remaining: {strTime(tsRemaining)}")); } //Close the csv file, write message to screen sw.Close(); _logger.LogInformation(string.Format($"// FarmmapsKPI:")); _logger.LogInformation($"Done! Written all KPI for all fields in '{fnKPIinput}' to output file '{KPIItemPathCsv}'"); } private async Task Process(List roots, KPIInput input, StreamWriter sw) { List cropfieldChildren; List crprecChildren; KPIOutput kpio; KPIOutput kpioPrevious = new KPIOutput(); //creates a new empty //If KPI E1 is calculated, write these sub kpi's to output string[] mbp_KPIvariables = new string[] { "aquaticLife", "groundWater", "soilLife" }; string mbp_KPIvalue; string downloadFolder = input.DownloadFolder; if (string.IsNullOrEmpty(downloadFolder)) { downloadFolder = "Downloads"; } if (!Directory.Exists(downloadFolder)) Directory.CreateDirectory(downloadFolder); // !!specify if you are using an already created cropfield: bool useExistingCropfieldWithChildren = input.UseExistingCropfieldWithChildren; int cropYear = input.CropYear; string fieldName = input.fieldName; string fieldGeom = input.GeometryJson.ToString(Formatting.None); //Settings string settingsfile = $"Settings_{fieldName}.json"; LoadSettings(settingsfile); var uploadedRoot = roots.SingleOrDefault(r => r.Name == "USER_IN"); if (uploadedRoot == null) { _logger.LogError("Could not find a needed root item"); return; } var myDriveRoot = roots.SingleOrDefault(r => r.Name == "USER_FILES"); if (myDriveRoot == null) { _logger.LogError("Could not find a needed root item"); return; } // Use already created cropfield or create new one, added a Data input, with field specific data for the KPI calculation Item cropfieldItem; //1 useExistingCropfieldWithChildren = false -> create new //2 useExistingCropfieldWithChildren = true && input.CropfieldItemCode = "" or absent -> read from settings json //3 useExistingCropfieldWithChildren = true && input.CropfieldItemCode like "deb48a74c5b54299bb852f17288010e9" in KPIinput -> use this one if (useExistingCropfieldWithChildren == false) { _logger.LogInformation($"Creating cropfield with name '{fieldName}'"); cropfieldItem = await _generalService.CreateCropfieldItemAsync(myDriveRoot.Code, $"{fieldName}", cropYear, input.GeometryJson.ToString(Formatting.None), input.DataCropfield.ToString(Formatting.None), input.StartDate, input.EndDate); _settings.CropfieldItemCode = cropfieldItem.Code; SaveSettings(settingsfile); } else if (string.IsNullOrEmpty(input.CropfieldItemCode)) { _logger.LogInformation("reading CropfieldItemCode from settings file"); cropfieldItem = await _farmmapsApiService.GetItemAsync(_settings.CropfieldItemCode); } else { _logger.LogInformation("reading CropfieldItemCode from KPIinput json"); cropfieldItem = await _farmmapsApiService.GetItemAsync(input.CropfieldItemCode); SaveSettings(settingsfile); } // The cropfieldCharacteristicItem is used to enter crop yields Item cropfieldCharacteristicItem; //1 useExistingCropfieldWithChildren = false -> create new //2 useExistingCropfieldWithChildren = true && input.CropfieldCharacteristicItemCode = "" or absent -> read from settings json //3 useExistingCropfieldWithChildren = true && input.CropfieldCharacteristicItemCode like "deb48a74c5b54299bb852f17288010e9" in KPIinput -> use this one if (useExistingCropfieldWithChildren == false) { _logger.LogInformation("CreateCropfieldCharacteristicItemAsync ..."); cropfieldCharacteristicItem = await _generalService.CreateCropfieldCharacteristicItemAsync(cropfieldItem.Code, cropYear, fieldGeom, input.DataCropfieldCharacteristic.ToString(Formatting.None)); _settings.CropfieldCharacteristicItemCode = cropfieldCharacteristicItem.Code; SaveSettings(settingsfile); } else if (string.IsNullOrEmpty(input.CropfieldItemCode)) { _logger.LogInformation("reading OperationItemCode from settings file"); cropfieldCharacteristicItem = await _farmmapsApiService.GetItemAsync(_settings.CropfieldCharacteristicItemCode); } else { _logger.LogInformation("reading CropfieldCharacteristicItemCode from KPIinput json"); cropfieldCharacteristicItem = await _farmmapsApiService.GetItemAsync(input.CropfieldCharacteristicItemCode); SaveSettings(settingsfile); } // Now we can do a first KPI calculation and get the polygon area from the KPI output (where it is based on geometry) // We need that because for operations, you need to provide the area on which the operation was applied // And if we put that to the crop area, then we neatly get everything on a per ha basis _logger.LogInformation($"Getting polygon area (ha))"); List KPIItems = await _generalService.GetKpiItemsForCropField(cropfieldItem); _logger.LogInformation($"Found {KPIItems.Count} KPI items"); kpio = JsonConvert.DeserializeObject(KPIItems[0].Data.ToString()); string area_ha = kpio.data.area; // turn the area into a JObject for later merging with operation data; string strJarea = JsonConvert.SerializeObject(new { area = area_ha }); JObject Jarea = JObject.Parse(strJarea); if (useExistingCropfieldWithChildren == false) { //Retreiving BOFEK. A cropfield has 1 soil //Have a look at the cropfieldChildren before and after running this task, see one child (i.e. soil) has been added) //cropfieldChildren = await _farmmapsApiService.GetItemChildrenAsync(cropfieldItem.Code); _logger.LogInformation("Get BOFEK for field"); Item bofekItem = await _generalService.RunBofekTask(cropfieldItem); if (bofekItem == null) { _logger.LogError("Something went wrong while obtaining the BOFEK data"); return; } //cropfieldChildren = await _farmmapsApiService.GetItemChildrenAsync(cropfieldItem.Code); } else { _logger.LogInformation("For existing cropfield we assume it already has BOFEK soil data"); } // A cropfield has 1 crop recording and the crop recording has 0:many operations //So first at the crop recording Item crprecItem; //1 useExistingCropfieldWithChildren = false -> create new //2 useExistingCropfieldWithChildren = true && input.CropRecordingItemCode = "" or absent -> read from settings json //3 useExistingCropfieldWithChildren = true && input.CropRecordingItemCode like "deb48a74c5b54299bb852f17288010e9" in KPIinput -> use this one if (useExistingCropfieldWithChildren == false) { _logger.LogInformation("RunCropRecordingTask ..."); crprecItem = await _generalService.RunCropRecordingTask(cropfieldItem); _settings.CropRecordingItemCode = crprecItem.Code; SaveSettings(settingsfile); } else if (string.IsNullOrEmpty(input.CropfieldItemCode)) { _logger.LogInformation("reading CropRecordingItemCode from settings file"); crprecItem = await _farmmapsApiService.GetItemAsync(_settings.CropRecordingItemCode); } else { _logger.LogInformation("reading CropRecordingItemCode from KPIinput json"); crprecItem = await _farmmapsApiService.GetItemAsync(input.CropRecordingItemCode); SaveSettings(settingsfile); } // Now add the operations List crpOperationItems = new List { }; List crpOperationItemCodes = new List { }; Item crpOperationItem; string dataOperation; string codeOperation; //1 useExistingCropfieldWithChildren = false -> create new //2 useExistingCropfieldWithChildren = true && input.OperationItemCode = "" or absent -> read from settings json //3 useExistingCropfieldWithChildren = true && input.OperationItemCode like "deb48a74c5b54299bb852f17288010e9" in KPIinput -> use this one if (useExistingCropfieldWithChildren == false) { for (int i = 0; i < input.DataOperations.Length; i++) { dataOperation = input.DataOperations[i].ToString(Formatting.None); dynamic data = JObject.Parse(dataOperation); _logger.LogInformation($"CreateOperationItemAsync ... for operation {i}: '{data.name}', on date '{data.from}'"); // Now check if the operation has a field called area string? opArea = data["area"]; if (string.IsNullOrEmpty(opArea)) { // if not having field area: add it to dataOperation ... input.DataOperations[i].Merge(Jarea); // ... and update the string dataOperation = input.DataOperations[i].ToString(Formatting.None); } else { // if yes having field area: compare with polygon area. If not the same, throw warning if (data.area != area_ha) { double differencePercent = 100.0*(Convert.ToDouble(area_ha) / Convert.ToDouble(data.area) - 1.0); _logger.LogWarning($"cropfield has area {area_ha}, but in your KPIinput.json or file like that, you have an operation with area {data.area}" + $" Difference is {area_ha} / {data.area} - 100% = {differencePercent}%." + $" Is that correct? Example if operation was applied in part of field, e.g. in case of variable rate (VRA) application." + $" Or did you accidentally fill in area in the KPIinput json? To use cropfield area, omit field 'area' from json" + $" then the KPI applicataion will fill in area calculated from geometry"); } } //Now after optionally adding the area, add the Operation to the crop recording crpOperationItem = await _generalService.CreateOperationItemAsync(crprecItem.Code, cropYear, fieldGeom, dataOperation); crpOperationItems.Add(crpOperationItem); crpOperationItemCodes.Add(crpOperationItem.Code); } _settings.OperationItemCodes = crpOperationItemCodes.ToArray(); SaveSettings(settingsfile); } else if (string.IsNullOrEmpty(input.CropfieldItemCode)) { _logger.LogInformation("reading OperationItemCode from settings file"); for (int i = 0; i < _settings.OperationItemCodes.Length; i++) { codeOperation = _settings.OperationItemCodes[i]; crpOperationItem = await _farmmapsApiService.GetItemAsync(codeOperation); crpOperationItems.Add(crpOperationItem); crpOperationItemCodes.Add(crpOperationItem.Code); } } else { _logger.LogInformation("reading OperationItemCodes from KPIinput json"); for (int i = 0; i < input.OperationItemCodes.Length; i++) { codeOperation = input.OperationItemCodes[i]; crpOperationItem = await _farmmapsApiService.GetItemAsync(codeOperation); crpOperationItems.Add(crpOperationItem); crpOperationItemCodes.Add(crpOperationItem.Code); } _settings.OperationItemCodes = crpOperationItemCodes.ToArray(); SaveSettings(settingsfile); } // Inspect the children and grandchildren. If all is well, cropfield will have: // one crprec, with 0-many operations as children. And the Data of an operation will have specification of how much fertilizer / cropprotection agent was applied // one edicrop.characteristic (with yield in the data) cropfieldChildren = await _farmmapsApiService.GetItemChildrenAsync(cropfieldItem.Code); crprecChildren = await _farmmapsApiService.GetItemChildrenAsync(crprecItem.Code); //Now get the KPIs for this cropfield, mounted with operations & cropyield // Get KPI data for saving it in a file, here the generalsedrvice is called to get the KPI data _logger.LogInformation($"GetKpiItemsForCropField('{cropfieldItem.Code}')"); //Pesticide KPI's D1 and E1 are retreived from API's of CTBG and CLM and may take a bit longer to retrieve int targetKPIitemsCount = 8; //if we know we should be getting 8 int maxtries = 5; // but don't keep on trying forever; there is a maximum number of tries int trycnt = 0; _logger.LogInformation($"Firing calls GetKpiItemsForCropField() until we have {targetKPIitemsCount}, but don't keep firing forever, stop after {maxtries} calls"); while (KPIItems.Count < targetKPIitemsCount & trycnt < maxtries) { KPIItems = await _generalService.GetKpiItemsForCropField(cropfieldItem, 5); _logger.LogInformation($"Found {KPIItems.Count} KPI items"); trycnt ++; } if (KPIItems.Count < targetKPIitemsCount) { _logger.LogWarning($"Found {KPIItems.Count} KPIItems while you were aiming for {targetKPIitemsCount} KPIItems"); } //Download KPI's into a json output file for this specific cropfield (with unique cropfieldItem.Code) string KPIItemPathJson = Path.Combine(downloadFolder, $"KPIItems_{cropfieldItem.Code}.json"); var count = 0; foreach (Item item in KPIItems) { //Console.WriteLine($"KPI item #{count}: {item.Name}"); File.AppendAllText(KPIItemPathJson, item.Data + Environment.NewLine); count++; } _logger.LogInformation($"Downloaded file {KPIItemPathJson}"); //Write to the csv file that collects all KPI's for all the crop fields List dataList; foreach (Item item in KPIItems) { kpio = JsonConvert.DeserializeObject(item.Data.ToString()); if (kpio.id != null) { if (kpio.id != kpioPrevious.id) { if (kpio.id != "E1") { //Make a new dataList = new line to be written dataList = new List { }; //Fill the datalist with this kpi dataList.Add(kpio.parentName); dataList.Add(kpio.data.area); dataList.Add(kpio.data.cropTypeCode); dataList.Add(kpio.data.cropTypeName); dataList.Add(kpio.id); dataList.Add(kpio.quantity); // in KPI output quantity is what we call KPIvariable in headerlist of csv file dataList.Add(kpio.value); dataList.Add(kpio.unit); dataList.Add(kpio.targetValue); dataList.Add(kpio.thresholdValue); //Write the datalist to a line to the streamwrieter sw for the output csv file sw.WriteLine(string.Join(",", dataList)); } else { //for E1, environmentMeasureData is an array of elements with 1 per pesticide. Inside each element there are multiple mbp_KPIvariables to be written foreach (KPIenvironmentMeasureData e in kpio.data.environmentMeasureData) { foreach (string mbp_KPIvariable in mbp_KPIvariables) { if (mbp_KPIvariable == "aquaticLife") { mbp_KPIvalue = e.aquaticLife; } else if (mbp_KPIvariable == "groundWater") { mbp_KPIvalue = e.groundWater; } else if (mbp_KPIvariable == "soilLife") { mbp_KPIvalue = e.soilLife; } else { mbp_KPIvalue = ""; } //Make a new dataList = new line to be written dataList = new List { }; //Fill the datalist with this kpi dataList.Add(kpio.parentName); dataList.Add(kpio.data.area); dataList.Add(kpio.data.cropTypeCode); dataList.Add(kpio.data.cropTypeName); dataList.Add(kpio.id); //"E1" dataList.Add(kpio.quantity); // "mbp" dataList.Add(""); // not here the value dataList.Add(""); // not here KPIunit for this indicator dataList.Add(""); // not here KPItargetvalue for this indicator dataList.Add(""); // not here KPIthresholdValue for this indicator dataList.Add(e.productCode); dataList.Add(e.productName); dataList.Add(e.quantity); dataList.Add(e.unitCode); dataList.Add(e.date); dataList.Add(mbp_KPIvariable); dataList.Add(mbp_KPIvalue); //Write the datalist to a line to the streamwrieter sw for the output csv file sw.WriteLine(string.Join(",", dataList)); } } } } } kpioPrevious = kpio; } //Total N applied double totalNapplied = 0.0; double operationNapplied; for (int i = 0; i < crpOperationItemCodes.Count; i++) { codeOperation = crpOperationItemCodes[i]; crpOperationItem = await _farmmapsApiService.GetItemAsync(codeOperation); dynamic data = JObject.Parse(crpOperationItem.Data.ToString(Formatting.None)); if (data.n != null) { operationNapplied = crpOperationItem.Data.GetValue("n").ToObject(); totalNapplied = totalNapplied + operationNapplied; } } //Also add totalNapplied to the csv dataList = new List { }; //Seems sometimes duplicate KPI items are returned. So check that here and only write if this kpio is different from previous dataList.Add(kpioPrevious.parentName); try { dataList.Add(kpioPrevious.data.area); } catch { dataList.Add(""); }; try { dataList.Add(kpioPrevious.data.cropTypeCode); } catch { dataList.Add(""); }; try { dataList.Add(kpioPrevious.data.cropTypeName); } catch { dataList.Add(""); }; dataList.Add(""); dataList.Add("totalNapplied"); dataList.Add(totalNapplied.ToString()); dataList.Add("kg/ha"); dataList.Add(""); dataList.Add(""); sw.WriteLine(string.Join(",", dataList)); //Clean up. Only newly created fields //Look up instruction in Swagger / api / v1 / items /{ code} if (useExistingCropfieldWithChildren == false && input.DeleteNewlyCreatedAfterCalc == true) { // Tested and found that if I delete the parent (cropfieldItem) then // automatically deletes children (crprecItem, cropfieldCharacteristicItem, KPIcontainer) // and automatically deletes grand children (crprecItem: operations, KPIcontainer: KPI items) await _farmmapsApiService.DeleteItemAsync(cropfieldItem.Code); //Do we need to wait here for the task to be completed? Or does it continue in the background? //await Task.Delay(60000); // wait 60 secs for task to be completed //Before the GetItem you will see cropfieldItem, crprecItem etc exist (with data). //After the GetItem these cropfieldItem, crprecItem etc are null which means they have been succesfully deleted. //// Check if below works, i.e. is item really deleted? //cropfieldItem = await _farmmapsApiService.GetItemAsync(cropfieldItem.Code); //// Check if croprecording has also been deleted. If not also delete it //crprecItem = await _farmmapsApiService.GetItemAsync(crprecItem.Code); //// Check if 1 operation has also been deleted. If not also delete it //crpOperationItem = await _farmmapsApiService.GetItemAsync(crpOperationItemCodes[0]); //// Check if 1 cropfieldCharacteristicItem has also been deleted. If not also delete it //cropfieldCharacteristicItem = await _farmmapsApiService.GetItemAsync(cropfieldCharacteristicItem.Code); } } // Functions to save previously created cropfields private void LoadSettings(string file) { if (File.Exists(file)) { var jsonText = File.ReadAllText(file); _settings = JsonConvert.DeserializeObject(jsonText); } else { _settings = new Settings(); } } private void SaveSettings(string file) { if (_settings == null) return; var json = JsonConvert.SerializeObject(_settings); File.WriteAllText(file, json); } private void SaveInfo(string file) { if (_settings == null) return; var json = JsonConvert.SerializeObject(_settings); File.WriteAllText(file, json); } private string strTime(TimeSpan ts) { return String.Format("{0:00}:{1:00}:{2:00}", ts.Hours, ts.Minutes, ts.Seconds); } } }