FarmMapsApiClient_KB34_MAST/FarmmapsKPI/KPIApplication.cs

using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Net.NetworkInformation;
using System.Threading.Tasks;
using System.Transactions;
using FarmmapsApi;
using FarmmapsApi.Models;
using FarmmapsApi.Services;
using FarmmapsKPI.Models;
using Microsoft.Extensions.Logging;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using static FarmmapsApiSamples.Constants;

namespace FarmmapsKPI
{
    public class KPIApplication : IApplication
    {
        //private const string DownloadFolder = "Downloads";
        private const string SettingsFile = "settings.json";

        private readonly ILogger<KPIApplication> _logger;
        private readonly FarmmapsApiService _farmmapsApiService;
        private readonly KPIService _kpiService;
        private readonly GeneralService _generalService;

        private Settings _settings;

        public KPIApplication(ILogger<KPIApplication> logger, FarmmapsApiService farmmapsApiService,
            GeneralService generalService, KPIService kpiService)
        {
            _logger = logger;
            _farmmapsApiService = farmmapsApiService;
            _generalService = generalService;
            _kpiService = kpiService;
        }

        public async Task RunAsync()
        {
            KPIInput input;
            string fnKPIinput;

            //Console.WriteLine("Type name of input json file. Example: KPIinput.json (in same directory as FarmmapsKPI.exe) or for example like this: C:/temp/KPIinputChemieTmp.json");
            //fnKPIinput = Console.ReadLine();
            //if (string.IsNullOrEmpty(fnKPIinput))
            //{
            //fnKPIinput = "KPIinput.json";
            fnKPIinput = "C:\\git\\FarmMapsApiClient_KB34_MAST\\FarmmapsKPI\\KPIinput.json";
            //}

            var fieldsInputJson = File.ReadAllText(fnKPIinput);

            List<KPIInput> fieldsInputs = JsonConvert.DeserializeObject<List<KPIInput>>(fieldsInputJson);

            // !! this call is needed the first time an api is called with a fresh clientid and secret !!
            await _farmmapsApiService.GetCurrentUserCodeAsync();
            var roots = await _farmmapsApiService.GetCurrentUserRootsAsync();

            //Where to write the output
            string downloadFolder = fieldsInputs[0].DownloadFolder;
            if (string.IsNullOrEmpty(downloadFolder))
            {
                downloadFolder = "Downloads";
            }
            if (!Directory.Exists(downloadFolder))
                Directory.CreateDirectory(downloadFolder);


            //Write the same info to a single csv file. Note this means existing file will be overwritten!
            StreamWriter sw;
            string KPIItemCsv = Path.GetFileNameWithoutExtension(fnKPIinput) + "_Items.csv";
            string KPIItemPathCsv = Path.Combine(downloadFolder, KPIItemCsv);
            List<string> headerList = new List<string> { "run","parentName", "cropfieldcode", "area_ha", "cropTypeCode", "cropTypeName", "KPIid", "KPIvariable", "KPIvalue", "KPIunit", "KPItargetvalue", "KPIthresholdValue",
                "mbp_productCode","mbp_productName","mbp_quantity","mbp_unitCode","mbp_date","mbp_KPIvariable","mbp_KPIvalue"};
            //Create a new csv file. Means if existing then overwritten !!!
            sw = new StreamWriter(KPIItemPathCsv);
            Configuration cf = await _farmmapsApiService.GetConfiguration();
            string endPoint = cf.Endpoint;
            sw.WriteLine($"Using FarmmapsKPI application in FarmmapsApSamples.sln. Input file: '{fnKPIinput}'. Download DateTime: '{DateTime.Now}'. Calculations on: '{endPoint}'");
            sw.WriteLine();
            sw.WriteLine(string.Join(",", headerList));

            // For each input download all KPI's. Keep track to time, important when doing bulk calculations
            var watch = System.Diagnostics.Stopwatch.StartNew();
            TimeSpan tsSofar = new TimeSpan();
            TimeSpan tsRemaining;
            TimeSpan tsTotalEstimated;

            //Per default just 1 run per field. For debugging check if when we run multiple times do we get (should be) always the same output?
            int nrun = 1;
            for (int run = 1; run <= nrun; run++) {
                for (int i = 0; i < fieldsInputs.Count; i++)
                //for (int i = 3; i < 4; i++) // for testing
                {
                    watch.Restart();
                    input = fieldsInputs[i];
                    _logger.LogInformation(string.Format($"// FarmmapsKPI: Run {run}: Downloading KPI's for field {i + 1} out of {fieldsInputs.Count} to single csv file {KPIItemPathCsv}"));
                    try
                    {
                        await Process(roots, input, sw, run);
                    }
                    catch (Exception ex)
                    {
                        _logger.LogError(ex.Message);
                    }
                    watch.Stop();
                    tsSofar = tsSofar + watch.Elapsed;
                    tsTotalEstimated = tsSofar / (i + 1) * fieldsInputs.Count;
                    tsRemaining = tsTotalEstimated - tsSofar;
                    _logger.LogInformation(string.Format($"// Time (hh:mm:ss): this field: {strTime(watch.Elapsed)}. Sofar: {strTime(tsSofar)}. Estimated total: {strTime(tsTotalEstimated)}. Remaining: {strTime(tsRemaining)}"));
                } 
            }
            //Close the csv file, write message to screen
            sw.Close();
            _logger.LogInformation(string.Format($"// FarmmapsKPI:"));
            _logger.LogInformation($"Done! Written all KPI for all fields in '{fnKPIinput}' to output file '{KPIItemPathCsv}'");
        }

        private async Task Process(List<UserRoot> roots, KPIInput input, StreamWriter sw, int run)
        {
            List<Item> cropfieldChildren;
            List<Item> crprecChildren;
            KPIOutput kpio;
            KPIOutput kpioPrevious = new KPIOutput();   //creates a new empty
            //If KPI E1 is calculated, write these sub kpi's to output
            string[] mbp_KPIvariables = new string[] { "aquaticLife", "groundWater", "soilLife" };
            string mbp_KPIvalue;
            int targetKPIitemsCount;   //if we know we should be getting 8 KPI items (A1,B1,B2,C1,D1,E1,F1,F2)
            int maxtries = 5;   // but don't keep on trying forever; there is a maximum number of tries
            int trycnt;
            //double totalNferiliserInput = 0;
            //double totalNferiliserCropfield = 0;

            string downloadFolder = input.DownloadFolder;
            if (string.IsNullOrEmpty(downloadFolder)) {
                downloadFolder = "Downloads";
            }
            if (!Directory.Exists(downloadFolder))
                Directory.CreateDirectory(downloadFolder);

            // !!specify if you are using an already created cropfield:
            bool useExistingCropfieldWithChildren = input.UseExistingCropfieldWithChildren;
            int cropYear = input.CropYear;
            string fieldName = input.fieldName;
            string fieldGeom = input.GeometryJson.ToString(Formatting.None);

            //Settings
            string settingsfile = $"Settings_{fieldName}.json";
            LoadSettings(settingsfile);

            var uploadedRoot = roots.SingleOrDefault(r => r.Name == "USER_IN");
            if (uploadedRoot == null)
            {
                _logger.LogError("Could not find a needed root item");
                return;
            }

            var myDriveRoot = roots.SingleOrDefault(r => r.Name == "USER_FILES");
            if (myDriveRoot == null)
            {
                _logger.LogError("Could not find a needed root item");
                return;
            }

            // Use already created cropfield or create new one, added a Data input, with field specific data for the KPI calculation
            Item cropfieldItem;
            //1 useExistingCropfieldWithChildren = false -> create new
            //2 useExistingCropfieldWithChildren = true && input.CropfieldItemCode = "" or absent -> read from settings json
            //3 useExistingCropfieldWithChildren = true && input.CropfieldItemCode like "deb48a74c5b54299bb852f17288010e9" in KPIinput -> use this one
            if (useExistingCropfieldWithChildren == false)
            {
                _logger.LogInformation($"Creating cropfield with name '{fieldName}'");
                cropfieldItem = await _generalService.CreateCropfieldItemAsync(myDriveRoot.Code,
                    $"{fieldName}", cropYear, input.GeometryJson.ToString(Formatting.None), input.DataCropfield.ToString(Formatting.None), input.StartDate, input.EndDate);
                _settings.CropfieldItemCode = cropfieldItem.Code;
                SaveSettings(settingsfile);
            }
            else if (string.IsNullOrEmpty(input.CropfieldItemCode))
            {
                _logger.LogInformation("reading CropfieldItemCode from settings file");
                cropfieldItem = await _farmmapsApiService.GetItemAsync(_settings.CropfieldItemCode);
            }
            else
            {
                _logger.LogInformation("reading CropfieldItemCode from KPIinput json");
                cropfieldItem = await _farmmapsApiService.GetItemAsync(input.CropfieldItemCode);
                SaveSettings(settingsfile);
            }

            // The cropfieldCharacteristicItem is used to enter crop yields and/or straw yields
            Item cropfieldCharacteristicItem;
            string dataCropfieldCharacteristic;
            //1 useExistingCropfieldWithChildren = false -> create new
            //2 useExistingCropfieldWithChildren = true && input.CropfieldCharacteristicItemCode = "" or absent -> read from settings json
            //3 useExistingCropfieldWithChildren = true && input.CropfieldCharacteristicItemCode like "deb48a74c5b54299bb852f17288010e9" in KPIinput -> use this one
            if (useExistingCropfieldWithChildren == false)
            {
                for (int i = 0; i < input.DataCropfieldCharacteristics.Length; i++)
                {
                    dataCropfieldCharacteristic = input.DataCropfieldCharacteristics[i].ToString(Formatting.None);
                    dynamic data = JObject.Parse(dataCropfieldCharacteristic);
                    _logger.LogInformation($"CreateCropfieldCharacteristicItemAsync ... for cropfieldCharacteristic {i}: '{data.label}', value '{data.value}'");
                    cropfieldCharacteristicItem = await _generalService.CreateCropfieldCharacteristicItemAsync(cropfieldItem.Code, cropYear, fieldGeom, dataCropfieldCharacteristic);
                    _settings.CropfieldCharacteristicItemCode = cropfieldCharacteristicItem.Code;
                    SaveSettings(settingsfile);
                }
            }
            else if (string.IsNullOrEmpty(input.CropfieldItemCode))
            {
                _logger.LogInformation("reading CropfieldCharacteristicItemCode from settings file");
                cropfieldCharacteristicItem = await _farmmapsApiService.GetItemAsync(_settings.CropfieldCharacteristicItemCode);
            }
            else
            {
                _logger.LogInformation("reading CropfieldCharacteristicItemCode from KPIinput json");
                cropfieldCharacteristicItem = await _farmmapsApiService.GetItemAsync(input.CropfieldCharacteristicItemCode);
                SaveSettings(settingsfile);
            }

            // Now we can do a first KPI calculation and get the polygon area from the KPI output (where it is based on geometry)
            // We need that because for operations, you need to provide the area on which the operation was applied
            // And if we put that to the crop area, then we neatly get everything on a per ha basis
            _logger.LogInformation($"Getting polygon area (ha))");
            List<Item> KPIItemsArea = await _generalService.GetKpiItemsForCropField(cropfieldItem, 3); 
            trycnt = 1;
            targetKPIitemsCount = 3; // here for the area we need at least 3, but not more than that
            while (KPIItemsArea.Count < targetKPIitemsCount & trycnt < maxtries) 
            {
                KPIItemsArea = await _generalService.GetKpiItemsForCropField(cropfieldItem,3);
                _logger.LogInformation($"Found {KPIItemsArea.Count} KPI items");
                trycnt++;
            }
            kpio = JsonConvert.DeserializeObject<KPIOutput>(KPIItemsArea[0].Data.ToString());
            string area_ha = kpio.data.area;
            // turn the area into a JObject for later merging with operation data;
            string strJarea = JsonConvert.SerializeObject(new { area = area_ha });
            JObject Jarea = JObject.Parse(strJarea);

            if (useExistingCropfieldWithChildren == false)
            {
                //Retreiving BOFEK. A cropfield has 1 soil
                //Have a look at the cropfieldChildren before and after running this task, see one child (i.e. soil) has been added)
                //cropfieldChildren = await _farmmapsApiService.GetItemChildrenAsync(cropfieldItem.Code);
                _logger.LogInformation("Get BOFEK for field");
                Item bofekItem = await _generalService.RunBofekTask(cropfieldItem);
                if (bofekItem == null)
                {
                    _logger.LogError("Something went wrong while obtaining the BOFEK data");
                    return;
                }
                //cropfieldChildren = await _farmmapsApiService.GetItemChildrenAsync(cropfieldItem.Code);
            }
            else
            {
                _logger.LogInformation("For existing cropfield we assume it already has BOFEK soil data");
            }

            // A cropfield has 1 crop recording and the crop recording has 0:many operations
            //So first at the crop recording
            Item crprecItem;
            //1 useExistingCropfieldWithChildren = false -> create new
            //2 useExistingCropfieldWithChildren = true && input.CropRecordingItemCode = "" or absent -> read from settings json
            //3 useExistingCropfieldWithChildren = true && input.CropRecordingItemCode like "deb48a74c5b54299bb852f17288010e9" in KPIinput -> use this one
            if (useExistingCropfieldWithChildren == false)
            {
                _logger.LogInformation("RunCropRecordingTask ...");
                crprecItem = await _generalService.RunCropRecordingTask(cropfieldItem);
                _settings.CropRecordingItemCode = crprecItem.Code;
                SaveSettings(settingsfile);
            }
            else if (string.IsNullOrEmpty(input.CropfieldItemCode))
            {
                _logger.LogInformation("reading CropRecordingItemCode from settings file");
                crprecItem = await _farmmapsApiService.GetItemAsync(_settings.CropRecordingItemCode);
            }
            else
            {
                _logger.LogInformation("reading CropRecordingItemCode from KPIinput json");
                crprecItem = await _farmmapsApiService.GetItemAsync(input.CropRecordingItemCode);
                SaveSettings(settingsfile);
            }

            // Now add the operations
            List<Item> crpOperationItems = new List<Item> { };
            List<string> crpOperationItemCodes = new List<string> { };
            Item crpOperationItem;
            string dataOperation;
            string codeOperation;
            //1 useExistingCropfieldWithChildren = false -> create new
            //2 useExistingCropfieldWithChildren = true && input.OperationItemCode = "" or absent -> read from settings json
            //3 useExistingCropfieldWithChildren = true && input.OperationItemCode like "deb48a74c5b54299bb852f17288010e9" in KPIinput -> use this one
            if (useExistingCropfieldWithChildren == false)
            {
                for (int i = 0; i < input.DataOperations.Length; i++)
                {
                    dataOperation = input.DataOperations[i].ToString(Formatting.None);
                    dynamic data = JObject.Parse(dataOperation);
                    _logger.LogInformation($"CreateOperationItemAsync ... for operation {i}: '{data.name}', on date '{data.from}'");
                    // Now check if the operation has a field called area
                    string? opArea = data["area"];
                    if (string.IsNullOrEmpty(opArea))
                    {
                        // if not having field area: add it to dataOperation ...
                        input.DataOperations[i].Merge(Jarea);
                        // ... and update the string
                        dataOperation = input.DataOperations[i].ToString(Formatting.None);
                    }
                    else
                    {
                        // if yes having field area: compare with polygon area. If not the same, throw warning
                        if (data.area != area_ha)
                        {
                            double differencePercent = 100.0*(Convert.ToDouble(area_ha) / Convert.ToDouble(data.area) - 1.0);
                            _logger.LogWarning($"cropfield has area {area_ha}, but in your KPIinput.json or file like that, you have an operation with area {data.area}" +
                                $" Difference is {area_ha} / {data.area} - 100% = {differencePercent}%." +
                                $" Is that correct? Example if operation was applied in part of field, e.g. in case of variable rate (VRA) application." +
                                $" Or did you accidentally fill in area in the KPIinput json? To use cropfield area, omit field 'area' from json" +
                                $" then the KPI applicataion will fill in area calculated from geometry");
                        }
                    }
                    //Now after optionally adding the area, add the Operation to the crop recording
                    crpOperationItem = await _generalService.CreateOperationItemAsync(crprecItem.Code, cropYear, fieldGeom, dataOperation);
                    crpOperationItems.Add(crpOperationItem);
                    crpOperationItemCodes.Add(crpOperationItem.Code);
                    //Keep track of totalNferiliserInput 
                    //totalNferiliserInput = totalNferiliserInput + (double)data.n; / this causes a problem if N content of the fertilizer is missing. As this check is no longer needed I just commented it out
                }
                _settings.OperationItemCodes = crpOperationItemCodes.ToArray();
                SaveSettings(settingsfile);
            }
            else if (string.IsNullOrEmpty(input.CropfieldItemCode))
            {
                _logger.LogInformation("reading OperationItemCode from settings file");
                for (int i = 0; i < _settings.OperationItemCodes.Length; i++)
                {
                    codeOperation = _settings.OperationItemCodes[i];
                    crpOperationItem = await _farmmapsApiService.GetItemAsync(codeOperation);
                    crpOperationItems.Add(crpOperationItem);
                    crpOperationItemCodes.Add(crpOperationItem.Code);
                }
            }
            else
            {
                _logger.LogInformation("reading OperationItemCodes from KPIinput json");
                for (int i = 0; i < input.OperationItemCodes.Length; i++)
                {
                    codeOperation = input.OperationItemCodes[i];
                    crpOperationItem = await _farmmapsApiService.GetItemAsync(codeOperation);
                    crpOperationItems.Add(crpOperationItem);
                    crpOperationItemCodes.Add(crpOperationItem.Code);
                }
                _settings.OperationItemCodes = crpOperationItemCodes.ToArray();
                SaveSettings(settingsfile);
            }

            // Inspect the children and grandchildren. If all is well, cropfield will have:
            // one crprec, with 0-many operations as children. And the Data of an operation will have specification of how much fertilizer / cropprotection agent was applied
            // one edicrop.characteristic (with yield in the data)
            cropfieldChildren = await _farmmapsApiService.GetItemChildrenAsync(cropfieldItem.Code);
            crprecChildren = await _farmmapsApiService.GetItemChildrenAsync(crprecItem.Code);

            //Check total operations
            int nOperationsInput = input.DataOperations.Length;
            int nOperationsCropfield = crprecChildren.Count;
            if (nOperationsInput != nOperationsCropfield)
            {
                _logger.LogWarning(String.Format($"1. run {run}: nOperationsInput != nOperationsCropfield?!"));
                //throw new Exception(String.Format($"run {run}: nOperationsInput != nOperationsCropfield?!"));
            }
                

            //Check totalNfertiliser input and Cropfield, 
            //for (int o = 0; o < crprecChildren.Count; o++)
            //{
            //    dynamic data = JObject.Parse(crprecChildren[o].Data.ToString());
            //    totalNferiliserCropfield = totalNferiliserCropfield + (double)data.n;
            //}
            //if (totalNferiliserInput != totalNferiliserCropfield)
            //{
            //    _logger.LogWarning(String.Format($"2. run {run}: totalNferiliserInput != totalNferiliserCropfield?!"));
            //    //throw new Exception(String.Format($"run {run}: totalNferiliserInput != totalNferiliserCropfield?!"));
            //}

            //Now get the KPIs for this cropfield, mounted with operations & cropyield
            List<Item> KPIItems = await _generalService.GetKpiItemsForCropField(cropfieldItem, 3);
            _logger.LogInformation($"GetKpiItemsForCropField('{cropfieldItem.Code}')");
            //Pesticide KPI's D1 and E1 are retreived from API's of CTBG and CLM and may take a bit longer to retrieve
            targetKPIitemsCount = 8;   //if we know we should be getting 8 KPI items (A1,B1,B2,C1,D1,E1,F1,F2)
            trycnt = 0;
            bool boolAquaticLife = false;
            _logger.LogInformation($"Firing calls GetKpiItemsForCropField() until we have {targetKPIitemsCount} KPIitems, but don't keep firing forever, stop after {maxtries} calls");
            _logger.LogInformation($"Before we start:");
            //_logger.LogInformation($"* KPIItems.Count = {KPIItems.Count}");
            _logger.LogInformation($"* trycnt = {trycnt}");
            _logger.LogInformation($"* boolAquaticLife = {boolAquaticLife}");
            //additional criterion for while loop: check if it really contains the E1 mbp elements.
            //while (trycnt == 0 || ((KPIItems.Count < targetKPIitemsCount || boolAquaticLife == false) & trycnt < maxtries))
            targetKPIitemsCount = 8;   //if we know we should be getting 8 KPI items (A1,B1,B2,C1,D1,E1,F1,F2)
            while (trycnt == 0 || (KPIItems.Count < targetKPIitemsCount & trycnt < maxtries))
            {
                _logger.LogInformation($"Call nr {trycnt + 1}");
                KPIItems = await _generalService.GetKpiItemsForCropField(cropfieldItem, 30);    //number after comma is how many seconds per try, should be between 30 seconds and 5 minutes
                _logger.LogInformation($"Found {KPIItems.Count} KPI items");
                //boolAquaticLife = GetBoolAquaticLife(KPIItems);
                trycnt ++;
            }
            if (KPIItems.Count < targetKPIitemsCount) {
                _logger.LogWarning($"Found {KPIItems.Count} KPIItems while you were aiming for {targetKPIitemsCount} KPIItems");
            }

            //Download KPI's into a json output file for this specific cropfield (with unique cropfieldItem.Code)
            string KPIItemPathJson = Path.Combine(downloadFolder, $"KPIItems_{cropfieldItem.Code}.json");
            var count = 0;
            foreach (Item item in KPIItems)
            {
                //Console.WriteLine($"KPI item #{count}: {item.Name}");
                File.AppendAllText(KPIItemPathJson, item.Data + Environment.NewLine);
                count++;
            }
            _logger.LogInformation($"Downloaded file {KPIItemPathJson}");
            _logger.LogInformation($"");


            //Remove duplicate KPI items 
            List<Item> KPIItemsClean = new List<Item>();
            Item itemi;
            string id, idNext;
            double value, valueNext; 
            int DuplicateCounter;
            value = 0;
            id = "";
          DuplicateCounter = 1;
            for (int i = 0; i < KPIItems.Count; i++)
            {
                itemi = KPIItems[i];
                idNext = JsonConvert.DeserializeObject<KPIOutput>(itemi.Data.ToString()).id;
                valueNext = Convert.ToDouble(JsonConvert.DeserializeObject<KPIOutput>(itemi.Data.ToString()).value); 
                if (idNext != null)
                {
                    if (id != idNext)
                    {
                        KPIItemsClean.Add(itemi);
                    }
                    else 
                    {
                        if (valueNext >= value)
                        {
                            //Remove the previous element from this list with same id but wrong value (valueNext > value) or duplicate value (valueNext == value). Presumes list is always sorted by kpiid, e.g. we may havce "
                            KPIItemsClean.RemoveAt(i - DuplicateCounter); //added a duplicate counter to deal with calculations where there is more 1 duplicate.
                            KPIItemsClean.Add(itemi);
                            DuplicateCounter = DuplicateCounter + 1; 
                        }
                        else
                        {
                            //Previous element was correct and already added so do nothing here
                        }
                    }
                    id = idNext;
                    value = valueNext;
                }
            }

            ////Order again from A zo Z.
            //List<Item> KPIItemsCleanSorted = new List<Item>();
            //for (int i = KPIItemsClean.Count - 1; i >= 0; i--)
            //{
            //    KPIItemsCleanSorted.Add(KPIItemsClean[i]);
            //}

            //Write to the csv file that collects all KPI's for all the crop fields
            List<string> dataList; 
            foreach (Item item in KPIItemsClean)
                {
                kpio = JsonConvert.DeserializeObject<KPIOutput>(item.Data.ToString());
                if (kpio.id != null)
                {
                    KPIelementsOfBalance kPIelementsOfBalance = kpio.data.values;
                    if (kpio.id == "A1")    //TtD
                    {
                        //Make a new dataList = new line to be written
                        //Fill the datalist with this kpi
                        dataList = new List<string>
                            {
                                run.ToString(),
                                kpio.parentName,
                                cropfieldItem.Code,
                                kpio.data.area,
                                kpio.data.cropTypeCode,
                                kpio.data.cropTypeName,
                                kpio.id,
                                kpio.quantity,    // in KPI output quantity is what we call KPIvariable in headerlist of csv file
                                kpio.value,
                                kpio.unit,
                                kpio.targetValue,
                                kpio.thresholdValue
                            };
                        //Write the datalist to a line to the streamwrieter sw for the output csv file
                        sw.WriteLine(string.Join(",", dataList));
                        foreach (string elementName in kpio.A1elements)
                        {
                            // get A1element from the element called values
                            // TtD 20240311: note elements of A1 are in the data structure elements of kpio.data and not as below for B1, B2, etc as kpio.data.values
                            string elementValue = (string)kpio.data.GetType().GetProperty(elementName).GetValue(kpio.data, null);
                            dataList = new List<string>
                                {
                                    run.ToString(),
                                    kpio.parentName,
                                    cropfieldItem.Code,
                                    kpio.data.area,
                                    kpio.data.cropTypeCode,
                                    kpio.data.cropTypeName,
                                    kpio.id,
                                    elementName,    // specific output variable name for A1 element
                                    elementValue,    // specific output value name for A1 element
                                    kpio.unit,
                                    "",
                                    ""
                                };
                            //Write the datalist to a line to the streamwrieter sw for the output csv file
                            sw.WriteLine(string.Join(",", dataList));
                        }
                    }
                    else if (kpio.id == "B1")
                    {
                        //Make a new dataList = new line to be written
                        //Fill the datalist with this kpi
                        dataList = new List<string>
                            {
                                run.ToString(),
                                kpio.parentName,
                                cropfieldItem.Code,
                                kpio.data.area,
                                kpio.data.cropTypeCode,
                                kpio.data.cropTypeName,
                                kpio.id,
                                kpio.quantity,    // in KPI output quantity is what we call KPIvariable in headerlist of csv file
                                kpio.value,
                                kpio.unit,
                                kpio.targetValue,
                                kpio.thresholdValue
                            };
                        //Write the datalist to a line to the streamwrieter sw for the output csv file
                        sw.WriteLine(string.Join(",", dataList));

                        foreach (string elementName in kpio.B1elements)
                        {
                            // get B1element from the element called values
                            string elementValue = (string)kPIelementsOfBalance.GetType().GetProperty(elementName).GetValue(kPIelementsOfBalance, null);
                            dataList = new List<string>
                                {
                                    run.ToString(),
                                    kpio.parentName,
                                    cropfieldItem.Code,
                                    kpio.data.area,
                                    kpio.data.cropTypeCode,
                                    kpio.data.cropTypeName,
                                    kpio.id,
                                    elementName,    // specific output variable name for B1 element
                                    elementValue,    // specific output value name for B1 element
                                    kpio.unit,
                                    "",
                                    ""
                                };
                            //Write the datalist to a line to the streamwrieter sw for the output csv file
                            sw.WriteLine(string.Join(",", dataList));

                            //Check if totalNferiliserInput is equal to what comes out of the KPI calculation
                            //if (elementName == "nFertilizerNKgHa")
                            //{
                            //    if (totalNferiliserInput != Convert.ToDouble(elementValue))
                            //    {
                            //        _logger.LogWarning(String.Format($"3. run {run}: totalNferiliserInput != nFertilizerNKgHa"));
                            //        //throw new Exception(String.Format($"run {run}: totalNferiliserInput != nFertilizerNKgHa"));
                            //    }
                            //}
                        }
                    }
                    else if (kpio.id == "B2")
                    {
                        //Make a new dataList = new line to be written
                        dataList = new List<string>
                            {
                                //Fill the datalist with this kpi
                                run.ToString(),
                                kpio.parentName,
                                cropfieldItem.Code,
                                kpio.data.area,
                                kpio.data.cropTypeCode,
                                kpio.data.cropTypeName,
                                kpio.id,
                                kpio.quantity,    // in KPI output quantity is what we call KPIvariable in headerlist of csv file
                                kpio.value,
                                kpio.unit,
                                kpio.targetValue,
                                kpio.thresholdValue
                            };
                        //Write the datalist to a line to the streamwrieter sw for the output csv file
                        sw.WriteLine(string.Join(",", dataList));

                        foreach (string elementName in kpio.B2elements)
                        {
                            // get B1element from the element called values
                            string elementValue = (string)kPIelementsOfBalance.GetType().GetProperty(elementName).GetValue(kPIelementsOfBalance, null);
                            dataList = new List<string>
                                {
                                    run.ToString(),
                                    kpio.parentName,
                                    cropfieldItem.Code,
                                    kpio.data.area,
                                    kpio.data.cropTypeCode,
                                    kpio.data.cropTypeName,
                                    kpio.id,
                                    elementName,    // specific output variable name for B2 element
                                    elementValue,    // specific output value name for B2 element
                                    kpio.unit,
                                    "",
                                    ""
                                };
                            //Write the datalist to a line to the streamwrieter sw for the output csv file
                            sw.WriteLine(string.Join(",", dataList));
                        }
                    }

                    else if (kpio.id == "C1")  //TtD
                    {
                        //Make a new dataList = new line to be written
                        dataList = new List<string>
                            {
                                run.ToString(),
                                kpio.parentName,
                                cropfieldItem.Code,
                                kpio.data.area,
                                kpio.data.cropTypeCode,
                                kpio.data.cropTypeName,
                                kpio.id,
                                kpio.quantity,    // in KPI output quantity is what we call KPIvariable in headerlist of csv file
                                kpio.value,
                                kpio.unit,
                                kpio.targetValue,
                                kpio.thresholdValue
                            };
                        //Write the datalist to a line to the streamwrieter sw for the output csv file
                        sw.WriteLine(string.Join(",", dataList));

                        foreach (string elementName in kpio.C1elements)
                        {
                            // get C1element from the element called values
                            string elementValue = (string)kPIelementsOfBalance.GetType().GetProperty(elementName).GetValue(kPIelementsOfBalance, null);
                            dataList = new List<string>
                                {
                                    run.ToString(),
                                    kpio.parentName,
                                    cropfieldItem.Code,
                                    kpio.data.area,
                                    kpio.data.cropTypeCode,
                                    kpio.data.cropTypeName,
                                    kpio.id,
                                    elementName,    // specific output variable name for C1 element
                                    elementValue,    // specific output value name for C1 element
                                    kpio.unit,
                                    "",
                                    ""
                                };
                            //Write the datalist to a line to the streamwrieter sw for the output csv file
                            sw.WriteLine(string.Join(",", dataList));
                        }
                    }
                    else if (kpio.id == "E1")
                    {
                        //for E1, environmentMeasureData is an array of elements with 1 per pesticide. Inside each element there are multiple mbp_KPIvariables to be written
                        foreach (KPIenvironmentMeasureData e in kpio.data.environmentMeasureData)
                        {
                            foreach (string mbp_KPIvariable in mbp_KPIvariables)
                            {
                                if (mbp_KPIvariable == "aquaticLife") { mbp_KPIvalue = e.aquaticLife; } else if (mbp_KPIvariable == "groundWater") { mbp_KPIvalue = e.groundWater; } else if (mbp_KPIvariable == "soilLife") { mbp_KPIvalue = e.soilLife; } else { mbp_KPIvalue = ""; }
                                //Make a new dataList = new line to be written
                                dataList = new List<string>
                                    {
                                        run.ToString(),
                                        kpio.parentName,
                                        cropfieldItem.Code,
                                        kpio.data.area,
                                        kpio.data.cropTypeCode,
                                        kpio.data.cropTypeName,
                                        kpio.id,  //"E1"
                                        kpio.quantity,    // "mbp"
                                        "",    // not here the value
                                        "",    // not here KPIunit for this indicator
                                        "",    // not here KPItargetvalue for this indicator
                                        "",    // not here KPIthresholdValue for this indicator
                                        e.productCode,
                                        e.productName,
                                        e.quantity,
                                        e.unitCode,
                                        e.date,
                                        mbp_KPIvariable,
                                        mbp_KPIvalue
                                    };
                                //Write the datalist to a line to the streamwrieter sw for the output csv file
                                sw.WriteLine(string.Join(",", dataList));
                            }
                        }
                    }
                    else if (kpio.id == "G1")  //TtD
                    {
                        //Make a new dataList = new line to be written
                        dataList = new List<string>
                            {
                                run.ToString(),
                                kpio.parentName,
                                cropfieldItem.Code,
                                kpio.data.area,
                                kpio.data.cropTypeCode,
                                kpio.data.cropTypeName,
                                kpio.id,
                                kpio.quantity,    // in KPI output quantity is what we call KPIvariable in headerlist of csv file
                                kpio.value,
                                kpio.unit,
                                kpio.targetValue,
                                kpio.thresholdValue
                            };
                        //Write the datalist to a line to the streamwrieter sw for the output csv file
                        sw.WriteLine(string.Join(",", dataList));

                        foreach (string elementName in kpio.G1elements)
                        {
                            // get G1element from the element called values
                            string elementValue = (string)kPIelementsOfBalance.GetType().GetProperty(elementName).GetValue(kPIelementsOfBalance, null);
                            dataList = new List<string>
                                {
                                    run.ToString(),
                                    kpio.parentName,
                                    cropfieldItem.Code,
                                    kpio.data.area,
                                    kpio.data.cropTypeCode,
                                    kpio.data.cropTypeName,
                                    kpio.id,
                                    elementName,    // specific output variable name for G1 element
                                    elementValue,    // specific output value name for G1 element
                                    kpio.unit,
                                    "",
                                    ""
                                };
                            //Write the datalist to a line to the streamwrieter sw for the output csv file
                            sw.WriteLine(string.Join(",", dataList));
                        }
                    }
                    else
                    {
                        //Any other KPI, (D1, F1, and F2), with just 1 record to write
                        dataList = new List<string>
                            {
                                run.ToString(),
                                kpio.parentName,
                                cropfieldItem.Code,
                                kpio.data.area,
                                kpio.data.cropTypeCode,
                                kpio.data.cropTypeName,
                                kpio.id,
                                kpio.quantity,
                                kpio.value,
                                kpio.unit,
                                kpio.targetValue,
                                kpio.thresholdValue
                            };
                        //Write the datalist to a line to the streamwrieter sw for the output csv file
                        sw.WriteLine(string.Join(",", dataList));
                    }
                }
            }

            ////Total N applied from input
            //double totalNapplied = 0.0;
            //double operationNapplied;
            //for (int i = 0; i < crpOperationItemCodes.Count; i++)
            //{
            //    codeOperation = crpOperationItemCodes[i];
            //    crpOperationItem = await _farmmapsApiService.GetItemAsync(codeOperation);
            //    dynamic data = JObject.Parse(crpOperationItem.Data.ToString(Formatting.None));
            //    if (data.n != null)
            //    {
            //        operationNapplied = crpOperationItem.Data.GetValue("n").ToObject<double>();
            //        totalNapplied = totalNapplied + operationNapplied;
            //    }
            //}
            ////Also add totalNapplied to the csv
            //dataList = new List<string> { };
            //dataList.Add(kpioPrevious.parentName);
            //dataList.Add(cropfieldItem.Code);
            //try { dataList.Add(kpioPrevious.data.area); } catch { dataList.Add(""); };
            //try { dataList.Add(kpioPrevious.data.cropTypeCode); } catch { dataList.Add(""); };
            //try { dataList.Add(kpioPrevious.data.cropTypeName); } catch { dataList.Add(""); };
            //dataList.Add("");
            //dataList.Add("totalNapplied");
            //dataList.Add(totalNapplied.ToString());
            //dataList.Add("kg/ha");
            //dataList.Add("");
            //dataList.Add("");
            //sw.WriteLine(string.Join(",", dataList));

            //Clean up. Only newly created fields 
            //Look up instruction in Swagger / api / v1 / items /{ code}
            if (useExistingCropfieldWithChildren == false && input.DeleteNewlyCreatedAfterCalc == true)
            {
                // Tested and found that if I delete the parent (cropfieldItem) then
                // automatically deletes children (crprecItem, cropfieldCharacteristicItem, KPIcontainer)
                // and automatically deletes grand children (crprecItem: operations, KPIcontainer: KPI items)
                await _farmmapsApiService.DeleteItemAsync(cropfieldItem.Code);
                //Do we need to wait here for the task to be completed? Or does it continue in the background?
                //await Task.Delay(60000); // wait 60 secs for task to be completed
                //Before the GetItem you will see cropfieldItem, crprecItem etc exist (with data).
                //After the GetItem these  cropfieldItem, crprecItem etc are null which means they have been succesfully deleted.
                //// Check if below works, i.e. is item really deleted?
                //cropfieldItem = await _farmmapsApiService.GetItemAsync(cropfieldItem.Code);
                //// Check if croprecording has also been deleted. If not also delete it
                //crprecItem = await _farmmapsApiService.GetItemAsync(crprecItem.Code);
                //// Check if 1 operation has also been deleted. If not also delete it
                //crpOperationItem = await _farmmapsApiService.GetItemAsync(crpOperationItemCodes[0]);
                //// Check if 1 cropfieldCharacteristicItem has also been deleted. If not also delete it
                //cropfieldCharacteristicItem = await _farmmapsApiService.GetItemAsync(cropfieldCharacteristicItem.Code);
            }
        }

        // Functions to save previously created cropfields
        private void LoadSettings(string file)
        {
            if (File.Exists(file))
            {
                var jsonText = File.ReadAllText(file);
                _settings = JsonConvert.DeserializeObject<Settings>(jsonText);
            }
            else
            {
                _settings = new Settings();
            }
        }

        private void SaveSettings(string file)
        {
            if (_settings == null)
                return;

            var json = JsonConvert.SerializeObject(_settings);
            File.WriteAllText(file, json);
        }
        private void SaveInfo(string file)
        {
            if (_settings == null)
                return;

            var json = JsonConvert.SerializeObject(_settings);
            File.WriteAllText(file, json);

        }
        private string strTime(TimeSpan ts)
        {
            return String.Format("{0:00}:{1:00}:{2:00}", ts.Hours, ts.Minutes, ts.Seconds);
        }

        private bool GetBoolAquaticLife(List<Item> KPIItems)
        {
            bool found = false;
            foreach (Item item in KPIItems)
            {
                KPIOutput kpio = JsonConvert.DeserializeObject<KPIOutput>(item.Data.ToString());
                if (kpio.id == "E1")
                {
                    try
                    {
                        foreach (KPIenvironmentMeasureData e in kpio.data.environmentMeasureData)
                        {
                            if (e.aquaticLife != null) 
                            {
                                found = true;
                                _logger.LogInformation($"Found e.aquaticLife = {e.aquaticLife}");
                                _logger.LogInformation("");
                            }
                        }
                    }
                    catch (Exception ex)
                    {
                        _logger.LogWarning($"NOT Found e.aquaticLife"); 
                        _logger.LogError(ex.Message);
                    }
                }
            }

            return found;
        }
    }
}