FarmMapsApiClient_KB34_MAST/FarmmapsDataDownload/DataDownloadApplication.cs

using System;
using System.Collections.Generic;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Threading.Tasks;
using FarmmapsApi;
using FarmmapsApi.Models;
using FarmmapsApi.Services;
using FarmmapsDataDownload.Models;
using Microsoft.Extensions.Logging;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using static FarmmapsApiSamples.Constants;

namespace FarmmapsDataDownload
{
    public class DataDownloadApplication : IApplication
    {
        //private const string DownloadFolder = "Downloads";
        private const string SettingsFile = "settings.json";

        private readonly ILogger<DataDownloadApplication> _logger;
        private readonly FarmmapsApiService _farmmapsApiService;
        private readonly DataDownloadService _dataDownloadService;
        private readonly GeneralService _generalService;

        private Settings _settings;

        public DataDownloadApplication(ILogger<DataDownloadApplication> logger, FarmmapsApiService farmmapsApiService,
            GeneralService generalService, DataDownloadService dataDownloadService)
        {
            _logger = logger;
            _farmmapsApiService = farmmapsApiService;
            _generalService = generalService;
            _dataDownloadService = dataDownloadService;
        }

        public async Task RunAsync()
        {
            string fieldsInputJson = File.ReadAllText("DataDownloadInput.json");

            List<DataDownloadInput> fieldsInputs = JsonConvert.DeserializeObject<List<DataDownloadInput>>(fieldsInputJson);

            // !! this call is needed the first time an api is called with a fresh clientid and secret !!
            await _farmmapsApiService.GetCurrentUserCodeAsync();
            var roots = await _farmmapsApiService.GetCurrentUserRootsAsync();

            foreach (var input in fieldsInputs)
            {
                try
                {
                    await Process(roots, input);
                }
                catch (Exception ex)
                {
                    _logger.LogError(ex.Message);
                }
            }
        }

        private async Task Process(List<UserRoot> roots, DataDownloadInput input)
        {
            //PO20220311: first time a call is made to download satellite images or statistics, an empty list is returned
            //If we wait a bit longer, e.g. 10 secs, then e.g. a list of 3 images may be returned
            //If we wait still longer, maybe 4 images.
            //The solution implemented below is to fire calls as long as the number of images returned keeps increasing
            //While in between each call, sleep for sleepSecs
            //Continue this until the number no longer increases or the maximum number of calls has been reached
            //Out of politeness, don't be too impatient. Don't set sleepSecs to 5 or 10 or 30 secs. Just accept this may take a while, have a coffee, we suggest sleepSecs = 60;
            int sleepSecs = 60;
            int callCntMax = 4;
            //For example we may set: "sleepSecs = 10;" and "callCntMax = 24;" and following result:
            //Call no: 1. Giving FarmMaps 10 seconds to get SatelliteItems...
            //Call no: 1: Received 2 images
            //Call no: 2. Giving FarmMaps 10 seconds to get SatelliteItems...
            //Call no: 2: Received 7 images
            //Call no: 3. Giving FarmMaps 10 seconds to get SatelliteItems...
            //Call no: 3: Received 7 images
            //And the firing of calls would stop because the number of images returned is no longer increasing
            //In the worst case, this could would lead to a total sleeping period of "sleepSecsSum = sleepSecs * callCntMax" seconds. After that we give up

            //This is an ugly fix. Neater would if FarmMaps would just take a bit longer and then do always deliver all satellite images on first call.
            //Once this has been fixed on the side of FarmMaps we can set callCntMax = 0 and the code below will work smoothly without any sleeping

            string downloadFolder = input.DownloadFolder;
            if (string.IsNullOrEmpty(downloadFolder)) {
                downloadFolder = "Downloads";
            }
            if (!Directory.Exists(downloadFolder))
                Directory.CreateDirectory(downloadFolder);

            // !!specify if you are using an already created cropfield:
            bool useCreatedCropfield = input.UseCreatedCropfield;
            var cropYear = input.CropYear;
            var fieldName = input.fieldName;
            bool storeSatelliteStatistics = input.StoreSatelliteStatisticsSingleImage;
            bool storeSatelliteStatisticsCropYear = input.StoreSatelliteStatisticsCropYear;
            //List<string> SatelliteBands = new List<string>(1) { input.SatelliteBand };
            List<string> satelliteBands = input.SatelliteBands;
            string headerLineStats = $"FieldName,satelliteDate,satelliteBand,max,min,mean,mode,median,stddev,minPlus,curtosis,maxMinus,skewness,variance,populationCount,variationCoefficient,confidenceIntervalLow, confidenceIntervalHigh,confidenceIntervalErrorMargin" + Environment.NewLine;


            string settingsfile = $"Settings_{fieldName}.json";

            LoadSettings(settingsfile);

            var uploadedRoot = roots.SingleOrDefault(r => r.Name == "USER_IN");
            if (uploadedRoot == null)
            {
                _logger.LogError("Could not find a needed root item");
                return;
            }

            var myDriveRoot = roots.SingleOrDefault(r => r.Name == "USER_FILES");
            if (myDriveRoot == null)
            {
                _logger.LogError("Could not find a needed root item");
                return;
            }

            // Use already created cropfield or create new one
            Item cropfieldItem;
            if (useCreatedCropfield == false || string.IsNullOrEmpty(_settings.CropfieldItemCode))
            {
                _logger.LogInformation("Creating cropfield");
                cropfieldItem = await _generalService.CreateCropfieldItemAsync(myDriveRoot.Code,
                    $"DataCropfield {input.OutputFileName}", cropYear, input.GeometryJson.ToString(Formatting.None));
                _settings.CropfieldItemCode = cropfieldItem.Code;
                SaveSettings(settingsfile);
            }
            else
            {
                _logger.LogInformation("Cropfield already exists, trying to get it");
                cropfieldItem = await _farmmapsApiService.GetItemAsync(_settings.CropfieldItemCode);
            }

            //Get croprecordings
            if (input.GetCropRecordings)
            {
                var crprecItem = input.CrprecItem;
                _logger.LogInformation($"Trying to get crop recordings of croprecording: {crprecItem}");

                var cropRec = await _farmmapsApiService.GetItemChildrenAsync(crprecItem, CROPREC_ITEMTYPE); 
                if (cropRec == null)
                {
                    _logger.LogError("Something went wrong while obtaining the croprecordings");
                    return;
                }

                var cropRecPath = Path.Combine(downloadFolder, $"croprecordings_{crprecItem}.json");
                _logger.LogInformation($"Found {cropRec.Count} crop recordings");
                var count = 0;
                await Task.Delay(500);
                foreach (var item in cropRec)
                {
                    Console.WriteLine($"Crop recording #{count}: {item.Name}");
                    File.AppendAllText(cropRecPath, item.Data +Environment.NewLine);
                    count++;
                }
                _logger.LogInformation($"Downloaded file {cropRecPath}");
            }

            // Get shadow data
            if (input.GetShadowData)
            {
                _logger.LogInformation("Calculate shadow map for field");
                var shadowItem = await _generalService.RunShadowTask(cropfieldItem);
                if (shadowItem == null)
                {
                    _logger.LogError("Something went wrong while obtaining the shadow map");
                    return;
                }

                _logger.LogInformation("Downloading shadow map");
                await _farmmapsApiService.DownloadItemAsync(shadowItem.Code,
                    Path.Combine(downloadFolder, $"{input.OutputFileName}_shadow.zip"));
            }


            // Get satellite data
            if (input.GetSatelliteData)
            {
                // check if satellite task not yet done, do here and save taskcode
                if (useCreatedCropfield == false || string.IsNullOrEmpty(_settings.SatelliteTaskCode))
                {
                    var satelliteTaskCode = await _generalService.RunSatelliteTask(cropfieldItem);
                    _settings.SatelliteTaskCode = satelliteTaskCode;
                    SaveSettings(settingsfile);
                }

                //Call first time
                int callCnt = 1;
                int sleepSecsSum = 0;
                //if callCntMax == 0 then don't sleep
                //if callCntMax = 1 then sleep first 1x
                if (callCntMax > 0)
                {
                    _logger.LogInformation($"Call no: {callCnt}. Giving FarmMaps {sleepSecs} seconds to get SatelliteItems...");
                    System.Threading.Thread.Sleep(1000 * sleepSecs);
                    sleepSecsSum = sleepSecsSum + sleepSecs;
                }
                List<Item> satelliteItemsCropYear = await _generalService.FindSatelliteItems(cropfieldItem, _settings.SatelliteTaskCode);
                int satelliteItemsCropYearCntPrev = satelliteItemsCropYear.Count;
                _logger.LogInformation($"Call no: {callCnt}. Received {satelliteItemsCropYearCntPrev} images");
                callCnt++;
                int satelliteItemsCropYearCnt = satelliteItemsCropYearCntPrev;
                //if callCntMax > 1 then sleep untill (1) no more increase in number of images received OR (2) maximum number of calls reached
                if (callCntMax > 1)
                {
                    //Call second time
                    _logger.LogInformation($"Call no: {callCnt}. Giving FarmMaps another {sleepSecs} seconds to get SatelliteItems...");
                    System.Threading.Thread.Sleep(1000 * sleepSecs);
                    satelliteItemsCropYear = await _generalService.FindSatelliteItems(cropfieldItem, _settings.SatelliteTaskCode);
                    satelliteItemsCropYearCnt = satelliteItemsCropYear.Count;
                    _logger.LogInformation($"Call no: {callCnt}. Received {satelliteItemsCropYearCnt} images");
                    sleepSecsSum = sleepSecsSum + sleepSecs;
                    //As long as there is progress, keep calling
                    callCnt++;
                    while (callCnt <= callCntMax && (satelliteItemsCropYearCnt == 0 || satelliteItemsCropYearCnt > satelliteItemsCropYearCntPrev))
                    {
                        _logger.LogInformation($"Surprise! The longer we wait, the more images we get. Sleep and call once more");
                        satelliteItemsCropYearCntPrev = satelliteItemsCropYearCnt;
                        _logger.LogInformation($"Call no: {callCnt} (max: {callCntMax}). Giving FarmMaps another {sleepSecs} seconds to get SatelliteItems...");
                        System.Threading.Thread.Sleep(1000 * sleepSecs);
                        satelliteItemsCropYear = await _generalService.FindSatelliteItems(cropfieldItem, _settings.SatelliteTaskCode);
                        satelliteItemsCropYearCnt = satelliteItemsCropYear.Count;
                        _logger.LogInformation($"Call no: {callCnt}. Received {satelliteItemsCropYearCnt} images");
                        callCnt++;
                        sleepSecsSum = sleepSecsSum + sleepSecs;
                    }
                }

                if (satelliteItemsCropYearCnt == 0)
                {
                    _logger.LogWarning($"DataDownloadApplication.cs: after calling one or more times and " +
                    $"sleeping in total {sleepSecsSum} seconds, still no images found. " +
                    $"Please check your settings for parameters callCntMax and sleepSecs in DataDownloadApplication.cs or contact FarmMaps");
                }

                satelliteItemsCropYear = satelliteItemsCropYear.OrderBy(x => x.DataDate).ToList();

                if (input.StoreSatelliteStatisticsSingleImage == true && satelliteItemsCropYearCnt > 0) {
                    _logger.LogInformation("Available satellite images:");
                    var count = 0;
                    TimeSpan.FromSeconds(0.5);
                    foreach (var item in satelliteItemsCropYear)
                    {

                        Console.WriteLine($"Satellite image #{count}: {item.DataDate}");
                        count++;
                    }

                    _logger.LogInformation("Enter satellite image number");
                    int element = Int32.Parse(Console.ReadLine());
                    var selectedSatelliteItem = satelliteItemsCropYear[element];

                    var SatelliteDate = selectedSatelliteItem.DataDate.Value.ToString("yyyyMMdd");
                    string fileName = string.Format($"satelliteGeotiff_{fieldName}_{SatelliteDate}");   // no need to add satelliteBand in the name because the tif contains all bands
                    string fileNameZip = Path.Combine(downloadFolder, string.Format($"{fileName}.zip"));
                    await _farmmapsApiService.DownloadItemAsync(selectedSatelliteItem.Code, fileNameZip);

                    // Download a csv file with stats
                    List<Item> selectedSatelliteItems = new List<Item>(1) { selectedSatelliteItem };
                    string fileNameStats = Path.Combine(downloadFolder, string.Format($"satelliteStats_{fieldName}_{SatelliteDate}.csv"));
                    _logger.LogInformation($"First call to get DownloadSatelliteStats for selected image...");
                    string downloadedStats = await _generalService.DownloadSatelliteStats(selectedSatelliteItems, fieldName, satelliteBands, downloadFolder);

                    //rename the csv file with stats
                    //if the targe file already exists, delete it
                    File.Delete(fileNameStats);
                    //rename
                    File.Move(downloadedStats, fileNameStats);

                    // name the tif file
                    string fileNameTifzipped = Path.Combine(downloadFolder, string.Format($"sentinelhub_{SatelliteDate}.tif")); 
                    string fileNameGeotiff = Path.Combine(downloadFolder, string.Format($"sentinelhub_{fieldName}_{SatelliteDate}.tif"));
                    // download the geotiffs. Returns a zip file with always these two files: 
                    // thumbnail.jpg
                    // sentinelhub_yyyyMMdd.tif. Contains 4 layers: (1) ndvi, (2) wdvi, (3) ci-red and (4) natural. Natural has 3 layers inside: redBand, blueBand and greenBand
                    if (true)
                    {
                        // Extract the file fileNameTifzipped from zip, rename it to fileNameGeotiff
                        ZipFile.ExtractToDirectory(fileNameZip, downloadFolder, true);
                        //if the targe file already exists, delete it
                        File.Delete(fileNameGeotiff);
                        //rename
                        File.Move(fileNameTifzipped, fileNameGeotiff);

                        // Cleanup
                        File.Delete(fileNameZip);
                        File.Delete(Path.Combine(downloadFolder, "thumbnail.jpg"));
                    }
                    //_logger.LogInformation($"Downloaded files {fileNameGeotiff} and {fileNameStats} to {downloadFolder}");
                    _logger.LogInformation($"Downloaded files to {downloadFolder}");

                }
                if (input.StoreSatelliteStatisticsCropYear == true) {
                    string fileNameStats = Path.Combine(downloadFolder, string.Format($"satelliteStats_{fieldName}_{cropYear}.csv"));
                    File.Delete(fileNameStats);
                    _logger.LogInformation($"First call to get DownloadSatelliteStats for whole cropYear...");
                    string downloadedStats = await _generalService.DownloadSatelliteStats(satelliteItemsCropYear, fieldName, satelliteBands, downloadFolder);
                    File.Move(downloadedStats, fileNameStats);
                    _logger.LogInformation($"Downloaded file {fileNameStats} with stats for field '{fieldName}', cropyear {cropYear}");
                }
            }



            // Get vanDerSat data
            if (input.GetVanDerSatData)
            {
                // check if satellite task not yet done, do here and save taskcode
                if (useCreatedCropfield == false || string.IsNullOrEmpty(_settings.VanDerSatTaskCode))
                {
                    var vanDerSatTaskCode = await _generalService.RunVanDerSatTask(cropfieldItem);
                    _settings.VanDerSatTaskCode = vanDerSatTaskCode;
                    SaveSettings(settingsfile);
                }

                // Select a particular satellite item from satelliteTask
                Item vanDerSatItem = await _generalService.FindVanDerSatItem(cropfieldItem, _settings.VanDerSatTaskCode, fieldName,  input.StoreVanDerSatStatistics);                
            }


        }

        // Functions to save previously created cropfields
        private void LoadSettings(string file)
        {
            if (File.Exists(file))
            {
                var jsonText = File.ReadAllText(file);
                _settings = JsonConvert.DeserializeObject<Settings>(jsonText);
            }
            else
            {
                _settings = new Settings();
            }
        }

        private void SaveSettings(string file)
        {
            if (_settings == null)
                return;

            var json = JsonConvert.SerializeObject(_settings);
            File.WriteAllText(file, json);
        }
        private void SaveInfo(string file)
        {
            if (_settings == null)
                return;

            var json = JsonConvert.SerializeObject(_settings);
            File.WriteAllText(file, json);

        }

    }
}