using System; using System.Collections.Generic; using System.IO; using System.IO.Compression; using System.Linq; using System.Threading.Tasks; using FarmmapsApi; using FarmmapsApi.Models; using FarmmapsApi.Services; using FarmmapsDataDownload.Models; using Microsoft.Extensions.Logging; using Newtonsoft.Json; using Newtonsoft.Json.Linq; using static FarmmapsApiSamples.Constants; namespace FarmmapsDataDownload { public class DataDownloadApplication : IApplication { //private const string DownloadFolder = "Downloads"; private const string SettingsFile = "settings.json"; private readonly ILogger _logger; private readonly FarmmapsApiService _farmmapsApiService; private readonly DataDownloadService _dataDownloadService; private readonly GeneralService _generalService; private Settings _settings; public DataDownloadApplication(ILogger logger, FarmmapsApiService farmmapsApiService, GeneralService generalService, DataDownloadService dataDownloadService) { _logger = logger; _farmmapsApiService = farmmapsApiService; _generalService = generalService; _dataDownloadService = dataDownloadService; } public async Task RunAsync() { string fieldsInputJson = File.ReadAllText("DataDownloadInput.json"); List fieldsInputs = JsonConvert.DeserializeObject>(fieldsInputJson); // !! this call is needed the first time an api is called with a fresh clientid and secret !! await _farmmapsApiService.GetCurrentUserCodeAsync(); var roots = await _farmmapsApiService.GetCurrentUserRootsAsync(); foreach (var input in fieldsInputs) { try { await Process(roots, input); } catch (Exception ex) { _logger.LogError(ex.Message); } } } private async Task Process(List roots, DataDownloadInput input) { //PO20220311: first time a call is made to download satellite images or statistics, an empty list is returned //If we wait a bit longer, e.g. 10 secs, then e.g. a list of 3 images may be returned //If we wait still longer, maybe 4 images. //The solution implemented below is to fire calls as long as the number of images returned keeps increasing //While in between each call, sleep for sleepSecs //Continue this until the number no longer increases or the maximum number of calls has been reached //Out of politeness, don't be too impatient. Don't set sleepSecs to 5 or 10 or 30 secs. Just accept this may take a while, have a coffee, we suggest sleepSecs = 60; int sleepSecs = 60; int callCntMax = 4; //For example we may set: "sleepSecs = 10;" and "callCntMax = 24;" and following result: //Call no: 1. Giving FarmMaps 10 seconds to get SatelliteItems... //Call no: 1: Received 2 images //Call no: 2. Giving FarmMaps 10 seconds to get SatelliteItems... //Call no: 2: Received 7 images //Call no: 3. Giving FarmMaps 10 seconds to get SatelliteItems... //Call no: 3: Received 7 images //And the firing of calls would stop because the number of images returned is no longer increasing //In the worst case, this could would lead to a total sleeping period of "sleepSecsSum = sleepSecs * callCntMax" seconds. After that we give up //This is an ugly fix. Neater would if FarmMaps would just take a bit longer and then do always deliver all satellite images on first call. //Once this has been fixed on the side of FarmMaps we can set callCntMax = 0 and the code below will work smoothly without any sleeping string downloadFolder = input.DownloadFolder; if (string.IsNullOrEmpty(downloadFolder)) { downloadFolder = "Downloads"; } if (!Directory.Exists(downloadFolder)) Directory.CreateDirectory(downloadFolder); // !!specify if you are using an already created cropfield: bool useCreatedCropfield = input.UseCreatedCropfield; var cropYear = input.CropYear; var fieldName = input.fieldName; bool storeSatelliteStatistics = input.StoreSatelliteStatisticsSingleImage; bool storeSatelliteStatisticsCropYear = input.StoreSatelliteStatisticsCropYear; //List SatelliteBands = new List(1) { input.SatelliteBand }; List satelliteBands = input.SatelliteBands; string headerLineStats = $"FieldName,satelliteDate,satelliteBand,max,min,mean,mode,median,stddev,minPlus,curtosis,maxMinus,skewness,variance,populationCount,variationCoefficient,confidenceIntervalLow, confidenceIntervalHigh,confidenceIntervalErrorMargin" + Environment.NewLine; string settingsfile = $"Settings_{fieldName}.json"; LoadSettings(settingsfile); var uploadedRoot = roots.SingleOrDefault(r => r.Name == "USER_IN"); if (uploadedRoot == null) { _logger.LogError("Could not find a needed root item"); return; } var myDriveRoot = roots.SingleOrDefault(r => r.Name == "USER_FILES"); if (myDriveRoot == null) { _logger.LogError("Could not find a needed root item"); return; } // Use already created cropfield or create new one Item cropfieldItem; if (useCreatedCropfield == false || string.IsNullOrEmpty(_settings.CropfieldItemCode)) { _logger.LogInformation("Creating cropfield"); cropfieldItem = await _generalService.CreateCropfieldItemAsync(myDriveRoot.Code, $"DataCropfield {input.OutputFileName}", cropYear, input.GeometryJson.ToString(Formatting.None)); _settings.CropfieldItemCode = cropfieldItem.Code; SaveSettings(settingsfile); } else { _logger.LogInformation("Cropfield already exists, trying to get it"); cropfieldItem = await _farmmapsApiService.GetItemAsync(_settings.CropfieldItemCode); } //Get croprecordings if (input.GetCropRecordings) { var crprecItem = input.CrprecItem; _logger.LogInformation($"Trying to get crop recordings of croprecording: {crprecItem}"); var cropRec = await _farmmapsApiService.GetItemChildrenAsync(crprecItem, CROPREC_ITEMTYPE); if (cropRec == null) { _logger.LogError("Something went wrong while obtaining the croprecordings"); return; } var cropRecPath = Path.Combine(downloadFolder, $"croprecordings_{crprecItem}.json"); _logger.LogInformation($"Found {cropRec.Count} crop recordings"); var count = 0; await Task.Delay(500); foreach (var item in cropRec) { Console.WriteLine($"Crop recording #{count}: {item.Name}"); File.AppendAllText(cropRecPath, item.Data +Environment.NewLine); count++; } _logger.LogInformation($"Downloaded file {cropRecPath}"); } // Get shadow data if (input.GetShadowData) { _logger.LogInformation("Calculate shadow map for field"); var shadowItem = await _generalService.RunShadowTask(cropfieldItem); if (shadowItem == null) { _logger.LogError("Something went wrong while obtaining the shadow map"); return; } _logger.LogInformation("Downloading shadow map"); await _farmmapsApiService.DownloadItemAsync(shadowItem.Code, Path.Combine(downloadFolder, $"{input.OutputFileName}_shadow.zip")); } // Get satellite data if (input.GetSatelliteData) { // check if satellite task not yet done, do here and save taskcode if (useCreatedCropfield == false || string.IsNullOrEmpty(_settings.SatelliteTaskCode)) { var satelliteTaskCode = await _generalService.RunSatelliteTask(cropfieldItem); _settings.SatelliteTaskCode = satelliteTaskCode; SaveSettings(settingsfile); } //Call first time int callCnt = 1; int sleepSecsSum = 0; //if callCntMax == 0 then don't sleep //if callCntMax = 1 then sleep first 1x if (callCntMax > 0) { _logger.LogInformation($"Call no: {callCnt}. Giving FarmMaps {sleepSecs} seconds to get SatelliteItems..."); System.Threading.Thread.Sleep(1000 * sleepSecs); sleepSecsSum = sleepSecsSum + sleepSecs; } List satelliteItemsCropYear = await _generalService.FindSatelliteItems(cropfieldItem, _settings.SatelliteTaskCode); int satelliteItemsCropYearCntPrev = satelliteItemsCropYear.Count; _logger.LogInformation($"Call no: {callCnt}. Received {satelliteItemsCropYearCntPrev} images"); callCnt++; int satelliteItemsCropYearCnt = satelliteItemsCropYearCntPrev; //if callCntMax > 1 then sleep untill (1) no more increase in number of images received OR (2) maximum number of calls reached if (callCntMax > 1) { //Call second time _logger.LogInformation($"Call no: {callCnt}. Giving FarmMaps another {sleepSecs} seconds to get SatelliteItems..."); System.Threading.Thread.Sleep(1000 * sleepSecs); satelliteItemsCropYear = await _generalService.FindSatelliteItems(cropfieldItem, _settings.SatelliteTaskCode); satelliteItemsCropYearCnt = satelliteItemsCropYear.Count; _logger.LogInformation($"Call no: {callCnt}. Received {satelliteItemsCropYearCnt} images"); sleepSecsSum = sleepSecsSum + sleepSecs; //As long as there is progress, keep calling callCnt++; while (callCnt <= callCntMax && (satelliteItemsCropYearCnt == 0 || satelliteItemsCropYearCnt > satelliteItemsCropYearCntPrev)) { _logger.LogInformation($"Surprise! The longer we wait, the more images we get. Sleep and call once more"); satelliteItemsCropYearCntPrev = satelliteItemsCropYearCnt; _logger.LogInformation($"Call no: {callCnt} (max: {callCntMax}). Giving FarmMaps another {sleepSecs} seconds to get SatelliteItems..."); System.Threading.Thread.Sleep(1000 * sleepSecs); satelliteItemsCropYear = await _generalService.FindSatelliteItems(cropfieldItem, _settings.SatelliteTaskCode); satelliteItemsCropYearCnt = satelliteItemsCropYear.Count; _logger.LogInformation($"Call no: {callCnt}. Received {satelliteItemsCropYearCnt} images"); callCnt++; sleepSecsSum = sleepSecsSum + sleepSecs; } } if (satelliteItemsCropYearCnt == 0) { _logger.LogWarning($"DataDownloadApplication.cs: after calling one or more times and " + $"sleeping in total {sleepSecsSum} seconds, still no images found. " + $"Please check your settings for parameters callCntMax and sleepSecs in DataDownloadApplication.cs or contact FarmMaps"); } satelliteItemsCropYear = satelliteItemsCropYear.OrderBy(x => x.DataDate).ToList(); if (input.StoreSatelliteStatisticsSingleImage == true && satelliteItemsCropYearCnt > 0) { _logger.LogInformation("Available satellite images:"); var count = 0; TimeSpan.FromSeconds(0.5); foreach (var item in satelliteItemsCropYear) { Console.WriteLine($"Satellite image #{count}: {item.DataDate}"); count++; } _logger.LogInformation("Enter satellite image number"); int element = Int32.Parse(Console.ReadLine()); var selectedSatelliteItem = satelliteItemsCropYear[element]; var SatelliteDate = selectedSatelliteItem.DataDate.Value.ToString("yyyyMMdd"); string fileName = string.Format($"satelliteGeotiff_{fieldName}_{SatelliteDate}"); // no need to add satelliteBand in the name because the tif contains all bands string fileNameZip = Path.Combine(downloadFolder, string.Format($"{fileName}.zip")); await _farmmapsApiService.DownloadItemAsync(selectedSatelliteItem.Code, fileNameZip); // Download a csv file with stats List selectedSatelliteItems = new List(1) { selectedSatelliteItem }; string fileNameStats = Path.Combine(downloadFolder, string.Format($"satelliteStats_{fieldName}_{SatelliteDate}.csv")); _logger.LogInformation($"First call to get DownloadSatelliteStats for selected image..."); string downloadedStats = await _generalService.DownloadSatelliteStats(selectedSatelliteItems, fieldName, satelliteBands, downloadFolder); //rename the csv file with stats //if the targe file already exists, delete it File.Delete(fileNameStats); //rename File.Move(downloadedStats, fileNameStats); // name the tif file string fileNameTifzipped = Path.Combine(downloadFolder, string.Format($"sentinelhub_{SatelliteDate}.tif")); string fileNameGeotiff = Path.Combine(downloadFolder, string.Format($"sentinelhub_{fieldName}_{SatelliteDate}.tif")); // download the geotiffs. Returns a zip file with always these two files: // thumbnail.jpg // sentinelhub_yyyyMMdd.tif. Contains 4 layers: (1) ndvi, (2) wdvi, (3) ci-red and (4) natural. Natural has 3 layers inside: redBand, blueBand and greenBand if (true) { // Extract the file fileNameTifzipped from zip, rename it to fileNameGeotiff ZipFile.ExtractToDirectory(fileNameZip, downloadFolder, true); //if the targe file already exists, delete it File.Delete(fileNameGeotiff); //rename File.Move(fileNameTifzipped, fileNameGeotiff); // Cleanup File.Delete(fileNameZip); File.Delete(Path.Combine(downloadFolder, "thumbnail.jpg")); } //_logger.LogInformation($"Downloaded files {fileNameGeotiff} and {fileNameStats} to {downloadFolder}"); _logger.LogInformation($"Downloaded files to {downloadFolder}"); } if (input.StoreSatelliteStatisticsCropYear == true) { string fileNameStats = Path.Combine(downloadFolder, string.Format($"satelliteStats_{fieldName}_{cropYear}.csv")); File.Delete(fileNameStats); _logger.LogInformation($"First call to get DownloadSatelliteStats for whole cropYear..."); string downloadedStats = await _generalService.DownloadSatelliteStats(satelliteItemsCropYear, fieldName, satelliteBands, downloadFolder); File.Move(downloadedStats, fileNameStats); _logger.LogInformation($"Downloaded file {fileNameStats} with stats for field '{fieldName}', cropyear {cropYear}"); } } // Get vanDerSat data if (input.GetVanDerSatData) { // check if satellite task not yet done, do here and save taskcode if (useCreatedCropfield == false || string.IsNullOrEmpty(_settings.VanDerSatTaskCode)) { var vanDerSatTaskCode = await _generalService.RunVanDerSatTask(cropfieldItem); _settings.VanDerSatTaskCode = vanDerSatTaskCode; SaveSettings(settingsfile); } // Select a particular satellite item from satelliteTask Item vanDerSatItem = await _generalService.FindVanDerSatItem(cropfieldItem, _settings.VanDerSatTaskCode, fieldName, input.StoreVanDerSatStatistics); } } // Functions to save previously created cropfields private void LoadSettings(string file) { if (File.Exists(file)) { var jsonText = File.ReadAllText(file); _settings = JsonConvert.DeserializeObject(jsonText); } else { _settings = new Settings(); } } private void SaveSettings(string file) { if (_settings == null) return; var json = JsonConvert.SerializeObject(_settings); File.WriteAllText(file, json); } private void SaveInfo(string file) { if (_settings == null) return; var json = JsonConvert.SerializeObject(_settings); File.WriteAllText(file, json); } } }