using System; using System.Collections.Generic; using System.IO; using System.IO.Compression; using System.Linq; using System.Threading.Tasks; using FarmmapsApi; using FarmmapsApi.Models; using FarmmapsApi.Services; using FarmmapsDataDownload.Models; using Microsoft.Extensions.Logging; using Newtonsoft.Json; using Newtonsoft.Json.Linq; using static FarmmapsApiSamples.Constants; namespace FarmmapsDataDownload { public class DataDownloadApplication : IApplication { //private const string DownloadFolder = "Downloads"; private const string SettingsFile = "settings.json"; private readonly ILogger _logger; private readonly FarmmapsApiService _farmmapsApiService; private readonly DataDownloadService _dataDownloadService; private readonly GeneralService _generalService; private Settings _settings; public DataDownloadApplication(ILogger logger, FarmmapsApiService farmmapsApiService, GeneralService generalService, DataDownloadService dataDownloadService) { _logger = logger; _farmmapsApiService = farmmapsApiService; _generalService = generalService; _dataDownloadService = dataDownloadService; } public async Task RunAsync() { var fieldsInputJson = File.ReadAllText("DataDownloadInput.json"); List fieldsInputs = JsonConvert.DeserializeObject>(fieldsInputJson); // !! this call is needed the first time an api is called with a fresh clientid and secret !! await _farmmapsApiService.GetCurrentUserCodeAsync(); var roots = await _farmmapsApiService.GetCurrentUserRootsAsync(); foreach (var input in fieldsInputs) { try { await Process(roots, input); } catch (Exception ex) { _logger.LogError(ex.Message); } } } private async Task Process(List roots, DataDownloadInput input) { string downloadFolder = input.DownloadFolder; if (string.IsNullOrEmpty(downloadFolder)) { downloadFolder = "Downloads"; } if (!Directory.Exists(downloadFolder)) Directory.CreateDirectory(downloadFolder); // !!specify if you are using an already created cropfield: bool useCreatedCropfield = input.UseCreatedCropfield; var cropYear = input.CropYear; var fieldName = input.fieldName; bool storeSatelliteStatistics = input.StoreSatelliteStatisticsSingleImage; bool storeSatelliteStatisticsCropYear = input.StoreSatelliteStatisticsCropYear; List SatelliteBands = new List(1) { input.SatelliteBand }; string headerLineStats = $"FieldName,satelliteDate,satelliteBand,max,min,mean,mode,median,stddev,minPlus,curtosis,maxMinus,skewness,variance,populationCount,variationCoefficient,confidenceIntervalLow, confidenceIntervalHigh,confidenceIntervalErrorMargin" + Environment.NewLine; string settingsfile = $"Settings_{fieldName}.json"; LoadSettings(settingsfile); var uploadedRoot = roots.SingleOrDefault(r => r.Name == "USER_IN"); if (uploadedRoot == null) { _logger.LogError("Could not find a needed root item"); return; } var myDriveRoot = roots.SingleOrDefault(r => r.Name == "USER_FILES"); if (myDriveRoot == null) { _logger.LogError("Could not find a needed root item"); return; } // Use already created cropfield or create new one Item cropfieldItem; if (useCreatedCropfield == false || string.IsNullOrEmpty(_settings.CropfieldItemCode)) { _logger.LogInformation("Creating cropfield"); cropfieldItem = await _generalService.CreateCropfieldItemAsync(myDriveRoot.Code, $"DataCropfield {input.OutputFileName}", cropYear, input.GeometryJson.ToString(Formatting.None)); _settings.CropfieldItemCode = cropfieldItem.Code; SaveSettings(settingsfile); } else { _logger.LogInformation("Cropfield already exists, trying to get it"); cropfieldItem = await _farmmapsApiService.GetItemAsync(_settings.CropfieldItemCode); } //Get croprecordings if (input.GetCropRecordings) { var crprecItem = input.CrprecItem; _logger.LogInformation($"Trying to get crop recordings of croprecording: {crprecItem}"); var cropRec = await _farmmapsApiService.GetItemChildrenAsync(crprecItem, CROPREC_ITEMTYPE); if (cropRec == null) { _logger.LogError("Something went wrong while obtaining the croprecordings"); return; } var cropRecPath = Path.Combine(downloadFolder, $"croprecordings_{crprecItem}.json"); _logger.LogInformation($"Found {cropRec.Count} crop recordings"); var count = 0; await Task.Delay(500); foreach (var item in cropRec) { Console.WriteLine($"Crop recording #{count}: {item.Name}"); File.AppendAllText(cropRecPath, item.Data +Environment.NewLine); count++; } _logger.LogInformation($"Downloaded file {cropRecPath}"); } // Get shadow data if (input.GetShadowData) { _logger.LogInformation("Calculate shadow map for field"); var shadowItem = await _generalService.RunShadowTask(cropfieldItem); if (shadowItem == null) { _logger.LogError("Something went wrong while obtaining the shadow map"); return; } _logger.LogInformation("Downloading shadow map"); await _farmmapsApiService.DownloadItemAsync(shadowItem.Code, Path.Combine(downloadFolder, $"{input.OutputFileName}_shadow.zip")); } // Get satellite data if (input.GetSatelliteData) { // check if satellite task not yet done, do here and save taskcode if (useCreatedCropfield == false || string.IsNullOrEmpty(_settings.SatelliteTaskCode)) { var satelliteTaskCode = await _generalService.RunSatelliteTask(cropfieldItem); _settings.SatelliteTaskCode = satelliteTaskCode; SaveSettings(settingsfile); } // Select all satellite items List satelliteItemsCropYear = await _generalService.FindSatelliteItems(cropfieldItem, _settings.SatelliteTaskCode); satelliteItemsCropYear = satelliteItemsCropYear.OrderBy(x => x.DataDate).ToList(); if (input.StoreSatelliteStatisticsSingleImage == true) { _logger.LogInformation("Available satellite images:"); var count = 0; TimeSpan.FromSeconds(0.5); foreach (var item in satelliteItemsCropYear) { Console.WriteLine($"Satellite image #{count}: {item.DataDate}"); count++; } _logger.LogInformation("Enter satellite image number"); int element = Int32.Parse(Console.ReadLine()); var selectedSatelliteItem = satelliteItemsCropYear[element]; var SatelliteDate = selectedSatelliteItem.DataDate.Value.ToString("yyyyMMdd"); string fileName = string.Format($"satelliteGeotiff_{fieldName}_{SatelliteDate}"); // no need to add satelliteBand in the name because the tif contains all bands string fileNameZip = string.Format($"{fileName}.zip"); string fileNameGeotiff = string.Format($"{fileName}.tif"); await _farmmapsApiService.DownloadItemAsync(selectedSatelliteItem.Code, Path.Combine(downloadFolder, fileNameZip)); // Download a csv file with stats List selectedSatalliteItems = new List(1) { selectedSatelliteItem }; string fileNameStats = Path.Combine(downloadFolder, string.Format($"satelliteStats_{fieldName}_{SatelliteDate}.csv")); string downloadedStats = await _generalService.DownloadSatelliteStats(selectedSatalliteItems, fieldName, SatelliteBands, downloadFolder); //rename the csv file with stats File.Delete(fileNameStats); File.Move(downloadedStats, fileNameStats); // wenr.tif. Contains 5 layers: (1) ndvi, (2) wdvi, (3) Red, (4) Green and (5) Blue // download the geotiffs. Returns a zip file with always these three files: // data.dat.aux.xml // thumbnail.jpg // wenr.tif. Contains 5 layers: (1) ndvi, (2) wdvi, (3) Red, (4) Green and (5) Blue if (true) { // Extract the file "wenr.tif" from zip, rename it to fileNameGeotiff ZipFile.ExtractToDirectory(Path.Combine(downloadFolder, fileNameZip), downloadFolder, true); File.Delete(Path.Combine(downloadFolder, fileNameGeotiff)); // Delete the fileNameGeotiff file if exists File.Move(Path.Combine(downloadFolder, "wenr.tif"), Path.Combine(downloadFolder, fileNameGeotiff)); // Rename the oldFileName into newFileName // Cleanup string[] filesToDelete = new string[] { fileNameZip, "wenr.tif", "thumbnail.jpg", "data.dat.aux.xml" }; foreach (string f in filesToDelete) { File.Delete(Path.Combine(downloadFolder, f)); } } _logger.LogInformation($"Downloaded files {fileNameGeotiff} and {fileNameStats} to {downloadFolder}"); } if (input.StoreSatelliteStatisticsCropYear == true) { string fileNameStats = Path.Combine(downloadFolder, string.Format($"satelliteStats_{fieldName}_{cropYear}.csv")); File.Delete(fileNameStats); string downloadedStats = await _generalService.DownloadSatelliteStats(satelliteItemsCropYear, fieldName, SatelliteBands, downloadFolder); File.Move(downloadedStats, fileNameStats); _logger.LogInformation($"Downloaded file {fileNameStats} with stats for field '{fieldName}', cropyear {cropYear}"); } } // Get vanDerSat data if (input.GetVanDerSatData) { // check if satellite task not yet done, do here and save taskcode if (useCreatedCropfield == false || string.IsNullOrEmpty(_settings.VanDerSatTaskCode)) { var vanDerSatTaskCode = await _generalService.RunVanDerSatTask(cropfieldItem); _settings.VanDerSatTaskCode = vanDerSatTaskCode; SaveSettings(settingsfile); } // Select a particular satellite item from satelliteTask Item vanDerSatItem = await _generalService.FindVanDerSatItem(cropfieldItem, _settings.VanDerSatTaskCode, fieldName, input.StoreVanDerSatStatistics); } } // Functions to save previously created cropfields private void LoadSettings(string file) { if (File.Exists(file)) { var jsonText = File.ReadAllText(file); _settings = JsonConvert.DeserializeObject(jsonText); } else { _settings = new Settings(); } } private void SaveSettings(string file) { if (_settings == null) return; var json = JsonConvert.SerializeObject(_settings); File.WriteAllText(file, json); } private void SaveInfo(string file) { if (_settings == null) return; var json = JsonConvert.SerializeObject(_settings); File.WriteAllText(file, json); } } }