FarmMapsApiClient/FarmmapsDataDownload/DataDownloadApplication.cs

287 lines
13 KiB
C#
Raw Normal View History

using System;
using System.Collections.Generic;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Threading.Tasks;
using FarmmapsApi;
using FarmmapsApi.Models;
using FarmmapsApi.Services;
using FarmmapsDataDownload.Models;
using Microsoft.Extensions.Logging;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using static FarmmapsApiSamples.Constants;
namespace FarmmapsDataDownload
{
public class DataDownloadApplication : IApplication
{
//private const string DownloadFolder = "Downloads";
private const string SettingsFile = "settings.json";
private readonly ILogger<DataDownloadApplication> _logger;
private readonly FarmmapsApiService _farmmapsApiService;
private readonly DataDownloadService _dataDownloadService;
private readonly GeneralService _generalService;
private Settings _settings;
public DataDownloadApplication(ILogger<DataDownloadApplication> logger, FarmmapsApiService farmmapsApiService,
GeneralService generalService, DataDownloadService dataDownloadService)
{
_logger = logger;
_farmmapsApiService = farmmapsApiService;
_generalService = generalService;
_dataDownloadService = dataDownloadService;
}
public async Task RunAsync()
{
var fieldsInputJson = File.ReadAllText("DataDownloadInput.json");
List<DataDownloadInput> fieldsInputs = JsonConvert.DeserializeObject<List<DataDownloadInput>>(fieldsInputJson);
// !! this call is needed the first time an api is called with a fresh clientid and secret !!
await _farmmapsApiService.GetCurrentUserCodeAsync();
var roots = await _farmmapsApiService.GetCurrentUserRootsAsync();
foreach (var input in fieldsInputs)
{
try
{
await Process(roots, input);
}
catch (Exception ex)
{
_logger.LogError(ex.Message);
}
}
}
private async Task Process(List<UserRoot> roots, DataDownloadInput input)
{
string downloadFolder = input.DownloadFolder;
if (string.IsNullOrEmpty(downloadFolder)) {
downloadFolder = "Downloads";
}
if (!Directory.Exists(downloadFolder))
Directory.CreateDirectory(downloadFolder);
// !!specify if you are using an already created cropfield:
bool useCreatedCropfield = input.UseCreatedCropfield;
var cropYear = input.CropYear;
var fieldName = input.fieldName;
bool storeSatelliteStatistics = input.StoreSatelliteStatisticsSingleImage;
bool storeSatelliteStatisticsCropYear = input.StoreSatelliteStatisticsCropYear;
List<string> SatelliteBands = new List<string>(1) { input.SatelliteBand };
string headerLineStats = $"FieldName,satelliteDate,satelliteBand,max,min,mean,mode,median,stddev,minPlus,curtosis,maxMinus,skewness,variance,populationCount,variationCoefficient,confidenceIntervalLow, confidenceIntervalHigh,confidenceIntervalErrorMargin" + Environment.NewLine;
string settingsfile = $"Settings_{fieldName}.json";
LoadSettings(settingsfile);
2022-02-28 13:29:56 +00:00
var uploadedRoot = roots.SingleOrDefault(r => r.Name == "USER_IN");
if (uploadedRoot == null)
{
_logger.LogError("Could not find a needed root item");
return;
}
2022-02-28 13:29:56 +00:00
var myDriveRoot = roots.SingleOrDefault(r => r.Name == "USER_FILES");
if (myDriveRoot == null)
{
_logger.LogError("Could not find a needed root item");
return;
}
// Use already created cropfield or create new one
Item cropfieldItem;
if (useCreatedCropfield == false || string.IsNullOrEmpty(_settings.CropfieldItemCode))
{
_logger.LogInformation("Creating cropfield");
cropfieldItem = await _generalService.CreateCropfieldItemAsync(myDriveRoot.Code,
$"DataCropfield {input.OutputFileName}", cropYear, input.GeometryJson.ToString(Formatting.None));
_settings.CropfieldItemCode = cropfieldItem.Code;
SaveSettings(settingsfile);
}
else
{
_logger.LogInformation("Cropfield already exists, trying to get it");
cropfieldItem = await _farmmapsApiService.GetItemAsync(_settings.CropfieldItemCode);
}
//Get croprecordings
if (input.GetCropRecordings)
{
var crprecItem = input.CrprecItem;
_logger.LogInformation($"Trying to get crop recordings of croprecording: {crprecItem}");
var cropRec = await _farmmapsApiService.GetItemChildrenAsync(crprecItem, CROPREC_ITEMTYPE);
if (cropRec == null)
{
_logger.LogError("Something went wrong while obtaining the croprecordings");
return;
}
var cropRecPath = Path.Combine(downloadFolder, $"croprecordings_{crprecItem}.json");
_logger.LogInformation($"Found {cropRec.Count} crop recordings");
var count = 0;
2022-02-28 11:23:21 +00:00
await Task.Delay(500);
foreach (var item in cropRec)
{
Console.WriteLine($"Crop recording #{count}: {item.Name}");
File.AppendAllText(cropRecPath, item.Data +Environment.NewLine);
count++;
}
_logger.LogInformation($"Downloaded file {cropRecPath}");
}
// Get shadow data
if (input.GetShadowData)
{
_logger.LogInformation("Calculate shadow map for field");
var shadowItem = await _generalService.RunShadowTask(cropfieldItem);
if (shadowItem == null)
{
_logger.LogError("Something went wrong while obtaining the shadow map");
return;
}
_logger.LogInformation("Downloading shadow map");
await _farmmapsApiService.DownloadItemAsync(shadowItem.Code,
Path.Combine(downloadFolder, $"{input.OutputFileName}_shadow.zip"));
}
// Get satellite data
if (input.GetSatelliteData)
{
// check if satellite task not yet done, do here and save taskcode
if (useCreatedCropfield == false || string.IsNullOrEmpty(_settings.SatelliteTaskCode))
{
var satelliteTaskCode = await _generalService.RunSatelliteTask(cropfieldItem);
_settings.SatelliteTaskCode = satelliteTaskCode;
SaveSettings(settingsfile);
}
// Select all satellite items
List<Item> satelliteItemsCropYear = await _generalService.FindSatelliteItems(cropfieldItem, _settings.SatelliteTaskCode);
satelliteItemsCropYear = satelliteItemsCropYear.OrderBy(x => x.DataDate).ToList();
if (input.StoreSatelliteStatisticsSingleImage == true) {
_logger.LogInformation("Available satellite images:");
var count = 0;
TimeSpan.FromSeconds(0.5);
foreach (var item in satelliteItemsCropYear)
{
Console.WriteLine($"Satellite image #{count}: {item.DataDate}");
count++;
}
_logger.LogInformation("Enter satellite image number");
int element = Int32.Parse(Console.ReadLine());
var selectedSatelliteItem = satelliteItemsCropYear[element];
var SatelliteDate = selectedSatelliteItem.DataDate.Value.ToString("yyyyMMdd");
string fileName = string.Format($"satelliteGeotiff_{fieldName}_{SatelliteDate}"); // no need to add satelliteBand in the name because the tif contains all bands
string fileNameZip = string.Format($"{fileName}.zip");
string fileNameGeotiff = string.Format($"{fileName}.tif");
await _farmmapsApiService.DownloadItemAsync(selectedSatelliteItem.Code, Path.Combine(downloadFolder, fileNameZip));
// Download a csv file with stats
List<Item> selectedSatalliteItems = new List<Item>(1) { selectedSatelliteItem };
string fileNameStats = Path.Combine(downloadFolder, string.Format($"satelliteStats_{fieldName}_{SatelliteDate}.csv"));
string downloadedStats = await _generalService.DownloadSatelliteStats(selectedSatalliteItems, fieldName, SatelliteBands, downloadFolder);
//rename the csv file with stats
File.Delete(fileNameStats);
File.Move(downloadedStats, fileNameStats);
// wenr.tif. Contains 5 layers: (1) ndvi, (2) wdvi, (3) Red, (4) Green and (5) Blue
// download the geotiffs. Returns a zip file with always these three files:
// data.dat.aux.xml
// thumbnail.jpg
// wenr.tif. Contains 5 layers: (1) ndvi, (2) wdvi, (3) Red, (4) Green and (5) Blue
if (true)
{
// Extract the file "wenr.tif" from zip, rename it to fileNameGeotiff
ZipFile.ExtractToDirectory(Path.Combine(downloadFolder, fileNameZip), downloadFolder, true);
File.Delete(Path.Combine(downloadFolder, fileNameGeotiff)); // Delete the fileNameGeotiff file if exists
File.Move(Path.Combine(downloadFolder, "wenr.tif"), Path.Combine(downloadFolder, fileNameGeotiff)); // Rename the oldFileName into newFileName
// Cleanup
string[] filesToDelete = new string[] { fileNameZip, "wenr.tif", "thumbnail.jpg", "data.dat.aux.xml" };
foreach (string f in filesToDelete)
{
File.Delete(Path.Combine(downloadFolder, f));
}
}
_logger.LogInformation($"Downloaded files {fileNameGeotiff} and {fileNameStats} to {downloadFolder}");
}
if (input.StoreSatelliteStatisticsCropYear == true) {
string fileNameStats = Path.Combine(downloadFolder, string.Format($"satelliteStats_{fieldName}_{cropYear}.csv"));
File.Delete(fileNameStats);
string downloadedStats = await _generalService.DownloadSatelliteStats(satelliteItemsCropYear, fieldName, SatelliteBands, downloadFolder);
File.Move(downloadedStats, fileNameStats);
_logger.LogInformation($"Downloaded file {fileNameStats} with stats for field '{fieldName}', cropyear {cropYear}");
}
}
// Get vanDerSat data
if (input.GetVanDerSatData)
{
// check if satellite task not yet done, do here and save taskcode
if (useCreatedCropfield == false || string.IsNullOrEmpty(_settings.VanDerSatTaskCode))
{
var vanDerSatTaskCode = await _generalService.RunVanDerSatTask(cropfieldItem);
_settings.VanDerSatTaskCode = vanDerSatTaskCode;
SaveSettings(settingsfile);
}
// Select a particular satellite item from satelliteTask
Item vanDerSatItem = await _generalService.FindVanDerSatItem(cropfieldItem, _settings.VanDerSatTaskCode, fieldName, input.StoreVanDerSatStatistics);
}
}
// Functions to save previously created cropfields
private void LoadSettings(string file)
{
if (File.Exists(file))
{
var jsonText = File.ReadAllText(file);
_settings = JsonConvert.DeserializeObject<Settings>(jsonText);
}
else
{
_settings = new Settings();
}
}
private void SaveSettings(string file)
{
if (_settings == null)
return;
var json = JsonConvert.SerializeObject(_settings);
File.WriteAllText(file, json);
}
private void SaveInfo(string file)
{
if (_settings == null)
return;
var json = JsonConvert.SerializeObject(_settings);
File.WriteAllText(file, json);
}
}
}