forked from FarmMaps/FarmMapsApiClient
362 lines
18 KiB
C#
362 lines
18 KiB
C#
using System;
|
|
using System.Collections.Generic;
|
|
using System.IO;
|
|
using System.IO.Compression;
|
|
using System.Linq;
|
|
using System.Threading.Tasks;
|
|
using FarmmapsApi;
|
|
using FarmmapsApi.Models;
|
|
using FarmmapsApi.Services;
|
|
using FarmmapsDataDownload.Models;
|
|
using Microsoft.Extensions.Logging;
|
|
using Newtonsoft.Json;
|
|
using Newtonsoft.Json.Linq;
|
|
using static FarmmapsApiSamples.Constants;
|
|
|
|
namespace FarmmapsDataDownload
|
|
{
|
|
public class DataDownloadApplication : IApplication
|
|
{
|
|
//private const string DownloadFolder = "Downloads";
|
|
private const string SettingsFile = "settings.json";
|
|
|
|
private readonly ILogger<DataDownloadApplication> _logger;
|
|
private readonly FarmmapsApiService _farmmapsApiService;
|
|
private readonly DataDownloadService _dataDownloadService;
|
|
private readonly GeneralService _generalService;
|
|
|
|
private Settings _settings;
|
|
|
|
public DataDownloadApplication(ILogger<DataDownloadApplication> logger, FarmmapsApiService farmmapsApiService,
|
|
GeneralService generalService, DataDownloadService dataDownloadService)
|
|
{
|
|
_logger = logger;
|
|
_farmmapsApiService = farmmapsApiService;
|
|
_generalService = generalService;
|
|
_dataDownloadService = dataDownloadService;
|
|
}
|
|
|
|
public async Task RunAsync()
|
|
{
|
|
string fieldsInputJson = File.ReadAllText("DataDownloadInput.json");
|
|
|
|
List<DataDownloadInput> fieldsInputs = JsonConvert.DeserializeObject<List<DataDownloadInput>>(fieldsInputJson);
|
|
|
|
// !! this call is needed the first time an api is called with a fresh clientid and secret !!
|
|
await _farmmapsApiService.GetCurrentUserCodeAsync();
|
|
var roots = await _farmmapsApiService.GetCurrentUserRootsAsync();
|
|
|
|
foreach (var input in fieldsInputs)
|
|
{
|
|
try
|
|
{
|
|
await Process(roots, input);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
_logger.LogError(ex.Message);
|
|
}
|
|
}
|
|
}
|
|
|
|
private async Task Process(List<UserRoot> roots, DataDownloadInput input)
|
|
{
|
|
//PO20220311: first time a call is made to download satellite images or statistics, an empty list is returned
|
|
//If we wait a bit longer, e.g. 10 secs, then e.g. a list of 3 images may be returned
|
|
//If we wait still longer, maybe 4 images.
|
|
//The solution implemented below is to fire calls as long as the number of images returned keeps increasing
|
|
//While in between each call, sleep for sleepSecs
|
|
//Continue this until the number no longer increases or the maximum number of calls has been reached
|
|
//Out of politeness, don't be too impatient. Don't set sleepSecs to 5 or 10 or 30 secs. Just accept this may take a while, have a coffee, we suggest sleepSecs = 60;
|
|
int sleepSecs = 60;
|
|
int callCntMax = 4;
|
|
//For example we may set: "sleepSecs = 10;" and "callCntMax = 24;" and following result:
|
|
//Call no: 1. Giving FarmMaps 10 seconds to get SatelliteItems...
|
|
//Call no: 1: Received 2 images
|
|
//Call no: 2. Giving FarmMaps 10 seconds to get SatelliteItems...
|
|
//Call no: 2: Received 7 images
|
|
//Call no: 3. Giving FarmMaps 10 seconds to get SatelliteItems...
|
|
//Call no: 3: Received 7 images
|
|
//And the firing of calls would stop because the number of images returned is no longer increasing
|
|
//In the worst case, this could would lead to a total sleeping period of "sleepSecsSum = sleepSecs * callCntMax" seconds. After that we give up
|
|
|
|
//This is an ugly fix. Neater would if FarmMaps would just take a bit longer and then do always deliver all satellite images on first call.
|
|
//Once this has been fixed on the side of FarmMaps we can set callCntMax = 0 and the code below will work smoothly without any sleeping
|
|
|
|
string downloadFolder = input.DownloadFolder;
|
|
if (string.IsNullOrEmpty(downloadFolder)) {
|
|
downloadFolder = "Downloads";
|
|
}
|
|
if (!Directory.Exists(downloadFolder))
|
|
Directory.CreateDirectory(downloadFolder);
|
|
|
|
// !!specify if you are using an already created cropfield:
|
|
bool useCreatedCropfield = input.UseCreatedCropfield;
|
|
var cropYear = input.CropYear;
|
|
var fieldName = input.fieldName;
|
|
bool storeSatelliteStatistics = input.StoreSatelliteStatisticsSingleImage;
|
|
bool storeSatelliteStatisticsCropYear = input.StoreSatelliteStatisticsCropYear;
|
|
//List<string> SatelliteBands = new List<string>(1) { input.SatelliteBand };
|
|
List<string> satelliteBands = input.SatelliteBands;
|
|
string headerLineStats = $"FieldName,satelliteDate,satelliteBand,max,min,mean,mode,median,stddev,minPlus,curtosis,maxMinus,skewness,variance,populationCount,variationCoefficient,confidenceIntervalLow, confidenceIntervalHigh,confidenceIntervalErrorMargin" + Environment.NewLine;
|
|
|
|
|
|
string settingsfile = $"Settings_{fieldName}.json";
|
|
|
|
LoadSettings(settingsfile);
|
|
|
|
var uploadedRoot = roots.SingleOrDefault(r => r.Name == "USER_IN");
|
|
if (uploadedRoot == null)
|
|
{
|
|
_logger.LogError("Could not find a needed root item");
|
|
return;
|
|
}
|
|
|
|
var myDriveRoot = roots.SingleOrDefault(r => r.Name == "USER_FILES");
|
|
if (myDriveRoot == null)
|
|
{
|
|
_logger.LogError("Could not find a needed root item");
|
|
return;
|
|
}
|
|
|
|
// Use already created cropfield or create new one
|
|
Item cropfieldItem;
|
|
if (useCreatedCropfield == false || string.IsNullOrEmpty(_settings.CropfieldItemCode))
|
|
{
|
|
_logger.LogInformation("Creating cropfield");
|
|
cropfieldItem = await _generalService.CreateCropfieldItemAsync(myDriveRoot.Code,
|
|
$"DataCropfield {input.OutputFileName}", cropYear, input.GeometryJson.ToString(Formatting.None));
|
|
_settings.CropfieldItemCode = cropfieldItem.Code;
|
|
SaveSettings(settingsfile);
|
|
}
|
|
else
|
|
{
|
|
_logger.LogInformation("Cropfield already exists, trying to get it");
|
|
cropfieldItem = await _farmmapsApiService.GetItemAsync(_settings.CropfieldItemCode);
|
|
}
|
|
|
|
//Get croprecordings
|
|
if (input.GetCropRecordings)
|
|
{
|
|
var crprecItem = input.CrprecItem;
|
|
_logger.LogInformation($"Trying to get crop recordings of croprecording: {crprecItem}");
|
|
|
|
var cropRec = await _farmmapsApiService.GetItemChildrenAsync(crprecItem, CROPREC_ITEMTYPE);
|
|
if (cropRec == null)
|
|
{
|
|
_logger.LogError("Something went wrong while obtaining the croprecordings");
|
|
return;
|
|
}
|
|
|
|
var cropRecPath = Path.Combine(downloadFolder, $"croprecordings_{crprecItem}.json");
|
|
_logger.LogInformation($"Found {cropRec.Count} crop recordings");
|
|
var count = 0;
|
|
await Task.Delay(500);
|
|
foreach (var item in cropRec)
|
|
{
|
|
Console.WriteLine($"Crop recording #{count}: {item.Name}");
|
|
File.AppendAllText(cropRecPath, item.Data +Environment.NewLine);
|
|
count++;
|
|
}
|
|
_logger.LogInformation($"Downloaded file {cropRecPath}");
|
|
}
|
|
|
|
// Get shadow data
|
|
if (input.GetShadowData)
|
|
{
|
|
_logger.LogInformation("Calculate shadow map for field");
|
|
var shadowItem = await _generalService.RunShadowTask(cropfieldItem);
|
|
if (shadowItem == null)
|
|
{
|
|
_logger.LogError("Something went wrong while obtaining the shadow map");
|
|
return;
|
|
}
|
|
|
|
_logger.LogInformation("Downloading shadow map");
|
|
await _farmmapsApiService.DownloadItemAsync(shadowItem.Code,
|
|
Path.Combine(downloadFolder, $"{input.OutputFileName}_shadow.zip"));
|
|
}
|
|
|
|
|
|
// Get satellite data
|
|
if (input.GetSatelliteData)
|
|
{
|
|
// check if satellite task not yet done, do here and save taskcode
|
|
if (useCreatedCropfield == false || string.IsNullOrEmpty(_settings.SatelliteTaskCode))
|
|
{
|
|
var satelliteTaskCode = await _generalService.RunSatelliteTask(cropfieldItem);
|
|
_settings.SatelliteTaskCode = satelliteTaskCode;
|
|
SaveSettings(settingsfile);
|
|
}
|
|
|
|
//Call first time
|
|
int callCnt = 1;
|
|
int sleepSecsSum = 0;
|
|
//if callCntMax == 0 then don't sleep
|
|
//if callCntMax = 1 then sleep first 1x
|
|
if (callCntMax > 0)
|
|
{
|
|
_logger.LogInformation($"Call no: {callCnt}. Giving FarmMaps {sleepSecs} seconds to get SatelliteItems...");
|
|
System.Threading.Thread.Sleep(1000 * sleepSecs);
|
|
sleepSecsSum = sleepSecsSum + sleepSecs;
|
|
}
|
|
List<Item> satelliteItemsCropYear = await _generalService.FindSatelliteItems(cropfieldItem, _settings.SatelliteTaskCode);
|
|
int satelliteItemsCropYearCntPrev = satelliteItemsCropYear.Count;
|
|
_logger.LogInformation($"Call no: {callCnt}. Received {satelliteItemsCropYearCntPrev} images");
|
|
callCnt++;
|
|
int satelliteItemsCropYearCnt = satelliteItemsCropYearCntPrev;
|
|
//if callCntMax > 1 then sleep untill (1) no more increase in number of images received OR (2) maximum number of calls reached
|
|
if (callCntMax > 1)
|
|
{
|
|
//Call second time
|
|
_logger.LogInformation($"Call no: {callCnt}. Giving FarmMaps another {sleepSecs} seconds to get SatelliteItems...");
|
|
System.Threading.Thread.Sleep(1000 * sleepSecs);
|
|
satelliteItemsCropYear = await _generalService.FindSatelliteItems(cropfieldItem, _settings.SatelliteTaskCode);
|
|
satelliteItemsCropYearCnt = satelliteItemsCropYear.Count;
|
|
_logger.LogInformation($"Call no: {callCnt}. Received {satelliteItemsCropYearCnt} images");
|
|
sleepSecsSum = sleepSecsSum + sleepSecs;
|
|
//As long as there is progress, keep calling
|
|
callCnt++;
|
|
while (callCnt <= callCntMax && (satelliteItemsCropYearCnt == 0 || satelliteItemsCropYearCnt > satelliteItemsCropYearCntPrev))
|
|
{
|
|
_logger.LogInformation($"Surprise! The longer we wait, the more images we get. Sleep and call once more");
|
|
satelliteItemsCropYearCntPrev = satelliteItemsCropYearCnt;
|
|
_logger.LogInformation($"Call no: {callCnt} (max: {callCntMax}). Giving FarmMaps another {sleepSecs} seconds to get SatelliteItems...");
|
|
System.Threading.Thread.Sleep(1000 * sleepSecs);
|
|
satelliteItemsCropYear = await _generalService.FindSatelliteItems(cropfieldItem, _settings.SatelliteTaskCode);
|
|
satelliteItemsCropYearCnt = satelliteItemsCropYear.Count;
|
|
_logger.LogInformation($"Call no: {callCnt}. Received {satelliteItemsCropYearCnt} images");
|
|
callCnt++;
|
|
sleepSecsSum = sleepSecsSum + sleepSecs;
|
|
}
|
|
}
|
|
|
|
if (satelliteItemsCropYearCnt == 0)
|
|
{
|
|
_logger.LogWarning($"DataDownloadApplication.cs: after calling one or more times and " +
|
|
$"sleeping in total {sleepSecsSum} seconds, still no images found. " +
|
|
$"Please check your settings for parameters callCntMax and sleepSecs in DataDownloadApplication.cs or contact FarmMaps");
|
|
}
|
|
|
|
satelliteItemsCropYear = satelliteItemsCropYear.OrderBy(x => x.DataDate).ToList();
|
|
|
|
if (input.StoreSatelliteStatisticsSingleImage == true && satelliteItemsCropYearCnt > 0) {
|
|
_logger.LogInformation("Available satellite images:");
|
|
var count = 0;
|
|
TimeSpan.FromSeconds(0.5);
|
|
foreach (var item in satelliteItemsCropYear)
|
|
{
|
|
|
|
Console.WriteLine($"Satellite image #{count}: {item.DataDate}");
|
|
count++;
|
|
}
|
|
|
|
_logger.LogInformation("Enter satellite image number");
|
|
int element = Int32.Parse(Console.ReadLine());
|
|
var selectedSatelliteItem = satelliteItemsCropYear[element];
|
|
|
|
var SatelliteDate = selectedSatelliteItem.DataDate.Value.ToString("yyyyMMdd");
|
|
string fileName = string.Format($"satelliteGeotiff_{fieldName}_{SatelliteDate}"); // no need to add satelliteBand in the name because the tif contains all bands
|
|
string fileNameZip = Path.Combine(downloadFolder, string.Format($"{fileName}.zip"));
|
|
await _farmmapsApiService.DownloadItemAsync(selectedSatelliteItem.Code, fileNameZip);
|
|
|
|
// Download a csv file with stats
|
|
List<Item> selectedSatelliteItems = new List<Item>(1) { selectedSatelliteItem };
|
|
string fileNameStats = Path.Combine(downloadFolder, string.Format($"satelliteStats_{fieldName}_{SatelliteDate}.csv"));
|
|
_logger.LogInformation($"First call to get DownloadSatelliteStats for selected image...");
|
|
string downloadedStats = await _generalService.DownloadSatelliteStats(selectedSatelliteItems, fieldName, satelliteBands, downloadFolder);
|
|
|
|
//rename the csv file with stats
|
|
//if the targe file already exists, delete it
|
|
File.Delete(fileNameStats);
|
|
//rename
|
|
File.Move(downloadedStats, fileNameStats);
|
|
|
|
// name the tif file
|
|
string fileNameTifzipped = Path.Combine(downloadFolder, string.Format($"sentinelhub_{SatelliteDate}.tif"));
|
|
string fileNameGeotiff = Path.Combine(downloadFolder, string.Format($"sentinelhub_{fieldName}_{SatelliteDate}.tif"));
|
|
// download the geotiffs. Returns a zip file with always these two files:
|
|
// thumbnail.jpg
|
|
// sentinelhub_yyyyMMdd.tif. Contains 4 layers: (1) ndvi, (2) wdvi, (3) ci-red and (4) natural. Natural has 3 layers inside: redBand, blueBand and greenBand
|
|
if (true)
|
|
{
|
|
// Extract the file fileNameTifzipped from zip, rename it to fileNameGeotiff
|
|
ZipFile.ExtractToDirectory(fileNameZip, downloadFolder, true);
|
|
//if the targe file already exists, delete it
|
|
File.Delete(fileNameGeotiff);
|
|
//rename
|
|
File.Move(fileNameTifzipped, fileNameGeotiff);
|
|
|
|
// Cleanup
|
|
File.Delete(fileNameZip);
|
|
File.Delete(Path.Combine(downloadFolder, "thumbnail.jpg"));
|
|
}
|
|
//_logger.LogInformation($"Downloaded files {fileNameGeotiff} and {fileNameStats} to {downloadFolder}");
|
|
_logger.LogInformation($"Downloaded files to {downloadFolder}");
|
|
|
|
}
|
|
if (input.StoreSatelliteStatisticsCropYear == true) {
|
|
string fileNameStats = Path.Combine(downloadFolder, string.Format($"satelliteStats_{fieldName}_{cropYear}.csv"));
|
|
File.Delete(fileNameStats);
|
|
_logger.LogInformation($"First call to get DownloadSatelliteStats for whole cropYear...");
|
|
string downloadedStats = await _generalService.DownloadSatelliteStats(satelliteItemsCropYear, fieldName, satelliteBands, downloadFolder);
|
|
File.Move(downloadedStats, fileNameStats);
|
|
_logger.LogInformation($"Downloaded file {fileNameStats} with stats for field '{fieldName}', cropyear {cropYear}");
|
|
}
|
|
}
|
|
|
|
|
|
|
|
// Get vanDerSat data
|
|
if (input.GetVanDerSatData)
|
|
{
|
|
// check if satellite task not yet done, do here and save taskcode
|
|
if (useCreatedCropfield == false || string.IsNullOrEmpty(_settings.VanDerSatTaskCode))
|
|
{
|
|
var vanDerSatTaskCode = await _generalService.RunVanDerSatTask(cropfieldItem);
|
|
_settings.VanDerSatTaskCode = vanDerSatTaskCode;
|
|
SaveSettings(settingsfile);
|
|
}
|
|
|
|
// Select a particular satellite item from satelliteTask
|
|
Item vanDerSatItem = await _generalService.FindVanDerSatItem(cropfieldItem, _settings.VanDerSatTaskCode, fieldName, input.StoreVanDerSatStatistics);
|
|
}
|
|
|
|
|
|
}
|
|
|
|
// Functions to save previously created cropfields
|
|
private void LoadSettings(string file)
|
|
{
|
|
if (File.Exists(file))
|
|
{
|
|
var jsonText = File.ReadAllText(file);
|
|
_settings = JsonConvert.DeserializeObject<Settings>(jsonText);
|
|
}
|
|
else
|
|
{
|
|
_settings = new Settings();
|
|
}
|
|
}
|
|
|
|
private void SaveSettings(string file)
|
|
{
|
|
if (_settings == null)
|
|
return;
|
|
|
|
var json = JsonConvert.SerializeObject(_settings);
|
|
File.WriteAllText(file, json);
|
|
}
|
|
private void SaveInfo(string file)
|
|
{
|
|
if (_settings == null)
|
|
return;
|
|
|
|
var json = JsonConvert.SerializeObject(_settings);
|
|
File.WriteAllText(file, json);
|
|
|
|
}
|
|
|
|
}
|
|
}
|