pvanoort/FarmMapsApiClient_WUR
1
0
Fork 0
forked from FarmMaps/FarmMapsApiClient
Code Pull Requests Activity

fix BulkSatDownloadApplication: fire a number of calls awaiting satellite task, until number of images returned no longer increases

Browse Source
This commit is contained in:
Pepijn van Oort
2023-08-01 11:03:40 +02:00
parent 334d890f13
commit 166fd1f66d
2 changed files with 272 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

97
FarmmapsBulkSatDownload/BulkSatDownloadApplication.cs
View File

@@ -25,6 +25,7 @@ namespace FarmmapsBulkSatDownload
private readonly GeneralService _generalService;
public const string settingsfile = "Settings.json";
public const int firstAvailableYear = 2017;
private Settings _settings;
public BulkSatDownloadApplication(ILogger<BulkSatDownloadApplication> logger, FarmmapsApiService farmmapsApiService,
@@ -199,6 +200,27 @@ namespace FarmmapsBulkSatDownload
private async Task Process(List<UserRoot> roots, BulkSatDownloadInput input)
{
//PO20220311: first time a call is made to download satellite images or statistics, an empty list is returned
//If we wait a bit longer, e.g. 10 secs, then e.g. a list of 3 images may be returned
//If we wait still longer, maybe 4 images.
//The solution implemented below is to fire calls as long as the number of images returned keeps increasing
//While in between each call, sleep for sleepSecs
//Continue this until the number no longer increases or the maximum number of calls has been reached
//If you set sleepSecs to a very low value, e.g. 5 secs, then after 1 call you might get images and after 2nd call still zero images.
//to be on the safe side, better bit higher value.
//Just accept this may take a while, have a coffee, we suggest sleepSecs = 30;
int sleepSecs = 30;
int callCntMax = 4 * 60 / sleepSecs; //4*60 = max 4 minutes
//For example we may set: "sleepSecs = 10;" and "callCntMax = 24;" and following result:
//Call no: 1. Giving FarmMaps 10 seconds to get SatelliteItems...
//Call no: 1: Received 2 images
//Call no: 2. Giving FarmMaps 10 seconds to get SatelliteItems...
//Call no: 2: Received 7 images
//Call no: 3. Giving FarmMaps 10 seconds to get SatelliteItems...
//Call no: 3: Received 7 images
//And the firing of calls would stop because the number of images returned is no longer increasing
//In the worst case, this could would lead to a total sleeping period of "sleepSecsSum = sleepSecs * callCntMax" seconds. After that we give up
string cropfielditemcode;
string satellitetaskcode;
Item cropfieldItem;
@@ -230,6 +252,10 @@ namespace FarmmapsBulkSatDownload
DateTime lastDownloadedSatelliteDate = input.lastdownloadedimagedate;
cropfielditemcode = input.cropfielditemcode;
satellitetaskcode = input.satellitetaskcode;
int satelliteItemsCropYearCntPrev;
int satelliteItemsCropYearCnt;
int callCnt;
int sleepSecsSum;
LoadSettings(settingsfile);
@@ -317,23 +343,80 @@ namespace FarmmapsBulkSatDownload
// TODO also log satellitetaskcode to settings, how?
// SaveSettings(settingsfile);
// Getting satellite items
_logger.LogInformation(string.Format($"Running FindSatelliteItems for cropfieldItem.Code '{cropfieldItem.Code}', SatelliteTaskCode '{satellitetaskcode}'"));
satelliteItemsCropYear = await _generalService.FindSatelliteItems(cropfieldItem, satellitetaskcode);
// Getting satellite items. Only for years for which available
satelliteItemsCropYearCntPrev = 0;
satelliteItemsCropYearCnt = 0;
sleepSecsSum = 0;
satelliteItemsCropYear = null;
if (cropYear >= firstAvailableYear && cropYear <= DateTime.Now.Year)
{
_logger.LogInformation(string.Format($"Running FindSatelliteItems for cropfieldItem.Code '{cropfieldItem.Code}', SatelliteTaskCode '{satellitetaskcode}'"));
//Call first time
callCnt = 1;
//if callCntMax == 0 then don't sleep
//if callCntMax = 1 then sleep first 1x
if (callCntMax > 0)
{
_logger.LogInformation($"Call no: {callCnt}. Giving FarmMaps {sleepSecs} seconds to get SatelliteItems...");
System.Threading.Thread.Sleep(1000 * sleepSecs);
sleepSecsSum = sleepSecsSum + sleepSecs;
}
satelliteItemsCropYear = await _generalService.FindSatelliteItems(cropfieldItem, satellitetaskcode);
satelliteItemsCropYearCntPrev = satelliteItemsCropYear.Count;
_logger.LogInformation($"Call no: {callCnt}. Received {satelliteItemsCropYearCntPrev} images");
callCnt++;
satelliteItemsCropYearCnt = satelliteItemsCropYearCntPrev;
//if callCntMax > 1 then sleep untill (1) no more increase in number of images received OR (2) maximum number of calls reached
if (callCntMax > 1)
{
//Call second time
_logger.LogInformation($"Call no: {callCnt}. Giving FarmMaps another {sleepSecs} seconds to get SatelliteItems...");
System.Threading.Thread.Sleep(1000 * sleepSecs);
satelliteItemsCropYear = await _generalService.FindSatelliteItems(cropfieldItem, satellitetaskcode);
satelliteItemsCropYearCnt = satelliteItemsCropYear.Count;
_logger.LogInformation($"Call no: {callCnt}. Received {satelliteItemsCropYearCnt} images");
sleepSecsSum = sleepSecsSum + sleepSecs;
//As long as there is progress, keep calling
callCnt++;
while (callCnt <= callCntMax && (satelliteItemsCropYearCnt == 0 || satelliteItemsCropYearCnt > satelliteItemsCropYearCntPrev))
{
_logger.LogInformation($"Surprise! The longer we wait, the more images we get. Sleep and call once more");
satelliteItemsCropYearCntPrev = satelliteItemsCropYearCnt;
_logger.LogInformation($"Call no: {callCnt} (max: {callCntMax}). Giving FarmMaps another {sleepSecs} seconds to get SatelliteItems...");
System.Threading.Thread.Sleep(1000 * sleepSecs);
satelliteItemsCropYear = await _generalService.FindSatelliteItems(cropfieldItem, satellitetaskcode);
satelliteItemsCropYearCnt = satelliteItemsCropYear.Count;
_logger.LogInformation($"Call no: {callCnt}. Received {satelliteItemsCropYearCnt} images");
callCnt++;
sleepSecsSum = sleepSecsSum + sleepSecs;
}
}
}
else
{
_logger.LogWarning($"// FarmmapsBulkSatDownload: crop year {cropYear} is out of bounds. No stats will be written!");
}
if (satelliteItemsCropYearCnt == 0)
{
_logger.LogWarning($"// FarmmapsBulkSatDownload: after calling one or more times and " +
$"sleeping in total {sleepSecsSum} seconds, still no images found. " +
$"Please check your settings for parameters callCntMax and sleepSecs in FarmmapsBulkSatDownload.cs or contact FarmMaps");
}
// Checking if satellite items found
satelliteItemsAvailable = true;
if (satelliteItemsCropYear == null)
{
satelliteItemsAvailable = false;
_logger.LogInformation($"No satellite tiffs found for fieldName '{fieldName}', cropYear {cropYear}, cropfielditemcode '{cropfielditemcode}'");
_logger.LogInformation($"// FarmmapsBulkSatDownload: No satellite tiffs found for fieldName '{fieldName}', cropYear {cropYear}, cropfielditemcode '{cropfielditemcode}'");
}
else
{
if (satelliteItemsCropYear.Count == 0)
{
satelliteItemsAvailable = false;
_logger.LogInformation($"No satellite tiffs found for fieldName '{fieldName}', cropYear {cropYear}, cropfielditemcode '{cropfielditemcode}'");
_logger.LogInformation($"// FarmmapsBulkSatDownload: No satellite tiffs found for fieldName '{fieldName}', cropYear {cropYear}, cropfielditemcode '{cropfielditemcode}'");
}
}
@@ -343,10 +426,10 @@ namespace FarmmapsBulkSatDownload
// Download statistics to a single csv file
if (satelliteItemsAvailable && downloadFolder != null && fileNameStats != null)
{
// Write statistics for all images for all fieldNane and cropYear to a single csv file, fileNameStats
// Write statistics for all images for all fieldName and cropYear to a single csv file, fileNameStats
_logger.LogInformation($"Downloading stats for field '{fieldName}' in cropyear {cropYear} to {fileNameStats}");
string downloadedStats = await _generalService.DownloadSatelliteStats(satelliteItemsCropYear, fieldName, satelliteBands, downloadFolder);
// Add contents of this csv file to thee single large csv file
// Add contents of this csv file to the single large csv file
var retainedLines = File.ReadAllLines(downloadedStats).Skip(1);
File.AppendAllLines(fileNameStats, retainedLines);
File.Delete(downloadedStats);