forked from FarmMaps/FarmMapsApiClient
fix BulkSatDownloadApplication: fire a number of calls awaiting satellite task, until number of images returned no longer increases
This commit is contained in:
parent
334d890f13
commit
166fd1f66d
@ -25,6 +25,7 @@ namespace FarmmapsBulkSatDownload
|
||||
private readonly GeneralService _generalService;
|
||||
|
||||
public const string settingsfile = "Settings.json";
|
||||
public const int firstAvailableYear = 2017;
|
||||
private Settings _settings;
|
||||
|
||||
public BulkSatDownloadApplication(ILogger<BulkSatDownloadApplication> logger, FarmmapsApiService farmmapsApiService,
|
||||
@ -199,6 +200,27 @@ namespace FarmmapsBulkSatDownload
|
||||
|
||||
private async Task Process(List<UserRoot> roots, BulkSatDownloadInput input)
|
||||
{
|
||||
//PO20220311: first time a call is made to download satellite images or statistics, an empty list is returned
|
||||
//If we wait a bit longer, e.g. 10 secs, then e.g. a list of 3 images may be returned
|
||||
//If we wait still longer, maybe 4 images.
|
||||
//The solution implemented below is to fire calls as long as the number of images returned keeps increasing
|
||||
//While in between each call, sleep for sleepSecs
|
||||
//Continue this until the number no longer increases or the maximum number of calls has been reached
|
||||
//If you set sleepSecs to a very low value, e.g. 5 secs, then after 1 call you might get images and after 2nd call still zero images.
|
||||
//to be on the safe side, better bit higher value.
|
||||
//Just accept this may take a while, have a coffee, we suggest sleepSecs = 30;
|
||||
int sleepSecs = 30;
|
||||
int callCntMax = 4 * 60 / sleepSecs; //4*60 = max 4 minutes
|
||||
//For example we may set: "sleepSecs = 10;" and "callCntMax = 24;" and following result:
|
||||
//Call no: 1. Giving FarmMaps 10 seconds to get SatelliteItems...
|
||||
//Call no: 1: Received 2 images
|
||||
//Call no: 2. Giving FarmMaps 10 seconds to get SatelliteItems...
|
||||
//Call no: 2: Received 7 images
|
||||
//Call no: 3. Giving FarmMaps 10 seconds to get SatelliteItems...
|
||||
//Call no: 3: Received 7 images
|
||||
//And the firing of calls would stop because the number of images returned is no longer increasing
|
||||
//In the worst case, this could would lead to a total sleeping period of "sleepSecsSum = sleepSecs * callCntMax" seconds. After that we give up
|
||||
|
||||
string cropfielditemcode;
|
||||
string satellitetaskcode;
|
||||
Item cropfieldItem;
|
||||
@ -230,6 +252,10 @@ namespace FarmmapsBulkSatDownload
|
||||
DateTime lastDownloadedSatelliteDate = input.lastdownloadedimagedate;
|
||||
cropfielditemcode = input.cropfielditemcode;
|
||||
satellitetaskcode = input.satellitetaskcode;
|
||||
int satelliteItemsCropYearCntPrev;
|
||||
int satelliteItemsCropYearCnt;
|
||||
int callCnt;
|
||||
int sleepSecsSum;
|
||||
|
||||
LoadSettings(settingsfile);
|
||||
|
||||
@ -317,23 +343,80 @@ namespace FarmmapsBulkSatDownload
|
||||
// TODO also log satellitetaskcode to settings, how?
|
||||
// SaveSettings(settingsfile);
|
||||
|
||||
// Getting satellite items
|
||||
// Getting satellite items. Only for years for which available
|
||||
satelliteItemsCropYearCntPrev = 0;
|
||||
satelliteItemsCropYearCnt = 0;
|
||||
sleepSecsSum = 0;
|
||||
satelliteItemsCropYear = null;
|
||||
if (cropYear >= firstAvailableYear && cropYear <= DateTime.Now.Year)
|
||||
{
|
||||
_logger.LogInformation(string.Format($"Running FindSatelliteItems for cropfieldItem.Code '{cropfieldItem.Code}', SatelliteTaskCode '{satellitetaskcode}'"));
|
||||
//Call first time
|
||||
callCnt = 1;
|
||||
//if callCntMax == 0 then don't sleep
|
||||
//if callCntMax = 1 then sleep first 1x
|
||||
if (callCntMax > 0)
|
||||
{
|
||||
_logger.LogInformation($"Call no: {callCnt}. Giving FarmMaps {sleepSecs} seconds to get SatelliteItems...");
|
||||
System.Threading.Thread.Sleep(1000 * sleepSecs);
|
||||
sleepSecsSum = sleepSecsSum + sleepSecs;
|
||||
}
|
||||
satelliteItemsCropYear = await _generalService.FindSatelliteItems(cropfieldItem, satellitetaskcode);
|
||||
satelliteItemsCropYearCntPrev = satelliteItemsCropYear.Count;
|
||||
_logger.LogInformation($"Call no: {callCnt}. Received {satelliteItemsCropYearCntPrev} images");
|
||||
callCnt++;
|
||||
satelliteItemsCropYearCnt = satelliteItemsCropYearCntPrev;
|
||||
//if callCntMax > 1 then sleep untill (1) no more increase in number of images received OR (2) maximum number of calls reached
|
||||
if (callCntMax > 1)
|
||||
{
|
||||
//Call second time
|
||||
_logger.LogInformation($"Call no: {callCnt}. Giving FarmMaps another {sleepSecs} seconds to get SatelliteItems...");
|
||||
System.Threading.Thread.Sleep(1000 * sleepSecs);
|
||||
satelliteItemsCropYear = await _generalService.FindSatelliteItems(cropfieldItem, satellitetaskcode);
|
||||
satelliteItemsCropYearCnt = satelliteItemsCropYear.Count;
|
||||
_logger.LogInformation($"Call no: {callCnt}. Received {satelliteItemsCropYearCnt} images");
|
||||
sleepSecsSum = sleepSecsSum + sleepSecs;
|
||||
//As long as there is progress, keep calling
|
||||
callCnt++;
|
||||
while (callCnt <= callCntMax && (satelliteItemsCropYearCnt == 0 || satelliteItemsCropYearCnt > satelliteItemsCropYearCntPrev))
|
||||
{
|
||||
_logger.LogInformation($"Surprise! The longer we wait, the more images we get. Sleep and call once more");
|
||||
satelliteItemsCropYearCntPrev = satelliteItemsCropYearCnt;
|
||||
_logger.LogInformation($"Call no: {callCnt} (max: {callCntMax}). Giving FarmMaps another {sleepSecs} seconds to get SatelliteItems...");
|
||||
System.Threading.Thread.Sleep(1000 * sleepSecs);
|
||||
satelliteItemsCropYear = await _generalService.FindSatelliteItems(cropfieldItem, satellitetaskcode);
|
||||
satelliteItemsCropYearCnt = satelliteItemsCropYear.Count;
|
||||
_logger.LogInformation($"Call no: {callCnt}. Received {satelliteItemsCropYearCnt} images");
|
||||
callCnt++;
|
||||
sleepSecsSum = sleepSecsSum + sleepSecs;
|
||||
}
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
_logger.LogWarning($"// FarmmapsBulkSatDownload: crop year {cropYear} is out of bounds. No stats will be written!");
|
||||
}
|
||||
|
||||
if (satelliteItemsCropYearCnt == 0)
|
||||
{
|
||||
_logger.LogWarning($"// FarmmapsBulkSatDownload: after calling one or more times and " +
|
||||
$"sleeping in total {sleepSecsSum} seconds, still no images found. " +
|
||||
$"Please check your settings for parameters callCntMax and sleepSecs in FarmmapsBulkSatDownload.cs or contact FarmMaps");
|
||||
}
|
||||
|
||||
// Checking if satellite items found
|
||||
satelliteItemsAvailable = true;
|
||||
if (satelliteItemsCropYear == null)
|
||||
{
|
||||
satelliteItemsAvailable = false;
|
||||
_logger.LogInformation($"No satellite tiffs found for fieldName '{fieldName}', cropYear {cropYear}, cropfielditemcode '{cropfielditemcode}'");
|
||||
_logger.LogInformation($"// FarmmapsBulkSatDownload: No satellite tiffs found for fieldName '{fieldName}', cropYear {cropYear}, cropfielditemcode '{cropfielditemcode}'");
|
||||
}
|
||||
else
|
||||
{
|
||||
if (satelliteItemsCropYear.Count == 0)
|
||||
{
|
||||
satelliteItemsAvailable = false;
|
||||
_logger.LogInformation($"No satellite tiffs found for fieldName '{fieldName}', cropYear {cropYear}, cropfielditemcode '{cropfielditemcode}'");
|
||||
_logger.LogInformation($"// FarmmapsBulkSatDownload: No satellite tiffs found for fieldName '{fieldName}', cropYear {cropYear}, cropfielditemcode '{cropfielditemcode}'");
|
||||
}
|
||||
}
|
||||
|
||||
@ -343,10 +426,10 @@ namespace FarmmapsBulkSatDownload
|
||||
// Download statistics to a single csv file
|
||||
if (satelliteItemsAvailable && downloadFolder != null && fileNameStats != null)
|
||||
{
|
||||
// Write statistics for all images for all fieldNane and cropYear to a single csv file, fileNameStats
|
||||
// Write statistics for all images for all fieldName and cropYear to a single csv file, fileNameStats
|
||||
_logger.LogInformation($"Downloading stats for field '{fieldName}' in cropyear {cropYear} to {fileNameStats}");
|
||||
string downloadedStats = await _generalService.DownloadSatelliteStats(satelliteItemsCropYear, fieldName, satelliteBands, downloadFolder);
|
||||
// Add contents of this csv file to thee single large csv file
|
||||
// Add contents of this csv file to the single large csv file
|
||||
var retainedLines = File.ReadAllLines(downloadedStats).Skip(1);
|
||||
File.AppendAllLines(fileNameStats, retainedLines);
|
||||
File.Delete(downloadedStats);
|
||||
|
||||
@ -1,9 +1,34 @@
|
||||
[
|
||||
{
|
||||
"fieldName": "BvdT FieldlabG92", // FarmMaps minimum needs are: fieldName, cropYear & geometryJson
|
||||
"cropYear": 2022, // for testing same field last year
|
||||
"fieldName": "MyField_1", // FarmMaps minimum needs are: fieldName, cropYear & geometryJson
|
||||
"cropYear": 2023,
|
||||
"fieldID": 1,
|
||||
"SatelliteBands": [ "wdvi", "ndvi" ], // ["ndvi"] or ["wdvi"] or both: [ "wdvi", "ndvi" ]
|
||||
"SatelliteBands": [ "ci-red", "ndvi", "wdvi" ], // ["ndvi"] or ["wdvi"] or both: [ "wdvi", "ndvi" ]
|
||||
"lastdownloadedimagedate": "2023-01-01", //downloads images from this date till end of the year
|
||||
"geometryJson": {
|
||||
"type": "Polygon",
|
||||
"coordinates": [
|
||||
[
|
||||
[ 5.563472073408009, 52.547554398144172 ],
|
||||
[ 5.567425915520115, 52.547725375100377 ],
|
||||
[ 5.563878143678981, 52.54048022658143 ],
|
||||
[ 5.563878143678981, 52.54048022658143 ],
|
||||
[ 5.563472073408009, 52.547554398144172 ]
|
||||
]
|
||||
]
|
||||
},
|
||||
"downloadFolder": "C:\\workdir\\groenmonitor\\", //if not yet existing this folder will be created
|
||||
"fileNameStats": "BulkSatDownload.csv", //if file exists, probably will be overwritten. Check code in BulkSatDownloadApplication.cs
|
||||
"database": null, // keep null to work with json and csv. Check code in BulkSatDownloadApplication.cs if reading/writing to/from database
|
||||
"schemaname": null,
|
||||
"cropfieldtable": null,
|
||||
"satelllitetable": null
|
||||
},
|
||||
{
|
||||
"fieldName": "MyField_2", // FarmMaps minimum needs are: fieldName, cropYear & geometryJson
|
||||
"cropYear": 2022, // for testing same field last year (i.e. 2021)
|
||||
"fieldID": 2,
|
||||
"SatelliteBands": [ "ci-red", "ndvi", "wdvi" ], // ["ndvi"] or ["wdvi"] or both: [ "wdvi", "ndvi" ]
|
||||
"lastdownloadedimagedate": "2022-01-01", //downloads images from this date till end of the year
|
||||
"geometryJson": {
|
||||
"type": "Polygon",
|
||||
@ -25,10 +50,10 @@
|
||||
"satelllitetable": null
|
||||
},
|
||||
{
|
||||
"fieldName": "BvdT FieldlabG92", // FarmMaps minimum needs are: fieldName, cropYear & geometryJson
|
||||
"cropYear": 2021, // for testing same field last year (i.e. 2021)
|
||||
"fieldID": 1,
|
||||
"SatelliteBands": [ "wdvi", "ndvi" ], // ["ndvi"] or ["wdvi"] or both: [ "wdvi", "ndvi" ]
|
||||
"fieldName": "MyField_3", // FarmMaps minimum needs are: fieldName, cropYear & geometryJson
|
||||
"cropYear": 2021,
|
||||
"fieldID": 3,
|
||||
"SatelliteBands": [ "ci-red", "ndvi", "wdvi" ], // ["ndvi"] or ["wdvi"] or both: [ "wdvi", "ndvi" ]
|
||||
"lastdownloadedimagedate": "2021-01-01", //downloads images from this date till end of the year
|
||||
"geometryJson": {
|
||||
"type": "Polygon",
|
||||
@ -48,5 +73,155 @@
|
||||
"schemaname": null,
|
||||
"cropfieldtable": null,
|
||||
"satelllitetable": null
|
||||
},
|
||||
{
|
||||
"fieldName": "MyField_4", // FarmMaps minimum needs are: fieldName, cropYear & geometryJson
|
||||
"cropYear": 2020,
|
||||
"fieldID": 4,
|
||||
"SatelliteBands": [ "ci-red", "ndvi", "wdvi" ], // ["ndvi"] or ["wdvi"] or both: [ "wdvi", "ndvi" ]
|
||||
"lastdownloadedimagedate": "2020-01-01", //downloads images from this date till end of the year
|
||||
"geometryJson": {
|
||||
"type": "Polygon",
|
||||
"coordinates": [
|
||||
[
|
||||
[ 5.563472073408009, 52.547554398144172 ],
|
||||
[ 5.567425915520115, 52.547725375100377 ],
|
||||
[ 5.563878143678981, 52.54048022658143 ],
|
||||
[ 5.563878143678981, 52.54048022658143 ],
|
||||
[ 5.563472073408009, 52.547554398144172 ]
|
||||
]
|
||||
]
|
||||
},
|
||||
"downloadFolder": "C:\\workdir\\groenmonitor\\",
|
||||
"fileNameStats": "BulkSatDownload.csv",
|
||||
"database": null,
|
||||
"schemaname": null,
|
||||
"cropfieldtable": null,
|
||||
"satelllitetable": null
|
||||
},
|
||||
{
|
||||
"fieldName": "MyField_5", // FarmMaps minimum needs are: fieldName, cropYear & geometryJson
|
||||
"cropYear": 2019,
|
||||
"fieldID": 5,
|
||||
"SatelliteBands": [ "ci-red", "ndvi", "wdvi" ], // ["ndvi"] or ["wdvi"] or both: [ "wdvi", "ndvi" ]
|
||||
"lastdownloadedimagedate": "2019-01-01", //downloads images from this date till end of the year
|
||||
"geometryJson": {
|
||||
"type": "Polygon",
|
||||
"coordinates": [
|
||||
[
|
||||
[ 5.563472073408009, 52.547554398144172 ],
|
||||
[ 5.567425915520115, 52.547725375100377 ],
|
||||
[ 5.563878143678981, 52.54048022658143 ],
|
||||
[ 5.563878143678981, 52.54048022658143 ],
|
||||
[ 5.563472073408009, 52.547554398144172 ]
|
||||
]
|
||||
]
|
||||
},
|
||||
"downloadFolder": "C:\\workdir\\groenmonitor\\",
|
||||
"fileNameStats": "BulkSatDownload.csv",
|
||||
"database": null,
|
||||
"schemaname": null,
|
||||
"cropfieldtable": null,
|
||||
"satelllitetable": null
|
||||
},
|
||||
{
|
||||
"fieldName": "MyField_6", // FarmMaps minimum needs are: fieldName, cropYear & geometryJson
|
||||
"cropYear": 2018,
|
||||
"fieldID": 6,
|
||||
"SatelliteBands": [ "ci-red", "ndvi", "wdvi" ], // ["ndvi"] or ["wdvi"] or both: [ "wdvi", "ndvi" ]
|
||||
"lastdownloadedimagedate": "2018-01-01", //downloads images from this date till end of the year
|
||||
"geometryJson": {
|
||||
"type": "Polygon",
|
||||
"coordinates": [
|
||||
[
|
||||
[ 5.563472073408009, 52.547554398144172 ],
|
||||
[ 5.567425915520115, 52.547725375100377 ],
|
||||
[ 5.563878143678981, 52.54048022658143 ],
|
||||
[ 5.563878143678981, 52.54048022658143 ],
|
||||
[ 5.563472073408009, 52.547554398144172 ]
|
||||
]
|
||||
]
|
||||
},
|
||||
"downloadFolder": "C:\\workdir\\groenmonitor\\",
|
||||
"fileNameStats": "BulkSatDownload.csv",
|
||||
"database": null,
|
||||
"schemaname": null,
|
||||
"cropfieldtable": null,
|
||||
"satelllitetable": null
|
||||
},
|
||||
{
|
||||
"fieldName": "MyField_6", // FarmMaps minimum needs are: fieldName, cropYear & geometryJson
|
||||
"cropYear": 2017,
|
||||
"fieldID": 7,
|
||||
"SatelliteBands": [ "ci-red", "ndvi", "wdvi" ], // ["ndvi"] or ["wdvi"] or both: [ "wdvi", "ndvi" ]
|
||||
"lastdownloadedimagedate": "2017-01-01", //downloads images from this date till end of the year
|
||||
"geometryJson": {
|
||||
"type": "Polygon",
|
||||
"coordinates": [
|
||||
[
|
||||
[ 5.563472073408009, 52.547554398144172 ],
|
||||
[ 5.567425915520115, 52.547725375100377 ],
|
||||
[ 5.563878143678981, 52.54048022658143 ],
|
||||
[ 5.563878143678981, 52.54048022658143 ],
|
||||
[ 5.563472073408009, 52.547554398144172 ]
|
||||
]
|
||||
]
|
||||
},
|
||||
"downloadFolder": "C:\\workdir\\groenmonitor\\",
|
||||
"fileNameStats": "BulkSatDownload.csv",
|
||||
"database": null,
|
||||
"schemaname": null,
|
||||
"cropfieldtable": null,
|
||||
"satelllitetable": null
|
||||
},
|
||||
{
|
||||
"fieldName": "MyField_8", // FarmMaps minimum needs are: fieldName, cropYear & geometryJson. Actually no Satellite data before 2017. This here to illustrate the program does not crash
|
||||
"cropYear": 2016,
|
||||
"fieldID": 8,
|
||||
"SatelliteBands": [ "ci-red", "ndvi", "wdvi" ], // ["ndvi"] or ["wdvi"] or both: [ "wdvi", "ndvi" ]
|
||||
"lastdownloadedimagedate": "2016-01-01", //downloads images from this date till end of the year
|
||||
"geometryJson": {
|
||||
"type": "Polygon",
|
||||
"coordinates": [
|
||||
[
|
||||
[ 5.563472073408009, 52.547554398144172 ],
|
||||
[ 5.567425915520115, 52.547725375100377 ],
|
||||
[ 5.563878143678981, 52.54048022658143 ],
|
||||
[ 5.563878143678981, 52.54048022658143 ],
|
||||
[ 5.563472073408009, 52.547554398144172 ]
|
||||
]
|
||||
]
|
||||
},
|
||||
"downloadFolder": "C:\\workdir\\groenmonitor\\",
|
||||
"fileNameStats": "BulkSatDownload.csv",
|
||||
"database": null,
|
||||
"schemaname": null,
|
||||
"cropfieldtable": null,
|
||||
"satelllitetable": null
|
||||
},
|
||||
{
|
||||
"fieldName": "MyField_9", // FarmMaps minimum needs are: fieldName, cropYear & geometryJson. Actually no Satellite data before 2017. This here to illustrate the program does not crash
|
||||
"cropYear": 2015,
|
||||
"fieldID": 9,
|
||||
"SatelliteBands": [ "ci-red", "ndvi", "wdvi" ], // ["ndvi"] or ["wdvi"] or both: [ "wdvi", "ndvi" ]
|
||||
"lastdownloadedimagedate": "2015-01-01", //downloads images from this date till end of the year
|
||||
"geometryJson": {
|
||||
"type": "Polygon",
|
||||
"coordinates": [
|
||||
[
|
||||
[ 5.563472073408009, 52.547554398144172 ],
|
||||
[ 5.567425915520115, 52.547725375100377 ],
|
||||
[ 5.563878143678981, 52.54048022658143 ],
|
||||
[ 5.563878143678981, 52.54048022658143 ],
|
||||
[ 5.563472073408009, 52.547554398144172 ]
|
||||
]
|
||||
]
|
||||
},
|
||||
"downloadFolder": "C:\\workdir\\groenmonitor\\",
|
||||
"fileNameStats": "BulkSatDownload.csv",
|
||||
"database": null,
|
||||
"schemaname": null,
|
||||
"cropfieldtable": null,
|
||||
"satelllitetable": null
|
||||
}
|
||||
]
|
||||
Loading…
Reference in New Issue
Block a user