From 166fd1f66db22bfcb8b25cc1c02c7116feeebe75 Mon Sep 17 00:00:00 2001 From: Pepijn van Oort Date: Tue, 1 Aug 2023 11:03:40 +0200 Subject: [PATCH] fix BulkSatDownloadApplication: fire a number of calls awaiting satellite task, until number of images returned no longer increases --- .../BulkSatDownloadApplication.cs | 97 ++++++++- .../BulkSatDownloadInput.json | 189 +++++++++++++++++- 2 files changed, 272 insertions(+), 14 deletions(-) diff --git a/FarmmapsBulkSatDownload/BulkSatDownloadApplication.cs b/FarmmapsBulkSatDownload/BulkSatDownloadApplication.cs index ba0e346..6b14367 100644 --- a/FarmmapsBulkSatDownload/BulkSatDownloadApplication.cs +++ b/FarmmapsBulkSatDownload/BulkSatDownloadApplication.cs @@ -25,6 +25,7 @@ namespace FarmmapsBulkSatDownload private readonly GeneralService _generalService; public const string settingsfile = "Settings.json"; + public const int firstAvailableYear = 2017; private Settings _settings; public BulkSatDownloadApplication(ILogger logger, FarmmapsApiService farmmapsApiService, @@ -199,6 +200,27 @@ namespace FarmmapsBulkSatDownload private async Task Process(List roots, BulkSatDownloadInput input) { + //PO20220311: first time a call is made to download satellite images or statistics, an empty list is returned + //If we wait a bit longer, e.g. 10 secs, then e.g. a list of 3 images may be returned + //If we wait still longer, maybe 4 images. + //The solution implemented below is to fire calls as long as the number of images returned keeps increasing + //While in between each call, sleep for sleepSecs + //Continue this until the number no longer increases or the maximum number of calls has been reached + //If you set sleepSecs to a very low value, e.g. 5 secs, then after 1 call you might get images and after 2nd call still zero images. + //to be on the safe side, better bit higher value. + //Just accept this may take a while, have a coffee, we suggest sleepSecs = 30; + int sleepSecs = 30; + int callCntMax = 4 * 60 / sleepSecs; //4*60 = max 4 minutes + //For example we may set: "sleepSecs = 10;" and "callCntMax = 24;" and following result: + //Call no: 1. Giving FarmMaps 10 seconds to get SatelliteItems... + //Call no: 1: Received 2 images + //Call no: 2. Giving FarmMaps 10 seconds to get SatelliteItems... + //Call no: 2: Received 7 images + //Call no: 3. Giving FarmMaps 10 seconds to get SatelliteItems... + //Call no: 3: Received 7 images + //And the firing of calls would stop because the number of images returned is no longer increasing + //In the worst case, this could would lead to a total sleeping period of "sleepSecsSum = sleepSecs * callCntMax" seconds. After that we give up + string cropfielditemcode; string satellitetaskcode; Item cropfieldItem; @@ -230,6 +252,10 @@ namespace FarmmapsBulkSatDownload DateTime lastDownloadedSatelliteDate = input.lastdownloadedimagedate; cropfielditemcode = input.cropfielditemcode; satellitetaskcode = input.satellitetaskcode; + int satelliteItemsCropYearCntPrev; + int satelliteItemsCropYearCnt; + int callCnt; + int sleepSecsSum; LoadSettings(settingsfile); @@ -317,23 +343,80 @@ namespace FarmmapsBulkSatDownload // TODO also log satellitetaskcode to settings, how? // SaveSettings(settingsfile); - // Getting satellite items - _logger.LogInformation(string.Format($"Running FindSatelliteItems for cropfieldItem.Code '{cropfieldItem.Code}', SatelliteTaskCode '{satellitetaskcode}'")); - satelliteItemsCropYear = await _generalService.FindSatelliteItems(cropfieldItem, satellitetaskcode); + // Getting satellite items. Only for years for which available + satelliteItemsCropYearCntPrev = 0; + satelliteItemsCropYearCnt = 0; + sleepSecsSum = 0; + satelliteItemsCropYear = null; + if (cropYear >= firstAvailableYear && cropYear <= DateTime.Now.Year) + { + _logger.LogInformation(string.Format($"Running FindSatelliteItems for cropfieldItem.Code '{cropfieldItem.Code}', SatelliteTaskCode '{satellitetaskcode}'")); + //Call first time + callCnt = 1; + //if callCntMax == 0 then don't sleep + //if callCntMax = 1 then sleep first 1x + if (callCntMax > 0) + { + _logger.LogInformation($"Call no: {callCnt}. Giving FarmMaps {sleepSecs} seconds to get SatelliteItems..."); + System.Threading.Thread.Sleep(1000 * sleepSecs); + sleepSecsSum = sleepSecsSum + sleepSecs; + } + satelliteItemsCropYear = await _generalService.FindSatelliteItems(cropfieldItem, satellitetaskcode); + satelliteItemsCropYearCntPrev = satelliteItemsCropYear.Count; + _logger.LogInformation($"Call no: {callCnt}. Received {satelliteItemsCropYearCntPrev} images"); + callCnt++; + satelliteItemsCropYearCnt = satelliteItemsCropYearCntPrev; + //if callCntMax > 1 then sleep untill (1) no more increase in number of images received OR (2) maximum number of calls reached + if (callCntMax > 1) + { + //Call second time + _logger.LogInformation($"Call no: {callCnt}. Giving FarmMaps another {sleepSecs} seconds to get SatelliteItems..."); + System.Threading.Thread.Sleep(1000 * sleepSecs); + satelliteItemsCropYear = await _generalService.FindSatelliteItems(cropfieldItem, satellitetaskcode); + satelliteItemsCropYearCnt = satelliteItemsCropYear.Count; + _logger.LogInformation($"Call no: {callCnt}. Received {satelliteItemsCropYearCnt} images"); + sleepSecsSum = sleepSecsSum + sleepSecs; + //As long as there is progress, keep calling + callCnt++; + while (callCnt <= callCntMax && (satelliteItemsCropYearCnt == 0 || satelliteItemsCropYearCnt > satelliteItemsCropYearCntPrev)) + { + _logger.LogInformation($"Surprise! The longer we wait, the more images we get. Sleep and call once more"); + satelliteItemsCropYearCntPrev = satelliteItemsCropYearCnt; + _logger.LogInformation($"Call no: {callCnt} (max: {callCntMax}). Giving FarmMaps another {sleepSecs} seconds to get SatelliteItems..."); + System.Threading.Thread.Sleep(1000 * sleepSecs); + satelliteItemsCropYear = await _generalService.FindSatelliteItems(cropfieldItem, satellitetaskcode); + satelliteItemsCropYearCnt = satelliteItemsCropYear.Count; + _logger.LogInformation($"Call no: {callCnt}. Received {satelliteItemsCropYearCnt} images"); + callCnt++; + sleepSecsSum = sleepSecsSum + sleepSecs; + } + } + } + else + { + _logger.LogWarning($"// FarmmapsBulkSatDownload: crop year {cropYear} is out of bounds. No stats will be written!"); + } + + if (satelliteItemsCropYearCnt == 0) + { + _logger.LogWarning($"// FarmmapsBulkSatDownload: after calling one or more times and " + + $"sleeping in total {sleepSecsSum} seconds, still no images found. " + + $"Please check your settings for parameters callCntMax and sleepSecs in FarmmapsBulkSatDownload.cs or contact FarmMaps"); + } // Checking if satellite items found satelliteItemsAvailable = true; if (satelliteItemsCropYear == null) { satelliteItemsAvailable = false; - _logger.LogInformation($"No satellite tiffs found for fieldName '{fieldName}', cropYear {cropYear}, cropfielditemcode '{cropfielditemcode}'"); + _logger.LogInformation($"// FarmmapsBulkSatDownload: No satellite tiffs found for fieldName '{fieldName}', cropYear {cropYear}, cropfielditemcode '{cropfielditemcode}'"); } else { if (satelliteItemsCropYear.Count == 0) { satelliteItemsAvailable = false; - _logger.LogInformation($"No satellite tiffs found for fieldName '{fieldName}', cropYear {cropYear}, cropfielditemcode '{cropfielditemcode}'"); + _logger.LogInformation($"// FarmmapsBulkSatDownload: No satellite tiffs found for fieldName '{fieldName}', cropYear {cropYear}, cropfielditemcode '{cropfielditemcode}'"); } } @@ -343,10 +426,10 @@ namespace FarmmapsBulkSatDownload // Download statistics to a single csv file if (satelliteItemsAvailable && downloadFolder != null && fileNameStats != null) { - // Write statistics for all images for all fieldNane and cropYear to a single csv file, fileNameStats + // Write statistics for all images for all fieldName and cropYear to a single csv file, fileNameStats _logger.LogInformation($"Downloading stats for field '{fieldName}' in cropyear {cropYear} to {fileNameStats}"); string downloadedStats = await _generalService.DownloadSatelliteStats(satelliteItemsCropYear, fieldName, satelliteBands, downloadFolder); - // Add contents of this csv file to thee single large csv file + // Add contents of this csv file to the single large csv file var retainedLines = File.ReadAllLines(downloadedStats).Skip(1); File.AppendAllLines(fileNameStats, retainedLines); File.Delete(downloadedStats); diff --git a/FarmmapsBulkSatDownload/BulkSatDownloadInput.json b/FarmmapsBulkSatDownload/BulkSatDownloadInput.json index 4505680..07a9d0b 100644 --- a/FarmmapsBulkSatDownload/BulkSatDownloadInput.json +++ b/FarmmapsBulkSatDownload/BulkSatDownloadInput.json @@ -1,9 +1,34 @@ [ { - "fieldName": "BvdT FieldlabG92", // FarmMaps minimum needs are: fieldName, cropYear & geometryJson - "cropYear": 2022, // for testing same field last year + "fieldName": "MyField_1", // FarmMaps minimum needs are: fieldName, cropYear & geometryJson + "cropYear": 2023, "fieldID": 1, - "SatelliteBands": [ "wdvi", "ndvi" ], // ["ndvi"] or ["wdvi"] or both: [ "wdvi", "ndvi" ] + "SatelliteBands": [ "ci-red", "ndvi", "wdvi" ], // ["ndvi"] or ["wdvi"] or both: [ "wdvi", "ndvi" ] + "lastdownloadedimagedate": "2023-01-01", //downloads images from this date till end of the year + "geometryJson": { + "type": "Polygon", + "coordinates": [ + [ + [ 5.563472073408009, 52.547554398144172 ], + [ 5.567425915520115, 52.547725375100377 ], + [ 5.563878143678981, 52.54048022658143 ], + [ 5.563878143678981, 52.54048022658143 ], + [ 5.563472073408009, 52.547554398144172 ] + ] + ] + }, + "downloadFolder": "C:\\workdir\\groenmonitor\\", //if not yet existing this folder will be created + "fileNameStats": "BulkSatDownload.csv", //if file exists, probably will be overwritten. Check code in BulkSatDownloadApplication.cs + "database": null, // keep null to work with json and csv. Check code in BulkSatDownloadApplication.cs if reading/writing to/from database + "schemaname": null, + "cropfieldtable": null, + "satelllitetable": null + }, + { + "fieldName": "MyField_2", // FarmMaps minimum needs are: fieldName, cropYear & geometryJson + "cropYear": 2022, // for testing same field last year (i.e. 2021) + "fieldID": 2, + "SatelliteBands": [ "ci-red", "ndvi", "wdvi" ], // ["ndvi"] or ["wdvi"] or both: [ "wdvi", "ndvi" ] "lastdownloadedimagedate": "2022-01-01", //downloads images from this date till end of the year "geometryJson": { "type": "Polygon", @@ -25,10 +50,10 @@ "satelllitetable": null }, { - "fieldName": "BvdT FieldlabG92", // FarmMaps minimum needs are: fieldName, cropYear & geometryJson - "cropYear": 2021, // for testing same field last year (i.e. 2021) - "fieldID": 1, - "SatelliteBands": [ "wdvi", "ndvi" ], // ["ndvi"] or ["wdvi"] or both: [ "wdvi", "ndvi" ] + "fieldName": "MyField_3", // FarmMaps minimum needs are: fieldName, cropYear & geometryJson + "cropYear": 2021, + "fieldID": 3, + "SatelliteBands": [ "ci-red", "ndvi", "wdvi" ], // ["ndvi"] or ["wdvi"] or both: [ "wdvi", "ndvi" ] "lastdownloadedimagedate": "2021-01-01", //downloads images from this date till end of the year "geometryJson": { "type": "Polygon", @@ -48,5 +73,155 @@ "schemaname": null, "cropfieldtable": null, "satelllitetable": null + }, + { + "fieldName": "MyField_4", // FarmMaps minimum needs are: fieldName, cropYear & geometryJson + "cropYear": 2020, + "fieldID": 4, + "SatelliteBands": [ "ci-red", "ndvi", "wdvi" ], // ["ndvi"] or ["wdvi"] or both: [ "wdvi", "ndvi" ] + "lastdownloadedimagedate": "2020-01-01", //downloads images from this date till end of the year + "geometryJson": { + "type": "Polygon", + "coordinates": [ + [ + [ 5.563472073408009, 52.547554398144172 ], + [ 5.567425915520115, 52.547725375100377 ], + [ 5.563878143678981, 52.54048022658143 ], + [ 5.563878143678981, 52.54048022658143 ], + [ 5.563472073408009, 52.547554398144172 ] + ] + ] + }, + "downloadFolder": "C:\\workdir\\groenmonitor\\", + "fileNameStats": "BulkSatDownload.csv", + "database": null, + "schemaname": null, + "cropfieldtable": null, + "satelllitetable": null + }, + { + "fieldName": "MyField_5", // FarmMaps minimum needs are: fieldName, cropYear & geometryJson + "cropYear": 2019, + "fieldID": 5, + "SatelliteBands": [ "ci-red", "ndvi", "wdvi" ], // ["ndvi"] or ["wdvi"] or both: [ "wdvi", "ndvi" ] + "lastdownloadedimagedate": "2019-01-01", //downloads images from this date till end of the year + "geometryJson": { + "type": "Polygon", + "coordinates": [ + [ + [ 5.563472073408009, 52.547554398144172 ], + [ 5.567425915520115, 52.547725375100377 ], + [ 5.563878143678981, 52.54048022658143 ], + [ 5.563878143678981, 52.54048022658143 ], + [ 5.563472073408009, 52.547554398144172 ] + ] + ] + }, + "downloadFolder": "C:\\workdir\\groenmonitor\\", + "fileNameStats": "BulkSatDownload.csv", + "database": null, + "schemaname": null, + "cropfieldtable": null, + "satelllitetable": null + }, + { + "fieldName": "MyField_6", // FarmMaps minimum needs are: fieldName, cropYear & geometryJson + "cropYear": 2018, + "fieldID": 6, + "SatelliteBands": [ "ci-red", "ndvi", "wdvi" ], // ["ndvi"] or ["wdvi"] or both: [ "wdvi", "ndvi" ] + "lastdownloadedimagedate": "2018-01-01", //downloads images from this date till end of the year + "geometryJson": { + "type": "Polygon", + "coordinates": [ + [ + [ 5.563472073408009, 52.547554398144172 ], + [ 5.567425915520115, 52.547725375100377 ], + [ 5.563878143678981, 52.54048022658143 ], + [ 5.563878143678981, 52.54048022658143 ], + [ 5.563472073408009, 52.547554398144172 ] + ] + ] + }, + "downloadFolder": "C:\\workdir\\groenmonitor\\", + "fileNameStats": "BulkSatDownload.csv", + "database": null, + "schemaname": null, + "cropfieldtable": null, + "satelllitetable": null + }, + { + "fieldName": "MyField_6", // FarmMaps minimum needs are: fieldName, cropYear & geometryJson + "cropYear": 2017, + "fieldID": 7, + "SatelliteBands": [ "ci-red", "ndvi", "wdvi" ], // ["ndvi"] or ["wdvi"] or both: [ "wdvi", "ndvi" ] + "lastdownloadedimagedate": "2017-01-01", //downloads images from this date till end of the year + "geometryJson": { + "type": "Polygon", + "coordinates": [ + [ + [ 5.563472073408009, 52.547554398144172 ], + [ 5.567425915520115, 52.547725375100377 ], + [ 5.563878143678981, 52.54048022658143 ], + [ 5.563878143678981, 52.54048022658143 ], + [ 5.563472073408009, 52.547554398144172 ] + ] + ] + }, + "downloadFolder": "C:\\workdir\\groenmonitor\\", + "fileNameStats": "BulkSatDownload.csv", + "database": null, + "schemaname": null, + "cropfieldtable": null, + "satelllitetable": null + }, + { + "fieldName": "MyField_8", // FarmMaps minimum needs are: fieldName, cropYear & geometryJson. Actually no Satellite data before 2017. This here to illustrate the program does not crash + "cropYear": 2016, + "fieldID": 8, + "SatelliteBands": [ "ci-red", "ndvi", "wdvi" ], // ["ndvi"] or ["wdvi"] or both: [ "wdvi", "ndvi" ] + "lastdownloadedimagedate": "2016-01-01", //downloads images from this date till end of the year + "geometryJson": { + "type": "Polygon", + "coordinates": [ + [ + [ 5.563472073408009, 52.547554398144172 ], + [ 5.567425915520115, 52.547725375100377 ], + [ 5.563878143678981, 52.54048022658143 ], + [ 5.563878143678981, 52.54048022658143 ], + [ 5.563472073408009, 52.547554398144172 ] + ] + ] + }, + "downloadFolder": "C:\\workdir\\groenmonitor\\", + "fileNameStats": "BulkSatDownload.csv", + "database": null, + "schemaname": null, + "cropfieldtable": null, + "satelllitetable": null + }, + { + "fieldName": "MyField_9", // FarmMaps minimum needs are: fieldName, cropYear & geometryJson. Actually no Satellite data before 2017. This here to illustrate the program does not crash + "cropYear": 2015, + "fieldID": 9, + "SatelliteBands": [ "ci-red", "ndvi", "wdvi" ], // ["ndvi"] or ["wdvi"] or both: [ "wdvi", "ndvi" ] + "lastdownloadedimagedate": "2015-01-01", //downloads images from this date till end of the year + "geometryJson": { + "type": "Polygon", + "coordinates": [ + [ + [ 5.563472073408009, 52.547554398144172 ], + [ 5.567425915520115, 52.547725375100377 ], + [ 5.563878143678981, 52.54048022658143 ], + [ 5.563878143678981, 52.54048022658143 ], + [ 5.563472073408009, 52.547554398144172 ] + ] + ] + }, + "downloadFolder": "C:\\workdir\\groenmonitor\\", + "fileNameStats": "BulkSatDownload.csv", + "database": null, + "schemaname": null, + "cropfieldtable": null, + "satelllitetable": null } ] \ No newline at end of file