fix BulkSatDownloadApplication: fire a number of calls awaiting satellite task, until number of images returned no longer increases

FarmmapsBulkSatDownload/BulkSatDownloadApplication.cs

@@ -25,6 +25,7 @@ namespace FarmmapsBulkSatDownload
         private readonly GeneralService _generalService;
 
         public const string settingsfile = "Settings.json";
+        public const int firstAvailableYear = 2017;
         private Settings _settings;
 
         public BulkSatDownloadApplication(ILogger<BulkSatDownloadApplication> logger, FarmmapsApiService farmmapsApiService,
@@ -199,6 +200,27 @@ namespace FarmmapsBulkSatDownload
 
         private async Task Process(List<UserRoot> roots, BulkSatDownloadInput input)
         {
+            //PO20220311: first time a call is made to download satellite images or statistics, an empty list is returned
+            //If we wait a bit longer, e.g. 10 secs, then e.g. a list of 3 images may be returned
+            //If we wait still longer, maybe 4 images.
+            //The solution implemented below is to fire calls as long as the number of images returned keeps increasing
+            //While in between each call, sleep for sleepSecs
+            //Continue this until the number no longer increases or the maximum number of calls has been reached
+            //If you set sleepSecs to a very low value, e.g. 5 secs, then after 1 call you might get images and after 2nd call still zero images.
+            //to be on the safe side, better bit higher value.
+            //Just accept this may take a while, have a coffee, we suggest sleepSecs = 30;
+            int sleepSecs = 30;
+            int callCntMax = 4 * 60 / sleepSecs;   //4*60 = max 4 minutes
+            //For example we may set: "sleepSecs = 10;" and "callCntMax = 24;" and following result:
+            //Call no: 1. Giving FarmMaps 10 seconds to get SatelliteItems...
+            //Call no: 1: Received 2 images
+            //Call no: 2. Giving FarmMaps 10 seconds to get SatelliteItems...
+            //Call no: 2: Received 7 images
+            //Call no: 3. Giving FarmMaps 10 seconds to get SatelliteItems...
+            //Call no: 3: Received 7 images
+            //And the firing of calls would stop because the number of images returned is no longer increasing
+            //In the worst case, this could would lead to a total sleeping period of "sleepSecsSum = sleepSecs * callCntMax" seconds. After that we give up
+
             string cropfielditemcode;
             string satellitetaskcode;
             Item cropfieldItem;
@@ -230,6 +252,10 @@ namespace FarmmapsBulkSatDownload
             DateTime lastDownloadedSatelliteDate = input.lastdownloadedimagedate;
             cropfielditemcode = input.cropfielditemcode;
             satellitetaskcode = input.satellitetaskcode;
+            int satelliteItemsCropYearCntPrev;
+            int satelliteItemsCropYearCnt;
+            int callCnt;
+            int sleepSecsSum;
 
             LoadSettings(settingsfile);
 
@@ -317,23 +343,80 @@ namespace FarmmapsBulkSatDownload
             // TODO also log satellitetaskcode to settings, how?
             // SaveSettings(settingsfile);
 
-            // Getting satellite items
+            // Getting satellite items. Only for years for which available
+            satelliteItemsCropYearCntPrev = 0;
+            satelliteItemsCropYearCnt = 0;
+            sleepSecsSum = 0;
+            satelliteItemsCropYear = null;
+            if (cropYear >= firstAvailableYear && cropYear <= DateTime.Now.Year)
+            {
                 _logger.LogInformation(string.Format($"Running FindSatelliteItems for cropfieldItem.Code '{cropfieldItem.Code}', SatelliteTaskCode '{satellitetaskcode}'"));
+                //Call first time
+                callCnt = 1;
+                //if callCntMax == 0 then don't sleep
+                //if callCntMax = 1 then sleep first 1x
+                if (callCntMax > 0)
+                {
+                    _logger.LogInformation($"Call no: {callCnt}. Giving FarmMaps {sleepSecs} seconds to get SatelliteItems...");
+                    System.Threading.Thread.Sleep(1000 * sleepSecs);
+                    sleepSecsSum = sleepSecsSum + sleepSecs;
+                }
                 satelliteItemsCropYear = await _generalService.FindSatelliteItems(cropfieldItem, satellitetaskcode);
+                satelliteItemsCropYearCntPrev = satelliteItemsCropYear.Count;
+                _logger.LogInformation($"Call no: {callCnt}. Received {satelliteItemsCropYearCntPrev} images");
+                callCnt++;
+                satelliteItemsCropYearCnt = satelliteItemsCropYearCntPrev;
+                //if callCntMax > 1 then sleep untill (1) no more increase in number of images received OR (2) maximum number of calls reached
+                if (callCntMax > 1)
+                {
+                    //Call second time
+                    _logger.LogInformation($"Call no: {callCnt}. Giving FarmMaps another {sleepSecs} seconds to get SatelliteItems...");
+                    System.Threading.Thread.Sleep(1000 * sleepSecs);
+                    satelliteItemsCropYear = await _generalService.FindSatelliteItems(cropfieldItem, satellitetaskcode);
+                    satelliteItemsCropYearCnt = satelliteItemsCropYear.Count;
+                    _logger.LogInformation($"Call no: {callCnt}. Received {satelliteItemsCropYearCnt} images");
+                    sleepSecsSum = sleepSecsSum + sleepSecs;
+                    //As long as there is progress, keep calling
+                    callCnt++;
+                    while (callCnt <= callCntMax && (satelliteItemsCropYearCnt == 0 || satelliteItemsCropYearCnt > satelliteItemsCropYearCntPrev))
+                    {
+                        _logger.LogInformation($"Surprise! The longer we wait, the more images we get. Sleep and call once more");
+                        satelliteItemsCropYearCntPrev = satelliteItemsCropYearCnt;
+                        _logger.LogInformation($"Call no: {callCnt} (max: {callCntMax}). Giving FarmMaps another {sleepSecs} seconds to get SatelliteItems...");
+                        System.Threading.Thread.Sleep(1000 * sleepSecs);
+                        satelliteItemsCropYear = await _generalService.FindSatelliteItems(cropfieldItem, satellitetaskcode);
+                        satelliteItemsCropYearCnt = satelliteItemsCropYear.Count;
+                        _logger.LogInformation($"Call no: {callCnt}. Received {satelliteItemsCropYearCnt} images");
+                        callCnt++;
+                        sleepSecsSum = sleepSecsSum + sleepSecs;
+                    }
+                }
+            }
+            else
+            {
+                _logger.LogWarning($"// FarmmapsBulkSatDownload: crop year {cropYear} is out of bounds. No stats will be written!");
+            }
+
+            if (satelliteItemsCropYearCnt == 0)
+            {
+                _logger.LogWarning($"// FarmmapsBulkSatDownload: after calling one or more times and " +
+                $"sleeping in total {sleepSecsSum} seconds, still no images found. " +
+                $"Please check your settings for parameters callCntMax and sleepSecs in FarmmapsBulkSatDownload.cs or contact FarmMaps");
+            }
 
             // Checking if satellite items found
             satelliteItemsAvailable = true;
             if (satelliteItemsCropYear == null)
             {
                 satelliteItemsAvailable = false;
-                _logger.LogInformation($"No satellite tiffs found for fieldName '{fieldName}', cropYear {cropYear}, cropfielditemcode '{cropfielditemcode}'");
+                _logger.LogInformation($"// FarmmapsBulkSatDownload: No satellite tiffs found for fieldName '{fieldName}', cropYear {cropYear}, cropfielditemcode '{cropfielditemcode}'");
             }
             else
             {
                 if (satelliteItemsCropYear.Count == 0)
                 {
                     satelliteItemsAvailable = false;
-                    _logger.LogInformation($"No satellite tiffs found for fieldName '{fieldName}', cropYear {cropYear}, cropfielditemcode '{cropfielditemcode}'");
+                    _logger.LogInformation($"// FarmmapsBulkSatDownload: No satellite tiffs found for fieldName '{fieldName}', cropYear {cropYear}, cropfielditemcode '{cropfielditemcode}'");
                 }
             }
 
@@ -343,10 +426,10 @@ namespace FarmmapsBulkSatDownload
             // Download statistics to a single csv file
             if (satelliteItemsAvailable && downloadFolder != null && fileNameStats != null)
             {
-                // Write statistics for all images for all fieldNane and cropYear to a single csv file, fileNameStats
+                // Write statistics for all images for all fieldName and cropYear to a single csv file, fileNameStats
                 _logger.LogInformation($"Downloading stats for field '{fieldName}' in cropyear {cropYear} to {fileNameStats}");
                 string downloadedStats = await _generalService.DownloadSatelliteStats(satelliteItemsCropYear, fieldName, satelliteBands, downloadFolder);
-                // Add contents of this csv file to thee single large csv file
+                // Add contents of this csv file to the single large csv file
                 var retainedLines = File.ReadAllLines(downloadedStats).Skip(1);
                 File.AppendAllLines(fileNameStats, retainedLines);
                 File.Delete(downloadedStats);

FarmmapsBulkSatDownload/BulkSatDownloadInput.json

@@ -1,9 +1,34 @@
 [
   {
-    "fieldName": "BvdT FieldlabG92", // FarmMaps minimum needs are: fieldName, cropYear & geometryJson
+    "fieldName": "MyField_1", // FarmMaps minimum needs are: fieldName, cropYear & geometryJson
-    "cropYear": 2022, // for testing same field last year
+    "cropYear": 2023,
     "fieldID": 1,
-    "SatelliteBands": [ "wdvi", "ndvi" ], // ["ndvi"] or ["wdvi"] or both: [ "wdvi", "ndvi" ]
+    "SatelliteBands": [ "ci-red", "ndvi", "wdvi" ], // ["ndvi"] or ["wdvi"] or both: [ "wdvi", "ndvi" ]
+    "lastdownloadedimagedate": "2023-01-01", //downloads images from this date till end of the year
+    "geometryJson": {
+      "type": "Polygon",
+      "coordinates": [
+        [
+          [ 5.563472073408009, 52.547554398144172 ],
+          [ 5.567425915520115, 52.547725375100377 ],
+          [ 5.563878143678981, 52.54048022658143 ],
+          [ 5.563878143678981, 52.54048022658143 ],
+          [ 5.563472073408009, 52.547554398144172 ]
+        ]
+      ]
+    },
+    "downloadFolder": "C:\\workdir\\groenmonitor\\", //if not yet existing this folder will be created
+    "fileNameStats": "BulkSatDownload.csv", //if file exists, probably will be overwritten. Check code in BulkSatDownloadApplication.cs
+    "database": null, // keep null to work with json and csv. Check code in BulkSatDownloadApplication.cs if reading/writing to/from database
+    "schemaname": null,
+    "cropfieldtable": null,
+    "satelllitetable": null
+  },
+  {
+    "fieldName": "MyField_2", // FarmMaps minimum needs are: fieldName, cropYear & geometryJson
+    "cropYear": 2022, // for testing same field last year (i.e. 2021)
+    "fieldID": 2,
+    "SatelliteBands": [ "ci-red", "ndvi", "wdvi" ], // ["ndvi"] or ["wdvi"] or both: [ "wdvi", "ndvi" ]
     "lastdownloadedimagedate": "2022-01-01", //downloads images from this date till end of the year
     "geometryJson": {
       "type": "Polygon",
@@ -25,10 +50,10 @@
     "satelllitetable": null
   },
   {
-    "fieldName": "BvdT FieldlabG92", // FarmMaps minimum needs are: fieldName, cropYear & geometryJson
+    "fieldName": "MyField_3", // FarmMaps minimum needs are: fieldName, cropYear & geometryJson
-    "cropYear": 2021, // for testing same field last year (i.e. 2021)
+    "cropYear": 2021,
-    "fieldID": 1,
+    "fieldID": 3,
-    "SatelliteBands": [ "wdvi", "ndvi" ], // ["ndvi"] or ["wdvi"] or both: [ "wdvi", "ndvi" ]
+    "SatelliteBands": [ "ci-red", "ndvi", "wdvi" ], // ["ndvi"] or ["wdvi"] or both: [ "wdvi", "ndvi" ]
     "lastdownloadedimagedate": "2021-01-01", //downloads images from this date till end of the year
     "geometryJson": {
       "type": "Polygon",
@@ -48,5 +73,155 @@
     "schemaname": null,
     "cropfieldtable": null,
     "satelllitetable": null
+  },
+  {
+    "fieldName": "MyField_4", // FarmMaps minimum needs are: fieldName, cropYear & geometryJson
+    "cropYear": 2020,
+    "fieldID": 4,
+    "SatelliteBands": [ "ci-red", "ndvi", "wdvi" ], // ["ndvi"] or ["wdvi"] or both: [ "wdvi", "ndvi" ]
+    "lastdownloadedimagedate": "2020-01-01", //downloads images from this date till end of the year
+    "geometryJson": {
+      "type": "Polygon",
+      "coordinates": [
+        [
+          [ 5.563472073408009, 52.547554398144172 ],
+          [ 5.567425915520115, 52.547725375100377 ],
+          [ 5.563878143678981, 52.54048022658143 ],
+          [ 5.563878143678981, 52.54048022658143 ],
+          [ 5.563472073408009, 52.547554398144172 ]
+        ]
+      ]
+    },
+    "downloadFolder": "C:\\workdir\\groenmonitor\\",
+    "fileNameStats": "BulkSatDownload.csv",
+    "database": null,
+    "schemaname": null,
+    "cropfieldtable": null,
+    "satelllitetable": null
+  },
+  {
+    "fieldName": "MyField_5", // FarmMaps minimum needs are: fieldName, cropYear & geometryJson
+    "cropYear": 2019,
+    "fieldID": 5,
+    "SatelliteBands": [ "ci-red", "ndvi", "wdvi" ], // ["ndvi"] or ["wdvi"] or both: [ "wdvi", "ndvi" ]
+    "lastdownloadedimagedate": "2019-01-01", //downloads images from this date till end of the year
+    "geometryJson": {
+      "type": "Polygon",
+      "coordinates": [
+        [
+          [ 5.563472073408009, 52.547554398144172 ],
+          [ 5.567425915520115, 52.547725375100377 ],
+          [ 5.563878143678981, 52.54048022658143 ],
+          [ 5.563878143678981, 52.54048022658143 ],
+          [ 5.563472073408009, 52.547554398144172 ]
+        ]
+      ]
+    },
+    "downloadFolder": "C:\\workdir\\groenmonitor\\",
+    "fileNameStats": "BulkSatDownload.csv",
+    "database": null,
+    "schemaname": null,
+    "cropfieldtable": null,
+    "satelllitetable": null
+  },
+  {
+    "fieldName": "MyField_6", // FarmMaps minimum needs are: fieldName, cropYear & geometryJson
+    "cropYear": 2018,
+    "fieldID": 6,
+    "SatelliteBands": [ "ci-red", "ndvi", "wdvi" ], // ["ndvi"] or ["wdvi"] or both: [ "wdvi", "ndvi" ]
+    "lastdownloadedimagedate": "2018-01-01", //downloads images from this date till end of the year
+    "geometryJson": {
+      "type": "Polygon",
+      "coordinates": [
+        [
+          [ 5.563472073408009, 52.547554398144172 ],
+          [ 5.567425915520115, 52.547725375100377 ],
+          [ 5.563878143678981, 52.54048022658143 ],
+          [ 5.563878143678981, 52.54048022658143 ],
+          [ 5.563472073408009, 52.547554398144172 ]
+        ]
+      ]
+    },
+    "downloadFolder": "C:\\workdir\\groenmonitor\\",
+    "fileNameStats": "BulkSatDownload.csv",
+    "database": null,
+    "schemaname": null,
+    "cropfieldtable": null,
+    "satelllitetable": null
+  },
+  {
+    "fieldName": "MyField_6", // FarmMaps minimum needs are: fieldName, cropYear & geometryJson
+    "cropYear": 2017,
+    "fieldID": 7,
+    "SatelliteBands": [ "ci-red", "ndvi", "wdvi" ], // ["ndvi"] or ["wdvi"] or both: [ "wdvi", "ndvi" ]
+    "lastdownloadedimagedate": "2017-01-01", //downloads images from this date till end of the year
+    "geometryJson": {
+      "type": "Polygon",
+      "coordinates": [
+        [
+          [ 5.563472073408009, 52.547554398144172 ],
+          [ 5.567425915520115, 52.547725375100377 ],
+          [ 5.563878143678981, 52.54048022658143 ],
+          [ 5.563878143678981, 52.54048022658143 ],
+          [ 5.563472073408009, 52.547554398144172 ]
+        ]
+      ]
+    },
+    "downloadFolder": "C:\\workdir\\groenmonitor\\",
+    "fileNameStats": "BulkSatDownload.csv",
+    "database": null,
+    "schemaname": null,
+    "cropfieldtable": null,
+    "satelllitetable": null
+  },
+  {
+    "fieldName": "MyField_8", // FarmMaps minimum needs are: fieldName, cropYear & geometryJson. Actually no Satellite data before 2017. This here to illustrate the program does not crash
+    "cropYear": 2016,
+    "fieldID": 8,
+    "SatelliteBands": [ "ci-red", "ndvi", "wdvi" ], // ["ndvi"] or ["wdvi"] or both: [ "wdvi", "ndvi" ]
+    "lastdownloadedimagedate": "2016-01-01", //downloads images from this date till end of the year
+    "geometryJson": {
+      "type": "Polygon",
+      "coordinates": [
+        [
+          [ 5.563472073408009, 52.547554398144172 ],
+          [ 5.567425915520115, 52.547725375100377 ],
+          [ 5.563878143678981, 52.54048022658143 ],
+          [ 5.563878143678981, 52.54048022658143 ],
+          [ 5.563472073408009, 52.547554398144172 ]
+        ]
+      ]
+    },
+    "downloadFolder": "C:\\workdir\\groenmonitor\\",
+    "fileNameStats": "BulkSatDownload.csv",
+    "database": null,
+    "schemaname": null,
+    "cropfieldtable": null,
+    "satelllitetable": null
+  },
+  {
+    "fieldName": "MyField_9", // FarmMaps minimum needs are: fieldName, cropYear & geometryJson. Actually no Satellite data before 2017. This here to illustrate the program does not crash
+    "cropYear": 2015,
+    "fieldID": 9,
+    "SatelliteBands": [ "ci-red", "ndvi", "wdvi" ], // ["ndvi"] or ["wdvi"] or both: [ "wdvi", "ndvi" ]
+    "lastdownloadedimagedate": "2015-01-01", //downloads images from this date till end of the year
+    "geometryJson": {
+      "type": "Polygon",
+      "coordinates": [
+        [
+          [ 5.563472073408009, 52.547554398144172 ],
+          [ 5.567425915520115, 52.547725375100377 ],
+          [ 5.563878143678981, 52.54048022658143 ],
+          [ 5.563878143678981, 52.54048022658143 ],
+          [ 5.563472073408009, 52.547554398144172 ]
+        ]
+      ]
+    },
+    "downloadFolder": "C:\\workdir\\groenmonitor\\",
+    "fileNameStats": "BulkSatDownload.csv",
+    "database": null,
+    "schemaname": null,
+    "cropfieldtable": null,
+    "satelllitetable": null
   }
 ]