Merge pull request 'master' (#15) from jits/FarmMapsApiClient_WURtest:master into master

Reviewed-on: #15

FarmMapsBlight/BlightApplication.cs

@@ -44,14 +44,14 @@ namespace FarmMapsBlight
             await _farmmapsApiService.GetCurrentUserCodeAsync();
             var roots = await _farmmapsApiService.GetCurrentUserRootsAsync();
 
-            var myDrive = roots.SingleOrDefault(r => r.Name == "My drive");
+            var myDrive = roots.SingleOrDefault(r => r.Name == "USER_FILES");
             if (myDrive == null)
             {
                 _logger.LogError("Could not find a needed root item");
                 return;
             }
 
-            var uploadedRoot = roots.SingleOrDefault(r => r.Name == "Uploaded");
+            var uploadedRoot = roots.SingleOrDefault(r => r.Name == "USER_IN");
             if (uploadedRoot == null)
             {
                 _logger.LogError("Could not find a needed root item");

FarmmapsApi/Constants.cs

@@ -12,13 +12,14 @@ namespace FarmmapsApiSamples
         public const string SHAPE_ITEMTYPE = "vnd.farmmaps.itemtype.shape";
         public const string GEOJSON_ITEMTYPE = "vnd.farmmaps.itemtype.geojson";
         public const string BLIGHT_ITEMTYPE = "vnd.farmmaps.itemtype.blight";
+        public const string CROPREC_ITEMTYPE = "vnd.farmmaps.itemtype.crprec.operation";
 
         public const string VRANBS_TASK = "vnd.farmmaps.task.vranbs";
         public const string VRAHERBICIDE_TASK = "vnd.farmmaps.task.vraherbicide";
         public const string VRAHAULMKILLING_TASK = "vnd.farmmaps.task.vrahaulmkilling";
         public const string VRAPLANTING_TASK = "vnd.farmmaps.task.vrapoten";
         public const string VRAZONERING_TASK = "vnd.farmmaps.task.vrazonering";
-        public const string SATELLITE_TASK = "vnd.farmmaps.task.satellite";
+        public const string SATELLITE_TASK = "vnd.farmmaps.task.sentinelhub";   //"vnd.farmmaps.task.satellite";
         public const string VANDERSAT_TASK = "vnd.farmmaps.task.vandersat";
         public const string TASKMAP_TASK = "vnd.farmmaps.task.taskmap";
         public const string WORKFLOW_TASK = "vnd.farmmaps.task.workflow";

FarmmapsApi/Services/FarmmapsApiService.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Collections.Generic;
 using System.IO;
+using System.Linq;
 using System.Net;
 using System.Net.Http;
 using System.Net.Http.Headers;
@@ -95,7 +96,19 @@ namespace FarmmapsApi.Services
 
             var jsonString = await response.Content.ReadAsStringAsync();
 
-            return JsonConvert.DeserializeObject<List<UserRoot>>(jsonString);
+            var roots = JsonConvert.DeserializeObject<List<UserRoot>>(jsonString);
+            roots.ForEach(v =>
+            {
+                v.Name = GetSuffixFromSystemItemCode(v.Code);
+            });
+            
+            return roots;
+        }
+        
+        private static string GetSuffixFromSystemItemCode(string itemCode)
+        {
+            string[] strArray = itemCode.Split(":");
+            return strArray.Length == 2 ? strArray[1] : throw new ArgumentException("No system item code :" + itemCode);
         }
 
         public async Task<Item> GetItemAsync(string itemCode, string itemType = null, JObject dataFilter = null)

FarmmapsApi/Services/GeneralService.cs

@@ -23,7 +23,7 @@ namespace FarmmapsApi.Services
 
         public async Task<Item> CreateCropfieldItemAsync(string parentItemCode, string name, int year,
             string fieldGeomJson, string data = "{}") {
-            var currentYear = new DateTime(year, 1, 1);
+            var currentYear = new DateTime(year, 1, 1, 0, 0, 0, DateTimeKind.Utc);
             var cropfieldItemRequest = new ItemRequest() {
                 ParentCode = parentItemCode,
                 ItemType = CROPFIELD_ITEMTYPE,

FarmmapsBulkSatDownload/BulkSatDownloadApplication.cs

@@ -25,6 +25,7 @@ namespace FarmmapsBulkSatDownload
         private readonly GeneralService _generalService;
 
         public const string settingsfile = "Settings.json";
+        public const int firstAvailableYear = 2017;
         private Settings _settings;
 
         public BulkSatDownloadApplication(ILogger<BulkSatDownloadApplication> logger, FarmmapsApiService farmmapsApiService,
@@ -51,95 +52,103 @@ namespace FarmmapsBulkSatDownload
             DateTime lastdownloadedimagedate;
             int cropYear;
 
-            // Option 1: When using database need to (1) fill in database data in DBsettings.secrets.json; (2) write tailor made SELECT query for fieldinputs in following lines; 
-            // (3) Write tailor made INSERT INTO query in Task Process() below;
-            // Initialize databases. Username, password etc stored in file "DBsettings.secrets.json". 
-            // Crashes if "DBsettings.secrets.json" is absent or empty
-            DB dbparcels = JsonConvert.DeserializeObject<DB>(File.ReadAllText("DBsettings.secrets.json"));
-            string schemaname = "bigdata";
-            string parceltablename = "parcel_bollenrevolutie_tulips2020"; //"parcelsijbrandij" "parcel"; "parcel_flowerbulbs"; "parcel_disac"; ""parcel_bollenrevolutie_tulips2020""
-            string groenmonitortablename = "groenmonitor_bollenrevolutie_tulips2020";   //"groenmonitorsijbrandij" "groenmonitor" "groenmonitor_flowerbulbs" "groenmonitor_disac" "groenmonitor_bollenrevolutie_tulips2020"
-            // The  view 'groenmonitorlatestviewname' contains per parcelid (arbid) the year in which it "exists" and the date of the latest image downloaded. It is used to prevent unneccessary downloading of image statistics already in the database
-            string groenmonitorlatestviewname = "groenmonitorlatest_bollenrevolutie_tulips2020"; //"groenmonitorsijbrandijlatest" "groenmonitorlatest" "groenmonitorlatest_flowerbulbs" "groenmonitorlatest_disac" "groenmonitorlatest_bollenrevolutie_tulips2020"
-
-            // Database query and connection. Geometry must be in WGS84 coordinate system, EPSG 4326
-            // Apparently the FarmmapsApi cannot handle MultiPolygon, so we need to convert to single Polygon
-            // In case database returns a MultiPolygon use ST_NumGeometries(pt.geom) to count the number of polygons
-            // If necessary use WHERE ST_NumGeometries(pt.geom) = 1 to select only single polygons
-            //
-            // FarmMaps get's its satellite images from www.groenmonitor.nl through the https://agrodatacube.wur.nl/. 
-            // Many images are available at www.groenmonitor.nl, the https://agrodatacube.wur.nl/ serves only the clean images, 10-30 per year, 2019 onwards. Possibly more images will be added for earlier years
-            // For other images contact www.groenmonitor.nl, gerbert.roerink@wur.nl
-            bulkSatDownloadInputListDB = new List<BulkSatDownloadInput>();
-            List<string> satelliteBands = new List<string> { "wdvi", "ndvi" };
-            string connectionString = dbparcels.GetConnectionString();
-            string readSql = string.Format(
-@"
-SELECT pt.arbid, pt.year, gml.lastwenrdate, ST_AsGeoJSON(ST_Transform((ST_DUMP(pt.geom)).geom::geometry(Polygon),4326)) AS geojson_polygon_wgs84, 
-    COALESCE(pt.cropfielditemcode,'') AS cropfielditemcode,
-    CASE WHEN pt.year >= DATE_PART('year', CURRENT_DATE) THEN '' ELSE COALESCE(pt.satellitetaskcode,'') END AS satellitetaskcode
-FROM {0}.{1} pt, {0}.{2} gml
-WHERE
-    pt.arbid = gml.arbid
-    AND pt.satellitetaskcode IS NULL
-ORDER BY pt.arbid
-LIMIT 5;", schemaname, parceltablename, groenmonitorlatestviewname);   //LIMIT x for testing
-
-            using (NpgsqlConnection connection = new NpgsqlConnection(connectionString))
+            //Use doDB to decide if reading from/writing to database (doDB = true) or 
+            //read from file BulkSatDownloadInput.json and write satellite statistics to file(s) specified in BulkSatDownloadInput.json
+            //note in case of doDB == true you will need to provide a file "DBsettings.secrets.json" with login details for the database
+            //see empty example "DBsettings.json"
+            bool doDB = false;
+            if (doDB == true)
             {
-                connection.Open();
+                // Option 1: When using database need to (1) fill in database data in DBsettings.secrets.json; (2) write tailor made SELECT query for fieldinputs in following lines; 
+                // (3) Write tailor made INSERT INTO query in Task Process() below;
+                // Initialize databases. Username, password etc stored in file "DBsettings.secrets.json". 
+                // Crashes if "DBsettings.secrets.json" is absent or empty
+                DB dbparcels = JsonConvert.DeserializeObject<DB>(File.ReadAllText("DBsettings.secrets.json"));
+                string schemaname = "bigdata";
+                string parceltablename = "parcel_bollenrevolutie_tulips2020"; //"parcelsijbrandij" "parcel"; "parcel_flowerbulbs"; "parcel_disac"; ""parcel_bollenrevolutie_tulips2020""
+                string groenmonitortablename = "groenmonitor_bollenrevolutie_tulips2020";   //"groenmonitorsijbrandij" "groenmonitor" "groenmonitor_flowerbulbs" "groenmonitor_disac" "groenmonitor_bollenrevolutie_tulips2020"
+                                                                                            // The  view 'groenmonitorlatestviewname' contains per parcelid (arbid) the year in which it "exists" and the date of the latest image downloaded. It is used to prevent unneccessary downloading of image statistics already in the database
+                string groenmonitorlatestviewname = "groenmonitorlatest_bollenrevolutie_tulips2020"; //"groenmonitorsijbrandijlatest" "groenmonitorlatest" "groenmonitorlatest_flowerbulbs" "groenmonitorlatest_disac" "groenmonitorlatest_bollenrevolutie_tulips2020"
 
-                // Read data (run query) = build a list of fields for which to download images
-                NpgsqlCommand command = connection.CreateCommand();
-                command.CommandText = readSql;
-                NpgsqlDataReader dr = command.ExecuteReader();
-                while (dr.Read())
+                // Database query and connection. Geometry must be in WGS84 coordinate system, EPSG 4326
+                // Apparently the FarmmapsApi cannot handle MultiPolygon, so we need to convert to single Polygon
+                // In case database returns a MultiPolygon use ST_NumGeometries(pt.geom) to count the number of polygons
+                // If necessary use WHERE ST_NumGeometries(pt.geom) = 1 to select only single polygons
+                //
+                // FarmMaps get's its satellite images from www.groenmonitor.nl through the https://agrodatacube.wur.nl/. 
+                // Many images are available at www.groenmonitor.nl, the https://agrodatacube.wur.nl/ serves only the clean images, 10-30 per year, 2019 onwards. Possibly more images will be added for earlier years
+                // For other images contact www.groenmonitor.nl, gerbert.roerink@wur.nl
+                bulkSatDownloadInputListDB = new List<BulkSatDownloadInput>();
+                List<string> satelliteBands = new List<string> { "wdvi", "ndvi" };
+                string connectionString = dbparcels.GetConnectionString();
+                string readSql = string.Format(
+    @"
+                SELECT pt.arbid, pt.year, gml.lastwenrdate, ST_AsGeoJSON(ST_Transform((ST_DUMP(pt.geom)).geom::geometry(Polygon),4326)) AS geojson_polygon_wgs84, 
+                    COALESCE(pt.cropfielditemcode,'') AS cropfielditemcode,
+                    CASE WHEN pt.year >= DATE_PART('year', CURRENT_DATE) THEN '' ELSE COALESCE(pt.satellitetaskcode,'') END AS satellitetaskcode
+                FROM {0}.{1} pt, {0}.{2} gml
+                WHERE
+                    pt.arbid = gml.arbid
+                    AND pt.satellitetaskcode IS NULL
+                ORDER BY pt.arbid
+                LIMIT 5;", schemaname, parceltablename, groenmonitorlatestviewname);   //LIMIT x for testing
+
+                using (NpgsqlConnection connection = new NpgsqlConnection(connectionString))
                 {
-                    bulkSatDownloadInput = new BulkSatDownloadInput();
-                    bulkSatDownloadInput.fieldID = dr.GetInt16(0);
-                    bulkSatDownloadInput.fieldName = string.Format($"{parceltablename}_{bulkSatDownloadInput.fieldID}");
-                    bulkSatDownloadInput.cropYear = dr.GetInt16(1); ;
-                    bulkSatDownloadInput.lastdownloadedimagedate = dr.GetDateTime(2);
-                    bulkSatDownloadInput.GeometryJson = JObject.Parse(dr.GetString(3));
-                    bulkSatDownloadInput.SatelliteBands = satelliteBands;
-                    bulkSatDownloadInput.cropfielditemcode = dr.GetString(4);
-                    bulkSatDownloadInput.satellitetaskcode = dr.GetString(5);
-                    bulkSatDownloadInput.database = dbparcels;
-                    bulkSatDownloadInput.schemaname = schemaname;
-                    bulkSatDownloadInput.cropfieldtable = parceltablename;
-                    bulkSatDownloadInput.satelllitetable = groenmonitortablename;
-                    bulkSatDownloadInputListDB.Add(bulkSatDownloadInput);
+                    connection.Open();
+
+                    // Read data (run query) = build a list of fields for which to download images
+                    NpgsqlCommand command = connection.CreateCommand();
+                    command.CommandText = readSql;
+                    NpgsqlDataReader dr = command.ExecuteReader();
+                    while (dr.Read())
+                    {
+                        bulkSatDownloadInput = new BulkSatDownloadInput();
+                        bulkSatDownloadInput.fieldID = dr.GetInt16(0);
+                        bulkSatDownloadInput.fieldName = string.Format($"{parceltablename}_{bulkSatDownloadInput.fieldID}");
+                        bulkSatDownloadInput.cropYear = dr.GetInt16(1); ;
+                        bulkSatDownloadInput.lastdownloadedimagedate = dr.GetDateTime(2);
+                        bulkSatDownloadInput.GeometryJson = JObject.Parse(dr.GetString(3));
+                        bulkSatDownloadInput.SatelliteBands = satelliteBands;
+                        bulkSatDownloadInput.cropfielditemcode = dr.GetString(4);
+                        bulkSatDownloadInput.satellitetaskcode = dr.GetString(5);
+                        bulkSatDownloadInput.database = dbparcels;
+                        bulkSatDownloadInput.schemaname = schemaname;
+                        bulkSatDownloadInput.cropfieldtable = parceltablename;
+                        bulkSatDownloadInput.satelllitetable = groenmonitortablename;
+                        bulkSatDownloadInputListDB.Add(bulkSatDownloadInput);
+                    }
+                    connection.Close();
                 }
-                connection.Close();
+                // Now choose which list you want to use
+                bulkSatDownloadInputList = bulkSatDownloadInputListDB;
             }
+            else
+            {
+                // Option 2: Example without database. Comment out this part if you want to use database
+                // Read cropfields "BulkSatDownloadInput.json" and write all stats to a single csv file
+                // Write all stats for multiple fields will be written to a single csv file
+                string downloadFolder;
+                string fileNameStats;
+                string headerLineStats = $"FieldName,satelliteDate,satelliteBand,max,min,mean,mode,median,stddev,minPlus,curtosis,maxMinus,skewness,variance,populationCount,variationCoefficient,confidenceIntervalLow, confidenceIntervalHigh,confidenceIntervalErrorMargin" + Environment.NewLine;
+                var fieldsInputJson = File.ReadAllText("BulkSatDownloadInput.json");
+                bulkSatDownloadInputListCsv = JsonConvert.DeserializeObject<List<BulkSatDownloadInput>>(fieldsInputJson);
+                for (int i = 0; i < bulkSatDownloadInputListCsv.Count; i++)
+                {
+                    downloadFolder = bulkSatDownloadInputListCsv[i].downloadFolder;
+                    fileNameStats = Path.Combine(downloadFolder, bulkSatDownloadInputListCsv[i].fileNameStats);
+                    if (!Directory.Exists(downloadFolder))
+                        Directory.CreateDirectory(downloadFolder);
+                    bulkSatDownloadInputListCsv[i].fileNameStats = fileNameStats;
+                    // Header same as in GeneralService.DownloadSatelliteStats
+                    // Delete fileNameStats if existing. Create a new file. Add a header to csv file
+                    File.Delete(fileNameStats);
+                    File.AppendAllText(fileNameStats, headerLineStats);
+                }
 
-            // Option 2: Example without database. Comment out this part if you want to use database
-            // Read cropfields "BulkSatDownloadInput.json" and write all stats to a single csv file
-            // Write all stats for multiple fields will be written to a single csv file
-            //string downloadFolder;
-            //string fileNameStats;
-            //string headerLineStats = $"FieldName,satelliteDate,satelliteBand,max,min,mean,mode,median,stddev,minPlus,curtosis,maxMinus,skewness,variance,populationCount,variationCoefficient,confidenceIntervalLow, confidenceIntervalHigh,confidenceIntervalErrorMargin" + Environment.NewLine;
-            //var fieldsInputJson = File.ReadAllText("BulkSatDownloadInput.json");
-            //bulkSatDownloadInputListCsv = JsonConvert.DeserializeObject<List<BulkSatDownloadInput>>(fieldsInputJson);
-            //for (int i = 0; i < bulkSatDownloadInputListCsv.Count; i++)
-            //{
-            //    downloadFolder = bulkSatDownloadInputListCsv[i].downloadFolder;
-            //    fileNameStats = Path.Combine(downloadFolder, bulkSatDownloadInputListCsv[i].fileNameStats);
-            //    if (!Directory.Exists(downloadFolder))
-            //        Directory.CreateDirectory(downloadFolder);
-            //    bulkSatDownloadInputListCsv[i].fileNameStats = fileNameStats;
-            //    // Header same as in GeneralService.DownloadSatelliteStats
-            //    // Delete fileNameStats if existing. Create a new file. Add a header to csv file
-            //    File.Delete(fileNameStats);
-            //    File.AppendAllText(fileNameStats, headerLineStats);
-            //}
-
-            // Now choose which list you want to use
-            bulkSatDownloadInputList = bulkSatDownloadInputListDB;  //bulkSatDownloadInputListDB;  //bulkSatDownloadInputListCsv;
-
-            // Whichever option (database or json/csv), continue here
-            // Delete the settingsfile
-            // File.Delete(settingsfile);
+                // Now choose which list you want to use
+                bulkSatDownloadInputList = bulkSatDownloadInputListCsv;
+            }
 
             // For each input download all images. Keep track to time, important when doing bulk downloads
             var watch = System.Diagnostics.Stopwatch.StartNew();
@@ -191,6 +200,33 @@ LIMIT 5;", schemaname, parceltablename, groenmonitorlatestviewname);   //LIMIT x
 
         private async Task Process(List<UserRoot> roots, BulkSatDownloadInput input)
         {
+            //PO20220311: first time a call is made to download satellite images or statistics, an empty list is returned
+            //If we wait a bit longer, e.g. 10 secs, then e.g. a list of 3 images may be returned
+            //If we wait still longer, maybe 4 images.
+            //The solution implemented below is to fire calls as long as the number of images returned keeps increasing
+            //While in between each call, sleep for sleepSecs
+            //Continue this until the number no longer increases or the maximum number of calls has been reached
+            //If you set sleepSecs to a very low value, e.g. 5 secs, then after 1 call you might get images and after 2nd call still zero images.
+            //to be on the safe side, better bit higher value.
+            //Just accept this may take a while, have a coffee, we suggest sleepSecs = 30;
+            int sleepSecs = 30;
+            int callCntMax = 4 * 60 / sleepSecs;   //4*60 = max 4 minutes
+            //For example we may set: "sleepSecs = 10;" and "callCntMax = 24;" and following result:
+            //Call no: 1. Giving FarmMaps 10 seconds to get SatelliteItems...
+            //Call no: 1: Received 2 images
+            //Call no: 2. Giving FarmMaps 10 seconds to get SatelliteItems...
+            //Call no: 2: Received 7 images
+            //Call no: 3. Giving FarmMaps 10 seconds to get SatelliteItems...
+            //Call no: 3: Received 7 images
+            //And the firing of calls would stop because the number of images returned is no longer increasing
+            //In the worst case, this could would lead to a total sleeping period of "sleepSecsSum = sleepSecs * callCntMax" seconds. After that we give up
+
+            //PO20230801: I also tried creating a cropfieldItem with startDate '2017-01-01' and endDate '2023-08-01'
+            //so that only one RunSatelliteTask would need to be executed and statistics would be returned for the e.g. 233 images in this period from start to end date
+            //instead of running 7 tasks (one per year) for a single field
+            //But code doing that returned much less images than those 233 records which  I should be getting.
+            //So in the end, stick to downloading stats per year, even if it is for the same location in multiple years
+
             string cropfielditemcode;
             string satellitetaskcode;
             Item cropfieldItem;
@@ -222,17 +258,21 @@ LIMIT 5;", schemaname, parceltablename, groenmonitorlatestviewname);   //LIMIT x
             DateTime lastDownloadedSatelliteDate = input.lastdownloadedimagedate;
             cropfielditemcode = input.cropfielditemcode;
             satellitetaskcode = input.satellitetaskcode;
+            int satelliteItemsCropYearCntPrev;
+            int satelliteItemsCropYearCnt;
+            int callCnt;
+            int sleepSecsSum;
 
             LoadSettings(settingsfile);
 
-            var uploadedRoot = roots.SingleOrDefault(r => r.Name == "Uploaded");
+            var uploadedRoot = roots.SingleOrDefault(r => r.Name == "USER_IN");
             if (uploadedRoot == null)
             {
                 _logger.LogError("Could not find a needed root item");
                 return;
             }
 
-            var myDriveRoot = roots.SingleOrDefault(r => r.Name == "My drive");
+            var myDriveRoot = roots.SingleOrDefault(r => r.Name == "USER_FILES");
             if (myDriveRoot == null)
             {
                 _logger.LogError("Could not find a needed root item");
@@ -309,23 +349,80 @@ LIMIT 5;", schemaname, parceltablename, groenmonitorlatestviewname);   //LIMIT x
             // TODO also log satellitetaskcode to settings, how?
             // SaveSettings(settingsfile);
 
-            // Getting satellite items
-            _logger.LogInformation(string.Format($"Running FindSatelliteItems for cropfieldItem.Code '{cropfieldItem.Code}', SatelliteTaskCode '{satellitetaskcode}'"));
-            satelliteItemsCropYear = await _generalService.FindSatelliteItems(cropfieldItem, satellitetaskcode);
+            // Getting satellite items. Only for years for which available
+            satelliteItemsCropYearCntPrev = 0;
+            satelliteItemsCropYearCnt = 0;
+            sleepSecsSum = 0;
+            satelliteItemsCropYear = null;
+            if (cropYear >= firstAvailableYear && cropYear <= DateTime.Now.Year)
+            {
+                _logger.LogInformation(string.Format($"Running FindSatelliteItems for cropfieldItem.Code '{cropfieldItem.Code}', SatelliteTaskCode '{satellitetaskcode}'"));
+                //Call first time
+                callCnt = 1;
+                //if callCntMax == 0 then don't sleep
+                //if callCntMax = 1 then sleep first 1x
+                if (callCntMax > 0)
+                {
+                    _logger.LogInformation($"Call no: {callCnt}. Giving FarmMaps {sleepSecs} seconds to get SatelliteItems...");
+                    System.Threading.Thread.Sleep(1000 * sleepSecs);
+                    sleepSecsSum = sleepSecsSum + sleepSecs;
+                }
+                satelliteItemsCropYear = await _generalService.FindSatelliteItems(cropfieldItem, satellitetaskcode);
+                satelliteItemsCropYearCntPrev = satelliteItemsCropYear.Count;
+                _logger.LogInformation($"Call no: {callCnt}. Received {satelliteItemsCropYearCntPrev} images");
+                callCnt++;
+                satelliteItemsCropYearCnt = satelliteItemsCropYearCntPrev;
+                //if callCntMax > 1 then sleep untill (1) no more increase in number of images received OR (2) maximum number of calls reached
+                if (callCntMax > 1)
+                {
+                    //Call second time
+                    _logger.LogInformation($"Call no: {callCnt}. Giving FarmMaps another {sleepSecs} seconds to get SatelliteItems...");
+                    System.Threading.Thread.Sleep(1000 * sleepSecs);
+                    satelliteItemsCropYear = await _generalService.FindSatelliteItems(cropfieldItem, satellitetaskcode);
+                    satelliteItemsCropYearCnt = satelliteItemsCropYear.Count;
+                    _logger.LogInformation($"Call no: {callCnt}. Received {satelliteItemsCropYearCnt} images");
+                    sleepSecsSum = sleepSecsSum + sleepSecs;
+                    //As long as there is progress, keep calling
+                    callCnt++;
+                    while (callCnt <= callCntMax && (satelliteItemsCropYearCnt == 0 || satelliteItemsCropYearCnt > satelliteItemsCropYearCntPrev))
+                    {
+                        _logger.LogInformation($"Surprise! The longer we wait, the more images we get. Sleep and call once more");
+                        satelliteItemsCropYearCntPrev = satelliteItemsCropYearCnt;
+                        _logger.LogInformation($"Call no: {callCnt} (max: {callCntMax}). Giving FarmMaps another {sleepSecs} seconds to get SatelliteItems...");
+                        System.Threading.Thread.Sleep(1000 * sleepSecs);
+                        satelliteItemsCropYear = await _generalService.FindSatelliteItems(cropfieldItem, satellitetaskcode);
+                        satelliteItemsCropYearCnt = satelliteItemsCropYear.Count;
+                        _logger.LogInformation($"Call no: {callCnt}. Received {satelliteItemsCropYearCnt} images");
+                        callCnt++;
+                        sleepSecsSum = sleepSecsSum + sleepSecs;
+                    }
+                }
+            }
+            else
+            {
+                _logger.LogWarning($"// FarmmapsBulkSatDownload: crop year {cropYear} is out of bounds. No stats will be written!");
+            }
+
+            if (satelliteItemsCropYearCnt == 0)
+            {
+                _logger.LogWarning($"// FarmmapsBulkSatDownload: after calling one or more times and " +
+                $"sleeping in total {sleepSecsSum} seconds, still no images found. " +
+                $"Please check your settings for parameters callCntMax and sleepSecs in FarmmapsBulkSatDownload.cs or contact FarmMaps");
+            }
 
             // Checking if satellite items found
             satelliteItemsAvailable = true;
             if (satelliteItemsCropYear == null)
             {
                 satelliteItemsAvailable = false;
-                _logger.LogInformation($"No satellite tiffs found for fieldName '{fieldName}', cropYear {cropYear}, cropfielditemcode '{cropfielditemcode}'");
+                _logger.LogInformation($"// FarmmapsBulkSatDownload: No satellite tiffs found for fieldName '{fieldName}', cropYear {cropYear}, cropfielditemcode '{cropfielditemcode}'");
             }
             else
             {
                 if (satelliteItemsCropYear.Count == 0)
                 {
                     satelliteItemsAvailable = false;
-                    _logger.LogInformation($"No satellite tiffs found for fieldName '{fieldName}', cropYear {cropYear}, cropfielditemcode '{cropfielditemcode}'");
+                    _logger.LogInformation($"// FarmmapsBulkSatDownload: No satellite tiffs found for fieldName '{fieldName}', cropYear {cropYear}, cropfielditemcode '{cropfielditemcode}'");
                 }
             }
 
@@ -335,10 +432,10 @@ LIMIT 5;", schemaname, parceltablename, groenmonitorlatestviewname);   //LIMIT x
             // Download statistics to a single csv file
             if (satelliteItemsAvailable && downloadFolder != null && fileNameStats != null)
             {
-                // Write statistics for all images for all fieldNane and cropYear to a single csv file, fileNameStats
+                // Write statistics for all images for all fieldName and cropYear to a single csv file, fileNameStats
                 _logger.LogInformation($"Downloading stats for field '{fieldName}' in cropyear {cropYear} to {fileNameStats}");
                 string downloadedStats = await _generalService.DownloadSatelliteStats(satelliteItemsCropYear, fieldName, satelliteBands, downloadFolder);
-                // Add contents of this csv file to thee single large csv file
+                // Add contents of this csv file to the single large csv file
                 var retainedLines = File.ReadAllLines(downloadedStats).Skip(1);
                 File.AppendAllLines(fileNameStats, retainedLines);
                 File.Delete(downloadedStats);

FarmmapsBulkSatDownload/BulkSatDownloadInput.json

@@ -1,46 +1,44 @@
 [
   {
-    "fieldName": "5641", // FarmMaps minimum needs are: fieldName, cropYear & geometryJson
-    "cropYear": 1999, //For testing a year for which we know no satellite data available, program shouldn't crash
-    "fieldID": 5641,
-    "SatelliteBands": [ "wdvi", "ndvi" ], // ["ndvi"] or ["wdvi"] or both: [ "wdvi", "ndvi" ]
-    "lastdownloadedimagedate": "1999-01-01", //downloads images from this date till end of the year
+    "fieldName": "MyField_1", // FarmMaps minimum needs are: fieldName, cropYear & geometryJson
+    "cropYear": 2023,
+    "fieldID": 1,
+    "SatelliteBands": [ "ci-red", "ndvi", "wdvi" ], // ["ndvi"] or ["wdvi"] or both: [ "wdvi", "ndvi" ]
+    "lastdownloadedimagedate": "2023-01-01", //downloads images from this date till end of the year
     "geometryJson": {
       "type": "Polygon",
       "coordinates": [
         [
-          [ 3.37837807779104, 51.3231095796538 ],
-          [ 3.38065689232502, 51.3212527499355 ],
-          [ 3.38022924592256, 51.3210683536359 ],
-          [ 3.37980548452565, 51.3208801127141 ],
-          [ 3.37959556105776, 51.3207540143696 ],
-          [ 3.3793691292654, 51.3205959677371 ],
-          [ 3.37822219207335, 51.3215667913007 ],
-          [ 3.37816999925795, 51.3216109809456 ],
-          [ 3.37646704574705, 51.3208025481261 ],
-          [ 3.37646695791282, 51.3208025061493 ],
-          [ 3.37608401443192, 51.3206231652693 ],
-          [ 3.37607169507628, 51.3206173959751 ],
-          [ 3.37606021048754, 51.320612017601 ],
-          [ 3.37582728410659, 51.3205029306946 ],
-          [ 3.37580409779263, 51.3206502985963 ],
-          [ 3.37575872019649, 51.3207993094705 ],
-          [ 3.37575476634361, 51.3208122883487 ],
-          [ 3.37571181656268, 51.3208797459348 ],
-          [ 3.3756624532907, 51.3209415238446 ],
-          [ 3.37557609963811, 51.3210110142077 ],
-          [ 3.37541089899821, 51.3211055871218 ],
-          [ 3.37477516102591, 51.3214102985009 ],
-          [ 3.37473173914127, 51.3214311108204 ],
-          [ 3.37455904622072, 51.3215138815012 ],
-          [ 3.37415098054777, 51.3217199232877 ],
-          [ 3.37313700916272, 51.3222422862785 ],
-          [ 3.37748824689601, 51.3242852920348 ],
-          [ 3.37749760805371, 51.3242713084009 ],
-          [ 3.37811903757028, 51.3233437635596 ],
-          [ 3.37818758851947, 51.3232647797363 ],
-          [ 3.37823803668144, 51.3232236798646 ],
-          [ 3.37837807779104, 51.3231095796538 ]
+          [ 5.563472073408009, 52.547554398144172 ],
+          [ 5.567425915520115, 52.547725375100377 ],
+          [ 5.563878143678981, 52.54048022658143 ],
+          [ 5.563878143678981, 52.54048022658143 ],
+          [ 5.563472073408009, 52.547554398144172 ]
+        ]
+      ]
+    },
+    "downloadFolder": "C:\\workdir\\groenmonitor\\", //if not yet existing this folder will be created
+    "fileNameStats": "BulkSatDownload.csv", //if file exists, probably will be overwritten. Check code in BulkSatDownloadApplication.cs
+    "database": null, // keep null to work with json and csv. Check code in BulkSatDownloadApplication.cs if reading/writing to/from database
+    "schemaname": null,
+    "cropfieldtable": null,
+    "satelllitetable": null
+  },
+  {
+    "fieldName": "MyField_2", // FarmMaps minimum needs are: fieldName, cropYear & geometryJson
+    "cropYear": 2022, // for testing same field last year (i.e. 2021)
+    "fieldID": 2,
+    "SatelliteBands": [ "ci-red", "ndvi", "wdvi" ], // ["ndvi"] or ["wdvi"] or both: [ "wdvi", "ndvi" ]
+    "lastdownloadedimagedate": "2022-01-01", //downloads images from this date till end of the year
+    "geometryJson": {
+      "type": "Polygon",
+      "coordinates": [
+        [
+          [ 5.563472073408009, 52.547554398144172 ],
+          [ 5.567425915520115, 52.547725375100377 ],
+          [ 5.563878143678981, 52.54048022658143 ],
+          [ 5.563878143678981, 52.54048022658143 ],
+          [ 5.563472073408009, 52.547554398144172 ]
         ]
       ]
     },
@@ -52,47 +50,20 @@
     "satelllitetable": null
   },
   {
-    "fieldName": "5641", // FarmMaps minimum needs are: fieldName, cropYear & geometryJson
-    "cropYear": 2021, //For testing a year for which we know no satellite data available, program shouldn't crash
-    "fieldID": 5641,
-    "SatelliteBands": [ "wdvi", "ndvi" ], // ["ndvi"] or ["wdvi"] or both: [ "wdvi", "ndvi" ]
+    "fieldName": "MyField_3", // FarmMaps minimum needs are: fieldName, cropYear & geometryJson
+    "cropYear": 2021,
+    "fieldID": 3,
+    "SatelliteBands": [ "ci-red", "ndvi", "wdvi" ], // ["ndvi"] or ["wdvi"] or both: [ "wdvi", "ndvi" ]
     "lastdownloadedimagedate": "2021-01-01", //downloads images from this date till end of the year
     "geometryJson": {
       "type": "Polygon",
       "coordinates": [
         [
-          [ 3.37837807779104, 51.3231095796538 ],
-          [ 3.38065689232502, 51.3212527499355 ],
-          [ 3.38022924592256, 51.3210683536359 ],
-          [ 3.37980548452565, 51.3208801127141 ],
-          [ 3.37959556105776, 51.3207540143696 ],
-          [ 3.3793691292654, 51.3205959677371 ],
-          [ 3.37822219207335, 51.3215667913007 ],
-          [ 3.37816999925795, 51.3216109809456 ],
-          [ 3.37646704574705, 51.3208025481261 ],
-          [ 3.37646695791282, 51.3208025061493 ],
-          [ 3.37608401443192, 51.3206231652693 ],
-          [ 3.37607169507628, 51.3206173959751 ],
-          [ 3.37606021048754, 51.320612017601 ],
-          [ 3.37582728410659, 51.3205029306946 ],
-          [ 3.37580409779263, 51.3206502985963 ],
-          [ 3.37575872019649, 51.3207993094705 ],
-          [ 3.37575476634361, 51.3208122883487 ],
-          [ 3.37571181656268, 51.3208797459348 ],
-          [ 3.3756624532907, 51.3209415238446 ],
-          [ 3.37557609963811, 51.3210110142077 ],
-          [ 3.37541089899821, 51.3211055871218 ],
-          [ 3.37477516102591, 51.3214102985009 ],
-          [ 3.37473173914127, 51.3214311108204 ],
-          [ 3.37455904622072, 51.3215138815012 ],
-          [ 3.37415098054777, 51.3217199232877 ],
-          [ 3.37313700916272, 51.3222422862785 ],
-          [ 3.37748824689601, 51.3242852920348 ],
-          [ 3.37749760805371, 51.3242713084009 ],
-          [ 3.37811903757028, 51.3233437635596 ],
-          [ 3.37818758851947, 51.3232647797363 ],
-          [ 3.37823803668144, 51.3232236798646 ],
-          [ 3.37837807779104, 51.3231095796538 ]
+          [ 5.563472073408009, 52.547554398144172 ],
+          [ 5.567425915520115, 52.547725375100377 ],
+          [ 5.563878143678981, 52.54048022658143 ],
+          [ 5.563878143678981, 52.54048022658143 ],
+          [ 5.563472073408009, 52.547554398144172 ]
         ]
       ]
     },
@@ -104,47 +75,20 @@
     "satelllitetable": null
   },
   {
-    "fieldName": "5641", // FarmMaps minimum needs are: fieldName, cropYear & geometryJson
+    "fieldName": "MyField_4", // FarmMaps minimum needs are: fieldName, cropYear & geometryJson
     "cropYear": 2020,
-    "fieldID": 5641,
-    "SatelliteBands": [ "wdvi", "ndvi" ], // ["ndvi"] or ["wdvi"] or both: [ "wdvi", "ndvi" ]
+    "fieldID": 4,
+    "SatelliteBands": [ "ci-red", "ndvi", "wdvi" ], // ["ndvi"] or ["wdvi"] or both: [ "wdvi", "ndvi" ]
     "lastdownloadedimagedate": "2020-01-01", //downloads images from this date till end of the year
     "geometryJson": {
       "type": "Polygon",
       "coordinates": [
         [
-          [ 3.37837807779104, 51.3231095796538 ],
-          [ 3.38065689232502, 51.3212527499355 ],
-          [ 3.38022924592256, 51.3210683536359 ],
-          [ 3.37980548452565, 51.3208801127141 ],
-          [ 3.37959556105776, 51.3207540143696 ],
-          [ 3.3793691292654, 51.3205959677371 ],
-          [ 3.37822219207335, 51.3215667913007 ],
-          [ 3.37816999925795, 51.3216109809456 ],
-          [ 3.37646704574705, 51.3208025481261 ],
-          [ 3.37646695791282, 51.3208025061493 ],
-          [ 3.37608401443192, 51.3206231652693 ],
-          [ 3.37607169507628, 51.3206173959751 ],
-          [ 3.37606021048754, 51.320612017601 ],
-          [ 3.37582728410659, 51.3205029306946 ],
-          [ 3.37580409779263, 51.3206502985963 ],
-          [ 3.37575872019649, 51.3207993094705 ],
-          [ 3.37575476634361, 51.3208122883487 ],
-          [ 3.37571181656268, 51.3208797459348 ],
-          [ 3.3756624532907, 51.3209415238446 ],
-          [ 3.37557609963811, 51.3210110142077 ],
-          [ 3.37541089899821, 51.3211055871218 ],
-          [ 3.37477516102591, 51.3214102985009 ],
-          [ 3.37473173914127, 51.3214311108204 ],
-          [ 3.37455904622072, 51.3215138815012 ],
-          [ 3.37415098054777, 51.3217199232877 ],
-          [ 3.37313700916272, 51.3222422862785 ],
-          [ 3.37748824689601, 51.3242852920348 ],
-          [ 3.37749760805371, 51.3242713084009 ],
-          [ 3.37811903757028, 51.3233437635596 ],
-          [ 3.37818758851947, 51.3232647797363 ],
-          [ 3.37823803668144, 51.3232236798646 ],
-          [ 3.37837807779104, 51.3231095796538 ]
+          [ 5.563472073408009, 52.547554398144172 ],
+          [ 5.567425915520115, 52.547725375100377 ],
+          [ 5.563878143678981, 52.54048022658143 ],
+          [ 5.563878143678981, 52.54048022658143 ],
+          [ 5.563472073408009, 52.547554398144172 ]
         ]
       ]
     },
@@ -156,47 +100,20 @@
     "satelllitetable": null
   },
   {
-    "fieldName": "5641", // FarmMaps minimum needs are: fieldName, cropYear & geometryJson
+    "fieldName": "MyField_5", // FarmMaps minimum needs are: fieldName, cropYear & geometryJson
     "cropYear": 2019,
-    "fieldID": 5641,
-    "SatelliteBands": [ "wdvi", "ndvi" ], // ["ndvi"] or ["wdvi"] or both: [ "wdvi", "ndvi" ]
+    "fieldID": 5,
+    "SatelliteBands": [ "ci-red", "ndvi", "wdvi" ], // ["ndvi"] or ["wdvi"] or both: [ "wdvi", "ndvi" ]
     "lastdownloadedimagedate": "2019-01-01", //downloads images from this date till end of the year
     "geometryJson": {
       "type": "Polygon",
       "coordinates": [
         [
-          [ 3.37837807779104, 51.3231095796538 ],
-          [ 3.38065689232502, 51.3212527499355 ],
-          [ 3.38022924592256, 51.3210683536359 ],
-          [ 3.37980548452565, 51.3208801127141 ],
-          [ 3.37959556105776, 51.3207540143696 ],
-          [ 3.3793691292654, 51.3205959677371 ],
-          [ 3.37822219207335, 51.3215667913007 ],
-          [ 3.37816999925795, 51.3216109809456 ],
-          [ 3.37646704574705, 51.3208025481261 ],
-          [ 3.37646695791282, 51.3208025061493 ],
-          [ 3.37608401443192, 51.3206231652693 ],
-          [ 3.37607169507628, 51.3206173959751 ],
-          [ 3.37606021048754, 51.320612017601 ],
-          [ 3.37582728410659, 51.3205029306946 ],
-          [ 3.37580409779263, 51.3206502985963 ],
-          [ 3.37575872019649, 51.3207993094705 ],
-          [ 3.37575476634361, 51.3208122883487 ],
-          [ 3.37571181656268, 51.3208797459348 ],
-          [ 3.3756624532907, 51.3209415238446 ],
-          [ 3.37557609963811, 51.3210110142077 ],
-          [ 3.37541089899821, 51.3211055871218 ],
-          [ 3.37477516102591, 51.3214102985009 ],
-          [ 3.37473173914127, 51.3214311108204 ],
-          [ 3.37455904622072, 51.3215138815012 ],
-          [ 3.37415098054777, 51.3217199232877 ],
-          [ 3.37313700916272, 51.3222422862785 ],
-          [ 3.37748824689601, 51.3242852920348 ],
-          [ 3.37749760805371, 51.3242713084009 ],
-          [ 3.37811903757028, 51.3233437635596 ],
-          [ 3.37818758851947, 51.3232647797363 ],
-          [ 3.37823803668144, 51.3232236798646 ],
-          [ 3.37837807779104, 51.3231095796538 ]
+          [ 5.563472073408009, 52.547554398144172 ],
+          [ 5.567425915520115, 52.547725375100377 ],
+          [ 5.563878143678981, 52.54048022658143 ],
+          [ 5.563878143678981, 52.54048022658143 ],
+          [ 5.563472073408009, 52.547554398144172 ]
         ]
       ]
     },
@@ -208,47 +125,95 @@
     "satelllitetable": null
   },
   {
-    "fieldName": "5641", // FarmMaps minimum needs are: fieldName, cropYear & geometryJson
-    "cropYear": 2018, //little to no images for 2018
-    "fieldID": 5641,
-    "SatelliteBands": [ "wdvi", "ndvi" ], // ["ndvi"] or ["wdvi"] or both: [ "wdvi", "ndvi" ]
+    "fieldName": "MyField_6", // FarmMaps minimum needs are: fieldName, cropYear & geometryJson
+    "cropYear": 2018,
+    "fieldID": 6,
+    "SatelliteBands": [ "ci-red", "ndvi", "wdvi" ], // ["ndvi"] or ["wdvi"] or both: [ "wdvi", "ndvi" ]
     "lastdownloadedimagedate": "2018-01-01", //downloads images from this date till end of the year
     "geometryJson": {
       "type": "Polygon",
       "coordinates": [
         [
-          [ 3.37837807779104, 51.3231095796538 ],
-          [ 3.38065689232502, 51.3212527499355 ],
-          [ 3.38022924592256, 51.3210683536359 ],
-          [ 3.37980548452565, 51.3208801127141 ],
-          [ 3.37959556105776, 51.3207540143696 ],
-          [ 3.3793691292654, 51.3205959677371 ],
-          [ 3.37822219207335, 51.3215667913007 ],
-          [ 3.37816999925795, 51.3216109809456 ],
-          [ 3.37646704574705, 51.3208025481261 ],
-          [ 3.37646695791282, 51.3208025061493 ],
-          [ 3.37608401443192, 51.3206231652693 ],
-          [ 3.37607169507628, 51.3206173959751 ],
-          [ 3.37606021048754, 51.320612017601 ],
-          [ 3.37582728410659, 51.3205029306946 ],
-          [ 3.37580409779263, 51.3206502985963 ],
-          [ 3.37575872019649, 51.3207993094705 ],
-          [ 3.37575476634361, 51.3208122883487 ],
-          [ 3.37571181656268, 51.3208797459348 ],
-          [ 3.3756624532907, 51.3209415238446 ],
-          [ 3.37557609963811, 51.3210110142077 ],
-          [ 3.37541089899821, 51.3211055871218 ],
-          [ 3.37477516102591, 51.3214102985009 ],
-          [ 3.37473173914127, 51.3214311108204 ],
-          [ 3.37455904622072, 51.3215138815012 ],
-          [ 3.37415098054777, 51.3217199232877 ],
-          [ 3.37313700916272, 51.3222422862785 ],
-          [ 3.37748824689601, 51.3242852920348 ],
-          [ 3.37749760805371, 51.3242713084009 ],
-          [ 3.37811903757028, 51.3233437635596 ],
-          [ 3.37818758851947, 51.3232647797363 ],
-          [ 3.37823803668144, 51.3232236798646 ],
-          [ 3.37837807779104, 51.3231095796538 ]
+          [ 5.563472073408009, 52.547554398144172 ],
+          [ 5.567425915520115, 52.547725375100377 ],
+          [ 5.563878143678981, 52.54048022658143 ],
+          [ 5.563878143678981, 52.54048022658143 ],
+          [ 5.563472073408009, 52.547554398144172 ]
+        ]
+      ]
+    },
+    "downloadFolder": "C:\\workdir\\groenmonitor\\",
+    "fileNameStats": "BulkSatDownload.csv",
+    "database": null,
+    "schemaname": null,
+    "cropfieldtable": null,
+    "satelllitetable": null
+  },
+  {
+    "fieldName": "MyField_6", // FarmMaps minimum needs are: fieldName, cropYear & geometryJson
+    "cropYear": 2017,
+    "fieldID": 7,
+    "SatelliteBands": [ "ci-red", "ndvi", "wdvi" ], // ["ndvi"] or ["wdvi"] or both: [ "wdvi", "ndvi" ]
+    "lastdownloadedimagedate": "2017-01-01", //downloads images from this date till end of the year
+    "geometryJson": {
+      "type": "Polygon",
+      "coordinates": [
+        [
+          [ 5.563472073408009, 52.547554398144172 ],
+          [ 5.567425915520115, 52.547725375100377 ],
+          [ 5.563878143678981, 52.54048022658143 ],
+          [ 5.563878143678981, 52.54048022658143 ],
+          [ 5.563472073408009, 52.547554398144172 ]
+        ]
+      ]
+    },
+    "downloadFolder": "C:\\workdir\\groenmonitor\\",
+    "fileNameStats": "BulkSatDownload.csv",
+    "database": null,
+    "schemaname": null,
+    "cropfieldtable": null,
+    "satelllitetable": null
+  },
+  {
+    "fieldName": "MyField_8", // FarmMaps minimum needs are: fieldName, cropYear & geometryJson. Actually no Satellite data before 2017. This here to illustrate the program does not crash
+    "cropYear": 2016,
+    "fieldID": 8,
+    "SatelliteBands": [ "ci-red", "ndvi", "wdvi" ], // ["ndvi"] or ["wdvi"] or both: [ "wdvi", "ndvi" ]
+    "lastdownloadedimagedate": "2016-01-01", //downloads images from this date till end of the year
+    "geometryJson": {
+      "type": "Polygon",
+      "coordinates": [
+        [
+          [ 5.563472073408009, 52.547554398144172 ],
+          [ 5.567425915520115, 52.547725375100377 ],
+          [ 5.563878143678981, 52.54048022658143 ],
+          [ 5.563878143678981, 52.54048022658143 ],
+          [ 5.563472073408009, 52.547554398144172 ]
+        ]
+      ]
+    },
+    "downloadFolder": "C:\\workdir\\groenmonitor\\",
+    "fileNameStats": "BulkSatDownload.csv",
+    "database": null,
+    "schemaname": null,
+    "cropfieldtable": null,
+    "satelllitetable": null
+  },
+  {
+    "fieldName": "MyField_9", // FarmMaps minimum needs are: fieldName, cropYear & geometryJson. Actually no Satellite data before 2017. This here to illustrate the program does not crash
+    "cropYear": 2015,
+    "fieldID": 9,
+    "SatelliteBands": [ "ci-red", "ndvi", "wdvi" ], // ["ndvi"] or ["wdvi"] or both: [ "wdvi", "ndvi" ]
+    "lastdownloadedimagedate": "2015-01-01", //downloads images from this date till end of the year
+    "geometryJson": {
+      "type": "Polygon",
+      "coordinates": [
+        [
+          [ 5.563472073408009, 52.547554398144172 ],
+          [ 5.567425915520115, 52.547725375100377 ],
+          [ 5.563878143678981, 52.54048022658143 ],
+          [ 5.563878143678981, 52.54048022658143 ],
+          [ 5.563472073408009, 52.547554398144172 ]
         ]
       ]
     },

FarmmapsBulkSatDownload/ShowGeotiff.r

@@ -1,178 +0,0 @@
-# ShowGeotiff.r
-# I downloaded and calculated the stats for the polygon defined in C:\git\FarmMapsApiClient_WURtest\FarmmapsDataDownload\DataDownloadInput.json
-# in which I set "SatelliteBand": "wdvi" and in which in the console I requested the image for date '2020-09-22'
-# FarmmapsBulkSatDownload generates many files. Here is what I tried when inputing the same field for a number of years using BulkSatDownloadInput.json
-# see list below
-
-library(raster)
-library(sf)
-library(rgdal)
-setwd("C:/workdir/groenmonitor/DataDownload/")
-
-# FarmmapsDataDownload and BulkSatDownload can be used to download zip files with inside two files:
-fileGeotiff <- "wenr.tif"
-fileJpg <- "thumbnail.jpg"
-# Here is what I tried when inputing the same field for a number of years:
-# fileGeotiff <- "wheat_fld5641_20210224.tif"	# 2 files for year 2021. This file is a nice example having in upperleft corner no data
-# fileGeotiff <- "wheat_fld5641_20210331.tif"	# 2 files for year 2021
-# fileGeotiff <- "wheat_fld5641_20200321.tif"	# 14 files for year 2020, earliest
-# fileGeotiff <- "wheat_fld5641_20200922.tif"	# 14 files for year 2020, latest
-# fileGeotiff <- "wheat_fld5641_20190121.tif"	# 9 files for year 2019, earliest
-# fileGeotiff <- "wheat_fld5641_20191117.tif"	# 9 files for year 2019, latest
-# 1 file for year 2018, with error message 'End of Central Directory record could not be found' and invalid wheat_fld5641_20180630.zip 
-# fileGeotiff <- "wheat_fld5641_20170526.tif"	# 1 file for year 2017
-# Zero files for 2016
-lenfilename <- nchar(fileGeotiff)
-year <- substr(fileGeotiff,lenfilename-11,lenfilename-8)
-imgdate <- substr(fileGeotiff,lenfilename-11,lenfilename-4)
-
-# The thumbnail has the polygon clipped out, has 1 layer, no crs and the mean value is not the mean wdvi we are looking for
-r.thumbnail <- raster(fileJpg)
-plot(r.thumbnail)
-crs(r.thumbnail)
-#CRS arguments: NA 
-cellStats(r.thumbnail,'mean')	#87.5128 # nonsense
-
-stk.wenr <- stack(x=fileGeotiff)
-# plot(stk.wenr) shows 5 plots (5 bands)
-# I think these are:
-# 1. ndvi (since it runs from 0 to 1)
-# 2. wdvi (since it runs from 0 to 0.5)
-# 3-5: RGB (since they run from 0 to 255)
-plot(stk.wenr)
-# CRS arguments:
-# +proj=sterea +lat_0=52.1561605555556 +lon_0=5.38763888888889 +k=0.9999079 +x_0=155000 +y_0=463000 +ellps=bessel +units=m +no_defs
-# Or use st_crs(stk.wenr) to get more info, but no EPSG code in there. 
-# Likely it is epsg:28992 (Amersfoort)
-crs(stk.wenr)
-stk.wenr <- projectRaster(stk.wenr, crs = CRS('+init=EPSG:28992'))
-crs(stk.wenr)
-# Looks the same but strangely, if we don't do projectRaster(stk.wenr, crs = CRS('+init=EPSG:28992')), we find below the bottom left corner of the polygon missing
-
-r.wenr.rd.wdvi <- subset(stk.wenr,2)
-dev.off()
-plot(r.wenr.rd.wdvi,main=paste("wdvi",imgdate),xlab="RDX",ylab="RDY")
-cellStats(r.wenr.rd.wdvi,'mean')	#0.1350561
-
-# Furthermore we can see
-# shows coordinates in RD
-# returns a rectangle, thus the shape of the polygon submitted is not clipped. 
-# The polygon was provided in WGS84. Let's draw it on top. 
-# First convert the raster to WGS84
-r.wenr.wgs84.wdvi <- projectRaster(r.wenr.rd.wdvi, crs = CRS('+init=EPSG:4326'))
-
-# Draw a polygon on top of the raster
-# Example polygon p1
-# p1 <- data.frame(id = 1, wkt = 'POLYGON((4.963 52.801, 4.966 52.801, 4.966 52.803, 4.963 52.803, 4.963 52.801))')
-# p1 <- st_as_sf(p1, wkt = 'wkt', crs = targetcrs)
-# plot(p1,add=TRUE, col="transparent",border="black")
-# Draw the polygon on top of the raster
-# Polygon p2 from C:\git\FarmMapsApiClient_WURtest\FarmmapsDataDownload\DataDownloadInput.json
-p2 <- data.frame(id = 1, wkt = gsub("\n","",'POLYGON((
-          4.960707146896585 52.800583669708487,
-          4.960645975538824 52.800470217610922,
-          4.962140695752897 52.799177147194797,
-          4.967523821195745 52.801502400041208,
-          4.966336768950911 52.802543735879809,
-          4.961711880764330 52.801009996856429,
-          4.960707146896585 52.800583669708487))'))
-# Polygon p2 from C:\git\FarmMapsApiClient_WURtest\FarmmapsBulkSatDownload\BulkSatDownloadInput.json
-p2 <- data.frame(id = 1, wkt = gsub("\n","",'POLYGON((
-			3.37837807779104   51.3231095796538,
-			3.38065689232502   51.3212527499355,
-			3.38022924592256   51.3210683536359,
-			3.37980548452565   51.3208801127141,
-			3.37959556105776   51.3207540143696,
-			3.3793691292654   51.3205959677371,
-			3.37822219207335   51.3215667913007,
-			3.37816999925795   51.3216109809456,
-			3.37646704574705   51.3208025481261,
-			3.37646695791282   51.3208025061493,
-			3.37608401443192   51.3206231652693,
-			3.37607169507628   51.3206173959751,
-			3.37606021048754   51.320612017601,
-			3.37582728410659   51.3205029306946,
-			3.37580409779263   51.3206502985963,
-			3.37575872019649   51.3207993094705,
-			3.37575476634361   51.3208122883487,
-			3.37571181656268   51.3208797459348,
-			3.3756624532907   51.3209415238446,
-			3.37557609963811   51.3210110142077,
-			3.37541089899821   51.3211055871218,
-			3.37477516102591   51.3214102985009,
-			3.37473173914127   51.3214311108204,
-			3.37455904622072   51.3215138815012,
-			3.37415098054777   51.3217199232877,
-			3.37313700916272   51.3222422862785,
-			3.37748824689601   51.3242852920348,
-			3.37749760805371   51.3242713084009,
-			3.37811903757028   51.3233437635596,
-			3.37818758851947   51.3232647797363,
-			3.37823803668144   51.3232236798646,
-			3.37837807779104   51.3231095796538))'))
-p2.wgs84 <- st_as_sf(p2, wkt = 'wkt', crs = CRS('+init=EPSG:4326'))
-# Or other way round, in RD Amersfoort. That looks ok
-p2.rd <- st_transform(p2.wgs84, "+init=epsg:28992")
-
-# Have a look at both
-# wg84
-dev.off()
-plot(r.wenr.wgs84.wdvi,main=paste("wdvi",imgdate),xlab="LON",ylab="LAT")
-plot(p2.wgs84,add=TRUE, col="transparent",border="red")
-# RD
-dev.off()
-plot(r.wenr.rd.wdvi,main=paste("wdvi",imgdate),xlab="RDX",ylab="RDY")
-plot(p2.rd,add=TRUE, col="transparent",border="red")
-
-#Let's clip the polygon
-r.wenr.rd.wdvi.pol <- mask(r.wenr.rd.wdvi,p2.rd)
-r.wenr.wgs84.wdvi.pol <- mask(r.wenr.wgs84.wdvi,p2.wgs84)
-dev.off()
-plot(r.wenr.wgs84.wdvi.pol,main=paste("wdvi",imgdate),xlab="LON",ylab="LAT")
-plot(p2.wgs84,add=TRUE, col="transparent",border="red")
-#That's what we want! Now compare the stats
-cellStats(r.wenr.rd.wdvi,'mean')	# [1] 0.1350561 # Stats from rectangle, RD
-cellStats(r.wenr.wgs84.wdvi,'mean')	# [1] 0.1351411 # Stats from rectangle, WGS84
-cellStats(r.wenr.rd.wdvi.pol,'mean')	# [1] 0.05723957 # Stats from raster clipped by polygon, RD
-cellStats(r.wenr.wgs84.wdvi.pol,'mean')	# [1] 0.05723607 # Stats from raster clipped by polygon, WGS84
-# file SatelliteDataStatistics_test_satData_wdvi_2020-09-22.csv
-# "mean": 0.057430520945401985	# SatelliteDataStatistics_test_satData_wdvi_2020.csv returns stats for the clipped raster (.pol). 'mean' almost the same, maybe 
-# cellStats cannot return median, just a few stats.
-cellStats(r.wenr.wgs84.wdvi.pol,'median')	# Error in .local(x, stat, ...) : invalid 'stat'. Should be sum, min, max, sd, mean, or 'countNA'
-r.wenr.wgs84.wdvi.vals <- values(r.wenr.wgs84.wdvi)
-median(r.wenr.wgs84.wdvi.vals)	# [1] NA
-median(r.wenr.wgs84.wdvi.vals,na.rm=TRUE)	# [1] 0.076
-r.wenr.wgs84.wdvi.pol.vals <- values(r.wenr.wgs84.wdvi.pol)
-median(r.wenr.wgs84.wdvi.pol.vals)	# [1] NA
-median(r.wenr.wgs84.wdvi.pol.vals,na.rm=TRUE)	# [1] 0.048
-# "median": 0.04800000041723251	# SatelliteDataStatistics_test_satData_wdvi_2020.csv returns stats for the clipped raster (.pol). 
-# An image may contain NA values. Check:
-cellStats(r.wenr.wgs84.wdvi,'countNA')	# [1] 22956
-ncell(r.wenr.wgs84.wdvi)	# [1] 221696
-cellStats(r.wenr.wgs84.wdvi,'countNA') / ncell(r.wenr.wgs84.wdvi)	# [1] 0.1035472	# 10% no data? doesn't show in the plot?
-cellStats(r.wenr.wgs84.wdvi.pol,'countNA')	# [1] 147387
-summary(r.wenr.wgs84.wdvi.pol.vals)	# shows the same: NA's: 147387
-ncell(r.wenr.wgs84.wdvi.pol)	# [1] 221696
-cellStats(r.wenr.wgs84.wdvi.pol,'countNA') / ncell(r.wenr.wgs84.wdvi.pol)	# [1] 0.6648158	# 66% no data? doesn't show in the plot?
-
-# The project FarmmapsNbs can generate a wenr.tif file, application.tif, uptake.tif (in rtest1.uptake.zip)and shape.shp (in rtest1.taskmap.zip)
-r.application <- raster("C:/git/FarmMapsApiClient_WURtest/FarmmapsNbs/bin/Debug/netcoreapp3.1/Downloads/application.tif")
-dev.off()
-plot(r.application)
-plot(p2.rd,add=TRUE, col="transparent",border="red")
-# The application.tif file is a rectangle (polygon not yet clipped), in projection Amersfoort RD New (EPSG:28992)
-
-r.uptake <- raster("C:/git/FarmMapsApiClient_WURtest/FarmmapsNbs/bin/Debug/netcoreapp3.1/Downloads/uptake.tif")
-dev.off()
-plot(r.uptake)
-plot(p2.rd,add=TRUE, col="transparent",border="red")
-# The uptake.tif file is a rectangle (polygon not yet clipped), in projection Amersfoort RD New (EPSG:28992)
-
-shp.wgs84 <- readOGR(dsn="C:/git/FarmMapsApiClient_WURtest/FarmmapsNbs/bin/Debug/netcoreapp3.1/Downloads", layer="shape")
-crs(shp.wgs84)
-# CRS arguments: +proj=longlat +datum=WGS84 +no_defs
-dev.off()
-plot(r.wenr.wgs84.wdvi,main="wdvi",xlab="LON",ylab="LAT")
-plot(shp.wgs84,add=TRUE, col="transparent",border="black")
-plot(p2.wgs84,add=TRUE, col="transparent",border="red")
-# The shape file is in WGS84

FarmmapsDataDownload/DataDownloadApplication.cs

@@ -38,7 +38,7 @@ namespace FarmmapsDataDownload
 
         public async Task RunAsync()
         {
-            var fieldsInputJson = File.ReadAllText("DataDownloadInput.json");
+            string fieldsInputJson = File.ReadAllText("DataDownloadInput.json");
 
             List<DataDownloadInput> fieldsInputs = JsonConvert.DeserializeObject<List<DataDownloadInput>>(fieldsInputJson);
 
@@ -61,6 +61,28 @@ namespace FarmmapsDataDownload
 
         private async Task Process(List<UserRoot> roots, DataDownloadInput input)
         {
+            //PO20220311: first time a call is made to download satellite images or statistics, an empty list is returned
+            //If we wait a bit longer, e.g. 10 secs, then e.g. a list of 3 images may be returned
+            //If we wait still longer, maybe 4 images.
+            //The solution implemented below is to fire calls as long as the number of images returned keeps increasing
+            //While in between each call, sleep for sleepSecs
+            //Continue this until the number no longer increases or the maximum number of calls has been reached
+            //Out of politeness, don't be too impatient. Don't set sleepSecs to 5 or 10 or 30 secs. Just accept this may take a while, have a coffee, we suggest sleepSecs = 60;
+            int sleepSecs = 60;
+            int callCntMax = 4;
+            //For example we may set: "sleepSecs = 10;" and "callCntMax = 24;" and following result:
+            //Call no: 1. Giving FarmMaps 10 seconds to get SatelliteItems...
+            //Call no: 1: Received 2 images
+            //Call no: 2. Giving FarmMaps 10 seconds to get SatelliteItems...
+            //Call no: 2: Received 7 images
+            //Call no: 3. Giving FarmMaps 10 seconds to get SatelliteItems...
+            //Call no: 3: Received 7 images
+            //And the firing of calls would stop because the number of images returned is no longer increasing
+            //In the worst case, this could would lead to a total sleeping period of "sleepSecsSum = sleepSecs * callCntMax" seconds. After that we give up
+
+            //This is an ugly fix. Neater would if FarmMaps would just take a bit longer and then do always deliver all satellite images on first call.
+            //Once this has been fixed on the side of FarmMaps we can set callCntMax = 0 and the code below will work smoothly without any sleeping
+
             string downloadFolder = input.DownloadFolder;
             if (string.IsNullOrEmpty(downloadFolder)) {
                 downloadFolder = "Downloads";
@@ -74,7 +96,8 @@ namespace FarmmapsDataDownload
             var fieldName = input.fieldName;
             bool storeSatelliteStatistics = input.StoreSatelliteStatisticsSingleImage;
             bool storeSatelliteStatisticsCropYear = input.StoreSatelliteStatisticsCropYear;
-            List<string> SatelliteBands = new List<string>(1) { input.SatelliteBand };
+            //List<string> SatelliteBands = new List<string>(1) { input.SatelliteBand };
+            List<string> satelliteBands = input.SatelliteBands;
             string headerLineStats = $"FieldName,satelliteDate,satelliteBand,max,min,mean,mode,median,stddev,minPlus,curtosis,maxMinus,skewness,variance,populationCount,variationCoefficient,confidenceIntervalLow, confidenceIntervalHigh,confidenceIntervalErrorMargin" + Environment.NewLine;
 
 
@@ -82,14 +105,14 @@ namespace FarmmapsDataDownload
 
             LoadSettings(settingsfile);
 
-            var uploadedRoot = roots.SingleOrDefault(r => r.Name == "Uploaded");
+            var uploadedRoot = roots.SingleOrDefault(r => r.Name == "USER_IN");
             if (uploadedRoot == null)
             {
                 _logger.LogError("Could not find a needed root item");
                 return;
             }
 
-            var myDriveRoot = roots.SingleOrDefault(r => r.Name == "My drive");
+            var myDriveRoot = roots.SingleOrDefault(r => r.Name == "USER_FILES");
             if (myDriveRoot == null)
             {
                 _logger.LogError("Could not find a needed root item");
@@ -112,9 +135,33 @@ namespace FarmmapsDataDownload
                 cropfieldItem = await _farmmapsApiService.GetItemAsync(_settings.CropfieldItemCode);
             }
 
+            //Get croprecordings
+            if (input.GetCropRecordings)
+            {
+                var crprecItem = input.CrprecItem;
+                _logger.LogInformation($"Trying to get crop recordings of croprecording: {crprecItem}");
+
+                var cropRec = await _farmmapsApiService.GetItemChildrenAsync(crprecItem, CROPREC_ITEMTYPE); 
+                if (cropRec == null)
+                {
+                    _logger.LogError("Something went wrong while obtaining the croprecordings");
+                    return;
+                }
+
+                var cropRecPath = Path.Combine(downloadFolder, $"croprecordings_{crprecItem}.json");
+                _logger.LogInformation($"Found {cropRec.Count} crop recordings");
+                var count = 0;
+                await Task.Delay(500);
+                foreach (var item in cropRec)
+                {
+                    Console.WriteLine($"Crop recording #{count}: {item.Name}");
+                    File.AppendAllText(cropRecPath, item.Data +Environment.NewLine);
+                    count++;
+                }
+                _logger.LogInformation($"Downloaded file {cropRecPath}");
+            }
 
             // Get shadow data
-
             if (input.GetShadowData)
             {
                 _logger.LogInformation("Calculate shadow map for field");
@@ -142,12 +189,58 @@ namespace FarmmapsDataDownload
                     SaveSettings(settingsfile);
                 }
 
-
-                // Select all satellite items
+                //Call first time
+                int callCnt = 1;
+                int sleepSecsSum = 0;
+                //if callCntMax == 0 then don't sleep
+                //if callCntMax = 1 then sleep first 1x
+                if (callCntMax > 0)
+                {
+                    _logger.LogInformation($"Call no: {callCnt}. Giving FarmMaps {sleepSecs} seconds to get SatelliteItems...");
+                    System.Threading.Thread.Sleep(1000 * sleepSecs);
+                    sleepSecsSum = sleepSecsSum + sleepSecs;
+                }
                 List<Item> satelliteItemsCropYear = await _generalService.FindSatelliteItems(cropfieldItem, _settings.SatelliteTaskCode);
+                int satelliteItemsCropYearCntPrev = satelliteItemsCropYear.Count;
+                _logger.LogInformation($"Call no: {callCnt}. Received {satelliteItemsCropYearCntPrev} images");
+                callCnt++;
+                int satelliteItemsCropYearCnt = satelliteItemsCropYearCntPrev;
+                //if callCntMax > 1 then sleep untill (1) no more increase in number of images received OR (2) maximum number of calls reached
+                if (callCntMax > 1)
+                {
+                    //Call second time
+                    _logger.LogInformation($"Call no: {callCnt}. Giving FarmMaps another {sleepSecs} seconds to get SatelliteItems...");
+                    System.Threading.Thread.Sleep(1000 * sleepSecs);
+                    satelliteItemsCropYear = await _generalService.FindSatelliteItems(cropfieldItem, _settings.SatelliteTaskCode);
+                    satelliteItemsCropYearCnt = satelliteItemsCropYear.Count;
+                    _logger.LogInformation($"Call no: {callCnt}. Received {satelliteItemsCropYearCnt} images");
+                    sleepSecsSum = sleepSecsSum + sleepSecs;
+                    //As long as there is progress, keep calling
+                    callCnt++;
+                    while (callCnt <= callCntMax && (satelliteItemsCropYearCnt == 0 || satelliteItemsCropYearCnt > satelliteItemsCropYearCntPrev))
+                    {
+                        _logger.LogInformation($"Surprise! The longer we wait, the more images we get. Sleep and call once more");
+                        satelliteItemsCropYearCntPrev = satelliteItemsCropYearCnt;
+                        _logger.LogInformation($"Call no: {callCnt} (max: {callCntMax}). Giving FarmMaps another {sleepSecs} seconds to get SatelliteItems...");
+                        System.Threading.Thread.Sleep(1000 * sleepSecs);
+                        satelliteItemsCropYear = await _generalService.FindSatelliteItems(cropfieldItem, _settings.SatelliteTaskCode);
+                        satelliteItemsCropYearCnt = satelliteItemsCropYear.Count;
+                        _logger.LogInformation($"Call no: {callCnt}. Received {satelliteItemsCropYearCnt} images");
+                        callCnt++;
+                        sleepSecsSum = sleepSecsSum + sleepSecs;
+                    }
+                }
+
+                if (satelliteItemsCropYearCnt == 0)
+                {
+                    _logger.LogWarning($"DataDownloadApplication.cs: after calling one or more times and " +
+                    $"sleeping in total {sleepSecsSum} seconds, still no images found. " +
+                    $"Please check your settings for parameters callCntMax and sleepSecs in DataDownloadApplication.cs or contact FarmMaps");
+                }
+
                 satelliteItemsCropYear = satelliteItemsCropYear.OrderBy(x => x.DataDate).ToList();
 
-                if (input.StoreSatelliteStatisticsSingleImage == true) {
+                if (input.StoreSatelliteStatisticsSingleImage == true && satelliteItemsCropYearCnt > 0) {
                     _logger.LogInformation("Available satellite images:");
                     var count = 0;
                     TimeSpan.FromSeconds(0.5);
@@ -164,43 +257,49 @@ namespace FarmmapsDataDownload
 
                     var SatelliteDate = selectedSatelliteItem.DataDate.Value.ToString("yyyyMMdd");
                     string fileName = string.Format($"satelliteGeotiff_{fieldName}_{SatelliteDate}");   // no need to add satelliteBand in the name because the tif contains all bands
-                    string fileNameZip = string.Format($"{fileName}.zip");
-                    string fileNameGeotiff = string.Format($"{fileName}.tif");
-                    await _farmmapsApiService.DownloadItemAsync(selectedSatelliteItem.Code, Path.Combine(downloadFolder, fileNameZip));
+                    string fileNameZip = Path.Combine(downloadFolder, string.Format($"{fileName}.zip"));
+                    await _farmmapsApiService.DownloadItemAsync(selectedSatelliteItem.Code, fileNameZip);
 
                     // Download a csv file with stats
-                    List<Item> selectedSatalliteItems = new List<Item>(1) { selectedSatelliteItem };
+                    List<Item> selectedSatelliteItems = new List<Item>(1) { selectedSatelliteItem };
                     string fileNameStats = Path.Combine(downloadFolder, string.Format($"satelliteStats_{fieldName}_{SatelliteDate}.csv"));
-                    string downloadedStats = await _generalService.DownloadSatelliteStats(selectedSatalliteItems, fieldName, SatelliteBands, downloadFolder);
+                    _logger.LogInformation($"First call to get DownloadSatelliteStats for selected image...");
+                    string downloadedStats = await _generalService.DownloadSatelliteStats(selectedSatelliteItems, fieldName, satelliteBands, downloadFolder);
+
                     //rename the csv file with stats
+                    //if the targe file already exists, delete it
                     File.Delete(fileNameStats);
+                    //rename
                     File.Move(downloadedStats, fileNameStats);
-                    // wenr.tif. Contains 5 layers: (1) ndvi, (2) wdvi, (3) Red, (4) Green and (5) Blue
-                    // download the geotiffs. Returns a zip file with always these three files: 
-                    // data.dat.aux.xml
+
+                    // name the tif file
+                    string fileNameTifzipped = Path.Combine(downloadFolder, string.Format($"sentinelhub_{SatelliteDate}.tif")); 
+                    string fileNameGeotiff = Path.Combine(downloadFolder, string.Format($"sentinelhub_{fieldName}_{SatelliteDate}.tif"));
+                    // download the geotiffs. Returns a zip file with always these two files: 
                     // thumbnail.jpg
-                    // wenr.tif. Contains 5 layers: (1) ndvi, (2) wdvi, (3) Red, (4) Green and (5) Blue
+                    // sentinelhub_yyyyMMdd.tif. Contains 4 layers: (1) ndvi, (2) wdvi, (3) ci-red and (4) natural. Natural has 3 layers inside: redBand, blueBand and greenBand
                     if (true)
                     {
-                        // Extract the file "wenr.tif" from zip, rename it to fileNameGeotiff
-                        ZipFile.ExtractToDirectory(Path.Combine(downloadFolder, fileNameZip), downloadFolder, true);
-                        File.Delete(Path.Combine(downloadFolder, fileNameGeotiff)); // Delete the fileNameGeotiff file if exists
-                        File.Move(Path.Combine(downloadFolder, "wenr.tif"), Path.Combine(downloadFolder, fileNameGeotiff)); // Rename the oldFileName into newFileName
+                        // Extract the file fileNameTifzipped from zip, rename it to fileNameGeotiff
+                        ZipFile.ExtractToDirectory(fileNameZip, downloadFolder, true);
+                        //if the targe file already exists, delete it
+                        File.Delete(fileNameGeotiff);
+                        //rename
+                        File.Move(fileNameTifzipped, fileNameGeotiff);
 
                         // Cleanup
-                        string[] filesToDelete = new string[] { fileNameZip, "wenr.tif", "thumbnail.jpg", "data.dat.aux.xml" };
-                        foreach (string f in filesToDelete)
-                        {
-                            File.Delete(Path.Combine(downloadFolder, f));
-                        }
+                        File.Delete(fileNameZip);
+                        File.Delete(Path.Combine(downloadFolder, "thumbnail.jpg"));
                     }
-                    _logger.LogInformation($"Downloaded files {fileNameGeotiff} and {fileNameStats} to {downloadFolder}");
+                    //_logger.LogInformation($"Downloaded files {fileNameGeotiff} and {fileNameStats} to {downloadFolder}");
+                    _logger.LogInformation($"Downloaded files to {downloadFolder}");
 
                 }
                 if (input.StoreSatelliteStatisticsCropYear == true) {
                     string fileNameStats = Path.Combine(downloadFolder, string.Format($"satelliteStats_{fieldName}_{cropYear}.csv"));
                     File.Delete(fileNameStats);
-                    string downloadedStats = await _generalService.DownloadSatelliteStats(satelliteItemsCropYear, fieldName, SatelliteBands, downloadFolder);
+                    _logger.LogInformation($"First call to get DownloadSatelliteStats for whole cropYear...");
+                    string downloadedStats = await _generalService.DownloadSatelliteStats(satelliteItemsCropYear, fieldName, satelliteBands, downloadFolder);
                     File.Move(downloadedStats, fileNameStats);
                     _logger.LogInformation($"Downloaded file {fileNameStats} with stats for field '{fieldName}', cropyear {cropYear}");
                 }
@@ -221,9 +320,6 @@ namespace FarmmapsDataDownload
 
                 // Select a particular satellite item from satelliteTask
                 Item vanDerSatItem = await _generalService.FindVanDerSatItem(cropfieldItem, _settings.VanDerSatTaskCode, fieldName,  input.StoreVanDerSatStatistics);                
-                
-
-
             }
 
 

FarmmapsDataDownload/DataDownloadInput.json

@@ -1,32 +1,30 @@
 [
   {
-    "UseCreatedCropfield": true,
-    "outputFileName": "testSatData2",
-    "fieldName": "test_satData2",
-    "DownloadFolder": "C:\\workdir\\groenmonitor\\", //"C:\\workdir\\groenmonitor\\", // "Downloads", -> if you just put  "Downloads" the program will download to somewhere in ..\FarmMapsApiClient_WURtest\FarmmapsDataDownload\bin\Debug\netcoreapp3.1\Downloads\
+    "UseCreatedCropfield": false, // if false, program will make new CropfieldItemCode and SatelliteTaskCode; if true, the program will read CropfieldItemCode and SatelliteTaskCode from a file called "..\FarmmapsDataDownload\bin\Debug\netcoreapp3.1\Settings_{fieldName}.json", which will be faster
+    "outputFileName": "test_BvdTFieldlabG92",
+    "fieldName": "test_BvdTFieldlabG92",
+    "DownloadFolder": "Downloads", //"C:\\workdir\\groenmonitor\\", // "Downloads", -> if you just put  "Downloads" the program will download to somewhere in ..\FarmMapsApiClient_WURtest\FarmmapsDataDownload\bin\Debug\netcoreapp3.1\Downloads\
+    "GetCropRecordings": false,
+    "CrprecItem": "...", //item code of de crop recording parrent - can be found by opening the crop recording page of a field. 
     "GetShadowData": false,
     "GetSatelliteData": true,
-    "SatelliteBand": "wdvi", // "natural", "ndvi" or "wdvi"
+    "SatelliteBands": [ "ndvi", "wdvi", "ci-red" ], // ["ndvi"] or ["wdvi"] or ["ci-red"] or multiple: [ "wdvi", "ndvi" ]
     "StoreSatelliteStatisticsSingleImage": true,
     "StoreSatelliteStatisticsCropYear": true,
     "GetVanDerSatData": false,
     "StoreVanDerSatStatistics": false,
-    "CropYear": 2020,
+    "CropYear": 2022,
     "geometryJson": {
       "type": "Polygon",
       "coordinates": [
         [
-          [ 4.960707146896585, 52.800583669708487 ],
-          [ 4.960645975538824, 52.800470217610922 ],
-          [ 4.962140695752897, 52.799177147194797 ],
-          [ 4.967523821195745, 52.801502400041208 ],
-          [ 4.966336768950911, 52.802543735879809 ],
-          [ 4.961711880764330, 52.801009996856429 ],
-          [ 4.960707146896585, 52.800583669708487 ]
+          [ 5.563472073408009, 52.547554398144172 ],
+          [ 5.567425915520115, 52.547725375100377 ],
+          [ 5.567917474269188, 52.540608459298582 ],
+          [ 5.563878143678981, 52.54048022658143 ],
+          [ 5.563472073408009, 52.547554398144172 ]
         ]
       ]
     }
   }
-    
-
 ]

FarmmapsDataDownload/FarmmapsDataDownload.csproj

@@ -2,7 +2,7 @@
 
     <PropertyGroup>
         <OutputType>Exe</OutputType>
-        <TargetFramework>netcoreapp3.0</TargetFramework>
+        <TargetFramework>netcoreapp3.1</TargetFramework>
     </PropertyGroup>
 
     <ItemGroup>

FarmmapsDataDownload/Models/DataDownloadInput.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Collections.Generic;
 using Newtonsoft.Json.Linq;
 
 namespace FarmmapsDataDownload.Models
@@ -16,13 +17,13 @@ namespace FarmmapsDataDownload.Models
         public string fieldName { get; set; }
         public bool GetSatelliteData  { get; set; }
         public bool GetVanDerSatData { get; set; }
-        public string SatelliteBand { get; set; }
+        public List<string> SatelliteBands { get; set; }
         public bool StoreSatelliteStatisticsSingleImage { get; set; }
         public bool StoreSatelliteStatisticsCropYear { get; set; }
         public bool StoreVanDerSatStatistics { get; set; }
         public bool GetShadowData { get; set; }
-
-
+        public bool GetCropRecordings { get; set; }
+        public string CrprecItem { get; set; }
 
     }
 }

FarmmapsDataDownload/ShowGeotiff.r

@@ -0,0 +1,114 @@
+# ShowGeotiff.r
+# Have a look at a downloaded satellite image and check if stats are correctly calculated
+# I downloaded and calculated the stats for the polygon defined in C:\git\FarmMapsApiClient_WURtest\FarmmapsDataDownload\DataDownloadInput.json
+# in which I set "SatelliteBand": "wdvi" and in which in the console I requested the image for date '2022-03-08'
+
+library(raster)
+library(sf)
+library(rgdal)
+setwd("C:/git/FarmMapsApiClient_WURtest/FarmmapsDataDownload/bin/Debug/netcoreapp3.1/Downloads")
+
+# FarmmapsDataDownload
+fileGeotiff <- "sentinelhub_test_BvdTFieldlabG92_20220308.tif"
+lenfilename <- nchar(fileGeotiff)
+year <- substr(fileGeotiff,lenfilename-11,lenfilename-8)
+imgdate <- substr(fileGeotiff,lenfilename-11,lenfilename-4)
+
+stk.sentinelhub <- stack(x=fileGeotiff)
+# plot(stk.sentinelhub) shows 6 plots (6 bands)
+# 1. ndvi
+# 2. wdvi Note wdvi-red
+# 3. ci-red
+# 4. natural: red
+# 5. natural: green
+# 6. natural: blue
+names(stk.sentinelhub) <- c("ndvi","wdvired","ci-red","red","green","blue")
+plot(stk.sentinelhub)
+crs(stk.sentinelhub)
+# CRS arguments: +proj=longlat +datum=WGS84 +no_defs
+stk.sentinelhub.rd <- projectRaster(stk.sentinelhub, crs = CRS('+init=EPSG:28992'))
+crs(stk.sentinelhub)
+
+r.sentinelhub.rd.wdvi <- subset(stk.sentinelhub.rd,2)
+dev.off()
+plot(r.sentinelhub.rd.wdvi,main=paste("wdvi",imgdate),xlab="RDX",ylab="RDY")
+cellStats(r.sentinelhub.rd.wdvi,'mean')	# 0.2252725
+
+# Convert the .rd.wdvi raster to WGS84
+r.sentinelhub.wgs84.wdvi <- projectRaster(r.sentinelhub.rd.wdvi, crs = CRS('+init=EPSG:4326'))
+
+# Draw a polygon on top of the raster
+# Polygon pol from C:\git\FarmMapsApiClient_WURtest\FarmmapsDataDownload\DataDownloadInput.json
+pol <- data.frame(id = 1, wkt = gsub("\n","",'POLYGON((
+      5.563472073408009 52.547554398144172,
+      5.567425915520115 52.547725375100377,
+      5.567917474269188 52.540608459298582,
+      5.563878143678981 52.54048022658143,
+      5.563472073408009 52.547554398144172
+      ))'))
+
+
+pol.wgs84 <- st_as_sf(pol, wkt = 'wkt', crs = CRS('+init=EPSG:4326'))
+pol.rd <- st_transform(pol.wgs84, "+init=epsg:28992")
+
+#Calculate approximate middle of polygon
+res <- as.data.frame(do.call("rbind", lapply(st_geometry(pol.wgs84), st_bbox)))
+res$latmid <- (res$ymax+res$ymin)/2.0
+res$lonmid <- (res$xmax+res$xmin)/2.0
+res
+      # xmin     ymin     xmax     ymax  latmid   lonmid
+# 1 5.563472 52.54048 5.567917 52.54773 52.5441 5.565695
+
+# Have a look at both polygons
+# wg84
+dev.off()
+plot(r.sentinelhub.wgs84.wdvi,main=paste("wdvi",imgdate),xlab="LON",ylab="LAT")
+plot(pol.wgs84,add=TRUE, col="transparent",border="red")
+# RD
+dev.off()
+plot(r.sentinelhub.rd.wdvi,main=paste("wdvi",imgdate),xlab="RDX",ylab="RDY")
+plot(pol.rd,add=TRUE, col="transparent",border="red")
+
+# Clip the polygon from the full rectangle figure
+r.sentinelhub.rd.wdvi.pol <- mask(r.sentinelhub.rd.wdvi,pol.rd)
+r.sentinelhub.wgs84.wdvi.pol <- mask(r.sentinelhub.wgs84.wdvi,pol.wgs84)
+dev.off()
+plot(r.sentinelhub.wgs84.wdvi.pol,main=paste("wdvi",imgdate),xlab="LON",ylab="LAT")
+plot(pol.wgs84,add=TRUE, col="transparent",border="red")
+#That's what we want! 
+
+# Now compare the stats
+cellStats(r.sentinelhub.wgs84.wdvi,'mean')	# [1] 0.2250987 # Stats from rectangle, WGS84
+cellStats(r.sentinelhub.rd.wdvi,'mean')	# [1] 0.2252725 # Stats from rectangle, RD. Almost but not exactly same as above
+cellStats(r.sentinelhub.wgs84.wdvi.pol,'mean')	# [1] 0.2275067 # Stats from raster clipped by polygon, WGS84
+cellStats(r.sentinelhub.rd.wdvi.pol,'mean')	# [1] 0.2275073 # Stats from raster clipped by polygon, RD. Almost but not exactly same as above
+# file satelliteStats_test_BvdTFieldlabG92_20220308.csv
+# "wdvi" "mean": 0.22744397204465
+# Mean in csv corresponds with cellStats calculated from clipped tif!
+# So while the tif returned is a non-clipped image, the downloaded statistics are from the clipped image
+# Exactly as we wanted.
+cellStats(r.sentinelhub.wgs84.wdvi.pol,'median')	# Error in .local(x, stat, ...) : invalid 'stat'. Should be sum, min, max, sd, mean, or 'countNA'
+r.sentinelhub.wgs84.wdvi.vals <- values(r.sentinelhub.wgs84.wdvi)
+median(r.sentinelhub.wgs84.wdvi.vals)	# [1] NA
+median(r.sentinelhub.wgs84.wdvi.vals,na.rm=TRUE)	# [1] 0.2318
+r.sentinelhub.wgs84.wdvi.pol.vals <- values(r.sentinelhub.wgs84.wdvi.pol)
+median(r.sentinelhub.wgs84.wdvi.pol.vals)	# [1] NA
+median(r.sentinelhub.wgs84.wdvi.pol.vals,na.rm=TRUE)	# [1] 0.2338
+# file satelliteStats_test_BvdTFieldlabG92_20220308.csv
+# "wdvi" "mean": 0.233799993991851
+# Median is same as for median(r.sentinelhub.wgs84.wdvi.pol.vals,na.rm=TRUE)
+#   in csv corresponds with cellStats calculated from clipped tif!
+# So while the tif returned is a non-clipped image, the downloaded statistics are from the clipped image
+# Exactly as we wanted.
+cellStats(r.sentinelhub.wgs84.wdvi,'countNA')	# [1] 27896
+ncell(r.sentinelhub.wgs84.wdvi)	# [1] 272718
+cellStats(r.sentinelhub.wgs84.wdvi,'countNA') / ncell(r.sentinelhub.wgs84.wdvi)	# [1] 0.1022888	# 10% no data? doesn't show in the plot?
+cellStats(r.sentinelhub.wgs84.wdvi.pol,'countNA')	# [1] 57625
+summary(r.sentinelhub.wgs84.wdvi.pol.vals)	# shows the same: NA's: 57625
+ncell(r.sentinelhub.wgs84.wdvi.pol)	# [1] 272718
+populationCount = ncell(r.sentinelhub.wgs84.wdvi.pol) - cellStats(r.sentinelhub.wgs84.wdvi.pol,'countNA')
+populationCount # [1] 215093
+# file satelliteStats_test_BvdTFieldlabG92_20220308.csv
+# "wdvi" "populationCount": 214688
+# similar but not same
+

FarmmapsHaulmkilling/HaulmkillingApplication.cs

@@ -45,14 +45,14 @@ namespace FarmmapsHaulmkilling
             await _farmmapsApiService.GetCurrentUserCodeAsync();
             var roots = await _farmmapsApiService.GetCurrentUserRootsAsync();
 
-            var myDrive = roots.SingleOrDefault(r => r.Name == "My drive");
+            var myDrive = roots.SingleOrDefault(r => r.Name == "USER_FILES");
             if (myDrive == null)
             {
                 _logger.LogError("Could not find a needed root item");
                 return;
             }
 
-            var uploadedRoot = roots.SingleOrDefault(r => r.Name == "Uploaded");
+            var uploadedRoot = roots.SingleOrDefault(r => r.Name == "USER_IN");
             if (uploadedRoot == null)
             {
                 _logger.LogError("Could not find a needed root item");

FarmmapsHerbicide/HerbicideApplication.cs

@@ -50,14 +50,14 @@ namespace FarmmapsHerbicide
 
         private async Task SingleLutumTiffFlow(List<UserRoot> roots)
         {
-            var myDrive = roots.SingleOrDefault(r => r.Name == "My drive");
+            var myDrive = roots.SingleOrDefault(r => r.Name == "USER_FILES");
             if (myDrive == null)
             {
                 _logger.LogError("Could not find a needed root item");
                 return;
             }
 
-            var uploadedRoot = roots.SingleOrDefault(r => r.Name == "Uploaded");
+            var uploadedRoot = roots.SingleOrDefault(r => r.Name == "USER_IN");
             if (uploadedRoot == null)
             {
                 _logger.LogError("Could not find a needed root item");
@@ -91,14 +91,14 @@ namespace FarmmapsHerbicide
 
         private async Task MultiVanDenBorneShapeFlow(List<UserRoot> roots)
         {
-            var myDrive = roots.SingleOrDefault(r => r.Name == "My drive");
+            var myDrive = roots.SingleOrDefault(r => r.Name == "USER_FILES");
             if (myDrive == null)
             {
                 _logger.LogError("Could not find a needed root item");
                 return;
             }
 
-            var uploadedRoot = roots.SingleOrDefault(r => r.Name == "Uploaded");
+            var uploadedRoot = roots.SingleOrDefault(r => r.Name == "USER_IN");
             if (uploadedRoot == null)
             {
                 _logger.LogError("Could not find a needed root item");

FarmmapsNbs/NbsApplication.cs

@@ -64,13 +64,13 @@ namespace FarmmapsNbs {
 
             LoadSettings(settingsfile);
 
-            var uploadedRoot = roots.SingleOrDefault(r => r.Name == "Uploaded");
+            var uploadedRoot = roots.SingleOrDefault(r => r.Name == "USER_IN");
             if (uploadedRoot == null) {
                 _logger.LogError("Could not find a needed root item");
                 return;
             }
 
-            var myDriveRoot = roots.SingleOrDefault(r => r.Name == "My drive");
+            var myDriveRoot = roots.SingleOrDefault(r => r.Name == "USER_FILES");
             if (myDriveRoot == null) {
                 _logger.LogError("Could not find a needed root item");
                 return;
@@ -188,7 +188,6 @@ namespace FarmmapsNbs {
                 ////End shape to geotiff transformation
             }
 
-
             _logger.LogInformation($"Calculating targetN with targetYield: {input.TargetYield}");
             var targetNItem = await _nitrogenService.CreateTargetNItem(cropfieldItem);
             var targetNData = await _nitrogenService.CalculateTargetN(cropfieldItem, targetNItem, plantingDate,

FarmmapsNbs/NitrogenInput.json

@@ -1,11 +1,11 @@
 [
   {
-    "UseCreatedCropfield": false,
+    "UseCreatedCropfield": true,
     "file": "Scan1_20190605_sample.json",
     "inputVariable": "irmi",
     "inputLayerName": "irmi",
-    "outputFileName": "vranbs_sample-fritsmeier-geojson",
-    "fieldName": "sampleJson_irmi",
+    "outputFileName": "vranbs_sample",
+    "fieldName": "sampleJson_irmi-usertest",
     "storeSatelliteStatistics": true,
     "plantingDate": "2019-04-18",
     "measurementDate": "2019-06-05",

FarmmapsPoten/PotenApplication.cs

@@ -61,13 +61,13 @@ namespace FarmmapsVRApoten
             bool countPerArea = input.CountPerArea;
             var inputLayerName = input.InputLayerName;
 
-            var myDrive = roots.SingleOrDefault(r => r.Name == "My drive");
+            var myDrive = roots.SingleOrDefault(r => r.Name == "USER_FILES");
             if (myDrive == null) {
                 _logger.LogError("Could not find a needed root item");
                 return;
             }
 
-            var uploadedRoot = roots.SingleOrDefault(r => r.Name == "Uploaded");
+            var uploadedRoot = roots.SingleOrDefault(r => r.Name == "USER_IN");
             if (uploadedRoot == null) {
                 _logger.LogError("Could not find a needed root item");
                 return;

FarmmapsZonering/ZoneringApplication.cs

@@ -66,7 +66,7 @@ namespace FarmmapsZonering
         
         private async Task ZoningAsync(List<UserRoot> roots, ZoneringInput input)
         {
-            var myDrive = roots.SingleOrDefault(r => r.Name == "My drive");
+            var myDrive = roots.SingleOrDefault(r => r.Name == "USER_FILES");
             if (myDrive == null)
             {
                 _logger.LogError("Could not find a needed root item");
@@ -83,7 +83,7 @@ namespace FarmmapsZonering
             // Load settings from previous cropfield
             LoadSettings(settingsfile);
 
-            var uploadedRoot = roots.SingleOrDefault(r => r.Name == "Uploaded");
+            var uploadedRoot = roots.SingleOrDefault(r => r.Name == "USER_IN");
             if (uploadedRoot == null)
             {
                 _logger.LogError("Could not find a needed root item");

README.MD

@@ -1,15 +1,27 @@
 ##### NOT PRODUCTION READY CODE, JUST AN EXAMPLE
 
-Put your clientId and clientSecret in a newly created appsettings.secrets.json file inside the root of the Secrets project.  
+Put your clientId and clientSecret or your Farmmaps-account data (username and password) in a newly created appsettings.secrets.json file inside the root of the Secrets project.  
 
 **appsettings.secrets.json**
 ```
 {
+  //API credential sign-in
   "ClientId": "<clientId>",
   "ClientSecret": "<clientSecret>"
 }
 ```
 
+Or when using a Farmmaps account to sing-in
+
+**appsettings.secrets.json**
+```
+{
+ //Sign-in using farmmaps account
+ "Username": "",
+ "Password": ""
+}
+```
+
 * Isn't 100% complete.  
 * Needs proper testing of all public api methods.
 * Needs documentation of all public api methods.

Secrets/appsettings.json

@@ -2,11 +2,11 @@
   //There are three farmmaps environments, uncomment the environemnt you want to use with this sample client
 
   //test environment
-//  "Authority": "https://accounts.test.farmmaps.eu/",
-//  "Endpoint": "https://test.farmmaps.eu/",
-//  "BasePath": "api/v1",
-//  "DiscoveryEndpointUrl": "https://accounts.test.farmmaps.eu/.well-known/openid-configuration",
-//  "GrantClientId": "farmmapstesteu",
+  "Authority": "https://accounts.test.farmmaps.eu/",
+  "Endpoint": "https://test.farmmaps.eu/",
+  "BasePath": "api/v1",
+  "DiscoveryEndpointUrl": "https://accounts.test.farmmaps.eu/.well-known/openid-configuration",
+  "GrantClientId": "farmmapstesteu",
 
   ////acceptance environment
   //"Authority": "https://accounts.acc.farmmaps.eu/",
@@ -15,12 +15,12 @@
   //"DiscoveryEndpointUrl": "https://accounts.acc.farmmaps.eu/.well-known/openid-configuration",
   //"GrantClientId": "farmmapsacceu",
 
-  //production environment
-  "authority": "https://accounts.farmmaps.eu/",
-  "endpoint": "https://farmmaps.eu/",
-  "basepath": "api/v1",
-  "discoveryendpointurl": "https://accounts.farmmaps.eu/.well-known/openid-configuration",
-  "GrantClientId": "farmmaps",
+  ////production environment
+  //"authority": "https://accounts.farmmaps.eu/",
+  //"endpoint": "https://farmmaps.eu/",
+  //"basepath": "api/v1",
+  //"discoveryendpointurl": "https://accounts.farmmaps.eu/.well-known/openid-configuration",
+  //"GrantClientId": "farmmaps",
 
   //overige info
   "RedirectUri": "http://example.nl/api",