Added project FarmmapsBulkSatDownload

FarmmapsApi/Services/GeneralService.cs

@@ -388,6 +388,29 @@ namespace FarmmapsApi.Services
             return selectedSatelliteItem;
         }
 
+        public async Task<List<Item>> FindSatelliteItemsAll(Item cropfieldItem, string satelliteTaskCode)
+        {
+
+            var taskStatus = await _farmmapsApiService.GetTaskStatusAsync(cropfieldItem.Code, satelliteTaskCode);
+
+
+            // find ndvi or wdvi satellite data geotiffs
+            var temporalItem = await FindChildItemAsync(cropfieldItem.Code, TEMPORAL_ITEMTYPE,
+                "Cropfield Satellite items", item => item.SourceTask == SATELLITE_TASK &&
+                                                                 taskStatus.Finished >= item.Created &&
+                                                                 taskStatus.Finished <= item.Created.Value.AddHours(1));
+
+
+            if (temporalItem == null)
+            {
+                _logger.LogError("Temporal item not found");
+
+            }
+
+            var satelliteTiffs = await _farmmapsApiService.GetItemChildrenAsync(temporalItem.Code);
+
+            return satelliteTiffs;
+        }
         //VanDerSat
         public async Task<string> RunVanDerSatTask(Item cropfieldItem) {
 

FarmmapsApiSamples.sln

@@ -23,7 +23,9 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FarmMapsBlight", "FarmMapsB
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FarmmapsZonering", "FarmmapsZonering\FarmmapsZonering.csproj", "{91A58C4A-4A80-4079-B43D-9B851206194F}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "FarmmapsDataDownload", "FarmmapsDataDownload\FarmmapsDataDownload.csproj", "{32ED9500-AAAB-4030-9C7A-F611A85DF890}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FarmmapsDataDownload", "FarmmapsDataDownload\FarmmapsDataDownload.csproj", "{32ED9500-AAAB-4030-9C7A-F611A85DF890}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FarmmapsBulkSatDownload", "FarmmapsBulkSatDownload\FarmmapsBulkSatDownload.csproj", "{772DBDCD-9FAA-40A7-8551-2C1620C4AB67}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
@@ -63,6 +65,10 @@ Global
 		{32ED9500-AAAB-4030-9C7A-F611A85DF890}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{32ED9500-AAAB-4030-9C7A-F611A85DF890}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{32ED9500-AAAB-4030-9C7A-F611A85DF890}.Release|Any CPU.Build.0 = Release|Any CPU
+		{772DBDCD-9FAA-40A7-8551-2C1620C4AB67}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{772DBDCD-9FAA-40A7-8551-2C1620C4AB67}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{772DBDCD-9FAA-40A7-8551-2C1620C4AB67}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{772DBDCD-9FAA-40A7-8551-2C1620C4AB67}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

FarmmapsBulkSatDownload/BulkSatDownloadApplication.cs

@@ -0,0 +1,242 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.IO.Compression;
+using System.Linq;
+using System.Threading.Tasks;
+using FarmmapsApi;
+using FarmmapsApi.Models;
+using FarmmapsApi.Services;
+using FarmmapsBulkSatDownload.Models;
+using Microsoft.Extensions.Logging;
+using Newtonsoft.Json;
+using Npgsql;
+using Newtonsoft.Json.Linq;
+using static FarmmapsApiSamples.Constants;
+
+namespace FarmmapsBulkSatDownload
+{
+    public class BulkSatDownloadApplication : IApplication
+    {
+        private const string SettingsFile = "settings.json";
+
+        private readonly ILogger<BulkSatDownloadApplication> _logger;
+        private readonly FarmmapsApiService _farmmapsApiService;
+        private readonly BulkSatDownloadService _dataDownloadService;
+        private readonly GeneralService _generalService;
+
+        private Settings _settings;
+        static DB dbparcels;
+
+        public BulkSatDownloadApplication(ILogger<BulkSatDownloadApplication> logger, FarmmapsApiService farmmapsApiService,
+            GeneralService generalService, BulkSatDownloadService dataDownloadService)
+        {
+            _logger = logger;
+            _farmmapsApiService = farmmapsApiService;
+            _generalService = generalService;
+            _dataDownloadService = dataDownloadService;
+        }
+        public async Task RunAsync()
+        {
+            // Check if we have permission
+            // !! this call is needed the first time an api is called with a fresh clientid and secret !!
+            await _farmmapsApiService.GetCurrentUserCodeAsync();
+            var roots = await _farmmapsApiService.GetCurrentUserRootsAsync();
+            // Initialize databases. Username, password etc stored in file "DBsettings.json"
+            dbparcels = JsonConvert.DeserializeObject<DB>(File.ReadAllText("DBsettings.json"));
+
+            //string date;
+            //string asterices = "*****************************************************************************";
+            //string bboxtiffWithTempPath;
+            //string parcelshp = "parcel.shp";
+            //string parcelshpWithTempPath;
+            //string gwoptions; //= String.Format("-overwrite -s_srs EPSG:28992 -t_srs EPSG:28992 -dstnodata 0 -cutline {0} -crop_to_cutline", parcelshp);
+            //string parceltiff = "parcel.tiff";
+            //string parceltiffWithTempPath;
+            //string[] vegetationindices = { "WDVI", "NDVI" };
+            //string[] sources = { "groenmonitor" }; // { "groenmonitor", "akkerwebwenr", "akkerwebneo" }; { "groenmonitor" };
+            string schemaname = "bigdata";
+            string parceltablename = "parcel_flowerbulbs";//"parcelsijbrandij" "parcel"; "parcel_flowerbulbs"
+            //string groenmonitortablename = "groenmonitor_flowerbulbs";//"groenmonitorsijbrandij" "groenmonitor" "groenmonitor_flowerbulbs" //PO20190605: "groenmonitormpt" contains groenmonitor data from multiple parcel tables (mpt)
+            // The  view 'groenmonitorlatestviewname' contains per parcelid (arbid) the year in which it "exists" and the date of the latest image downloaded. It is used to prevent downloading images already downloaded
+            string groenmonitorlatestviewname = "groenmonitorlatest_flowerbulbs"; //"groenmonitorsijbrandijlatest" "groenmonitorlatest" "groenmonitorlatest_flowerbulbs" //PO20190605: "v_groenmonitorlatest" contains latest available dates from groenmonitortablename = "groenmonitormpt"
+            //GroenmonitorTable gmtb;
+            //DateTime dt;
+            //DateTime dtfirst;
+            //DateTime dtlast;
+            //double scalingfactor;
+            BulkSatDownloadInput dataDownloadInput;
+            List<BulkSatDownloadInput> fieldsInputs = new List<BulkSatDownloadInput>();
+
+            // Database query and connection. Geometry must be in WGS84 coordinate system, EPSG 4326
+            // Apparently the FarmmapsApi cannot handle MultiPolygon, so we need to convert to single Polygon
+            // In case database returns a MultiPolygon use ST_NumGeometries(pt.geom) to count the number of polygons
+            // If necessary use WHERE T_NumGeometries(pt.geom) = 1 to use only single polygons
+            string connectionString = dbparcels.GetConnectionString();
+            string readSql = string.Format(
+@"
+SELECT pt.arbid, pt.crop, pt.year, gml.lastwenrdate, ST_AsGeoJSON(ST_Transform((ST_DUMP(pt.geom)).geom::geometry(Polygon),4326)) AS geojson_polygon_wgs84
+FROM {0}.{1} pt, {0}.{2} gml
+WHERE
+    pt.arbid = gml.arbid AND
+    pt.crop NOT IN('Tulp', 'Lelie') AND
+    pt.year = 2018
+LIMIT 10;", schemaname, parceltablename, groenmonitorlatestviewname);   //LIMIT 10 for testing
+
+            using (NpgsqlConnection connection = new NpgsqlConnection(connectionString))
+            {
+                connection.Open();
+
+                // Read data (run query) = build a list of fields for which to download images
+                NpgsqlCommand command = connection.CreateCommand();
+                command.CommandText = readSql;
+                NpgsqlDataReader dr = command.ExecuteReader();
+                while (dr.Read())
+                {
+                    dataDownloadInput = new BulkSatDownloadInput();
+                    dataDownloadInput.UseCreatedCropfield = false;
+                    dataDownloadInput.DownloadFolder = "C:\\workdir\\groenmonitor\\";
+                    dataDownloadInput.fieldID = dr.GetInt16(0);
+                    dataDownloadInput.fieldName = string.Format($"{parceltablename}_fld{dataDownloadInput.fieldID}");
+                    dataDownloadInput.cropName = dr.GetString(1);
+                    dataDownloadInput.cropYear = dr.GetInt16(2);
+                    dataDownloadInput.lastdownloadedimagedate = dr.GetDateTime(3);
+                    dataDownloadInput.GeometryJson = JObject.Parse(dr.GetString(4));
+                    fieldsInputs.Add(dataDownloadInput);
+                }
+                connection.Close();
+            }
+
+            // For testing using json input instead of database input
+            var fieldsInputJson = File.ReadAllText("BulkSatDownloadInput.json");
+            List<BulkSatDownloadInput> fieldsInputs2 = JsonConvert.DeserializeObject<List<BulkSatDownloadInput>>(fieldsInputJson);
+
+            // Now for each input download all images. Use fieldsInputs or fieldsInputs2 for testing
+            foreach (var input in fieldsInputs2)
+            {
+                try
+                {
+                    await Process(roots, input);
+                }
+                catch (Exception ex)
+                {
+                    _logger.LogError(ex.Message);
+                }
+            }
+        }
+
+        private async Task Process(List<UserRoot> roots, BulkSatDownloadInput input)
+        {
+            string DownloadFolder = input.DownloadFolder;
+            if (!Directory.Exists(DownloadFolder))
+                Directory.CreateDirectory(DownloadFolder);
+
+            // !!specify if you are using an already created cropfield:
+            bool useCreatedCropfield = input.UseCreatedCropfield;
+            string settingsfile = $"Settings_BulkSatDownloadApplication.json";
+
+            LoadSettings(settingsfile);
+
+            var uploadedRoot = roots.SingleOrDefault(r => r.Name == "Uploaded");
+            if (uploadedRoot == null)
+            {
+                _logger.LogError("Could not find a needed root item");
+                return;
+            }
+
+            var myDriveRoot = roots.SingleOrDefault(r => r.Name == "My drive");
+            if (myDriveRoot == null)
+            {
+                _logger.LogError("Could not find a needed root item");
+                return;
+            }
+
+            // Use already created cropfield or create new one
+            Item cropfieldItem;
+            if (useCreatedCropfield == false || string.IsNullOrEmpty(_settings.CropfieldItemCode))
+            {
+                _logger.LogInformation(string.Format($"Creating cropfield for a field in the year {input.cropYear}"));
+                //string geomjson = input.GeometryJson.ToString(Formatting.None); //"{\"type\":\"Polygon\",\"coordinates\":[[[4.960707146896585,52.80058366970849],[4.960645975538824,52.80047021761092],[4.962140695752897,52.7991771471948],[4.967523821195745,52.80150240004121],[4.966336768950911,52.80254373587981],[4.96171188076433,52.80100999685643],[4.960707146896585,52.80058366970849]]]}"
+                string geomjson = input.GeometryJson.ToString(Formatting.None); 
+                cropfieldItem = await _generalService.CreateCropfieldItemAsync(myDriveRoot.Code,
+                    $"DataCropfield {input.cropName}", input.cropYear, input.GeometryJson.ToString(Formatting.None));
+                _settings.CropfieldItemCode = cropfieldItem.Code;
+                SaveSettings(settingsfile);
+            }
+            else
+            {
+                _logger.LogInformation("Cropfield already exists, trying to get it");
+                cropfieldItem = await _farmmapsApiService.GetItemAsync(_settings.CropfieldItemCode);
+            }
+
+            // check if satellite task not yet done, do here and save taskcode
+            if (useCreatedCropfield == false || string.IsNullOrEmpty(_settings.SatelliteTaskCode))
+            {
+                var satelliteTaskCode = await _generalService.RunSatelliteTask(cropfieldItem);
+                _settings.SatelliteTaskCode = satelliteTaskCode;
+                SaveSettings(settingsfile);
+            }
+
+            // Now get the tiffs
+            List<Item> satelliteTiffs = await _generalService.FindSatelliteItemsAll(cropfieldItem, _settings.SatelliteTaskCode);
+            _logger.LogInformation(string.Format($"Downloading {satelliteTiffs.Count} geotiffs for a field in year {input.cropYear} ..."));
+            foreach (Item satalliteItem in satelliteTiffs) {
+                // download the geotiff. Returns a zip file with always these three files: 
+                // data.dat.aux.xml
+                // thumbnail.jpg
+                // wenr.tif. Contains 5 layers: (1) ndvi, (2) wdvi, (3) Red, (4) Green and (5) Blue
+                DateTime SatelliteImageDate = (DateTime)satalliteItem.DataDate;
+                string SatelliteDate = SatelliteImageDate.ToString("yyyyMMdd");
+                string fileName = string.Format($"{input.cropName}_{input.fieldName}_{SatelliteDate}"); // assuming the fieldName is good enough for an ID. One might add here _{input.fieldID} or _{input.cropName}
+                string fileNameZip = string.Format($"{fileName}.zip");
+                string fileNameGeotiff = string.Format($"{fileName}.tif");
+                await _farmmapsApiService.DownloadItemAsync(satalliteItem.Code, Path.Combine(DownloadFolder, fileNameZip));
+
+                // Extract the file "wenr.tif" from zip, rename it, delete "wenr.tif". overwriteFiles = true
+                ZipFile.ExtractToDirectory(Path.Combine(DownloadFolder, fileNameZip), DownloadFolder, true);
+                File.Delete(Path.Combine(DownloadFolder, fileNameGeotiff)); // Delete the fileNameGeotiff file if exists
+                File.Move(Path.Combine(DownloadFolder, "wenr.tif"), Path.Combine(DownloadFolder, fileNameGeotiff)); // Rename the oldFileName into newFileName
+
+                // Cleanup
+                string[] filesToDelete = new string[] { fileNameZip, "wenr.tif", "thumbnail.jpg", "data.dat.aux.xml" };
+                foreach (string f in filesToDelete)
+                {
+                    File.Delete(Path.Combine(DownloadFolder, f));
+                }
+            }
+        }
+
+        // Functions to save previously created cropfields
+        private void LoadSettings(string file)
+        {
+            if (File.Exists(file))
+            {
+                var jsonText = File.ReadAllText(file);
+                _settings = JsonConvert.DeserializeObject<Settings>(jsonText);
+            }
+            else
+            {
+                _settings = new Settings();
+            }
+        }
+
+        private void SaveSettings(string file)
+        {
+            if (_settings == null)
+                return;
+
+            var json = JsonConvert.SerializeObject(_settings);
+            File.WriteAllText(file, json);
+        }
+        private void SaveInfo(string file)
+        {
+            if (_settings == null)
+                return;
+
+            var json = JsonConvert.SerializeObject(_settings);
+            File.WriteAllText(file, json);
+
+        }
+
+    }
+}

FarmmapsBulkSatDownload/BulkSatDownloadInput.json

@@ -0,0 +1,293 @@
+[
+  {
+    "UseCreatedCropfield": false,
+    "fieldName": "fld5641", // FarmMaps won't do without a fieldName
+    "DownloadFolder": "C:\\workdir\\groenmonitor\\", //"C:\\workdir\\groenmonitor\\"; // "Downloads" -> if you just put  "Downloads" the program will download to somewhere in ..\FarmMapsApiClient_WURtest\FarmmapsDataDownload\bin\Debug\netcoreapp3.1\Downloads\
+    "cropName": "wheat",
+    "cropYear": 2021,
+    "fieldID": 5641,
+    "lastdownloadedimagedate": "2021-01-01",
+    "geometryJson": {
+      "type": "Polygon",
+      "coordinates": [
+        [
+          [ 3.37837807779104, 51.3231095796538 ],
+          [ 3.38065689232502, 51.3212527499355 ],
+          [ 3.38022924592256, 51.3210683536359 ],
+          [ 3.37980548452565, 51.3208801127141 ],
+          [ 3.37959556105776, 51.3207540143696 ],
+          [ 3.3793691292654, 51.3205959677371 ],
+          [ 3.37822219207335, 51.3215667913007 ],
+          [ 3.37816999925795, 51.3216109809456 ],
+          [ 3.37646704574705, 51.3208025481261 ],
+          [ 3.37646695791282, 51.3208025061493 ],
+          [ 3.37608401443192, 51.3206231652693 ],
+          [ 3.37607169507628, 51.3206173959751 ],
+          [ 3.37606021048754, 51.320612017601 ],
+          [ 3.37582728410659, 51.3205029306946 ],
+          [ 3.37580409779263, 51.3206502985963 ],
+          [ 3.37575872019649, 51.3207993094705 ],
+          [ 3.37575476634361, 51.3208122883487 ],
+          [ 3.37571181656268, 51.3208797459348 ],
+          [ 3.3756624532907, 51.3209415238446 ],
+          [ 3.37557609963811, 51.3210110142077 ],
+          [ 3.37541089899821, 51.3211055871218 ],
+          [ 3.37477516102591, 51.3214102985009 ],
+          [ 3.37473173914127, 51.3214311108204 ],
+          [ 3.37455904622072, 51.3215138815012 ],
+          [ 3.37415098054777, 51.3217199232877 ],
+          [ 3.37313700916272, 51.3222422862785 ],
+          [ 3.37748824689601, 51.3242852920348 ],
+          [ 3.37749760805371, 51.3242713084009 ],
+          [ 3.37811903757028, 51.3233437635596 ],
+          [ 3.37818758851947, 51.3232647797363 ],
+          [ 3.37823803668144, 51.3232236798646 ],
+          [ 3.37837807779104, 51.3231095796538 ]
+        ]
+      ]
+    }
+  },
+  {
+    "UseCreatedCropfield": false,
+    "fieldName": "fld5641", // FarmMaps won't do without a fieldName
+    "DownloadFolder": "C:\\workdir\\groenmonitor\\", //"C:\\workdir\\groenmonitor\\"; // "Downloads" -> if you just put  "Downloads" the program will download to somewhere in ..\FarmMapsApiClient_WURtest\FarmmapsDataDownload\bin\Debug\netcoreapp3.1\Downloads\
+    "cropName": "wheat",
+    "cropYear": 2020,
+    "fieldID": 5641,
+    "lastdownloadedimagedate": "2020-01-01",
+    "geometryJson": {
+      "type": "Polygon",
+      "coordinates": [
+        [
+          [ 3.37837807779104, 51.3231095796538 ],
+          [ 3.38065689232502, 51.3212527499355 ],
+          [ 3.38022924592256, 51.3210683536359 ],
+          [ 3.37980548452565, 51.3208801127141 ],
+          [ 3.37959556105776, 51.3207540143696 ],
+          [ 3.3793691292654, 51.3205959677371 ],
+          [ 3.37822219207335, 51.3215667913007 ],
+          [ 3.37816999925795, 51.3216109809456 ],
+          [ 3.37646704574705, 51.3208025481261 ],
+          [ 3.37646695791282, 51.3208025061493 ],
+          [ 3.37608401443192, 51.3206231652693 ],
+          [ 3.37607169507628, 51.3206173959751 ],
+          [ 3.37606021048754, 51.320612017601 ],
+          [ 3.37582728410659, 51.3205029306946 ],
+          [ 3.37580409779263, 51.3206502985963 ],
+          [ 3.37575872019649, 51.3207993094705 ],
+          [ 3.37575476634361, 51.3208122883487 ],
+          [ 3.37571181656268, 51.3208797459348 ],
+          [ 3.3756624532907, 51.3209415238446 ],
+          [ 3.37557609963811, 51.3210110142077 ],
+          [ 3.37541089899821, 51.3211055871218 ],
+          [ 3.37477516102591, 51.3214102985009 ],
+          [ 3.37473173914127, 51.3214311108204 ],
+          [ 3.37455904622072, 51.3215138815012 ],
+          [ 3.37415098054777, 51.3217199232877 ],
+          [ 3.37313700916272, 51.3222422862785 ],
+          [ 3.37748824689601, 51.3242852920348 ],
+          [ 3.37749760805371, 51.3242713084009 ],
+          [ 3.37811903757028, 51.3233437635596 ],
+          [ 3.37818758851947, 51.3232647797363 ],
+          [ 3.37823803668144, 51.3232236798646 ],
+          [ 3.37837807779104, 51.3231095796538 ]
+
+        ]
+      ]
+    }
+  },
+  {
+    "UseCreatedCropfield": false,
+    "fieldName": "fld5641", // FarmMaps won't do without a fieldName
+    "DownloadFolder": "C:\\workdir\\groenmonitor\\", //"C:\\workdir\\groenmonitor\\"; // "Downloads" -> if you just put  "Downloads" the program will download to somewhere in ..\FarmMapsApiClient_WURtest\FarmmapsDataDownload\bin\Debug\netcoreapp3.1\Downloads\
+    "cropName": "wheat",
+    "cropYear": 2019,
+    "fieldID": 5641,
+    "lastdownloadedimagedate": "2019-01-01",
+    "geometryJson": {
+      "type": "Polygon",
+      "coordinates": [
+        [
+          [ 3.37837807779104, 51.3231095796538 ],
+          [ 3.38065689232502, 51.3212527499355 ],
+          [ 3.38022924592256, 51.3210683536359 ],
+          [ 3.37980548452565, 51.3208801127141 ],
+          [ 3.37959556105776, 51.3207540143696 ],
+          [ 3.3793691292654, 51.3205959677371 ],
+          [ 3.37822219207335, 51.3215667913007 ],
+          [ 3.37816999925795, 51.3216109809456 ],
+          [ 3.37646704574705, 51.3208025481261 ],
+          [ 3.37646695791282, 51.3208025061493 ],
+          [ 3.37608401443192, 51.3206231652693 ],
+          [ 3.37607169507628, 51.3206173959751 ],
+          [ 3.37606021048754, 51.320612017601 ],
+          [ 3.37582728410659, 51.3205029306946 ],
+          [ 3.37580409779263, 51.3206502985963 ],
+          [ 3.37575872019649, 51.3207993094705 ],
+          [ 3.37575476634361, 51.3208122883487 ],
+          [ 3.37571181656268, 51.3208797459348 ],
+          [ 3.3756624532907, 51.3209415238446 ],
+          [ 3.37557609963811, 51.3210110142077 ],
+          [ 3.37541089899821, 51.3211055871218 ],
+          [ 3.37477516102591, 51.3214102985009 ],
+          [ 3.37473173914127, 51.3214311108204 ],
+          [ 3.37455904622072, 51.3215138815012 ],
+          [ 3.37415098054777, 51.3217199232877 ],
+          [ 3.37313700916272, 51.3222422862785 ],
+          [ 3.37748824689601, 51.3242852920348 ],
+          [ 3.37749760805371, 51.3242713084009 ],
+          [ 3.37811903757028, 51.3233437635596 ],
+          [ 3.37818758851947, 51.3232647797363 ],
+          [ 3.37823803668144, 51.3232236798646 ],
+          [ 3.37837807779104, 51.3231095796538 ]
+        ]
+      ]
+    }
+  },
+  {
+    "UseCreatedCropfield": false,
+    "fieldName": "fld5641", // FarmMaps won't do without a fieldName
+    "DownloadFolder": "C:\\workdir\\groenmonitor\\", //"C:\\workdir\\groenmonitor\\"; // "Downloads" -> if you just put  "Downloads" the program will download to somewhere in ..\FarmMapsApiClient_WURtest\FarmmapsDataDownload\bin\Debug\netcoreapp3.1\Downloads\
+    "cropName": "wheat",
+    "cropYear": 2018,
+    "fieldID": 5641,
+    "lastdownloadedimagedate": "2018-01-01",
+    "geometryJson": {
+      "type": "Polygon",
+      "coordinates": [
+        [
+          [ 3.37837807779104, 51.3231095796538 ],
+          [ 3.38065689232502, 51.3212527499355 ],
+          [ 3.38022924592256, 51.3210683536359 ],
+          [ 3.37980548452565, 51.3208801127141 ],
+          [ 3.37959556105776, 51.3207540143696 ],
+          [ 3.3793691292654, 51.3205959677371 ],
+          [ 3.37822219207335, 51.3215667913007 ],
+          [ 3.37816999925795, 51.3216109809456 ],
+          [ 3.37646704574705, 51.3208025481261 ],
+          [ 3.37646695791282, 51.3208025061493 ],
+          [ 3.37608401443192, 51.3206231652693 ],
+          [ 3.37607169507628, 51.3206173959751 ],
+          [ 3.37606021048754, 51.320612017601 ],
+          [ 3.37582728410659, 51.3205029306946 ],
+          [ 3.37580409779263, 51.3206502985963 ],
+          [ 3.37575872019649, 51.3207993094705 ],
+          [ 3.37575476634361, 51.3208122883487 ],
+          [ 3.37571181656268, 51.3208797459348 ],
+          [ 3.3756624532907, 51.3209415238446 ],
+          [ 3.37557609963811, 51.3210110142077 ],
+          [ 3.37541089899821, 51.3211055871218 ],
+          [ 3.37477516102591, 51.3214102985009 ],
+          [ 3.37473173914127, 51.3214311108204 ],
+          [ 3.37455904622072, 51.3215138815012 ],
+          [ 3.37415098054777, 51.3217199232877 ],
+          [ 3.37313700916272, 51.3222422862785 ],
+          [ 3.37748824689601, 51.3242852920348 ],
+          [ 3.37749760805371, 51.3242713084009 ],
+          [ 3.37811903757028, 51.3233437635596 ],
+          [ 3.37818758851947, 51.3232647797363 ],
+          [ 3.37823803668144, 51.3232236798646 ],
+          [ 3.37837807779104, 51.3231095796538 ]
+        ]
+      ]
+    }
+  },
+  {
+    "UseCreatedCropfield": false,
+    "fieldName": "fld5641", // FarmMaps won't do without a fieldName
+    "DownloadFolder": "C:\\workdir\\groenmonitor\\", //"C:\\workdir\\groenmonitor\\"; // "Downloads" -> if you just put  "Downloads" the program will download to somewhere in ..\FarmMapsApiClient_WURtest\FarmmapsDataDownload\bin\Debug\netcoreapp3.1\Downloads\
+    "cropName": "wheat",
+    "cropYear": 2017,
+    "fieldID": 5641,
+    "lastdownloadedimagedate": "2017-01-01",
+    "geometryJson": {
+      "type": "Polygon",
+      "coordinates": [
+        [
+          [ 3.37837807779104, 51.3231095796538 ],
+          [ 3.38065689232502, 51.3212527499355 ],
+          [ 3.38022924592256, 51.3210683536359 ],
+          [ 3.37980548452565, 51.3208801127141 ],
+          [ 3.37959556105776, 51.3207540143696 ],
+          [ 3.3793691292654, 51.3205959677371 ],
+          [ 3.37822219207335, 51.3215667913007 ],
+          [ 3.37816999925795, 51.3216109809456 ],
+          [ 3.37646704574705, 51.3208025481261 ],
+          [ 3.37646695791282, 51.3208025061493 ],
+          [ 3.37608401443192, 51.3206231652693 ],
+          [ 3.37607169507628, 51.3206173959751 ],
+          [ 3.37606021048754, 51.320612017601 ],
+          [ 3.37582728410659, 51.3205029306946 ],
+          [ 3.37580409779263, 51.3206502985963 ],
+          [ 3.37575872019649, 51.3207993094705 ],
+          [ 3.37575476634361, 51.3208122883487 ],
+          [ 3.37571181656268, 51.3208797459348 ],
+          [ 3.3756624532907, 51.3209415238446 ],
+          [ 3.37557609963811, 51.3210110142077 ],
+          [ 3.37541089899821, 51.3211055871218 ],
+          [ 3.37477516102591, 51.3214102985009 ],
+          [ 3.37473173914127, 51.3214311108204 ],
+          [ 3.37455904622072, 51.3215138815012 ],
+          [ 3.37415098054777, 51.3217199232877 ],
+          [ 3.37313700916272, 51.3222422862785 ],
+          [ 3.37748824689601, 51.3242852920348 ],
+          [ 3.37749760805371, 51.3242713084009 ],
+          [ 3.37811903757028, 51.3233437635596 ],
+          [ 3.37818758851947, 51.3232647797363 ],
+          [ 3.37823803668144, 51.3232236798646 ],
+          [ 3.37837807779104, 51.3231095796538 ]
+        ]
+      ]
+    }
+  },
+  {
+    "UseCreatedCropfield": false,
+    "fieldName": "fld5641", // FarmMaps won't do without a fieldName
+    "DownloadFolder": "C:\\workdir\\groenmonitor\\", //"C:\\workdir\\groenmonitor\\"; // "Downloads" -> if you just put  "Downloads" the program will download to somewhere in ..\FarmMapsApiClient_WURtest\FarmmapsDataDownload\bin\Debug\netcoreapp3.1\Downloads\
+    "cropName": "wheat",
+    "cropYear": 2016,
+    "fieldID": 5641,
+    "lastdownloadedimagedate": "2016-01-01",
+    "geometryJson": {
+      "type": "Polygon",
+      "coordinates": [
+        [
+          [ 3.37837807779104, 51.3231095796538 ],
+          [ 3.38065689232502, 51.3212527499355 ],
+          [ 3.38022924592256, 51.3210683536359 ],
+          [ 3.37980548452565, 51.3208801127141 ],
+          [ 3.37959556105776, 51.3207540143696 ],
+          [ 3.3793691292654, 51.3205959677371 ],
+          [ 3.37822219207335, 51.3215667913007 ],
+          [ 3.37816999925795, 51.3216109809456 ],
+          [ 3.37646704574705, 51.3208025481261 ],
+          [ 3.37646695791282, 51.3208025061493 ],
+          [ 3.37608401443192, 51.3206231652693 ],
+          [ 3.37607169507628, 51.3206173959751 ],
+          [ 3.37606021048754, 51.320612017601 ],
+          [ 3.37582728410659, 51.3205029306946 ],
+          [ 3.37580409779263, 51.3206502985963 ],
+          [ 3.37575872019649, 51.3207993094705 ],
+          [ 3.37575476634361, 51.3208122883487 ],
+          [ 3.37571181656268, 51.3208797459348 ],
+          [ 3.3756624532907, 51.3209415238446 ],
+          [ 3.37557609963811, 51.3210110142077 ],
+          [ 3.37541089899821, 51.3211055871218 ],
+          [ 3.37477516102591, 51.3214102985009 ],
+          [ 3.37473173914127, 51.3214311108204 ],
+          [ 3.37455904622072, 51.3215138815012 ],
+          [ 3.37415098054777, 51.3217199232877 ],
+          [ 3.37313700916272, 51.3222422862785 ],
+          [ 3.37748824689601, 51.3242852920348 ],
+          [ 3.37749760805371, 51.3242713084009 ],
+          [ 3.37811903757028, 51.3233437635596 ],
+          [ 3.37818758851947, 51.3232647797363 ],
+          [ 3.37823803668144, 51.3232236798646 ],
+          [ 3.37837807779104, 51.3231095796538 ]
+        ]
+      ]
+    }
+  }
+
+
+]

FarmmapsBulkSatDownload/BulkSatDownloadService.cs

@@ -0,0 +1,29 @@
+using System;
+using System.Globalization;
+using System.Threading.Tasks;
+using FarmmapsApi.Models;
+using FarmmapsApi.Services;
+using FarmmapsBulkSatDownload.Models;
+using Microsoft.Extensions.Logging;
+using static FarmmapsApi.Extensions;
+using static FarmmapsApiSamples.Constants;
+
+namespace FarmmapsBulkSatDownload
+{
+    public class BulkSatDownloadService
+    {
+        private readonly ILogger<BulkSatDownloadService> _logger;
+        private readonly FarmmapsApiService _farmmapsApiService;
+        private readonly GeneralService _generalService;
+
+        public BulkSatDownloadService(ILogger<BulkSatDownloadService> logger, FarmmapsApiService farmmapsApiService,
+            GeneralService generalService)
+        {
+            _logger = logger;
+            _farmmapsApiService = farmmapsApiService;
+            _generalService = generalService;
+        }
+
+       
+    }
+}

FarmmapsBulkSatDownload/DBsettings.json

@@ -0,0 +1,6 @@
+{
+  "User": "",
+  "Password": "",
+  "Database": "",
+  "Host": ""
+}

FarmmapsBulkSatDownload/FarmmapsBulkSatDownload.csproj

@@ -0,0 +1,31 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+    <PropertyGroup>
+        <OutputType>Exe</OutputType>
+        <TargetFramework>netcoreapp3.1</TargetFramework>
+    </PropertyGroup>
+
+    <ItemGroup>
+      <PackageReference Include="Npgsql" Version="5.0.5" />
+    </ItemGroup>
+
+    <ItemGroup>
+      <None Update="appsettings.json">
+        <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+      </None>
+      <None Update="Data\**\*">
+        <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+      </None>
+      <None Update="BulkSatDownloadInput.json">
+        <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+      </None>
+      <None Update="DBsettings.json">
+        <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+      </None>
+    </ItemGroup>
+
+    <ItemGroup>
+      <ProjectReference Include="..\FarmmapsApi\FarmmapsApi.csproj" />
+    </ItemGroup>
+
+</Project>

FarmmapsBulkSatDownload/Models/BulkSatDownloadInput.cs

@@ -0,0 +1,18 @@
+using System;
+using Newtonsoft.Json.Linq;
+
+namespace FarmmapsBulkSatDownload.Models
+{
+    public class BulkSatDownloadInput
+    {
+        public bool UseCreatedCropfield { get; set; }
+        public string fieldName { get; set; }
+        public string DownloadFolder { get; set; }
+        public string cropName { get; set; }
+        public int cropYear { get; set; }
+        public int fieldID { get; set; }
+        public DateTime lastdownloadedimagedate { get; set; }
+        public JObject GeometryJson { get; set; }
+        //public string fieldName { get { return string.Format($"{cropName}_fld{fieldID}"); } }
+    }
+}

FarmmapsBulkSatDownload/Models/DB.cs

@@ -0,0 +1,47 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+using Npgsql;
+
+namespace FarmmapsApi.Services
+{
+    public class DB
+    {
+        public string Database;
+        public string User;
+        public string Password;
+        public string Host;
+
+        public string GetConnectionString()
+        {
+            NpgsqlConnectionStringBuilder sb = new NpgsqlConnectionStringBuilder();
+            sb.Database = Database;
+            sb.Host = Host;
+            sb.Username = User;
+            sb.Password = Password;
+            sb.Port = 5432;
+
+            return sb.ConnectionString;
+        }
+
+        public int ExecuteNonQuery(string sql)
+        {
+            using (NpgsqlConnection conn = new NpgsqlConnection(GetConnectionString()))
+            {
+                conn.Open();
+                NpgsqlCommand command = conn.CreateCommand();
+                command.CommandText = sql;
+                int r = command.ExecuteNonQuery();
+                return r;
+            }
+        }
+
+        public NpgsqlConnection CreateConnection()
+        {
+            NpgsqlConnection conn = new NpgsqlConnection(GetConnectionString());
+            conn.Open();
+            return conn;
+        }
+
+    }
+}

FarmmapsBulkSatDownload/Models/GroenmonitorTable.cs

@@ -0,0 +1,28 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace FarmmapsBulkSatDownload.Models
+{
+    public class GroenmonitorTable
+    {
+        //public string parceltablename; //PO20190605: parceltablename added as a field (=column) in the GroenmonitorTable. The GroenmonitorTable contains data from multiple parceltables
+        public int parcelid;
+        public string date;
+        public string source;
+        public int wdvi_pixelcount; //count of pixels with data
+        public double wdvi_max;
+        public double wdvi_mean;
+        public double wdvi_min;
+        public double wdvi_stdev;
+        public double wdvi_median;
+        public double wdvi_p90;
+        public int ndvi_pixelcount; //count of pixels with data
+        public double ndvi_max;
+        public double ndvi_mean;
+        public double ndvi_min;
+        public double ndvi_stdev;
+        public double ndvi_median;
+        public double ndvi_p90;
+    }
+}

FarmmapsBulkSatDownload/Models/Settings.cs

@@ -0,0 +1,11 @@
+namespace FarmmapsBulkSatDownload
+{
+    public class Settings
+    {
+        public string CropfieldItemCode { get; set; }
+        public string SatelliteTaskCode { get; set; }
+        public string VanDerSatTaskCode { get; set; }
+        public string WatBalTaskCode { get; set; }
+
+    }
+}

FarmmapsBulkSatDownload/Program.cs

@@ -0,0 +1,21 @@
+using System.Threading.Tasks;
+using FarmmapsApi;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging;
+
+namespace FarmmapsBulkSatDownload
+{
+    class Program : FarmmapsProgram<BulkSatDownloadApplication>
+    {
+        private static async Task Main(string[] args)
+        {
+            await new Program().Start(args);
+        }
+
+        protected override void Configure(IServiceCollection serviceCollection)
+        {
+            serviceCollection.AddLogging()
+                .AddTransient<BulkSatDownloadService>();
+        }
+    }
+}

FarmmapsBulkSatDownload/ShowGeotiff.r

@@ -0,0 +1,152 @@
+# ShowGeotiff.r
+# I downloaded and calculated the stats for the polygon defined in C:\git\FarmMapsApiClient_WURtest\FarmmapsDataDownload\DataDownloadInput.json
+# in which I set "SatelliteBand": "wdvi" and in which in the console I requested image 8 for date '2020-06-14'
+
+library(raster)
+library(sf)
+library(rgdal)
+setwd("C:/workdir/groenmonitor/")
+# FarmmapsDataDownload generates two files:
+fileGeotiff <- "wenr.tif"
+# fileJpg <- "thumbnail.jpg"
+# FarmmapsBulkSatDownload generates many files. Here is what I tried when inputing the same field for a number of years using BulkSatDownloadInput.json
+# fileGeotiff <- "wheat_fld5641_20210224.tif"	# 2 files for year 2021. This file is a nice example having in upperleft corner no data
+# fileGeotiff <- "wheat_fld5641_20210331.tif"	# 2 files for year 2021
+# fileGeotiff <- "wheat_fld5641_20200321.tif"	# 14 files for year 2020, earliest
+# fileGeotiff <- "wheat_fld5641_20200922.tif"	# 14 files for year 2020, latest
+# fileGeotiff <- "wheat_fld5641_20190121.tif"	# 9 files for year 2019, earliest
+# fileGeotiff <- "wheat_fld5641_20191117.tif"	# 9 files for year 2019, latest
+# 1 file for year 2018, with error message 'End of Central Directory record could not be found' and invalid wheat_fld5641_20180630.zip 
+# fileGeotiff <- "wheat_fld5641_20170526.tif"	# 1 file for year 2017
+# Zero files for 2016
+lenfilename <- nchar(fileGeotiff)
+year <- substr(fileGeotiff,lenfilename-11,lenfilename-8)
+imgdate <- substr(fileGeotiff,lenfilename-11,lenfilename-4)
+
+# The thumbnail has the polygon clipped out, has 1 layer, no crs and the mean value is not the mean wdvi we are looking for
+# r.thumbnail <- raster(fileJpg)
+# plot(r.thumbnail)
+# crs(r.thumbnail)
+# CRS arguments: NA 
+# cellStats(r.thumbnail,'mean')	#59.91667
+
+stk.wenr <- stack(x=fileGeotiff)
+# plot(stk.wenr) shows 5 plots (5 bands)
+# I think these are:
+# 1. ndvi (since it runs from 0 to 1)
+# 2. wdvi (since it runs from 0 to 0.5)
+# 3-5: RGB (since they run from 0 to 255)
+plot(stk.wenr)
+# CRS arguments:
+# +proj=sterea +lat_0=52.1561605555556 +lon_0=5.38763888888889 +k=0.9999079 +x_0=155000 +y_0=463000 +ellps=bessel +units=m +no_defs
+# Or use st_crs(stk.wenr) to get more info, but no EPSG code in there. 
+# Likely it is epsg:28992 (Amersfoort)
+crs(stk.wenr)
+stk.wenr <- projectRaster(stk.wenr, crs = CRS('+init=EPSG:28992'))
+crs(stk.wenr)
+# Looks the same but strangely, if we don't do projectRaster(stk.wenr, crs = CRS('+init=EPSG:28992')), we find below the bottom left corner of the polygon missing
+
+r.wenr.rd.wdvi <- subset(stk.wenr,2)
+dev.off()
+plot(r.wenr.rd.wdvi,main=paste("wdvi",imgdate),xlab="RDX",ylab="RDY")
+cellStats(r.wenr.rd.wdvi,'mean')	#0.1654627
+# Same as in downloaded file SatelliteDataStatistics_test_satData_wdvi_2020-06-14.csv
+# "mean": 0.16564258790046743
+# So FindSatelliteItem in C:\git\FarmMapsApiClient_WURtest\FarmmapsApi\Services\GeneralService.cs returns a a stacked geotiff in 
+# i.e. the cellstats are calculated from the rectangle
+
+# Furthermore we can see
+# shows coordinates in RD
+# returns a rectangle, thus the shape of the polygon submitted is not clipped. 
+# The polygon was provided in WGS84. Let's draw it on top. 
+# First convert the raster to WGS84
+r.wenr.wgs84.wdvi <- projectRaster(r.wenr.rd.wdvi, crs = CRS('+init=EPSG:4326'))
+
+# Draw a polygon on top of the raster
+# Example polygon p1
+# p1 <- data.frame(id = 1, wkt = 'POLYGON((4.963 52.801, 4.966 52.801, 4.966 52.803, 4.963 52.803, 4.963 52.801))')
+# p1 <- st_as_sf(p1, wkt = 'wkt', crs = targetcrs)
+# plot(p1,add=TRUE, col="transparent",border="black")
+# Draw the polygon on top of the raster
+# Polygon p2 from C:\git\FarmMapsApiClient_WURtest\FarmmapsDataDownload\DataDownloadInput.json
+p2 <- data.frame(id = 1, wkt = gsub("\n","",'POLYGON((
+			3.37837807779104   51.3231095796538,
+			3.38065689232502   51.3212527499355,
+			3.38022924592256   51.3210683536359,
+			3.37980548452565   51.3208801127141,
+			3.37959556105776   51.3207540143696,
+			3.3793691292654   51.3205959677371,
+			3.37822219207335   51.3215667913007,
+			3.37816999925795   51.3216109809456,
+			3.37646704574705   51.3208025481261,
+			3.37646695791282   51.3208025061493,
+			3.37608401443192   51.3206231652693,
+			3.37607169507628   51.3206173959751,
+			3.37606021048754   51.320612017601,
+			3.37582728410659   51.3205029306946,
+			3.37580409779263   51.3206502985963,
+			3.37575872019649   51.3207993094705,
+			3.37575476634361   51.3208122883487,
+			3.37571181656268   51.3208797459348,
+			3.3756624532907   51.3209415238446,
+			3.37557609963811   51.3210110142077,
+			3.37541089899821   51.3211055871218,
+			3.37477516102591   51.3214102985009,
+			3.37473173914127   51.3214311108204,
+			3.37455904622072   51.3215138815012,
+			3.37415098054777   51.3217199232877,
+			3.37313700916272   51.3222422862785,
+			3.37748824689601   51.3242852920348,
+			3.37749760805371   51.3242713084009,
+			3.37811903757028   51.3233437635596,
+			3.37818758851947   51.3232647797363,
+			3.37823803668144   51.3232236798646,
+			3.37837807779104   51.3231095796538))'))
+p2.wgs84 <- st_as_sf(p2, wkt = 'wkt', crs = CRS('+init=EPSG:4326'))
+# Or other way round, in RD Amersfoort. That looks ok
+p2.rd <- st_transform(p2.wgs84, "+init=epsg:28992")
+
+# Have a look at both
+# wg84
+dev.off()
+plot(r.wenr.wgs84.wdvi,main=paste("wdvi",imgdate),xlab="LON",ylab="LAT")
+plot(p2.wgs84,add=TRUE, col="transparent",border="red")
+# RD
+dev.off()
+plot(r.wenr.rd.wdvi,main=paste("wdvi",imgdate),xlab="RDX",ylab="RDY")
+plot(p2.rd,add=TRUE, col="transparent",border="red")
+
+#Let's clip the polygon
+r.wenr.rd.wdvi.pol <- mask(r.wenr.rd.wdvi,p2.rd)
+r.wenr.wgs84.wdvi.pol <- mask(r.wenr.wgs84.wdvi,p2.wgs84)
+dev.off()
+plot(r.wenr.wgs84.wdvi.pol,main=paste("wdvi",imgdate),xlab="LON",ylab="LAT")
+plot(p2.wgs84,add=TRUE, col="transparent",border="red")
+#That's what we want! Now compare the stats
+cellStats(r.wenr.rd.wdvi,'mean')	#0.1654627 # Stats from rectangle, RD
+cellStats(r.wenr.wgs84.wdvi,'mean')	#0.1656399 # Stats from rectangle, WGS84
+cellStats(r.wenr.rd.wdvi.pol,'mean')	#0.1658702 # Stats from raster clipped by polygon, WGS84
+cellStats(r.wenr.wgs84.wdvi.pol,'mean')	#0.1658611 # Stats from raster clipped by polygon, WGS84
+# Conclusion: in this example little difference, but can be quite different!
+
+# The project FarmmapsNbs can generate a wenr.tif file, application.tif, uptake.tif (in rtest1.uptake.zip)and shape.shp (in rtest1.taskmap.zip)
+r.application <- raster("C:/git/FarmMapsApiClient_WURtest/FarmmapsNbs/bin/Debug/netcoreapp3.1/Downloads/application.tif")
+dev.off()
+plot(r.application)
+plot(p2.rd,add=TRUE, col="transparent",border="red")
+# The application.tif file is a rectangle (polygon not yet clipped), in projection Amersfoort RD New (EPSG:28992)
+
+r.uptake <- raster("C:/git/FarmMapsApiClient_WURtest/FarmmapsNbs/bin/Debug/netcoreapp3.1/Downloads/uptake.tif")
+dev.off()
+plot(r.uptake)
+plot(p2.rd,add=TRUE, col="transparent",border="red")
+# The uptake.tif file is a rectangle (polygon not yet clipped), in projection Amersfoort RD New (EPSG:28992)
+
+shp.wgs84 <- readOGR(dsn="C:/git/FarmMapsApiClient_WURtest/FarmmapsNbs/bin/Debug/netcoreapp3.1/Downloads", layer="shape")
+crs(shp.wgs84)
+# CRS arguments: +proj=longlat +datum=WGS84 +no_defs
+dev.off()
+plot(r.wenr.wgs84.wdvi,main="wdvi",xlab="LON",ylab="LAT")
+plot(shp.wgs84,add=TRUE, col="transparent",border="black")
+plot(p2.wgs84,add=TRUE, col="transparent",border="red")
+# The shape file is in WGS84

FarmmapsBulkSatDownload/appsettings.json

@@ -0,0 +1,10 @@
+{
+  "Authority": "https://accounts.test.farmmaps.eu/",
+  "Endpoint": "https://test.farmmaps.eu/",
+  "BasePath": "api/v1",
+  "DiscoveryEndpointUrl": "https://accounts.test.farmmaps.eu/.well-known/openid-configuration",
+  "RedirectUri": "http://example.nl/api",
+  "ClientId": "",
+  "ClientSecret": "",
+  "Scopes": [ "api" ]
+}