From 250020be7890dd5a85a0204621d036ed3366f8ed Mon Sep 17 00:00:00 2001 From: pepijn van oort Date: Tue, 18 May 2021 18:19:28 +0200 Subject: [PATCH] Added project FarmmapsBulkSatDownload --- FarmmapsApi/Services/GeneralService.cs | 23 ++ FarmmapsApiSamples.sln | 8 +- .../BulkSatDownloadApplication.cs | 242 +++++++++++++++ .../BulkSatDownloadInput.json | 293 ++++++++++++++++++ .../BulkSatDownloadService.cs | 29 ++ FarmmapsBulkSatDownload/DBsettings.json | 6 + .../FarmmapsBulkSatDownload.csproj | 31 ++ .../Models/BulkSatDownloadInput.cs | 18 ++ FarmmapsBulkSatDownload/Models/DB.cs | 47 +++ .../Models/GroenmonitorTable.cs | 28 ++ FarmmapsBulkSatDownload/Models/Settings.cs | 11 + FarmmapsBulkSatDownload/Program.cs | 21 ++ FarmmapsBulkSatDownload/ShowGeotiff.r | 152 +++++++++ FarmmapsBulkSatDownload/appsettings.json | 10 + 14 files changed, 918 insertions(+), 1 deletion(-) create mode 100644 FarmmapsBulkSatDownload/BulkSatDownloadApplication.cs create mode 100644 FarmmapsBulkSatDownload/BulkSatDownloadInput.json create mode 100644 FarmmapsBulkSatDownload/BulkSatDownloadService.cs create mode 100644 FarmmapsBulkSatDownload/DBsettings.json create mode 100644 FarmmapsBulkSatDownload/FarmmapsBulkSatDownload.csproj create mode 100644 FarmmapsBulkSatDownload/Models/BulkSatDownloadInput.cs create mode 100644 FarmmapsBulkSatDownload/Models/DB.cs create mode 100644 FarmmapsBulkSatDownload/Models/GroenmonitorTable.cs create mode 100644 FarmmapsBulkSatDownload/Models/Settings.cs create mode 100644 FarmmapsBulkSatDownload/Program.cs create mode 100644 FarmmapsBulkSatDownload/ShowGeotiff.r create mode 100644 FarmmapsBulkSatDownload/appsettings.json diff --git a/FarmmapsApi/Services/GeneralService.cs b/FarmmapsApi/Services/GeneralService.cs index e934f85..9cd3015 100644 --- a/FarmmapsApi/Services/GeneralService.cs +++ b/FarmmapsApi/Services/GeneralService.cs @@ -388,6 +388,29 @@ namespace FarmmapsApi.Services return selectedSatelliteItem; } + public async Task> FindSatelliteItemsAll(Item cropfieldItem, string satelliteTaskCode) + { + + var taskStatus = await _farmmapsApiService.GetTaskStatusAsync(cropfieldItem.Code, satelliteTaskCode); + + + // find ndvi or wdvi satellite data geotiffs + var temporalItem = await FindChildItemAsync(cropfieldItem.Code, TEMPORAL_ITEMTYPE, + "Cropfield Satellite items", item => item.SourceTask == SATELLITE_TASK && + taskStatus.Finished >= item.Created && + taskStatus.Finished <= item.Created.Value.AddHours(1)); + + + if (temporalItem == null) + { + _logger.LogError("Temporal item not found"); + + } + + var satelliteTiffs = await _farmmapsApiService.GetItemChildrenAsync(temporalItem.Code); + + return satelliteTiffs; + } //VanDerSat public async Task RunVanDerSatTask(Item cropfieldItem) { diff --git a/FarmmapsApiSamples.sln b/FarmmapsApiSamples.sln index 3fc9bd7..42962b5 100644 --- a/FarmmapsApiSamples.sln +++ b/FarmmapsApiSamples.sln @@ -23,7 +23,9 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FarmMapsBlight", "FarmMapsB EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FarmmapsZonering", "FarmmapsZonering\FarmmapsZonering.csproj", "{91A58C4A-4A80-4079-B43D-9B851206194F}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "FarmmapsDataDownload", "FarmmapsDataDownload\FarmmapsDataDownload.csproj", "{32ED9500-AAAB-4030-9C7A-F611A85DF890}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FarmmapsDataDownload", "FarmmapsDataDownload\FarmmapsDataDownload.csproj", "{32ED9500-AAAB-4030-9C7A-F611A85DF890}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FarmmapsBulkSatDownload", "FarmmapsBulkSatDownload\FarmmapsBulkSatDownload.csproj", "{772DBDCD-9FAA-40A7-8551-2C1620C4AB67}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -63,6 +65,10 @@ Global {32ED9500-AAAB-4030-9C7A-F611A85DF890}.Debug|Any CPU.Build.0 = Debug|Any CPU {32ED9500-AAAB-4030-9C7A-F611A85DF890}.Release|Any CPU.ActiveCfg = Release|Any CPU {32ED9500-AAAB-4030-9C7A-F611A85DF890}.Release|Any CPU.Build.0 = Release|Any CPU + {772DBDCD-9FAA-40A7-8551-2C1620C4AB67}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {772DBDCD-9FAA-40A7-8551-2C1620C4AB67}.Debug|Any CPU.Build.0 = Debug|Any CPU + {772DBDCD-9FAA-40A7-8551-2C1620C4AB67}.Release|Any CPU.ActiveCfg = Release|Any CPU + {772DBDCD-9FAA-40A7-8551-2C1620C4AB67}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/FarmmapsBulkSatDownload/BulkSatDownloadApplication.cs b/FarmmapsBulkSatDownload/BulkSatDownloadApplication.cs new file mode 100644 index 0000000..57dd71a --- /dev/null +++ b/FarmmapsBulkSatDownload/BulkSatDownloadApplication.cs @@ -0,0 +1,242 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Threading.Tasks; +using FarmmapsApi; +using FarmmapsApi.Models; +using FarmmapsApi.Services; +using FarmmapsBulkSatDownload.Models; +using Microsoft.Extensions.Logging; +using Newtonsoft.Json; +using Npgsql; +using Newtonsoft.Json.Linq; +using static FarmmapsApiSamples.Constants; + +namespace FarmmapsBulkSatDownload +{ + public class BulkSatDownloadApplication : IApplication + { + private const string SettingsFile = "settings.json"; + + private readonly ILogger _logger; + private readonly FarmmapsApiService _farmmapsApiService; + private readonly BulkSatDownloadService _dataDownloadService; + private readonly GeneralService _generalService; + + private Settings _settings; + static DB dbparcels; + + public BulkSatDownloadApplication(ILogger logger, FarmmapsApiService farmmapsApiService, + GeneralService generalService, BulkSatDownloadService dataDownloadService) + { + _logger = logger; + _farmmapsApiService = farmmapsApiService; + _generalService = generalService; + _dataDownloadService = dataDownloadService; + } + public async Task RunAsync() + { + // Check if we have permission + // !! this call is needed the first time an api is called with a fresh clientid and secret !! + await _farmmapsApiService.GetCurrentUserCodeAsync(); + var roots = await _farmmapsApiService.GetCurrentUserRootsAsync(); + // Initialize databases. Username, password etc stored in file "DBsettings.json" + dbparcels = JsonConvert.DeserializeObject(File.ReadAllText("DBsettings.json")); + + //string date; + //string asterices = "*****************************************************************************"; + //string bboxtiffWithTempPath; + //string parcelshp = "parcel.shp"; + //string parcelshpWithTempPath; + //string gwoptions; //= String.Format("-overwrite -s_srs EPSG:28992 -t_srs EPSG:28992 -dstnodata 0 -cutline {0} -crop_to_cutline", parcelshp); + //string parceltiff = "parcel.tiff"; + //string parceltiffWithTempPath; + //string[] vegetationindices = { "WDVI", "NDVI" }; + //string[] sources = { "groenmonitor" }; // { "groenmonitor", "akkerwebwenr", "akkerwebneo" }; { "groenmonitor" }; + string schemaname = "bigdata"; + string parceltablename = "parcel_flowerbulbs";//"parcelsijbrandij" "parcel"; "parcel_flowerbulbs" + //string groenmonitortablename = "groenmonitor_flowerbulbs";//"groenmonitorsijbrandij" "groenmonitor" "groenmonitor_flowerbulbs" //PO20190605: "groenmonitormpt" contains groenmonitor data from multiple parcel tables (mpt) + // The view 'groenmonitorlatestviewname' contains per parcelid (arbid) the year in which it "exists" and the date of the latest image downloaded. It is used to prevent downloading images already downloaded + string groenmonitorlatestviewname = "groenmonitorlatest_flowerbulbs"; //"groenmonitorsijbrandijlatest" "groenmonitorlatest" "groenmonitorlatest_flowerbulbs" //PO20190605: "v_groenmonitorlatest" contains latest available dates from groenmonitortablename = "groenmonitormpt" + //GroenmonitorTable gmtb; + //DateTime dt; + //DateTime dtfirst; + //DateTime dtlast; + //double scalingfactor; + BulkSatDownloadInput dataDownloadInput; + List fieldsInputs = new List(); + + // Database query and connection. Geometry must be in WGS84 coordinate system, EPSG 4326 + // Apparently the FarmmapsApi cannot handle MultiPolygon, so we need to convert to single Polygon + // In case database returns a MultiPolygon use ST_NumGeometries(pt.geom) to count the number of polygons + // If necessary use WHERE T_NumGeometries(pt.geom) = 1 to use only single polygons + string connectionString = dbparcels.GetConnectionString(); + string readSql = string.Format( +@" +SELECT pt.arbid, pt.crop, pt.year, gml.lastwenrdate, ST_AsGeoJSON(ST_Transform((ST_DUMP(pt.geom)).geom::geometry(Polygon),4326)) AS geojson_polygon_wgs84 +FROM {0}.{1} pt, {0}.{2} gml +WHERE + pt.arbid = gml.arbid AND + pt.crop NOT IN('Tulp', 'Lelie') AND + pt.year = 2018 +LIMIT 10;", schemaname, parceltablename, groenmonitorlatestviewname); //LIMIT 10 for testing + + using (NpgsqlConnection connection = new NpgsqlConnection(connectionString)) + { + connection.Open(); + + // Read data (run query) = build a list of fields for which to download images + NpgsqlCommand command = connection.CreateCommand(); + command.CommandText = readSql; + NpgsqlDataReader dr = command.ExecuteReader(); + while (dr.Read()) + { + dataDownloadInput = new BulkSatDownloadInput(); + dataDownloadInput.UseCreatedCropfield = false; + dataDownloadInput.DownloadFolder = "C:\\workdir\\groenmonitor\\"; + dataDownloadInput.fieldID = dr.GetInt16(0); + dataDownloadInput.fieldName = string.Format($"{parceltablename}_fld{dataDownloadInput.fieldID}"); + dataDownloadInput.cropName = dr.GetString(1); + dataDownloadInput.cropYear = dr.GetInt16(2); + dataDownloadInput.lastdownloadedimagedate = dr.GetDateTime(3); + dataDownloadInput.GeometryJson = JObject.Parse(dr.GetString(4)); + fieldsInputs.Add(dataDownloadInput); + } + connection.Close(); + } + + // For testing using json input instead of database input + var fieldsInputJson = File.ReadAllText("BulkSatDownloadInput.json"); + List fieldsInputs2 = JsonConvert.DeserializeObject>(fieldsInputJson); + + // Now for each input download all images. Use fieldsInputs or fieldsInputs2 for testing + foreach (var input in fieldsInputs2) + { + try + { + await Process(roots, input); + } + catch (Exception ex) + { + _logger.LogError(ex.Message); + } + } + } + + private async Task Process(List roots, BulkSatDownloadInput input) + { + string DownloadFolder = input.DownloadFolder; + if (!Directory.Exists(DownloadFolder)) + Directory.CreateDirectory(DownloadFolder); + + // !!specify if you are using an already created cropfield: + bool useCreatedCropfield = input.UseCreatedCropfield; + string settingsfile = $"Settings_BulkSatDownloadApplication.json"; + + LoadSettings(settingsfile); + + var uploadedRoot = roots.SingleOrDefault(r => r.Name == "Uploaded"); + if (uploadedRoot == null) + { + _logger.LogError("Could not find a needed root item"); + return; + } + + var myDriveRoot = roots.SingleOrDefault(r => r.Name == "My drive"); + if (myDriveRoot == null) + { + _logger.LogError("Could not find a needed root item"); + return; + } + + // Use already created cropfield or create new one + Item cropfieldItem; + if (useCreatedCropfield == false || string.IsNullOrEmpty(_settings.CropfieldItemCode)) + { + _logger.LogInformation(string.Format($"Creating cropfield for a field in the year {input.cropYear}")); + //string geomjson = input.GeometryJson.ToString(Formatting.None); //"{\"type\":\"Polygon\",\"coordinates\":[[[4.960707146896585,52.80058366970849],[4.960645975538824,52.80047021761092],[4.962140695752897,52.7991771471948],[4.967523821195745,52.80150240004121],[4.966336768950911,52.80254373587981],[4.96171188076433,52.80100999685643],[4.960707146896585,52.80058366970849]]]}" + string geomjson = input.GeometryJson.ToString(Formatting.None); + cropfieldItem = await _generalService.CreateCropfieldItemAsync(myDriveRoot.Code, + $"DataCropfield {input.cropName}", input.cropYear, input.GeometryJson.ToString(Formatting.None)); + _settings.CropfieldItemCode = cropfieldItem.Code; + SaveSettings(settingsfile); + } + else + { + _logger.LogInformation("Cropfield already exists, trying to get it"); + cropfieldItem = await _farmmapsApiService.GetItemAsync(_settings.CropfieldItemCode); + } + + // check if satellite task not yet done, do here and save taskcode + if (useCreatedCropfield == false || string.IsNullOrEmpty(_settings.SatelliteTaskCode)) + { + var satelliteTaskCode = await _generalService.RunSatelliteTask(cropfieldItem); + _settings.SatelliteTaskCode = satelliteTaskCode; + SaveSettings(settingsfile); + } + + // Now get the tiffs + List satelliteTiffs = await _generalService.FindSatelliteItemsAll(cropfieldItem, _settings.SatelliteTaskCode); + _logger.LogInformation(string.Format($"Downloading {satelliteTiffs.Count} geotiffs for a field in year {input.cropYear} ...")); + foreach (Item satalliteItem in satelliteTiffs) { + // download the geotiff. Returns a zip file with always these three files: + // data.dat.aux.xml + // thumbnail.jpg + // wenr.tif. Contains 5 layers: (1) ndvi, (2) wdvi, (3) Red, (4) Green and (5) Blue + DateTime SatelliteImageDate = (DateTime)satalliteItem.DataDate; + string SatelliteDate = SatelliteImageDate.ToString("yyyyMMdd"); + string fileName = string.Format($"{input.cropName}_{input.fieldName}_{SatelliteDate}"); // assuming the fieldName is good enough for an ID. One might add here _{input.fieldID} or _{input.cropName} + string fileNameZip = string.Format($"{fileName}.zip"); + string fileNameGeotiff = string.Format($"{fileName}.tif"); + await _farmmapsApiService.DownloadItemAsync(satalliteItem.Code, Path.Combine(DownloadFolder, fileNameZip)); + + // Extract the file "wenr.tif" from zip, rename it, delete "wenr.tif". overwriteFiles = true + ZipFile.ExtractToDirectory(Path.Combine(DownloadFolder, fileNameZip), DownloadFolder, true); + File.Delete(Path.Combine(DownloadFolder, fileNameGeotiff)); // Delete the fileNameGeotiff file if exists + File.Move(Path.Combine(DownloadFolder, "wenr.tif"), Path.Combine(DownloadFolder, fileNameGeotiff)); // Rename the oldFileName into newFileName + + // Cleanup + string[] filesToDelete = new string[] { fileNameZip, "wenr.tif", "thumbnail.jpg", "data.dat.aux.xml" }; + foreach (string f in filesToDelete) + { + File.Delete(Path.Combine(DownloadFolder, f)); + } + } + } + + // Functions to save previously created cropfields + private void LoadSettings(string file) + { + if (File.Exists(file)) + { + var jsonText = File.ReadAllText(file); + _settings = JsonConvert.DeserializeObject(jsonText); + } + else + { + _settings = new Settings(); + } + } + + private void SaveSettings(string file) + { + if (_settings == null) + return; + + var json = JsonConvert.SerializeObject(_settings); + File.WriteAllText(file, json); + } + private void SaveInfo(string file) + { + if (_settings == null) + return; + + var json = JsonConvert.SerializeObject(_settings); + File.WriteAllText(file, json); + + } + + } +} diff --git a/FarmmapsBulkSatDownload/BulkSatDownloadInput.json b/FarmmapsBulkSatDownload/BulkSatDownloadInput.json new file mode 100644 index 0000000..3463e41 --- /dev/null +++ b/FarmmapsBulkSatDownload/BulkSatDownloadInput.json @@ -0,0 +1,293 @@ +[ + { + "UseCreatedCropfield": false, + "fieldName": "fld5641", // FarmMaps won't do without a fieldName + "DownloadFolder": "C:\\workdir\\groenmonitor\\", //"C:\\workdir\\groenmonitor\\"; // "Downloads" -> if you just put "Downloads" the program will download to somewhere in ..\FarmMapsApiClient_WURtest\FarmmapsDataDownload\bin\Debug\netcoreapp3.1\Downloads\ + "cropName": "wheat", + "cropYear": 2021, + "fieldID": 5641, + "lastdownloadedimagedate": "2021-01-01", + "geometryJson": { + "type": "Polygon", + "coordinates": [ + [ + [ 3.37837807779104, 51.3231095796538 ], + [ 3.38065689232502, 51.3212527499355 ], + [ 3.38022924592256, 51.3210683536359 ], + [ 3.37980548452565, 51.3208801127141 ], + [ 3.37959556105776, 51.3207540143696 ], + [ 3.3793691292654, 51.3205959677371 ], + [ 3.37822219207335, 51.3215667913007 ], + [ 3.37816999925795, 51.3216109809456 ], + [ 3.37646704574705, 51.3208025481261 ], + [ 3.37646695791282, 51.3208025061493 ], + [ 3.37608401443192, 51.3206231652693 ], + [ 3.37607169507628, 51.3206173959751 ], + [ 3.37606021048754, 51.320612017601 ], + [ 3.37582728410659, 51.3205029306946 ], + [ 3.37580409779263, 51.3206502985963 ], + [ 3.37575872019649, 51.3207993094705 ], + [ 3.37575476634361, 51.3208122883487 ], + [ 3.37571181656268, 51.3208797459348 ], + [ 3.3756624532907, 51.3209415238446 ], + [ 3.37557609963811, 51.3210110142077 ], + [ 3.37541089899821, 51.3211055871218 ], + [ 3.37477516102591, 51.3214102985009 ], + [ 3.37473173914127, 51.3214311108204 ], + [ 3.37455904622072, 51.3215138815012 ], + [ 3.37415098054777, 51.3217199232877 ], + [ 3.37313700916272, 51.3222422862785 ], + [ 3.37748824689601, 51.3242852920348 ], + [ 3.37749760805371, 51.3242713084009 ], + [ 3.37811903757028, 51.3233437635596 ], + [ 3.37818758851947, 51.3232647797363 ], + [ 3.37823803668144, 51.3232236798646 ], + [ 3.37837807779104, 51.3231095796538 ] + ] + ] + } + }, + { + "UseCreatedCropfield": false, + "fieldName": "fld5641", // FarmMaps won't do without a fieldName + "DownloadFolder": "C:\\workdir\\groenmonitor\\", //"C:\\workdir\\groenmonitor\\"; // "Downloads" -> if you just put "Downloads" the program will download to somewhere in ..\FarmMapsApiClient_WURtest\FarmmapsDataDownload\bin\Debug\netcoreapp3.1\Downloads\ + "cropName": "wheat", + "cropYear": 2020, + "fieldID": 5641, + "lastdownloadedimagedate": "2020-01-01", + "geometryJson": { + "type": "Polygon", + "coordinates": [ + [ + [ 3.37837807779104, 51.3231095796538 ], + [ 3.38065689232502, 51.3212527499355 ], + [ 3.38022924592256, 51.3210683536359 ], + [ 3.37980548452565, 51.3208801127141 ], + [ 3.37959556105776, 51.3207540143696 ], + [ 3.3793691292654, 51.3205959677371 ], + [ 3.37822219207335, 51.3215667913007 ], + [ 3.37816999925795, 51.3216109809456 ], + [ 3.37646704574705, 51.3208025481261 ], + [ 3.37646695791282, 51.3208025061493 ], + [ 3.37608401443192, 51.3206231652693 ], + [ 3.37607169507628, 51.3206173959751 ], + [ 3.37606021048754, 51.320612017601 ], + [ 3.37582728410659, 51.3205029306946 ], + [ 3.37580409779263, 51.3206502985963 ], + [ 3.37575872019649, 51.3207993094705 ], + [ 3.37575476634361, 51.3208122883487 ], + [ 3.37571181656268, 51.3208797459348 ], + [ 3.3756624532907, 51.3209415238446 ], + [ 3.37557609963811, 51.3210110142077 ], + [ 3.37541089899821, 51.3211055871218 ], + [ 3.37477516102591, 51.3214102985009 ], + [ 3.37473173914127, 51.3214311108204 ], + [ 3.37455904622072, 51.3215138815012 ], + [ 3.37415098054777, 51.3217199232877 ], + [ 3.37313700916272, 51.3222422862785 ], + [ 3.37748824689601, 51.3242852920348 ], + [ 3.37749760805371, 51.3242713084009 ], + [ 3.37811903757028, 51.3233437635596 ], + [ 3.37818758851947, 51.3232647797363 ], + [ 3.37823803668144, 51.3232236798646 ], + [ 3.37837807779104, 51.3231095796538 ] + + ] + ] + } + }, + { + "UseCreatedCropfield": false, + "fieldName": "fld5641", // FarmMaps won't do without a fieldName + "DownloadFolder": "C:\\workdir\\groenmonitor\\", //"C:\\workdir\\groenmonitor\\"; // "Downloads" -> if you just put "Downloads" the program will download to somewhere in ..\FarmMapsApiClient_WURtest\FarmmapsDataDownload\bin\Debug\netcoreapp3.1\Downloads\ + "cropName": "wheat", + "cropYear": 2019, + "fieldID": 5641, + "lastdownloadedimagedate": "2019-01-01", + "geometryJson": { + "type": "Polygon", + "coordinates": [ + [ + [ 3.37837807779104, 51.3231095796538 ], + [ 3.38065689232502, 51.3212527499355 ], + [ 3.38022924592256, 51.3210683536359 ], + [ 3.37980548452565, 51.3208801127141 ], + [ 3.37959556105776, 51.3207540143696 ], + [ 3.3793691292654, 51.3205959677371 ], + [ 3.37822219207335, 51.3215667913007 ], + [ 3.37816999925795, 51.3216109809456 ], + [ 3.37646704574705, 51.3208025481261 ], + [ 3.37646695791282, 51.3208025061493 ], + [ 3.37608401443192, 51.3206231652693 ], + [ 3.37607169507628, 51.3206173959751 ], + [ 3.37606021048754, 51.320612017601 ], + [ 3.37582728410659, 51.3205029306946 ], + [ 3.37580409779263, 51.3206502985963 ], + [ 3.37575872019649, 51.3207993094705 ], + [ 3.37575476634361, 51.3208122883487 ], + [ 3.37571181656268, 51.3208797459348 ], + [ 3.3756624532907, 51.3209415238446 ], + [ 3.37557609963811, 51.3210110142077 ], + [ 3.37541089899821, 51.3211055871218 ], + [ 3.37477516102591, 51.3214102985009 ], + [ 3.37473173914127, 51.3214311108204 ], + [ 3.37455904622072, 51.3215138815012 ], + [ 3.37415098054777, 51.3217199232877 ], + [ 3.37313700916272, 51.3222422862785 ], + [ 3.37748824689601, 51.3242852920348 ], + [ 3.37749760805371, 51.3242713084009 ], + [ 3.37811903757028, 51.3233437635596 ], + [ 3.37818758851947, 51.3232647797363 ], + [ 3.37823803668144, 51.3232236798646 ], + [ 3.37837807779104, 51.3231095796538 ] + ] + ] + } + }, + { + "UseCreatedCropfield": false, + "fieldName": "fld5641", // FarmMaps won't do without a fieldName + "DownloadFolder": "C:\\workdir\\groenmonitor\\", //"C:\\workdir\\groenmonitor\\"; // "Downloads" -> if you just put "Downloads" the program will download to somewhere in ..\FarmMapsApiClient_WURtest\FarmmapsDataDownload\bin\Debug\netcoreapp3.1\Downloads\ + "cropName": "wheat", + "cropYear": 2018, + "fieldID": 5641, + "lastdownloadedimagedate": "2018-01-01", + "geometryJson": { + "type": "Polygon", + "coordinates": [ + [ + [ 3.37837807779104, 51.3231095796538 ], + [ 3.38065689232502, 51.3212527499355 ], + [ 3.38022924592256, 51.3210683536359 ], + [ 3.37980548452565, 51.3208801127141 ], + [ 3.37959556105776, 51.3207540143696 ], + [ 3.3793691292654, 51.3205959677371 ], + [ 3.37822219207335, 51.3215667913007 ], + [ 3.37816999925795, 51.3216109809456 ], + [ 3.37646704574705, 51.3208025481261 ], + [ 3.37646695791282, 51.3208025061493 ], + [ 3.37608401443192, 51.3206231652693 ], + [ 3.37607169507628, 51.3206173959751 ], + [ 3.37606021048754, 51.320612017601 ], + [ 3.37582728410659, 51.3205029306946 ], + [ 3.37580409779263, 51.3206502985963 ], + [ 3.37575872019649, 51.3207993094705 ], + [ 3.37575476634361, 51.3208122883487 ], + [ 3.37571181656268, 51.3208797459348 ], + [ 3.3756624532907, 51.3209415238446 ], + [ 3.37557609963811, 51.3210110142077 ], + [ 3.37541089899821, 51.3211055871218 ], + [ 3.37477516102591, 51.3214102985009 ], + [ 3.37473173914127, 51.3214311108204 ], + [ 3.37455904622072, 51.3215138815012 ], + [ 3.37415098054777, 51.3217199232877 ], + [ 3.37313700916272, 51.3222422862785 ], + [ 3.37748824689601, 51.3242852920348 ], + [ 3.37749760805371, 51.3242713084009 ], + [ 3.37811903757028, 51.3233437635596 ], + [ 3.37818758851947, 51.3232647797363 ], + [ 3.37823803668144, 51.3232236798646 ], + [ 3.37837807779104, 51.3231095796538 ] + ] + ] + } + }, + { + "UseCreatedCropfield": false, + "fieldName": "fld5641", // FarmMaps won't do without a fieldName + "DownloadFolder": "C:\\workdir\\groenmonitor\\", //"C:\\workdir\\groenmonitor\\"; // "Downloads" -> if you just put "Downloads" the program will download to somewhere in ..\FarmMapsApiClient_WURtest\FarmmapsDataDownload\bin\Debug\netcoreapp3.1\Downloads\ + "cropName": "wheat", + "cropYear": 2017, + "fieldID": 5641, + "lastdownloadedimagedate": "2017-01-01", + "geometryJson": { + "type": "Polygon", + "coordinates": [ + [ + [ 3.37837807779104, 51.3231095796538 ], + [ 3.38065689232502, 51.3212527499355 ], + [ 3.38022924592256, 51.3210683536359 ], + [ 3.37980548452565, 51.3208801127141 ], + [ 3.37959556105776, 51.3207540143696 ], + [ 3.3793691292654, 51.3205959677371 ], + [ 3.37822219207335, 51.3215667913007 ], + [ 3.37816999925795, 51.3216109809456 ], + [ 3.37646704574705, 51.3208025481261 ], + [ 3.37646695791282, 51.3208025061493 ], + [ 3.37608401443192, 51.3206231652693 ], + [ 3.37607169507628, 51.3206173959751 ], + [ 3.37606021048754, 51.320612017601 ], + [ 3.37582728410659, 51.3205029306946 ], + [ 3.37580409779263, 51.3206502985963 ], + [ 3.37575872019649, 51.3207993094705 ], + [ 3.37575476634361, 51.3208122883487 ], + [ 3.37571181656268, 51.3208797459348 ], + [ 3.3756624532907, 51.3209415238446 ], + [ 3.37557609963811, 51.3210110142077 ], + [ 3.37541089899821, 51.3211055871218 ], + [ 3.37477516102591, 51.3214102985009 ], + [ 3.37473173914127, 51.3214311108204 ], + [ 3.37455904622072, 51.3215138815012 ], + [ 3.37415098054777, 51.3217199232877 ], + [ 3.37313700916272, 51.3222422862785 ], + [ 3.37748824689601, 51.3242852920348 ], + [ 3.37749760805371, 51.3242713084009 ], + [ 3.37811903757028, 51.3233437635596 ], + [ 3.37818758851947, 51.3232647797363 ], + [ 3.37823803668144, 51.3232236798646 ], + [ 3.37837807779104, 51.3231095796538 ] + ] + ] + } + }, + { + "UseCreatedCropfield": false, + "fieldName": "fld5641", // FarmMaps won't do without a fieldName + "DownloadFolder": "C:\\workdir\\groenmonitor\\", //"C:\\workdir\\groenmonitor\\"; // "Downloads" -> if you just put "Downloads" the program will download to somewhere in ..\FarmMapsApiClient_WURtest\FarmmapsDataDownload\bin\Debug\netcoreapp3.1\Downloads\ + "cropName": "wheat", + "cropYear": 2016, + "fieldID": 5641, + "lastdownloadedimagedate": "2016-01-01", + "geometryJson": { + "type": "Polygon", + "coordinates": [ + [ + [ 3.37837807779104, 51.3231095796538 ], + [ 3.38065689232502, 51.3212527499355 ], + [ 3.38022924592256, 51.3210683536359 ], + [ 3.37980548452565, 51.3208801127141 ], + [ 3.37959556105776, 51.3207540143696 ], + [ 3.3793691292654, 51.3205959677371 ], + [ 3.37822219207335, 51.3215667913007 ], + [ 3.37816999925795, 51.3216109809456 ], + [ 3.37646704574705, 51.3208025481261 ], + [ 3.37646695791282, 51.3208025061493 ], + [ 3.37608401443192, 51.3206231652693 ], + [ 3.37607169507628, 51.3206173959751 ], + [ 3.37606021048754, 51.320612017601 ], + [ 3.37582728410659, 51.3205029306946 ], + [ 3.37580409779263, 51.3206502985963 ], + [ 3.37575872019649, 51.3207993094705 ], + [ 3.37575476634361, 51.3208122883487 ], + [ 3.37571181656268, 51.3208797459348 ], + [ 3.3756624532907, 51.3209415238446 ], + [ 3.37557609963811, 51.3210110142077 ], + [ 3.37541089899821, 51.3211055871218 ], + [ 3.37477516102591, 51.3214102985009 ], + [ 3.37473173914127, 51.3214311108204 ], + [ 3.37455904622072, 51.3215138815012 ], + [ 3.37415098054777, 51.3217199232877 ], + [ 3.37313700916272, 51.3222422862785 ], + [ 3.37748824689601, 51.3242852920348 ], + [ 3.37749760805371, 51.3242713084009 ], + [ 3.37811903757028, 51.3233437635596 ], + [ 3.37818758851947, 51.3232647797363 ], + [ 3.37823803668144, 51.3232236798646 ], + [ 3.37837807779104, 51.3231095796538 ] + ] + ] + } + } + + +] \ No newline at end of file diff --git a/FarmmapsBulkSatDownload/BulkSatDownloadService.cs b/FarmmapsBulkSatDownload/BulkSatDownloadService.cs new file mode 100644 index 0000000..5be9864 --- /dev/null +++ b/FarmmapsBulkSatDownload/BulkSatDownloadService.cs @@ -0,0 +1,29 @@ +using System; +using System.Globalization; +using System.Threading.Tasks; +using FarmmapsApi.Models; +using FarmmapsApi.Services; +using FarmmapsBulkSatDownload.Models; +using Microsoft.Extensions.Logging; +using static FarmmapsApi.Extensions; +using static FarmmapsApiSamples.Constants; + +namespace FarmmapsBulkSatDownload +{ + public class BulkSatDownloadService + { + private readonly ILogger _logger; + private readonly FarmmapsApiService _farmmapsApiService; + private readonly GeneralService _generalService; + + public BulkSatDownloadService(ILogger logger, FarmmapsApiService farmmapsApiService, + GeneralService generalService) + { + _logger = logger; + _farmmapsApiService = farmmapsApiService; + _generalService = generalService; + } + + + } +} \ No newline at end of file diff --git a/FarmmapsBulkSatDownload/DBsettings.json b/FarmmapsBulkSatDownload/DBsettings.json new file mode 100644 index 0000000..2f84ad7 --- /dev/null +++ b/FarmmapsBulkSatDownload/DBsettings.json @@ -0,0 +1,6 @@ +{ + "User": "", + "Password": "", + "Database": "", + "Host": "" +} \ No newline at end of file diff --git a/FarmmapsBulkSatDownload/FarmmapsBulkSatDownload.csproj b/FarmmapsBulkSatDownload/FarmmapsBulkSatDownload.csproj new file mode 100644 index 0000000..e3efb57 --- /dev/null +++ b/FarmmapsBulkSatDownload/FarmmapsBulkSatDownload.csproj @@ -0,0 +1,31 @@ + + + + Exe + netcoreapp3.1 + + + + + + + + + Always + + + Always + + + Always + + + Always + + + + + + + + diff --git a/FarmmapsBulkSatDownload/Models/BulkSatDownloadInput.cs b/FarmmapsBulkSatDownload/Models/BulkSatDownloadInput.cs new file mode 100644 index 0000000..f89833f --- /dev/null +++ b/FarmmapsBulkSatDownload/Models/BulkSatDownloadInput.cs @@ -0,0 +1,18 @@ +using System; +using Newtonsoft.Json.Linq; + +namespace FarmmapsBulkSatDownload.Models +{ + public class BulkSatDownloadInput + { + public bool UseCreatedCropfield { get; set; } + public string fieldName { get; set; } + public string DownloadFolder { get; set; } + public string cropName { get; set; } + public int cropYear { get; set; } + public int fieldID { get; set; } + public DateTime lastdownloadedimagedate { get; set; } + public JObject GeometryJson { get; set; } + //public string fieldName { get { return string.Format($"{cropName}_fld{fieldID}"); } } + } +} \ No newline at end of file diff --git a/FarmmapsBulkSatDownload/Models/DB.cs b/FarmmapsBulkSatDownload/Models/DB.cs new file mode 100644 index 0000000..2ff705f --- /dev/null +++ b/FarmmapsBulkSatDownload/Models/DB.cs @@ -0,0 +1,47 @@ +using System; +using System.Collections.Generic; +using System.Text; +using Npgsql; + +namespace FarmmapsApi.Services +{ + public class DB + { + public string Database; + public string User; + public string Password; + public string Host; + + public string GetConnectionString() + { + NpgsqlConnectionStringBuilder sb = new NpgsqlConnectionStringBuilder(); + sb.Database = Database; + sb.Host = Host; + sb.Username = User; + sb.Password = Password; + sb.Port = 5432; + + return sb.ConnectionString; + } + + public int ExecuteNonQuery(string sql) + { + using (NpgsqlConnection conn = new NpgsqlConnection(GetConnectionString())) + { + conn.Open(); + NpgsqlCommand command = conn.CreateCommand(); + command.CommandText = sql; + int r = command.ExecuteNonQuery(); + return r; + } + } + + public NpgsqlConnection CreateConnection() + { + NpgsqlConnection conn = new NpgsqlConnection(GetConnectionString()); + conn.Open(); + return conn; + } + + } +} diff --git a/FarmmapsBulkSatDownload/Models/GroenmonitorTable.cs b/FarmmapsBulkSatDownload/Models/GroenmonitorTable.cs new file mode 100644 index 0000000..8b48028 --- /dev/null +++ b/FarmmapsBulkSatDownload/Models/GroenmonitorTable.cs @@ -0,0 +1,28 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace FarmmapsBulkSatDownload.Models +{ + public class GroenmonitorTable + { + //public string parceltablename; //PO20190605: parceltablename added as a field (=column) in the GroenmonitorTable. The GroenmonitorTable contains data from multiple parceltables + public int parcelid; + public string date; + public string source; + public int wdvi_pixelcount; //count of pixels with data + public double wdvi_max; + public double wdvi_mean; + public double wdvi_min; + public double wdvi_stdev; + public double wdvi_median; + public double wdvi_p90; + public int ndvi_pixelcount; //count of pixels with data + public double ndvi_max; + public double ndvi_mean; + public double ndvi_min; + public double ndvi_stdev; + public double ndvi_median; + public double ndvi_p90; + } +} diff --git a/FarmmapsBulkSatDownload/Models/Settings.cs b/FarmmapsBulkSatDownload/Models/Settings.cs new file mode 100644 index 0000000..2d70cba --- /dev/null +++ b/FarmmapsBulkSatDownload/Models/Settings.cs @@ -0,0 +1,11 @@ +namespace FarmmapsBulkSatDownload +{ + public class Settings + { + public string CropfieldItemCode { get; set; } + public string SatelliteTaskCode { get; set; } + public string VanDerSatTaskCode { get; set; } + public string WatBalTaskCode { get; set; } + + } +} \ No newline at end of file diff --git a/FarmmapsBulkSatDownload/Program.cs b/FarmmapsBulkSatDownload/Program.cs new file mode 100644 index 0000000..b9ca73d --- /dev/null +++ b/FarmmapsBulkSatDownload/Program.cs @@ -0,0 +1,21 @@ +using System.Threading.Tasks; +using FarmmapsApi; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; + +namespace FarmmapsBulkSatDownload +{ + class Program : FarmmapsProgram + { + private static async Task Main(string[] args) + { + await new Program().Start(args); + } + + protected override void Configure(IServiceCollection serviceCollection) + { + serviceCollection.AddLogging() + .AddTransient(); + } + } +} \ No newline at end of file diff --git a/FarmmapsBulkSatDownload/ShowGeotiff.r b/FarmmapsBulkSatDownload/ShowGeotiff.r new file mode 100644 index 0000000..68e010c --- /dev/null +++ b/FarmmapsBulkSatDownload/ShowGeotiff.r @@ -0,0 +1,152 @@ +# ShowGeotiff.r +# I downloaded and calculated the stats for the polygon defined in C:\git\FarmMapsApiClient_WURtest\FarmmapsDataDownload\DataDownloadInput.json +# in which I set "SatelliteBand": "wdvi" and in which in the console I requested image 8 for date '2020-06-14' + +library(raster) +library(sf) +library(rgdal) +setwd("C:/workdir/groenmonitor/") +# FarmmapsDataDownload generates two files: +fileGeotiff <- "wenr.tif" +# fileJpg <- "thumbnail.jpg" +# FarmmapsBulkSatDownload generates many files. Here is what I tried when inputing the same field for a number of years using BulkSatDownloadInput.json +# fileGeotiff <- "wheat_fld5641_20210224.tif" # 2 files for year 2021. This file is a nice example having in upperleft corner no data +# fileGeotiff <- "wheat_fld5641_20210331.tif" # 2 files for year 2021 +# fileGeotiff <- "wheat_fld5641_20200321.tif" # 14 files for year 2020, earliest +# fileGeotiff <- "wheat_fld5641_20200922.tif" # 14 files for year 2020, latest +# fileGeotiff <- "wheat_fld5641_20190121.tif" # 9 files for year 2019, earliest +# fileGeotiff <- "wheat_fld5641_20191117.tif" # 9 files for year 2019, latest +# 1 file for year 2018, with error message 'End of Central Directory record could not be found' and invalid wheat_fld5641_20180630.zip +# fileGeotiff <- "wheat_fld5641_20170526.tif" # 1 file for year 2017 +# Zero files for 2016 +lenfilename <- nchar(fileGeotiff) +year <- substr(fileGeotiff,lenfilename-11,lenfilename-8) +imgdate <- substr(fileGeotiff,lenfilename-11,lenfilename-4) + +# The thumbnail has the polygon clipped out, has 1 layer, no crs and the mean value is not the mean wdvi we are looking for +# r.thumbnail <- raster(fileJpg) +# plot(r.thumbnail) +# crs(r.thumbnail) +# CRS arguments: NA +# cellStats(r.thumbnail,'mean') #59.91667 + +stk.wenr <- stack(x=fileGeotiff) +# plot(stk.wenr) shows 5 plots (5 bands) +# I think these are: +# 1. ndvi (since it runs from 0 to 1) +# 2. wdvi (since it runs from 0 to 0.5) +# 3-5: RGB (since they run from 0 to 255) +plot(stk.wenr) +# CRS arguments: +# +proj=sterea +lat_0=52.1561605555556 +lon_0=5.38763888888889 +k=0.9999079 +x_0=155000 +y_0=463000 +ellps=bessel +units=m +no_defs +# Or use st_crs(stk.wenr) to get more info, but no EPSG code in there. +# Likely it is epsg:28992 (Amersfoort) +crs(stk.wenr) +stk.wenr <- projectRaster(stk.wenr, crs = CRS('+init=EPSG:28992')) +crs(stk.wenr) +# Looks the same but strangely, if we don't do projectRaster(stk.wenr, crs = CRS('+init=EPSG:28992')), we find below the bottom left corner of the polygon missing + +r.wenr.rd.wdvi <- subset(stk.wenr,2) +dev.off() +plot(r.wenr.rd.wdvi,main=paste("wdvi",imgdate),xlab="RDX",ylab="RDY") +cellStats(r.wenr.rd.wdvi,'mean') #0.1654627 +# Same as in downloaded file SatelliteDataStatistics_test_satData_wdvi_2020-06-14.csv +# "mean": 0.16564258790046743 +# So FindSatelliteItem in C:\git\FarmMapsApiClient_WURtest\FarmmapsApi\Services\GeneralService.cs returns a a stacked geotiff in +# i.e. the cellstats are calculated from the rectangle + +# Furthermore we can see +# shows coordinates in RD +# returns a rectangle, thus the shape of the polygon submitted is not clipped. +# The polygon was provided in WGS84. Let's draw it on top. +# First convert the raster to WGS84 +r.wenr.wgs84.wdvi <- projectRaster(r.wenr.rd.wdvi, crs = CRS('+init=EPSG:4326')) + +# Draw a polygon on top of the raster +# Example polygon p1 +# p1 <- data.frame(id = 1, wkt = 'POLYGON((4.963 52.801, 4.966 52.801, 4.966 52.803, 4.963 52.803, 4.963 52.801))') +# p1 <- st_as_sf(p1, wkt = 'wkt', crs = targetcrs) +# plot(p1,add=TRUE, col="transparent",border="black") +# Draw the polygon on top of the raster +# Polygon p2 from C:\git\FarmMapsApiClient_WURtest\FarmmapsDataDownload\DataDownloadInput.json +p2 <- data.frame(id = 1, wkt = gsub("\n","",'POLYGON(( + 3.37837807779104 51.3231095796538, + 3.38065689232502 51.3212527499355, + 3.38022924592256 51.3210683536359, + 3.37980548452565 51.3208801127141, + 3.37959556105776 51.3207540143696, + 3.3793691292654 51.3205959677371, + 3.37822219207335 51.3215667913007, + 3.37816999925795 51.3216109809456, + 3.37646704574705 51.3208025481261, + 3.37646695791282 51.3208025061493, + 3.37608401443192 51.3206231652693, + 3.37607169507628 51.3206173959751, + 3.37606021048754 51.320612017601, + 3.37582728410659 51.3205029306946, + 3.37580409779263 51.3206502985963, + 3.37575872019649 51.3207993094705, + 3.37575476634361 51.3208122883487, + 3.37571181656268 51.3208797459348, + 3.3756624532907 51.3209415238446, + 3.37557609963811 51.3210110142077, + 3.37541089899821 51.3211055871218, + 3.37477516102591 51.3214102985009, + 3.37473173914127 51.3214311108204, + 3.37455904622072 51.3215138815012, + 3.37415098054777 51.3217199232877, + 3.37313700916272 51.3222422862785, + 3.37748824689601 51.3242852920348, + 3.37749760805371 51.3242713084009, + 3.37811903757028 51.3233437635596, + 3.37818758851947 51.3232647797363, + 3.37823803668144 51.3232236798646, + 3.37837807779104 51.3231095796538))')) +p2.wgs84 <- st_as_sf(p2, wkt = 'wkt', crs = CRS('+init=EPSG:4326')) +# Or other way round, in RD Amersfoort. That looks ok +p2.rd <- st_transform(p2.wgs84, "+init=epsg:28992") + +# Have a look at both +# wg84 +dev.off() +plot(r.wenr.wgs84.wdvi,main=paste("wdvi",imgdate),xlab="LON",ylab="LAT") +plot(p2.wgs84,add=TRUE, col="transparent",border="red") +# RD +dev.off() +plot(r.wenr.rd.wdvi,main=paste("wdvi",imgdate),xlab="RDX",ylab="RDY") +plot(p2.rd,add=TRUE, col="transparent",border="red") + +#Let's clip the polygon +r.wenr.rd.wdvi.pol <- mask(r.wenr.rd.wdvi,p2.rd) +r.wenr.wgs84.wdvi.pol <- mask(r.wenr.wgs84.wdvi,p2.wgs84) +dev.off() +plot(r.wenr.wgs84.wdvi.pol,main=paste("wdvi",imgdate),xlab="LON",ylab="LAT") +plot(p2.wgs84,add=TRUE, col="transparent",border="red") +#That's what we want! Now compare the stats +cellStats(r.wenr.rd.wdvi,'mean') #0.1654627 # Stats from rectangle, RD +cellStats(r.wenr.wgs84.wdvi,'mean') #0.1656399 # Stats from rectangle, WGS84 +cellStats(r.wenr.rd.wdvi.pol,'mean') #0.1658702 # Stats from raster clipped by polygon, WGS84 +cellStats(r.wenr.wgs84.wdvi.pol,'mean') #0.1658611 # Stats from raster clipped by polygon, WGS84 +# Conclusion: in this example little difference, but can be quite different! + +# The project FarmmapsNbs can generate a wenr.tif file, application.tif, uptake.tif (in rtest1.uptake.zip)and shape.shp (in rtest1.taskmap.zip) +r.application <- raster("C:/git/FarmMapsApiClient_WURtest/FarmmapsNbs/bin/Debug/netcoreapp3.1/Downloads/application.tif") +dev.off() +plot(r.application) +plot(p2.rd,add=TRUE, col="transparent",border="red") +# The application.tif file is a rectangle (polygon not yet clipped), in projection Amersfoort RD New (EPSG:28992) + +r.uptake <- raster("C:/git/FarmMapsApiClient_WURtest/FarmmapsNbs/bin/Debug/netcoreapp3.1/Downloads/uptake.tif") +dev.off() +plot(r.uptake) +plot(p2.rd,add=TRUE, col="transparent",border="red") +# The uptake.tif file is a rectangle (polygon not yet clipped), in projection Amersfoort RD New (EPSG:28992) + +shp.wgs84 <- readOGR(dsn="C:/git/FarmMapsApiClient_WURtest/FarmmapsNbs/bin/Debug/netcoreapp3.1/Downloads", layer="shape") +crs(shp.wgs84) +# CRS arguments: +proj=longlat +datum=WGS84 +no_defs +dev.off() +plot(r.wenr.wgs84.wdvi,main="wdvi",xlab="LON",ylab="LAT") +plot(shp.wgs84,add=TRUE, col="transparent",border="black") +plot(p2.wgs84,add=TRUE, col="transparent",border="red") +# The shape file is in WGS84 diff --git a/FarmmapsBulkSatDownload/appsettings.json b/FarmmapsBulkSatDownload/appsettings.json new file mode 100644 index 0000000..c5d440a --- /dev/null +++ b/FarmmapsBulkSatDownload/appsettings.json @@ -0,0 +1,10 @@ +{ + "Authority": "https://accounts.test.farmmaps.eu/", + "Endpoint": "https://test.farmmaps.eu/", + "BasePath": "api/v1", + "DiscoveryEndpointUrl": "https://accounts.test.farmmaps.eu/.well-known/openid-configuration", + "RedirectUri": "http://example.nl/api", + "ClientId": "", + "ClientSecret": "", + "Scopes": [ "api" ] +} \ No newline at end of file