FarmMaps/FarmMapsApiClient
11
0
Fork 3
Code Issues Pull Requests Releases Activity

Added project FarmmapsBulkSatDownload

Browse Source
This commit is contained in:
pepijn van oort 2021-05-18 18:19:28 +02:00
parent 245e82adbc
commit 250020be78
14 changed files with 918 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
FarmmapsApi/Services/GeneralService.cs
View File

@ -388,6 +388,29 @@ namespace FarmmapsApi.Services
return selectedSatelliteItem;
}
public async Task<List<Item>> FindSatelliteItemsAll(Item cropfieldItem, string satelliteTaskCode)
{
var taskStatus = await _farmmapsApiService.GetTaskStatusAsync(cropfieldItem.Code, satelliteTaskCode);
// find ndvi or wdvi satellite data geotiffs
var temporalItem = await FindChildItemAsync(cropfieldItem.Code, TEMPORAL_ITEMTYPE,
"Cropfield Satellite items", item => item.SourceTask == SATELLITE_TASK &&
taskStatus.Finished >= item.Created &&
taskStatus.Finished <= item.Created.Value.AddHours(1));
if (temporalItem == null)
{
_logger.LogError("Temporal item not found");
}
var satelliteTiffs = await _farmmapsApiService.GetItemChildrenAsync(temporalItem.Code);
return satelliteTiffs;
}
//VanDerSat
public async Task<string> RunVanDerSatTask(Item cropfieldItem) {

8
FarmmapsApiSamples.sln
View File

@ -23,7 +23,9 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FarmMapsBlight", "FarmMapsB
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FarmmapsZonering", "FarmmapsZonering\FarmmapsZonering.csproj", "{91A58C4A-4A80-4079-B43D-9B851206194F}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "FarmmapsDataDownload", "FarmmapsDataDownload\FarmmapsDataDownload.csproj", "{32ED9500-AAAB-4030-9C7A-F611A85DF890}"
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FarmmapsDataDownload", "FarmmapsDataDownload\FarmmapsDataDownload.csproj", "{32ED9500-AAAB-4030-9C7A-F611A85DF890}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FarmmapsBulkSatDownload", "FarmmapsBulkSatDownload\FarmmapsBulkSatDownload.csproj", "{772DBDCD-9FAA-40A7-8551-2C1620C4AB67}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
@ -63,6 +65,10 @@ Global
{32ED9500-AAAB-4030-9C7A-F611A85DF890}.Debug|Any CPU.Build.0 = Debug|Any CPU
{32ED9500-AAAB-4030-9C7A-F611A85DF890}.Release|Any CPU.ActiveCfg = Release|Any CPU
{32ED9500-AAAB-4030-9C7A-F611A85DF890}.Release|Any CPU.Build.0 = Release|Any CPU
{772DBDCD-9FAA-40A7-8551-2C1620C4AB67}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{772DBDCD-9FAA-40A7-8551-2C1620C4AB67}.Debug|Any CPU.Build.0 = Debug|Any CPU
{772DBDCD-9FAA-40A7-8551-2C1620C4AB67}.Release|Any CPU.ActiveCfg = Release|Any CPU
{772DBDCD-9FAA-40A7-8551-2C1620C4AB67}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE

242
FarmmapsBulkSatDownload/BulkSatDownloadApplication.cs Normal file
View File

@ -0,0 +1,242 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Threading.Tasks;
using FarmmapsApi;
using FarmmapsApi.Models;
using FarmmapsApi.Services;
using FarmmapsBulkSatDownload.Models;
using Microsoft.Extensions.Logging;
using Newtonsoft.Json;
using Npgsql;
using Newtonsoft.Json.Linq;
using static FarmmapsApiSamples.Constants;
namespace FarmmapsBulkSatDownload
{
public class BulkSatDownloadApplication : IApplication
{
private const string SettingsFile = "settings.json";
private readonly ILogger<BulkSatDownloadApplication> _logger;
private readonly FarmmapsApiService _farmmapsApiService;
private readonly BulkSatDownloadService _dataDownloadService;
private readonly GeneralService _generalService;
private Settings _settings;
static DB dbparcels;
public BulkSatDownloadApplication(ILogger<BulkSatDownloadApplication> logger, FarmmapsApiService farmmapsApiService,
GeneralService generalService, BulkSatDownloadService dataDownloadService)
{
_logger = logger;
_farmmapsApiService = farmmapsApiService;
_generalService = generalService;
_dataDownloadService = dataDownloadService;
}
public async Task RunAsync()
{
// Check if we have permission
// !! this call is needed the first time an api is called with a fresh clientid and secret !!
await _farmmapsApiService.GetCurrentUserCodeAsync();
var roots = await _farmmapsApiService.GetCurrentUserRootsAsync();
// Initialize databases. Username, password etc stored in file "DBsettings.json"
dbparcels = JsonConvert.DeserializeObject<DB>(File.ReadAllText("DBsettings.json"));
//string date;
//string asterices = "*****************************************************************************";
//string bboxtiffWithTempPath;
//string parcelshp = "parcel.shp";
//string parcelshpWithTempPath;
//string gwoptions; //= String.Format("-overwrite -s_srs EPSG:28992 -t_srs EPSG:28992 -dstnodata 0 -cutline {0} -crop_to_cutline", parcelshp);
//string parceltiff = "parcel.tiff";
//string parceltiffWithTempPath;
//string[] vegetationindices = { "WDVI", "NDVI" };
//string[] sources = { "groenmonitor" }; // { "groenmonitor", "akkerwebwenr", "akkerwebneo" }; { "groenmonitor" };
string schemaname = "bigdata";
string parceltablename = "parcel_flowerbulbs";//"parcelsijbrandij" "parcel"; "parcel_flowerbulbs"
//string groenmonitortablename = "groenmonitor_flowerbulbs";//"groenmonitorsijbrandij" "groenmonitor" "groenmonitor_flowerbulbs" //PO20190605: "groenmonitormpt" contains groenmonitor data from multiple parcel tables (mpt)
// The view 'groenmonitorlatestviewname' contains per parcelid (arbid) the year in which it "exists" and the date of the latest image downloaded. It is used to prevent downloading images already downloaded
string groenmonitorlatestviewname = "groenmonitorlatest_flowerbulbs"; //"groenmonitorsijbrandijlatest" "groenmonitorlatest" "groenmonitorlatest_flowerbulbs" //PO20190605: "v_groenmonitorlatest" contains latest available dates from groenmonitortablename = "groenmonitormpt"
//GroenmonitorTable gmtb;
//DateTime dt;
//DateTime dtfirst;
//DateTime dtlast;
//double scalingfactor;
BulkSatDownloadInput dataDownloadInput;
List<BulkSatDownloadInput> fieldsInputs = new List<BulkSatDownloadInput>();
// Database query and connection. Geometry must be in WGS84 coordinate system, EPSG 4326
// Apparently the FarmmapsApi cannot handle MultiPolygon, so we need to convert to single Polygon
// In case database returns a MultiPolygon use ST_NumGeometries(pt.geom) to count the number of polygons
// If necessary use WHERE T_NumGeometries(pt.geom) = 1 to use only single polygons
string connectionString = dbparcels.GetConnectionString();
string readSql = string.Format(
@"
SELECT pt.arbid, pt.crop, pt.year, gml.lastwenrdate, ST_AsGeoJSON(ST_Transform((ST_DUMP(pt.geom)).geom::geometry(Polygon),4326)) AS geojson_polygon_wgs84
FROM {0}.{1} pt, {0}.{2} gml
WHERE
pt.arbid = gml.arbid AND
pt.crop NOT IN('Tulp', 'Lelie') AND
pt.year = 2018
LIMIT 10;", schemaname, parceltablename, groenmonitorlatestviewname); //LIMIT 10 for testing
using (NpgsqlConnection connection = new NpgsqlConnection(connectionString))
{
connection.Open();
// Read data (run query) = build a list of fields for which to download images
NpgsqlCommand command = connection.CreateCommand();
command.CommandText = readSql;
NpgsqlDataReader dr = command.ExecuteReader();
while (dr.Read())
{
dataDownloadInput = new BulkSatDownloadInput();
dataDownloadInput.UseCreatedCropfield = false;
dataDownloadInput.DownloadFolder = "C:\\workdir\\groenmonitor\\";
dataDownloadInput.fieldID = dr.GetInt16(0);
dataDownloadInput.fieldName = string.Format($"{parceltablename}_fld{dataDownloadInput.fieldID}");
dataDownloadInput.cropName = dr.GetString(1);
dataDownloadInput.cropYear = dr.GetInt16(2);
dataDownloadInput.lastdownloadedimagedate = dr.GetDateTime(3);
dataDownloadInput.GeometryJson = JObject.Parse(dr.GetString(4));
fieldsInputs.Add(dataDownloadInput);
}
connection.Close();
}
// For testing using json input instead of database input
var fieldsInputJson = File.ReadAllText("BulkSatDownloadInput.json");
List<BulkSatDownloadInput> fieldsInputs2 = JsonConvert.DeserializeObject<List<BulkSatDownloadInput>>(fieldsInputJson);
// Now for each input download all images. Use fieldsInputs or fieldsInputs2 for testing
foreach (var input in fieldsInputs2)
{
try
{
await Process(roots, input);
}
catch (Exception ex)
{
_logger.LogError(ex.Message);
}
}
}
private async Task Process(List<UserRoot> roots, BulkSatDownloadInput input)
{
string DownloadFolder = input.DownloadFolder;
if (!Directory.Exists(DownloadFolder))
Directory.CreateDirectory(DownloadFolder);
// !!specify if you are using an already created cropfield:
bool useCreatedCropfield = input.UseCreatedCropfield;
string settingsfile = $"Settings_BulkSatDownloadApplication.json";
LoadSettings(settingsfile);
var uploadedRoot = roots.SingleOrDefault(r => r.Name == "Uploaded");
if (uploadedRoot == null)
{
_logger.LogError("Could not find a needed root item");
return;
}
var myDriveRoot = roots.SingleOrDefault(r => r.Name == "My drive");
if (myDriveRoot == null)
{
_logger.LogError("Could not find a needed root item");
return;
}
// Use already created cropfield or create new one
Item cropfieldItem;
if (useCreatedCropfield == false || string.IsNullOrEmpty(_settings.CropfieldItemCode))
{
_logger.LogInformation(string.Format($"Creating cropfield for a field in the year {input.cropYear}"));
//string geomjson = input.GeometryJson.ToString(Formatting.None); //"{\"type\":\"Polygon\",\"coordinates\":[[[4.960707146896585,52.80058366970849],[4.960645975538824,52.80047021761092],[4.962140695752897,52.7991771471948],[4.967523821195745,52.80150240004121],[4.966336768950911,52.80254373587981],[4.96171188076433,52.80100999685643],[4.960707146896585,52.80058366970849]]]}"
string geomjson = input.GeometryJson.ToString(Formatting.None);
cropfieldItem = await _generalService.CreateCropfieldItemAsync(myDriveRoot.Code,
$"DataCropfield {input.cropName}", input.cropYear, input.GeometryJson.ToString(Formatting.None));
_settings.CropfieldItemCode = cropfieldItem.Code;
SaveSettings(settingsfile);
}
else
{
_logger.LogInformation("Cropfield already exists, trying to get it");
cropfieldItem = await _farmmapsApiService.GetItemAsync(_settings.CropfieldItemCode);
}
// check if satellite task not yet done, do here and save taskcode
if (useCreatedCropfield == false || string.IsNullOrEmpty(_settings.SatelliteTaskCode))
{
var satelliteTaskCode = await _generalService.RunSatelliteTask(cropfieldItem);
_settings.SatelliteTaskCode = satelliteTaskCode;
SaveSettings(settingsfile);
}
// Now get the tiffs
List<Item> satelliteTiffs = await _generalService.FindSatelliteItemsAll(cropfieldItem, _settings.SatelliteTaskCode);
_logger.LogInformation(string.Format($"Downloading {satelliteTiffs.Count} geotiffs for a field in year {input.cropYear} ..."));
foreach (Item satalliteItem in satelliteTiffs) {
// download the geotiff. Returns a zip file with always these three files:
// data.dat.aux.xml
// thumbnail.jpg
// wenr.tif. Contains 5 layers: (1) ndvi, (2) wdvi, (3) Red, (4) Green and (5) Blue
DateTime SatelliteImageDate = (DateTime)satalliteItem.DataDate;
string SatelliteDate = SatelliteImageDate.ToString("yyyyMMdd");
string fileName = string.Format($"{input.cropName}_{input.fieldName}_{SatelliteDate}"); // assuming the fieldName is good enough for an ID. One might add here _{input.fieldID} or _{input.cropName}
string fileNameZip = string.Format($"{fileName}.zip");
string fileNameGeotiff = string.Format($"{fileName}.tif");
await _farmmapsApiService.DownloadItemAsync(satalliteItem.Code, Path.Combine(DownloadFolder, fileNameZip));
// Extract the file "wenr.tif" from zip, rename it, delete "wenr.tif". overwriteFiles = true
ZipFile.ExtractToDirectory(Path.Combine(DownloadFolder, fileNameZip), DownloadFolder, true);
File.Delete(Path.Combine(DownloadFolder, fileNameGeotiff)); // Delete the fileNameGeotiff file if exists
File.Move(Path.Combine(DownloadFolder, "wenr.tif"), Path.Combine(DownloadFolder, fileNameGeotiff)); // Rename the oldFileName into newFileName
// Cleanup
string[] filesToDelete = new string[] { fileNameZip, "wenr.tif", "thumbnail.jpg", "data.dat.aux.xml" };
foreach (string f in filesToDelete)
{
File.Delete(Path.Combine(DownloadFolder, f));
}
}
}
// Functions to save previously created cropfields
private void LoadSettings(string file)
{
if (File.Exists(file))
{
var jsonText = File.ReadAllText(file);
_settings = JsonConvert.DeserializeObject<Settings>(jsonText);
}
else
{
_settings = new Settings();
}
}
private void SaveSettings(string file)
{
if (_settings == null)
return;
var json = JsonConvert.SerializeObject(_settings);
File.WriteAllText(file, json);
}
private void SaveInfo(string file)
{
if (_settings == null)
return;
var json = JsonConvert.SerializeObject(_settings);
File.WriteAllText(file, json);
}
}
}

293
FarmmapsBulkSatDownload/BulkSatDownloadInput.json Normal file
View File

@ -0,0 +1,293 @@
[
{
"UseCreatedCropfield": false,
"fieldName": "fld5641", // FarmMaps won't do without a fieldName
"DownloadFolder": "C:\\workdir\\groenmonitor\\", //"C:\\workdir\\groenmonitor\\"; // "Downloads" -> if you just put "Downloads" the program will download to somewhere in ..\FarmMapsApiClient_WURtest\FarmmapsDataDownload\bin\Debug\netcoreapp3.1\Downloads\
"cropName": "wheat",
"cropYear": 2021,
"fieldID": 5641,
"lastdownloadedimagedate": "2021-01-01",
"geometryJson": {
"type": "Polygon",
"coordinates": [
[
[ 3.37837807779104, 51.3231095796538 ],
[ 3.38065689232502, 51.3212527499355 ],
[ 3.38022924592256, 51.3210683536359 ],
[ 3.37980548452565, 51.3208801127141 ],
[ 3.37959556105776, 51.3207540143696 ],
[ 3.3793691292654, 51.3205959677371 ],
[ 3.37822219207335, 51.3215667913007 ],
[ 3.37816999925795, 51.3216109809456 ],
[ 3.37646704574705, 51.3208025481261 ],
[ 3.37646695791282, 51.3208025061493 ],
[ 3.37608401443192, 51.3206231652693 ],
[ 3.37607169507628, 51.3206173959751 ],
[ 3.37606021048754, 51.320612017601 ],
[ 3.37582728410659, 51.3205029306946 ],
[ 3.37580409779263, 51.3206502985963 ],
[ 3.37575872019649, 51.3207993094705 ],
[ 3.37575476634361, 51.3208122883487 ],
[ 3.37571181656268, 51.3208797459348 ],
[ 3.3756624532907, 51.3209415238446 ],
[ 3.37557609963811, 51.3210110142077 ],
[ 3.37541089899821, 51.3211055871218 ],
[ 3.37477516102591, 51.3214102985009 ],
[ 3.37473173914127, 51.3214311108204 ],
[ 3.37455904622072, 51.3215138815012 ],
[ 3.37415098054777, 51.3217199232877 ],
[ 3.37313700916272, 51.3222422862785 ],
[ 3.37748824689601, 51.3242852920348 ],
[ 3.37749760805371, 51.3242713084009 ],
[ 3.37811903757028, 51.3233437635596 ],
[ 3.37818758851947, 51.3232647797363 ],
[ 3.37823803668144, 51.3232236798646 ],
[ 3.37837807779104, 51.3231095796538 ]
]
]
}
},
{
"UseCreatedCropfield": false,
"fieldName": "fld5641", // FarmMaps won't do without a fieldName
"DownloadFolder": "C:\\workdir\\groenmonitor\\", //"C:\\workdir\\groenmonitor\\"; // "Downloads" -> if you just put "Downloads" the program will download to somewhere in ..\FarmMapsApiClient_WURtest\FarmmapsDataDownload\bin\Debug\netcoreapp3.1\Downloads\
"cropName": "wheat",
"cropYear": 2020,
"fieldID": 5641,
"lastdownloadedimagedate": "2020-01-01",
"geometryJson": {
"type": "Polygon",
"coordinates": [
[
[ 3.37837807779104, 51.3231095796538 ],
[ 3.38065689232502, 51.3212527499355 ],
[ 3.38022924592256, 51.3210683536359 ],
[ 3.37980548452565, 51.3208801127141 ],
[ 3.37959556105776, 51.3207540143696 ],
[ 3.3793691292654, 51.3205959677371 ],
[ 3.37822219207335, 51.3215667913007 ],
[ 3.37816999925795, 51.3216109809456 ],
[ 3.37646704574705, 51.3208025481261 ],
[ 3.37646695791282, 51.3208025061493 ],
[ 3.37608401443192, 51.3206231652693 ],
[ 3.37607169507628, 51.3206173959751 ],
[ 3.37606021048754, 51.320612017601 ],
[ 3.37582728410659, 51.3205029306946 ],
[ 3.37580409779263, 51.3206502985963 ],
[ 3.37575872019649, 51.3207993094705 ],
[ 3.37575476634361, 51.3208122883487 ],
[ 3.37571181656268, 51.3208797459348 ],
[ 3.3756624532907, 51.3209415238446 ],
[ 3.37557609963811, 51.3210110142077 ],
[ 3.37541089899821, 51.3211055871218 ],
[ 3.37477516102591, 51.3214102985009 ],
[ 3.37473173914127, 51.3214311108204 ],
[ 3.37455904622072, 51.3215138815012 ],
[ 3.37415098054777, 51.3217199232877 ],
[ 3.37313700916272, 51.3222422862785 ],
[ 3.37748824689601, 51.3242852920348 ],
[ 3.37749760805371, 51.3242713084009 ],
[ 3.37811903757028, 51.3233437635596 ],
[ 3.37818758851947, 51.3232647797363 ],
[ 3.37823803668144, 51.3232236798646 ],
[ 3.37837807779104, 51.3231095796538 ]
]
]
}
},
{
"UseCreatedCropfield": false,
"fieldName": "fld5641", // FarmMaps won't do without a fieldName
"DownloadFolder": "C:\\workdir\\groenmonitor\\", //"C:\\workdir\\groenmonitor\\"; // "Downloads" -> if you just put "Downloads" the program will download to somewhere in ..\FarmMapsApiClient_WURtest\FarmmapsDataDownload\bin\Debug\netcoreapp3.1\Downloads\
"cropName": "wheat",
"cropYear": 2019,
"fieldID": 5641,
"lastdownloadedimagedate": "2019-01-01",
"geometryJson": {
"type": "Polygon",
"coordinates": [
[
[ 3.37837807779104, 51.3231095796538 ],
[ 3.38065689232502, 51.3212527499355 ],
[ 3.38022924592256, 51.3210683536359 ],
[ 3.37980548452565, 51.3208801127141 ],
[ 3.37959556105776, 51.3207540143696 ],
[ 3.3793691292654, 51.3205959677371 ],
[ 3.37822219207335, 51.3215667913007 ],
[ 3.37816999925795, 51.3216109809456 ],
[ 3.37646704574705, 51.3208025481261 ],
[ 3.37646695791282, 51.3208025061493 ],
[ 3.37608401443192, 51.3206231652693 ],
[ 3.37607169507628, 51.3206173959751 ],
[ 3.37606021048754, 51.320612017601 ],
[ 3.37582728410659, 51.3205029306946 ],
[ 3.37580409779263, 51.3206502985963 ],
[ 3.37575872019649, 51.3207993094705 ],
[ 3.37575476634361, 51.3208122883487 ],
[ 3.37571181656268, 51.3208797459348 ],
[ 3.3756624532907, 51.3209415238446 ],
[ 3.37557609963811, 51.3210110142077 ],
[ 3.37541089899821, 51.3211055871218 ],
[ 3.37477516102591, 51.3214102985009 ],
[ 3.37473173914127, 51.3214311108204 ],
[ 3.37455904622072, 51.3215138815012 ],
[ 3.37415098054777, 51.3217199232877 ],
[ 3.37313700916272, 51.3222422862785 ],
[ 3.37748824689601, 51.3242852920348 ],
[ 3.37749760805371, 51.3242713084009 ],
[ 3.37811903757028, 51.3233437635596 ],
[ 3.37818758851947, 51.3232647797363 ],
[ 3.37823803668144, 51.3232236798646 ],
[ 3.37837807779104, 51.3231095796538 ]
]
]
}
},
{
"UseCreatedCropfield": false,
"fieldName": "fld5641", // FarmMaps won't do without a fieldName
"DownloadFolder": "C:\\workdir\\groenmonitor\\", //"C:\\workdir\\groenmonitor\\"; // "Downloads" -> if you just put "Downloads" the program will download to somewhere in ..\FarmMapsApiClient_WURtest\FarmmapsDataDownload\bin\Debug\netcoreapp3.1\Downloads\
"cropName": "wheat",
"cropYear": 2018,
"fieldID": 5641,
"lastdownloadedimagedate": "2018-01-01",
"geometryJson": {
"type": "Polygon",
"coordinates": [
[
[ 3.37837807779104, 51.3231095796538 ],
[ 3.38065689232502, 51.3212527499355 ],
[ 3.38022924592256, 51.3210683536359 ],
[ 3.37980548452565, 51.3208801127141 ],
[ 3.37959556105776, 51.3207540143696 ],
[ 3.3793691292654, 51.3205959677371 ],
[ 3.37822219207335, 51.3215667913007 ],
[ 3.37816999925795, 51.3216109809456 ],
[ 3.37646704574705, 51.3208025481261 ],
[ 3.37646695791282, 51.3208025061493 ],
[ 3.37608401443192, 51.3206231652693 ],
[ 3.37607169507628, 51.3206173959751 ],
[ 3.37606021048754, 51.320612017601 ],
[ 3.37582728410659, 51.3205029306946 ],
[ 3.37580409779263, 51.3206502985963 ],
[ 3.37575872019649, 51.3207993094705 ],
[ 3.37575476634361, 51.3208122883487 ],
[ 3.37571181656268, 51.3208797459348 ],
[ 3.3756624532907, 51.3209415238446 ],
[ 3.37557609963811, 51.3210110142077 ],
[ 3.37541089899821, 51.3211055871218 ],
[ 3.37477516102591, 51.3214102985009 ],
[ 3.37473173914127, 51.3214311108204 ],
[ 3.37455904622072, 51.3215138815012 ],
[ 3.37415098054777, 51.3217199232877 ],
[ 3.37313700916272, 51.3222422862785 ],
[ 3.37748824689601, 51.3242852920348 ],
[ 3.37749760805371, 51.3242713084009 ],
[ 3.37811903757028, 51.3233437635596 ],
[ 3.37818758851947, 51.3232647797363 ],
[ 3.37823803668144, 51.3232236798646 ],
[ 3.37837807779104, 51.3231095796538 ]
]
]
}
},
{
"UseCreatedCropfield": false,
"fieldName": "fld5641", // FarmMaps won't do without a fieldName
"DownloadFolder": "C:\\workdir\\groenmonitor\\", //"C:\\workdir\\groenmonitor\\"; // "Downloads" -> if you just put "Downloads" the program will download to somewhere in ..\FarmMapsApiClient_WURtest\FarmmapsDataDownload\bin\Debug\netcoreapp3.1\Downloads\
"cropName": "wheat",
"cropYear": 2017,
"fieldID": 5641,
"lastdownloadedimagedate": "2017-01-01",
"geometryJson": {
"type": "Polygon",
"coordinates": [
[
[ 3.37837807779104, 51.3231095796538 ],
[ 3.38065689232502, 51.3212527499355 ],
[ 3.38022924592256, 51.3210683536359 ],
[ 3.37980548452565, 51.3208801127141 ],
[ 3.37959556105776, 51.3207540143696 ],
[ 3.3793691292654, 51.3205959677371 ],
[ 3.37822219207335, 51.3215667913007 ],
[ 3.37816999925795, 51.3216109809456 ],
[ 3.37646704574705, 51.3208025481261 ],
[ 3.37646695791282, 51.3208025061493 ],
[ 3.37608401443192, 51.3206231652693 ],
[ 3.37607169507628, 51.3206173959751 ],
[ 3.37606021048754, 51.320612017601 ],
[ 3.37582728410659, 51.3205029306946 ],
[ 3.37580409779263, 51.3206502985963 ],
[ 3.37575872019649, 51.3207993094705 ],
[ 3.37575476634361, 51.3208122883487 ],
[ 3.37571181656268, 51.3208797459348 ],
[ 3.3756624532907, 51.3209415238446 ],
[ 3.37557609963811, 51.3210110142077 ],
[ 3.37541089899821, 51.3211055871218 ],
[ 3.37477516102591, 51.3214102985009 ],
[ 3.37473173914127, 51.3214311108204 ],
[ 3.37455904622072, 51.3215138815012 ],
[ 3.37415098054777, 51.3217199232877 ],
[ 3.37313700916272, 51.3222422862785 ],
[ 3.37748824689601, 51.3242852920348 ],
[ 3.37749760805371, 51.3242713084009 ],
[ 3.37811903757028, 51.3233437635596 ],
[ 3.37818758851947, 51.3232647797363 ],
[ 3.37823803668144, 51.3232236798646 ],
[ 3.37837807779104, 51.3231095796538 ]
]
]
}
},
{
"UseCreatedCropfield": false,
"fieldName": "fld5641", // FarmMaps won't do without a fieldName
"DownloadFolder": "C:\\workdir\\groenmonitor\\", //"C:\\workdir\\groenmonitor\\"; // "Downloads" -> if you just put "Downloads" the program will download to somewhere in ..\FarmMapsApiClient_WURtest\FarmmapsDataDownload\bin\Debug\netcoreapp3.1\Downloads\
"cropName": "wheat",
"cropYear": 2016,
"fieldID": 5641,
"lastdownloadedimagedate": "2016-01-01",
"geometryJson": {
"type": "Polygon",
"coordinates": [
[
[ 3.37837807779104, 51.3231095796538 ],
[ 3.38065689232502, 51.3212527499355 ],
[ 3.38022924592256, 51.3210683536359 ],
[ 3.37980548452565, 51.3208801127141 ],
[ 3.37959556105776, 51.3207540143696 ],
[ 3.3793691292654, 51.3205959677371 ],
[ 3.37822219207335, 51.3215667913007 ],
[ 3.37816999925795, 51.3216109809456 ],
[ 3.37646704574705, 51.3208025481261 ],
[ 3.37646695791282, 51.3208025061493 ],
[ 3.37608401443192, 51.3206231652693 ],
[ 3.37607169507628, 51.3206173959751 ],
[ 3.37606021048754, 51.320612017601 ],
[ 3.37582728410659, 51.3205029306946 ],
[ 3.37580409779263, 51.3206502985963 ],
[ 3.37575872019649, 51.3207993094705 ],
[ 3.37575476634361, 51.3208122883487 ],
[ 3.37571181656268, 51.3208797459348 ],
[ 3.3756624532907, 51.3209415238446 ],
[ 3.37557609963811, 51.3210110142077 ],
[ 3.37541089899821, 51.3211055871218 ],
[ 3.37477516102591, 51.3214102985009 ],
[ 3.37473173914127, 51.3214311108204 ],
[ 3.37455904622072, 51.3215138815012 ],
[ 3.37415098054777, 51.3217199232877 ],
[ 3.37313700916272, 51.3222422862785 ],
[ 3.37748824689601, 51.3242852920348 ],
[ 3.37749760805371, 51.3242713084009 ],
[ 3.37811903757028, 51.3233437635596 ],
[ 3.37818758851947, 51.3232647797363 ],
[ 3.37823803668144, 51.3232236798646 ],
[ 3.37837807779104, 51.3231095796538 ]
]
]
}
}
]

29
FarmmapsBulkSatDownload/BulkSatDownloadService.cs Normal file
View File

@ -0,0 +1,29 @@
using System;
using System.Globalization;
using System.Threading.Tasks;
using FarmmapsApi.Models;
using FarmmapsApi.Services;
using FarmmapsBulkSatDownload.Models;
using Microsoft.Extensions.Logging;
using static FarmmapsApi.Extensions;
using static FarmmapsApiSamples.Constants;
namespace FarmmapsBulkSatDownload
{
public class BulkSatDownloadService
{
private readonly ILogger<BulkSatDownloadService> _logger;
private readonly FarmmapsApiService _farmmapsApiService;
private readonly GeneralService _generalService;
public BulkSatDownloadService(ILogger<BulkSatDownloadService> logger, FarmmapsApiService farmmapsApiService,
GeneralService generalService)
{
_logger = logger;
_farmmapsApiService = farmmapsApiService;
_generalService = generalService;
}
}
}

6
FarmmapsBulkSatDownload/DBsettings.json Normal file
View File

@ -0,0 +1,6 @@
{
"User": "",
"Password": "",
"Database": "",
"Host": ""
}

31
FarmmapsBulkSatDownload/FarmmapsBulkSatDownload.csproj Normal file
View File

@ -0,0 +1,31 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>netcoreapp3.1</TargetFramework>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Npgsql" Version="5.0.5" />
</ItemGroup>
<ItemGroup>
<None Update="appsettings.json">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</None>
<None Update="Data\**\*">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</None>
<None Update="BulkSatDownloadInput.json">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</None>
<None Update="DBsettings.json">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</None>
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\FarmmapsApi\FarmmapsApi.csproj" />
</ItemGroup>
</Project>

18
FarmmapsBulkSatDownload/Models/BulkSatDownloadInput.cs Normal file
View File

@ -0,0 +1,18 @@
using System;
using Newtonsoft.Json.Linq;
namespace FarmmapsBulkSatDownload.Models
{
public class BulkSatDownloadInput
{
public bool UseCreatedCropfield { get; set; }
public string fieldName { get; set; }
public string DownloadFolder { get; set; }
public string cropName { get; set; }
public int cropYear { get; set; }
public int fieldID { get; set; }
public DateTime lastdownloadedimagedate { get; set; }
public JObject GeometryJson { get; set; }
//public string fieldName { get { return string.Format($"{cropName}_fld{fieldID}"); } }
}
}

47
FarmmapsBulkSatDownload/Models/DB.cs Normal file
View File

@ -0,0 +1,47 @@
using System;
using System.Collections.Generic;
using System.Text;
using Npgsql;
namespace FarmmapsApi.Services
{
public class DB
{
public string Database;
public string User;
public string Password;
public string Host;
public string GetConnectionString()
{
NpgsqlConnectionStringBuilder sb = new NpgsqlConnectionStringBuilder();
sb.Database = Database;
sb.Host = Host;
sb.Username = User;
sb.Password = Password;
sb.Port = 5432;
return sb.ConnectionString;
}
public int ExecuteNonQuery(string sql)
{
using (NpgsqlConnection conn = new NpgsqlConnection(GetConnectionString()))
{
conn.Open();
NpgsqlCommand command = conn.CreateCommand();
command.CommandText = sql;
int r = command.ExecuteNonQuery();
return r;
}
}
public NpgsqlConnection CreateConnection()
{
NpgsqlConnection conn = new NpgsqlConnection(GetConnectionString());
conn.Open();
return conn;
}
}
}

28
FarmmapsBulkSatDownload/Models/GroenmonitorTable.cs Normal file
View File

@ -0,0 +1,28 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace FarmmapsBulkSatDownload.Models
{
public class GroenmonitorTable
{
//public string parceltablename; //PO20190605: parceltablename added as a field (=column) in the GroenmonitorTable. The GroenmonitorTable contains data from multiple parceltables
public int parcelid;
public string date;
public string source;
public int wdvi_pixelcount; //count of pixels with data
public double wdvi_max;
public double wdvi_mean;
public double wdvi_min;
public double wdvi_stdev;
public double wdvi_median;
public double wdvi_p90;
public int ndvi_pixelcount; //count of pixels with data
public double ndvi_max;
public double ndvi_mean;
public double ndvi_min;
public double ndvi_stdev;
public double ndvi_median;
public double ndvi_p90;
}
}

11
FarmmapsBulkSatDownload/Models/Settings.cs Normal file
View File

@ -0,0 +1,11 @@
namespace FarmmapsBulkSatDownload
{
public class Settings
{
public string CropfieldItemCode { get; set; }
public string SatelliteTaskCode { get; set; }
public string VanDerSatTaskCode { get; set; }
public string WatBalTaskCode { get; set; }
}
}

21
FarmmapsBulkSatDownload/Program.cs Normal file
View File

@ -0,0 +1,21 @@
using System.Threading.Tasks;
using FarmmapsApi;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
namespace FarmmapsBulkSatDownload
{
class Program : FarmmapsProgram<BulkSatDownloadApplication>
{
private static async Task Main(string[] args)
{
await new Program().Start(args);
}
protected override void Configure(IServiceCollection serviceCollection)
{
serviceCollection.AddLogging()
.AddTransient<BulkSatDownloadService>();
}
}
}

152
FarmmapsBulkSatDownload/ShowGeotiff.r Normal file
View File

@ -0,0 +1,152 @@
# ShowGeotiff.r
# I downloaded and calculated the stats for the polygon defined in C:\git\FarmMapsApiClient_WURtest\FarmmapsDataDownload\DataDownloadInput.json
# in which I set "SatelliteBand": "wdvi" and in which in the console I requested image 8 for date '2020-06-14'
library(raster)
library(sf)
library(rgdal)
setwd("C:/workdir/groenmonitor/")
# FarmmapsDataDownload generates two files:
fileGeotiff <- "wenr.tif"
# fileJpg <- "thumbnail.jpg"
# FarmmapsBulkSatDownload generates many files. Here is what I tried when inputing the same field for a number of years using BulkSatDownloadInput.json
# fileGeotiff <- "wheat_fld5641_20210224.tif" # 2 files for year 2021. This file is a nice example having in upperleft corner no data
# fileGeotiff <- "wheat_fld5641_20210331.tif" # 2 files for year 2021
# fileGeotiff <- "wheat_fld5641_20200321.tif" # 14 files for year 2020, earliest
# fileGeotiff <- "wheat_fld5641_20200922.tif" # 14 files for year 2020, latest
# fileGeotiff <- "wheat_fld5641_20190121.tif" # 9 files for year 2019, earliest
# fileGeotiff <- "wheat_fld5641_20191117.tif" # 9 files for year 2019, latest
# 1 file for year 2018, with error message 'End of Central Directory record could not be found' and invalid wheat_fld5641_20180630.zip
# fileGeotiff <- "wheat_fld5641_20170526.tif" # 1 file for year 2017
# Zero files for 2016
lenfilename <- nchar(fileGeotiff)
year <- substr(fileGeotiff,lenfilename-11,lenfilename-8)
imgdate <- substr(fileGeotiff,lenfilename-11,lenfilename-4)
# The thumbnail has the polygon clipped out, has 1 layer, no crs and the mean value is not the mean wdvi we are looking for
# r.thumbnail <- raster(fileJpg)
# plot(r.thumbnail)
# crs(r.thumbnail)
# CRS arguments: NA
# cellStats(r.thumbnail,'mean') #59.91667
stk.wenr <- stack(x=fileGeotiff)
# plot(stk.wenr) shows 5 plots (5 bands)
# I think these are:
# 1. ndvi (since it runs from 0 to 1)
# 2. wdvi (since it runs from 0 to 0.5)
# 3-5: RGB (since they run from 0 to 255)
plot(stk.wenr)
# CRS arguments:
# +proj=sterea +lat_0=52.1561605555556 +lon_0=5.38763888888889 +k=0.9999079 +x_0=155000 +y_0=463000 +ellps=bessel +units=m +no_defs
# Or use st_crs(stk.wenr) to get more info, but no EPSG code in there.
# Likely it is epsg:28992 (Amersfoort)
crs(stk.wenr)
stk.wenr <- projectRaster(stk.wenr, crs = CRS('+init=EPSG:28992'))
crs(stk.wenr)
# Looks the same but strangely, if we don't do projectRaster(stk.wenr, crs = CRS('+init=EPSG:28992')), we find below the bottom left corner of the polygon missing
r.wenr.rd.wdvi <- subset(stk.wenr,2)
dev.off()
plot(r.wenr.rd.wdvi,main=paste("wdvi",imgdate),xlab="RDX",ylab="RDY")
cellStats(r.wenr.rd.wdvi,'mean') #0.1654627
# Same as in downloaded file SatelliteDataStatistics_test_satData_wdvi_2020-06-14.csv
# "mean": 0.16564258790046743
# So FindSatelliteItem in C:\git\FarmMapsApiClient_WURtest\FarmmapsApi\Services\GeneralService.cs returns a a stacked geotiff in
# i.e. the cellstats are calculated from the rectangle
# Furthermore we can see
# shows coordinates in RD
# returns a rectangle, thus the shape of the polygon submitted is not clipped.
# The polygon was provided in WGS84. Let's draw it on top.
# First convert the raster to WGS84
r.wenr.wgs84.wdvi <- projectRaster(r.wenr.rd.wdvi, crs = CRS('+init=EPSG:4326'))
# Draw a polygon on top of the raster
# Example polygon p1
# p1 <- data.frame(id = 1, wkt = 'POLYGON((4.963 52.801, 4.966 52.801, 4.966 52.803, 4.963 52.803, 4.963 52.801))')
# p1 <- st_as_sf(p1, wkt = 'wkt', crs = targetcrs)
# plot(p1,add=TRUE, col="transparent",border="black")
# Draw the polygon on top of the raster
# Polygon p2 from C:\git\FarmMapsApiClient_WURtest\FarmmapsDataDownload\DataDownloadInput.json
p2 <- data.frame(id = 1, wkt = gsub("\n","",'POLYGON((
3.37837807779104 51.3231095796538,
3.38065689232502 51.3212527499355,
3.38022924592256 51.3210683536359,
3.37980548452565 51.3208801127141,
3.37959556105776 51.3207540143696,
3.3793691292654 51.3205959677371,
3.37822219207335 51.3215667913007,
3.37816999925795 51.3216109809456,
3.37646704574705 51.3208025481261,
3.37646695791282 51.3208025061493,
3.37608401443192 51.3206231652693,
3.37607169507628 51.3206173959751,
3.37606021048754 51.320612017601,
3.37582728410659 51.3205029306946,
3.37580409779263 51.3206502985963,
3.37575872019649 51.3207993094705,
3.37575476634361 51.3208122883487,
3.37571181656268 51.3208797459348,
3.3756624532907 51.3209415238446,
3.37557609963811 51.3210110142077,
3.37541089899821 51.3211055871218,
3.37477516102591 51.3214102985009,
3.37473173914127 51.3214311108204,
3.37455904622072 51.3215138815012,
3.37415098054777 51.3217199232877,
3.37313700916272 51.3222422862785,
3.37748824689601 51.3242852920348,
3.37749760805371 51.3242713084009,
3.37811903757028 51.3233437635596,
3.37818758851947 51.3232647797363,
3.37823803668144 51.3232236798646,
3.37837807779104 51.3231095796538))'))
p2.wgs84 <- st_as_sf(p2, wkt = 'wkt', crs = CRS('+init=EPSG:4326'))
# Or other way round, in RD Amersfoort. That looks ok
p2.rd <- st_transform(p2.wgs84, "+init=epsg:28992")
# Have a look at both
# wg84
dev.off()
plot(r.wenr.wgs84.wdvi,main=paste("wdvi",imgdate),xlab="LON",ylab="LAT")
plot(p2.wgs84,add=TRUE, col="transparent",border="red")
# RD
dev.off()
plot(r.wenr.rd.wdvi,main=paste("wdvi",imgdate),xlab="RDX",ylab="RDY")
plot(p2.rd,add=TRUE, col="transparent",border="red")
#Let's clip the polygon
r.wenr.rd.wdvi.pol <- mask(r.wenr.rd.wdvi,p2.rd)
r.wenr.wgs84.wdvi.pol <- mask(r.wenr.wgs84.wdvi,p2.wgs84)
dev.off()
plot(r.wenr.wgs84.wdvi.pol,main=paste("wdvi",imgdate),xlab="LON",ylab="LAT")
plot(p2.wgs84,add=TRUE, col="transparent",border="red")
#That's what we want! Now compare the stats
cellStats(r.wenr.rd.wdvi,'mean') #0.1654627 # Stats from rectangle, RD
cellStats(r.wenr.wgs84.wdvi,'mean') #0.1656399 # Stats from rectangle, WGS84
cellStats(r.wenr.rd.wdvi.pol,'mean') #0.1658702 # Stats from raster clipped by polygon, WGS84
cellStats(r.wenr.wgs84.wdvi.pol,'mean') #0.1658611 # Stats from raster clipped by polygon, WGS84
# Conclusion: in this example little difference, but can be quite different!
# The project FarmmapsNbs can generate a wenr.tif file, application.tif, uptake.tif (in rtest1.uptake.zip)and shape.shp (in rtest1.taskmap.zip)
r.application <- raster("C:/git/FarmMapsApiClient_WURtest/FarmmapsNbs/bin/Debug/netcoreapp3.1/Downloads/application.tif")
dev.off()
plot(r.application)
plot(p2.rd,add=TRUE, col="transparent",border="red")
# The application.tif file is a rectangle (polygon not yet clipped), in projection Amersfoort RD New (EPSG:28992)
r.uptake <- raster("C:/git/FarmMapsApiClient_WURtest/FarmmapsNbs/bin/Debug/netcoreapp3.1/Downloads/uptake.tif")
dev.off()
plot(r.uptake)
plot(p2.rd,add=TRUE, col="transparent",border="red")
# The uptake.tif file is a rectangle (polygon not yet clipped), in projection Amersfoort RD New (EPSG:28992)
shp.wgs84 <- readOGR(dsn="C:/git/FarmMapsApiClient_WURtest/FarmmapsNbs/bin/Debug/netcoreapp3.1/Downloads", layer="shape")
crs(shp.wgs84)
# CRS arguments: +proj=longlat +datum=WGS84 +no_defs
dev.off()
plot(r.wenr.wgs84.wdvi,main="wdvi",xlab="LON",ylab="LAT")
plot(shp.wgs84,add=TRUE, col="transparent",border="black")
plot(p2.wgs84,add=TRUE, col="transparent",border="red")
# The shape file is in WGS84

10
FarmmapsBulkSatDownload/appsettings.json Normal file
View File

@ -0,0 +1,10 @@
{
"Authority": "https://accounts.test.farmmaps.eu/",
"Endpoint": "https://test.farmmaps.eu/",
"BasePath": "api/v1",
"DiscoveryEndpointUrl": "https://accounts.test.farmmaps.eu/.well-known/openid-configuration",
"RedirectUri": "http://example.nl/api",
"ClientId": "",
"ClientSecret": "",
"Scopes": [ "api" ]
}