FarmMaps/FarmMapsApiClient
11
0
Fork 3
Code Issues Pull Requests Releases Activity

Veranderingen jits, fedde en pepijn. #9

Merged
mark merged 46 commits from jits/FarmMapsApiClient_WURtest:master into master 2021-06-10 08:25:25 +00:00
Conversation 0 Commits 46 Files Changed 56 +738 -479
14 changed files with 738 additions and 479 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 1bc326cfd2 - Show all commits

1
.gitignore vendored
View File

@ -3,3 +3,4 @@
bin/ bin/
obj/ obj/
appsettings.secrets.json appsettings.secrets.json
FarmmapsBulkSatDownload/DBsettings.secrets.json

29
FarmmapsApi/Models/SatelliteStatistics.cs Normal file
View File

@ -0,0 +1,29 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace FarmmapsApi.Models
{
public class SatelliteStatistics
{
public string fieldName;
public DateTime satelliteDate;
public string satelliteBand;
public double max;
public double min;
public double mean;
public double mode;
public double median;
public double stddev;
public double minPlus;
public double curtosis;
public double maxMinus;
public double skewness;
public double variance;
public int populationCount;
public double variationCoefficient;
public double confidenceIntervalLow;
public double confidenceIntervalHigh;
public double confidenceIntervalErrorMargin;
}
}

209
FarmmapsApi/Services/GeneralService.cs
View File

@ -6,6 +6,7 @@ using System.Threading.Tasks;
using FarmmapsApi.Models; using FarmmapsApi.Models;
using Google.Apis.Upload; using Google.Apis.Upload;
using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq; using Newtonsoft.Json.Linq;
using static FarmmapsApi.Extensions; using static FarmmapsApi.Extensions;
using static FarmmapsApiSamples.Constants; using static FarmmapsApiSamples.Constants;
@ -348,12 +349,10 @@ namespace FarmmapsApi.Services
} }
public async Task<Item> FindSatelliteItem(Item cropfieldItem, string satelliteTaskCode, string FieldName = null, int selectedLayer = 0, bool allItemsStatistics = false, string DownloadFolder = null) { public async Task<Item> FindSatelliteItem(Item cropfieldItem, string satelliteTaskCode) {
var taskStatus = await _farmmapsApiService.GetTaskStatusAsync(cropfieldItem.Code, satelliteTaskCode); var taskStatus = await _farmmapsApiService.GetTaskStatusAsync(cropfieldItem.Code, satelliteTaskCode);
// find ndvi or wdvi satellite data geotiffs
var temporalItem = await FindChildItemAsync(cropfieldItem.Code, TEMPORAL_ITEMTYPE, var temporalItem = await FindChildItemAsync(cropfieldItem.Code, TEMPORAL_ITEMTYPE,
"Cropfield Satellite items", item => item.SourceTask == SATELLITE_TASK && "Cropfield Satellite items", item => item.SourceTask == SATELLITE_TASK &&
taskStatus.Finished >= item.Created && taskStatus.Finished >= item.Created &&
@ -367,101 +366,149 @@ namespace FarmmapsApi.Services
var satelliteTiffs = await _farmmapsApiService.GetItemChildrenAsync(temporalItem.Code); var satelliteTiffs = await _farmmapsApiService.GetItemChildrenAsync(temporalItem.Code);
if (allItemsStatistics == false) { _logger.LogInformation("Available satellite images:");
_logger.LogInformation("Available satellite images:"); var count = 0;
var count = 0; TimeSpan.FromSeconds(0.5);
TimeSpan.FromSeconds(0.5); foreach (var item in satelliteTiffs) {
foreach (var item in satelliteTiffs) {
Console.WriteLine($"Satellite image #{count}: {item.DataDate}"); Console.WriteLine($"Satellite image #{count}: {item.DataDate}");
count++; count++;
}
_logger.LogInformation("Enter satellite image number for NBS application");
int element = Int32.Parse(Console.ReadLine());
var selectedSatelliteItem = satelliteTiffs[element];
if (selectedSatelliteItem == null) {
_logger.LogError("Satellite item not found");
return selectedSatelliteItem;
}
} }
if (allItemsStatistics == true ) { _logger.LogInformation("Enter satellite image number for NBS application");
var count = 0; int element = Int32.Parse(Console.ReadLine());
foreach (var item in satelliteTiffs) { var selectedSatelliteItem = satelliteTiffs[element];
var satellitetBand = item.Data["layers"][selectedLayer]["name"];
var SatelliteImageYear = (DateTime)item.DataDate;
var SatelliteYear = SatelliteImageYear.ToString("yyyy");
var SatelliteFile = $"{DownloadFolder}/SatelliteDataStatistics_{SatelliteYear}_{FieldName}_{satellitetBand}.csv";
var NewLineField = $"Field,Date,Mean,Min,Max,Standard deviation, ConfidenceInterval low, ConfidenceInterval high" + Environment.NewLine;
if (count == 0) {
File.AppendAllText(SatelliteFile, NewLineField);
var numbervandersat = satelliteTiffs.Count;
Console.WriteLine($"{numbervandersat} Satellite images found");
}
var SatelliteStatistics = item.Data["layers"][0]["renderer"]["band"]["statistics"]; if (selectedSatelliteItem == null)
var SatelliteImageDate = (DateTime)item.DataDate; {
var satelliteDate = SatelliteImageDate.ToString("yyyy-MM-dd");
var NewLineDate = $"\"date\":{satelliteDate}" + Environment.NewLine;
if (SatelliteStatistics == null) {
Console.WriteLine($"{satelliteDate} no statistics found");
//Console.WriteLine($"Available data: {item.Data}");
} else {
File.AppendAllText(SatelliteFile, $"{FieldName},{satelliteDate},{SatelliteStatistics["mean"]},{SatelliteStatistics["min"]},{SatelliteStatistics["max"]},{SatelliteStatistics["stddev"]},{SatelliteStatistics["confidenceIntervalLow"]},{SatelliteStatistics["confidenceIntervalHigh"]}" + Environment.NewLine);
}
if (true) {
// download the geotiff of needed inputtype
var selectedSatelliteItemDate = (DateTime)item.DataDate;
var SatelliteDate = selectedSatelliteItemDate.ToString("yyyyMMdd");
_logger.LogInformation("Downloading geotiff file");
await _farmmapsApiService.DownloadItemAsync(item.Code,
Path.Combine(DownloadFolder, $"satelliteGeotiff_{FieldName}_{satellitetBand}_{SatelliteDate}.zip"));
}
count++;
}
}
var selectedSatelliteItemTemp = satelliteTiffs[0];
if (selectedSatelliteItemTemp == null) {
_logger.LogError("Satellite item not found"); _logger.LogError("Satellite item not found");
return selectedSatelliteItemTemp;
} }
return selectedSatelliteItemTemp; return selectedSatelliteItem;
} }
public async Task<List<Item>> FindSatelliteItems(Item cropfieldItem, string satelliteTaskCode)
public async Task<List<Item>> FindSatelliteItemsAll(Item cropfieldItem, string satelliteTaskCode)
{ {
var taskStatus = await _farmmapsApiService.GetTaskStatusAsync(cropfieldItem.Code, satelliteTaskCode); var taskStatus = await _farmmapsApiService.GetTaskStatusAsync(cropfieldItem.Code, satelliteTaskCode);
if (taskStatus.State == ItemTaskState.Error)
// find ndvi or wdvi satellite data geotiffs
var temporalItem = await FindChildItemAsync(cropfieldItem.Code, TEMPORAL_ITEMTYPE,
"Cropfield Satellite items", item => item.SourceTask == SATELLITE_TASK &&
taskStatus.Finished >= item.Created &&
taskStatus.Finished <= item.Created.Value.AddHours(1));
if (temporalItem == null)
{ {
_logger.LogError("Temporal item not found"); _logger.LogWarning(taskStatus.Message);
return null;
}
else
{
// find satellite data geotiffs
var temporalItem = await FindChildItemAsync(cropfieldItem.Code, TEMPORAL_ITEMTYPE,
"Cropfield Satellite items", item => item.SourceTask == SATELLITE_TASK &&
taskStatus.Finished >= item.Created &&
taskStatus.Finished <= item.Created.Value.AddHours(1));
if (temporalItem == null)
{
_logger.LogWarning("Temporal item not found");
return null;
}
else
{
var satelliteTiffs = await _farmmapsApiService.GetItemChildrenAsync(temporalItem.Code);
return satelliteTiffs;
}
} }
var satelliteTiffs = await _farmmapsApiService.GetItemChildrenAsync(temporalItem.Code);
return satelliteTiffs;
} }
public async Task<string> DownloadSatelliteStats(List<Item> satelliteTiffs, string fieldName = null, List<string> satelliteBands = null, string downloadFolder = null)
{
string satelliteDataStatisticsFile = Path.Combine(downloadFolder, $"satelliteStats_{fieldName}.csv");
File.Delete(satelliteDataStatisticsFile); // Delete the SatelliteFile file if exists
List<int> selectedLayers = new List<int>();
foreach (string satelliteBand in satelliteBands)
{
if (satelliteBand == "ndvi") selectedLayers.Add(0);
if (satelliteBand == "wdvi") selectedLayers.Add(1);
};
string headerLineStats = $"FieldName,satelliteDate,satelliteBand,max,min,mean,mode,median,stddev,minPlus,curtosis,maxMinus,skewness,variance,populationCount,variationCoefficient,confidenceIntervalLow, confidenceIntervalHigh,confidenceIntervalErrorMargin" + Environment.NewLine;
File.AppendAllText(satelliteDataStatisticsFile, headerLineStats);
foreach (var item in satelliteTiffs)
foreach (int selectedLayer in selectedLayers)
{
{
var satelliteBand = item.Data["layers"][selectedLayer]["name"];
var satelliteImageDate = (DateTime)item.DataDate;
var satelliteStatisticsJtoken = item.Data["layers"][selectedLayer]["renderer"]["band"]["statistics"];
if (satelliteStatisticsJtoken == null)
{
_logger.LogWarning($"{satelliteImageDate.ToString("yyyy-MM-dd")} no statistics found");
//Console.WriteLine($"Available data: {item.Data}");
}
else
{
SatelliteStatistics satelliteStatistics = satelliteStatisticsJtoken.ToObject<SatelliteStatistics>();
satelliteStatistics.fieldName = fieldName;
satelliteStatistics.satelliteDate = satelliteImageDate;
satelliteStatistics.satelliteBand = satelliteBand.ToString();
File.AppendAllText(satelliteDataStatisticsFile, $"" +
$"{satelliteStatistics.fieldName}," +
$"{satelliteStatistics.satelliteDate.ToString("yyyy-MM-dd")}," +
$"{satelliteStatistics.satelliteBand}," +
$"{satelliteStatistics.max}," +
$"{satelliteStatistics.min}," +
$"{satelliteStatistics.mean}," +
$"{satelliteStatistics.mode}," +
$"{satelliteStatistics.median}," +
$"{satelliteStatistics.stddev}," +
$"{satelliteStatistics.minPlus}," +
$"{satelliteStatistics.curtosis}," +
$"{satelliteStatistics.maxMinus}," +
$"{satelliteStatistics.skewness}," +
$"{satelliteStatistics.variance}," +
$"{satelliteStatistics.populationCount}," +
$"{satelliteStatistics.variationCoefficient}," +
$"{satelliteStatistics.confidenceIntervalLow}," +
$"{satelliteStatistics.confidenceIntervalHigh}," +
$"{satelliteStatistics.confidenceIntervalErrorMargin}" +
Environment.NewLine);
}
}
}
return satelliteDataStatisticsFile;
}
public async Task<List<SatelliteStatistics>> ListSatelliteStatistics(Item satelliteTiff, List<string> satelliteBands = null, string fieldName = null)
mark commented 2021-05-26 13:24:14 +00:00
Review
Copy Link

Dit hoeft geen async methode te zijn (want niet async).

Dit hoeft geen async methode te zijn (want niet async).
{
SatelliteStatistics satelliteStatistics;
List<SatelliteStatistics> listSatelliteStatistics = new List<SatelliteStatistics>();
List<int> selectedLayers = new List<int>();
foreach (string satelliteBand in satelliteBands)
{
if (satelliteBand == "ndvi") selectedLayers.Add(0);
if (satelliteBand == "wdvi") selectedLayers.Add(1);
};
foreach (int selectedLayer in selectedLayers)
{
var satelliteBand = satelliteTiff.Data["layers"][selectedLayer]["name"];
var satelliteImageDate = (DateTime)satelliteTiff.DataDate;
var satelliteStatisticsJtoken = satelliteTiff.Data["layers"][selectedLayer]["renderer"]["band"]["statistics"];
if (satelliteStatisticsJtoken == null)
{
_logger.LogWarning($"{satelliteImageDate.ToString("yyyy-MM-dd")} no statistics found");
//Console.WriteLine($"Available data: {item.Data}");
}
else
{
satelliteStatistics = satelliteStatisticsJtoken.ToObject<SatelliteStatistics>();
satelliteStatistics.fieldName = fieldName.ToString();
satelliteStatistics.satelliteDate = satelliteImageDate;
satelliteStatistics.satelliteBand = satelliteBand.ToString();
listSatelliteStatistics.Add(satelliteStatistics);
}
}
return listSatelliteStatistics;
}
//VanDerSat //VanDerSat
public async Task<string> RunVanDerSatTask(Item cropfieldItem) { public async Task<string> RunVanDerSatTask(Item cropfieldItem) {
_logger.LogInformation("Gathering VanDerSat information for cropfield, this might take a while!"); _logger.LogInformation("Gathering VanDerSat information for cropfield, this might take a while!");

463
FarmmapsBulkSatDownload/BulkSatDownloadApplication.cs
View File

@ -13,28 +13,27 @@ using Newtonsoft.Json;
using Npgsql; using Npgsql;
using Newtonsoft.Json.Linq; using Newtonsoft.Json.Linq;
using static FarmmapsApiSamples.Constants; using static FarmmapsApiSamples.Constants;
using System.Text;
namespace FarmmapsBulkSatDownload namespace FarmmapsBulkSatDownload
{ {
public class BulkSatDownloadApplication : IApplication public class BulkSatDownloadApplication : IApplication
{ {
private const string SettingsFile = "settings.json";
private readonly ILogger<BulkSatDownloadApplication> _logger; private readonly ILogger<BulkSatDownloadApplication> _logger;
private readonly FarmmapsApiService _farmmapsApiService; private readonly FarmmapsApiService _farmmapsApiService;
private readonly BulkSatDownloadService _dataDownloadService; private readonly BulkSatDownloadService _bulkSatDownloadService;
private readonly GeneralService _generalService; private readonly GeneralService _generalService;
public const string settingsfile = "Settings.json";
private Settings _settings; private Settings _settings;
static DB dbparcels;
public BulkSatDownloadApplication(ILogger<BulkSatDownloadApplication> logger, FarmmapsApiService farmmapsApiService, public BulkSatDownloadApplication(ILogger<BulkSatDownloadApplication> logger, FarmmapsApiService farmmapsApiService,
GeneralService generalService, BulkSatDownloadService dataDownloadService) GeneralService generalService, BulkSatDownloadService bulkSatDownloadService)
{ {
_logger = logger; _logger = logger;
_farmmapsApiService = farmmapsApiService; _farmmapsApiService = farmmapsApiService;
_generalService = generalService; _generalService = generalService;
_dataDownloadService = dataDownloadService; _bulkSatDownloadService = bulkSatDownloadService;
} }
public async Task RunAsync() public async Task RunAsync()
{ {
@ -42,46 +41,47 @@ namespace FarmmapsBulkSatDownload
// !! this call is needed the first time an api is called with a fresh clientid and secret !! // !! this call is needed the first time an api is called with a fresh clientid and secret !!
await _farmmapsApiService.GetCurrentUserCodeAsync(); await _farmmapsApiService.GetCurrentUserCodeAsync();
var roots = await _farmmapsApiService.GetCurrentUserRootsAsync(); var roots = await _farmmapsApiService.GetCurrentUserRootsAsync();
// Initialize databases. Username, password etc stored in file "DBsettings.json"
dbparcels = JsonConvert.DeserializeObject<DB>(File.ReadAllText("DBsettings.json"));
//string date; BulkSatDownloadInput bulkSatDownloadInput;
//string asterices = "*****************************************************************************"; List<BulkSatDownloadInput> bulkSatDownloadInputList;
//string bboxtiffWithTempPath; // Below are two options to for bulk download: (1) from and to database or (2) inputs from json, output to csv
//string parcelshp = "parcel.shp"; // For illustration we make two lists bulkSatDownloadInputListDB and bulkSatDownloadInputListCsv and then choose which one we will use
//string parcelshpWithTempPath; List<BulkSatDownloadInput> bulkSatDownloadInputListDB;
//string gwoptions; //= String.Format("-overwrite -s_srs EPSG:28992 -t_srs EPSG:28992 -dstnodata 0 -cutline {0} -crop_to_cutline", parcelshp); List<BulkSatDownloadInput> bulkSatDownloadInputListCsv;
//string parceltiff = "parcel.tiff"; DateTime lastdownloadedimagedate;
//string parceltiffWithTempPath; int cropYear;
//string[] vegetationindices = { "WDVI", "NDVI" };
//string[] sources = { "groenmonitor" }; // { "groenmonitor", "akkerwebwenr", "akkerwebneo" }; { "groenmonitor" }; // Option 1: When using database need to (1) fill in database data in DBsettings.secrets.json; (2) write tailor made SELECT query for fieldinputs in following lines;
// (3) Write tailor made INSERT INTO query in Task Process() below;
// Initialize databases. Username, password etc stored in file "DBsettings.secrets.json".
// Crashes if "DBsettings.secrets.json" is absent or empty
DB dbparcels = JsonConvert.DeserializeObject<DB>(File.ReadAllText("DBsettings.secrets.json"));
string schemaname = "bigdata"; string schemaname = "bigdata";
string parceltablename = "parcel_flowerbulbs";//"parcelsijbrandij" "parcel"; "parcel_flowerbulbs" string parceltablename = "parcel_disac";//"parcelsijbrandij" "parcel"; "parcel_flowerbulbs"; "parcel_disac"
//string groenmonitortablename = "groenmonitor_flowerbulbs";//"groenmonitorsijbrandij" "groenmonitor" "groenmonitor_flowerbulbs" //PO20190605: "groenmonitormpt" contains groenmonitor data from multiple parcel tables (mpt) string groenmonitortablename = "groenmonitor_disac";//"groenmonitorsijbrandij" "groenmonitor" "groenmonitor_flowerbulbs" "groenmonitor_disac"
// The view 'groenmonitorlatestviewname' contains per parcelid (arbid) the year in which it "exists" and the date of the latest image downloaded. It is used to prevent downloading images already downloaded // The view 'groenmonitorlatestviewname' contains per parcelid (arbid) the year in which it "exists" and the date of the latest image downloaded. It is used to prevent unneccessary downloading of image statistics already in the database
string groenmonitorlatestviewname = "groenmonitorlatest_flowerbulbs"; //"groenmonitorsijbrandijlatest" "groenmonitorlatest" "groenmonitorlatest_flowerbulbs" //PO20190605: "v_groenmonitorlatest" contains latest available dates from groenmonitortablename = "groenmonitormpt" string groenmonitorlatestviewname = "groenmonitorlatest_disac"; //"groenmonitorsijbrandijlatest" "groenmonitorlatest" "groenmonitorlatest_flowerbulbs" "groenmonitorlatest_disac"
//GroenmonitorTable gmtb; //GroenmonitorTable gmtb;
//DateTime dt;
//DateTime dtfirst;
//DateTime dtlast;
//double scalingfactor;
BulkSatDownloadInput dataDownloadInput;
List<BulkSatDownloadInput> fieldsInputs = new List<BulkSatDownloadInput>();
// Database query and connection. Geometry must be in WGS84 coordinate system, EPSG 4326 // Database query and connection. Geometry must be in WGS84 coordinate system, EPSG 4326
// Apparently the FarmmapsApi cannot handle MultiPolygon, so we need to convert to single Polygon // Apparently the FarmmapsApi cannot handle MultiPolygon, so we need to convert to single Polygon
// In case database returns a MultiPolygon use ST_NumGeometries(pt.geom) to count the number of polygons // In case database returns a MultiPolygon use ST_NumGeometries(pt.geom) to count the number of polygons
// If necessary use WHERE T_NumGeometries(pt.geom) = 1 to use only single polygons // If necessary use WHERE T_NumGeometries(pt.geom) = 1 to select only single polygons
//
// FarmMaps get's its satellite images from www.groenmonitor.nl through the https://agrodatacube.wur.nl/.
// Many images are available at www.groenmonitor.nl, the https://agrodatacube.wur.nl/ serves only the clean images, 10-30 per year, 2019 onwards. Possibly more images will be added for earlier years
// For other images contact www.groenmonitor.nl, gerbert.roerink@wur.nl
bulkSatDownloadInputListDB = new List<BulkSatDownloadInput>();
List<string> satelliteBands = new List<string> { "wdvi", "ndvi" };
string connectionString = dbparcels.GetConnectionString(); string connectionString = dbparcels.GetConnectionString();
string readSql = string.Format( string readSql = string.Format(
@" @"
SELECT pt.arbid, pt.crop, pt.year, gml.lastwenrdate, ST_AsGeoJSON(ST_Transform((ST_DUMP(pt.geom)).geom::geometry(Polygon),4326)) AS geojson_polygon_wgs84 SELECT pt.arbid, pt.year, gml.lastwenrdate, ST_AsGeoJSON(ST_Transform((ST_DUMP(pt.geom)).geom::geometry(Polygon),4326)) AS geojson_polygon_wgs84, COALESCE(pt.cropfielditemcode,'') AS cropfielditemcode
FROM {0}.{1} pt, {0}.{2} gml FROM {0}.{1} pt, {0}.{2} gml
WHERE WHERE
pt.arbid = gml.arbid AND pt.arbid = gml.arbid
pt.crop NOT IN('Tulp', 'Lelie') AND ORDER BY pt.arbid
pt.year = 2018 LIMIT 15;", schemaname, parceltablename, groenmonitorlatestviewname); //LIMIT x for testing
LIMIT 10;", schemaname, parceltablename, groenmonitorlatestviewname); //LIMIT 10 for testing
using (NpgsqlConnection connection = new NpgsqlConnection(connectionString)) using (NpgsqlConnection connection = new NpgsqlConnection(connectionString))
{ {
@ -93,47 +93,113 @@ LIMIT 10;", schemaname, parceltablename, groenmonitorlatestviewname); //LIMIT
NpgsqlDataReader dr = command.ExecuteReader(); NpgsqlDataReader dr = command.ExecuteReader();
while (dr.Read()) while (dr.Read())
{ {
dataDownloadInput = new BulkSatDownloadInput(); bulkSatDownloadInput = new BulkSatDownloadInput();
dataDownloadInput.UseCreatedCropfield = false; bulkSatDownloadInput.fieldID = dr.GetInt16(0);
dataDownloadInput.DownloadFolder = "C:\\workdir\\groenmonitor\\"; bulkSatDownloadInput.fieldName = string.Format($"{parceltablename}_{bulkSatDownloadInput.fieldID}");
dataDownloadInput.fieldID = dr.GetInt16(0); bulkSatDownloadInput.cropYear = dr.GetInt16(1); ;
dataDownloadInput.fieldName = string.Format($"{parceltablename}_fld{dataDownloadInput.fieldID}"); bulkSatDownloadInput.lastdownloadedimagedate = dr.GetDateTime(2);
dataDownloadInput.cropName = dr.GetString(1); bulkSatDownloadInput.GeometryJson = JObject.Parse(dr.GetString(3));
dataDownloadInput.cropYear = dr.GetInt16(2); bulkSatDownloadInput.SatelliteBands = satelliteBands;
dataDownloadInput.lastdownloadedimagedate = dr.GetDateTime(3); bulkSatDownloadInput.cropfielditemcode = dr.GetString(4);
dataDownloadInput.GeometryJson = JObject.Parse(dr.GetString(4)); bulkSatDownloadInput.database = dbparcels;
fieldsInputs.Add(dataDownloadInput); bulkSatDownloadInput.schemaname = schemaname;
bulkSatDownloadInput.cropfieldtable = parceltablename;
bulkSatDownloadInput.satelllitetable = groenmonitortablename;
bulkSatDownloadInputListDB.Add(bulkSatDownloadInput);
} }
connection.Close(); connection.Close();
} }
// For testing using json input instead of database input // Option 2: Example without database. Comment out this part if you want to use database
// Read cropfields "BulkSatDownloadInput.json" and write all stats to a single csv file
// Write all stats for multiple fields will be written to a single csv file
string downloadFolder;
string fileNameStats;
string headerLineStats = $"FieldName,satelliteDate,satelliteBand,max,min,mean,mode,median,stddev,minPlus,curtosis,maxMinus,skewness,variance,populationCount,variationCoefficient,confidenceIntervalLow, confidenceIntervalHigh,confidenceIntervalErrorMargin" + Environment.NewLine;
var fieldsInputJson = File.ReadAllText("BulkSatDownloadInput.json"); var fieldsInputJson = File.ReadAllText("BulkSatDownloadInput.json");
List<BulkSatDownloadInput> fieldsInputs2 = JsonConvert.DeserializeObject<List<BulkSatDownloadInput>>(fieldsInputJson); bulkSatDownloadInputListCsv = JsonConvert.DeserializeObject<List<BulkSatDownloadInput>>(fieldsInputJson);
for (int i = 0; i < bulkSatDownloadInputListCsv.Count; i++)
// Now for each input download all images. Use fieldsInputs or fieldsInputs2 for testing
foreach (var input in fieldsInputs2)
{ {
downloadFolder = bulkSatDownloadInputListCsv[i].downloadFolder;
fileNameStats = Path.Combine(downloadFolder, bulkSatDownloadInputListCsv[i].fileNameStats);
if (!Directory.Exists(downloadFolder))
Directory.CreateDirectory(downloadFolder);
bulkSatDownloadInputListCsv[i].fileNameStats = fileNameStats;
// Header same as in GeneralService.DownloadSatelliteStats
// Delete fileNameStats if existing. Create a new file. Add a header to csv file
File.Delete(fileNameStats);
File.AppendAllText(fileNameStats, headerLineStats);
}
// Now choose which list you want to use
bulkSatDownloadInputList = bulkSatDownloadInputListDB; //bulkSatDownloadInputListDB; //bulkSatDownloadInputListCsv;
// Whichever option (database or json/csv), continue here
// Delete the settingsfile
File.Delete(settingsfile);
// For each input download all images. Keep track to time, important when doing bulk downloads
var watch = System.Diagnostics.Stopwatch.StartNew();
TimeSpan tsSofar = new TimeSpan();
TimeSpan tsRemaining;
TimeSpan tsTotalEstimated;
for (int i = 0; i < bulkSatDownloadInputList.Count; i++)
{
watch.Restart();
bulkSatDownloadInput = bulkSatDownloadInputList[i];
if (string.IsNullOrEmpty(bulkSatDownloadInput.fileNameStats) == false) _logger.LogInformation(string.Format($"// FarmmapsBulkSatDownload: Downloading stats for field {i+1} out of {bulkSatDownloadInputList.Count} to single csv file {bulkSatDownloadInput.fileNameStats}"));
if (bulkSatDownloadInput.database != null) _logger.LogInformation(string.Format($"// FarmmapsBulkSatDownload: Downloading stats for field {i+1} out of {bulkSatDownloadInputList.Count} to database {bulkSatDownloadInput.schemaname}.{bulkSatDownloadInput.satelllitetable}"));
try try
{ {
await Process(roots, input); await Process(roots, bulkSatDownloadInput);
} }
catch (Exception ex) catch (Exception ex)
{ {
_logger.LogError(ex.Message); _logger.LogError(ex.Message);
} }
watch.Stop();
tsSofar = tsSofar + watch.Elapsed;
tsTotalEstimated = tsSofar / (i + 1) * bulkSatDownloadInputList.Count;
tsRemaining = tsTotalEstimated - tsSofar;
_logger.LogInformation(string.Format($"// Time (hh:mm:ss): this field: {strTime(watch.Elapsed)}. Sofar: {strTime(tsSofar)}. Total: {strTime(tsTotalEstimated)}. Remaining: {strTime(tsRemaining)}"));
} }
string strExeFilePath = System.Reflection.Assembly.GetExecutingAssembly().Location;
string strWorkPath = Path.GetDirectoryName(strExeFilePath);
_logger.LogInformation(string.Format($"// FarmmapsBulkSatDownload:"));
_logger.LogInformation(string.Format($"// FarmmapsBulkSatDownload: Done! List of all downloaded cropfieldItems stored in {Path.Combine(strWorkPath,settingsfile)}"));
_logger.LogInformation(string.Format($"// FarmmapsBulkSatDownload: If you plan to rerun certain fields then adding cropfielditemcode to your input can greatly speed up your application!"));
} }
private async Task Process(List<UserRoot> roots, BulkSatDownloadInput input) private async Task Process(List<UserRoot> roots, BulkSatDownloadInput input)
{ {
string DownloadFolder = input.DownloadFolder; string cropfielditemcode;
if (!Directory.Exists(DownloadFolder)) Item cropfieldItem;
Directory.CreateDirectory(DownloadFolder); bool satelliteItemsAvailable;
bool statsAvailable;
DateTime dtSatelliteDate;
string strSatelliteDate;
List<Item> satelliteItemsCropYear;
GroenmonitorTable groenmonitorTable = new GroenmonitorTable();
List<SatelliteStatistics> listSatelliteStatistics;
SatelliteStatistics satelliteStatistics_wdvi;
SatelliteStatistics satelliteStatistics_ndvi;
int cntDatesDownloaded;
// !!specify if you are using an already created cropfield: string fieldName = input.fieldName;
bool useCreatedCropfield = input.UseCreatedCropfield; int cropYear = input.cropYear;
string settingsfile = $"Settings_BulkSatDownloadApplication.json"; List<string> satelliteBands = input.SatelliteBands;
string downloadFolder = input.downloadFolder;
string fileNameStats = input.fileNameStats;
DB database = input.database;
string schemaname = input.schemaname;
string cropfieldtable = input.cropfieldtable;
string satelllitetable = input.satelllitetable;
DateTime lastDownloadedSatelliteDate = input.lastdownloadedimagedate;
cropfielditemcode = input.cropfielditemcode;
string insertSql = InsertSQLfromClass(schemaname, satelllitetable);
LoadSettings(settingsfile); LoadSettings(settingsfile);
@ -151,61 +217,195 @@ LIMIT 10;", schemaname, parceltablename, groenmonitorlatestviewname); //LIMIT
return; return;
} }
// Use already created cropfield or create new one if (string.IsNullOrEmpty(cropfielditemcode))
Item cropfieldItem;
if (useCreatedCropfield == false || string.IsNullOrEmpty(_settings.CropfieldItemCode))
{ {
_logger.LogInformation(string.Format($"Creating cropfield for a field in the year {input.cropYear}")); _logger.LogInformation(string.Format($"Creating cropfield '{fieldName}' in the year {cropYear}"));
//string geomjson = input.GeometryJson.ToString(Formatting.None); //"{\"type\":\"Polygon\",\"coordinates\":[[[4.960707146896585,52.80058366970849],[4.960645975538824,52.80047021761092],[4.962140695752897,52.7991771471948],[4.967523821195745,52.80150240004121],[4.966336768950911,52.80254373587981],[4.96171188076433,52.80100999685643],[4.960707146896585,52.80058366970849]]]}"
string geomjson = input.GeometryJson.ToString(Formatting.None);
cropfieldItem = await _generalService.CreateCropfieldItemAsync(myDriveRoot.Code, cropfieldItem = await _generalService.CreateCropfieldItemAsync(myDriveRoot.Code,
$"DataCropfield {input.cropName}", input.cropYear, input.GeometryJson.ToString(Formatting.None)); $"DataCropfield {fieldName}", cropYear, input.GeometryJson.ToString(Formatting.None));
_settings.CropfieldItemCode = cropfieldItem.Code; cropfielditemcode = cropfieldItem.Code;
SaveSettings(settingsfile); // If working with a database, add this cropfieldItem.Code to the database so that next case same cropField is requested, will be faster
if (database != null)
{
// add this CropfieldItemCode to the parceltable
using (NpgsqlConnection connection = new NpgsqlConnection(database.GetConnectionString()))
{
connection.Open();
NpgsqlCommand updateCmd = connection.CreateCommand();
string updateSql = string.Format($"UPDATE {schemaname}.{cropfieldtable} SET cropfielditemcode = '{cropfieldItem.Code}' WHERE arbid = {input.fieldID};");
updateCmd.CommandText = updateSql;
//Console.WriteLine(insertCmd.CommandText);
int r = updateCmd.ExecuteNonQuery();
if (r != 1)
throw new Exception("// FarmmapsBulkSatDownload: Insert Failed");
connection.Close();
}
_logger.LogInformation($"// FarmmapsBulkSatDownload: Added cropfieldItem.Code '{cropfieldItem.Code}' for parcelid {input.fieldID} to {schemaname}.{cropfieldtable} ");
}
} }
else else
{ {
_logger.LogInformation("Cropfield already exists, trying to get it"); // WOULD IT BE POSSIBLE TO GET AVAILABLE Item MEMBER VALUES FOR A GIVEN cropfielditemcode?
cropfieldItem = await _farmmapsApiService.GetItemAsync(_settings.CropfieldItemCode); cropfieldItem = new Item();
cropfieldItem.Code = cropfielditemcode;
cropfieldItem.Name = "DataCropfield " + fieldName;
_logger.LogInformation($"// FarmmapsBulkSatDownload: CropfieldItem.Code for parcelid {input.fieldID} already there in {schemaname}.{cropfieldtable}: '{cropfieldItem.Code}'");
} }
_settings.cropFieldItems.Add(cropfieldItem);
SaveSettings(settingsfile);
// check if satellite task not yet done, do here and save taskcode //Create satelliteTaskCode & save satelliteTaskCode.Code to settingsfile for retracing last call (can be useful if failed)
if (useCreatedCropfield == false || string.IsNullOrEmpty(_settings.SatelliteTaskCode)) _logger.LogInformation(string.Format($"Running RunSatelliteTask for cropfieldItem '{cropfielditemcode}' and saving settings to {settingsfile}"));
var satelliteTaskCode = await _generalService.RunSatelliteTask(cropfieldItem);
// POSSIBLE & DESIRABLE TO ALSO LOG satelliteTaskCode?
// SaveSettings(settingsfile);
// Getting satellite items
_logger.LogInformation(string.Format($"Running FindSatelliteItemsCropYear for cropfieldItem.Code '{cropfieldItem.Code}', SatelliteTaskCode '{satelliteTaskCode}'"));
satelliteItemsCropYear = await _generalService.FindSatelliteItems(cropfieldItem, satelliteTaskCode);
// Checking if satellite items found
satelliteItemsAvailable = true;
if (satelliteItemsCropYear == null)
{ {
var satelliteTaskCode = await _generalService.RunSatelliteTask(cropfieldItem); satelliteItemsAvailable = false;
_settings.SatelliteTaskCode = satelliteTaskCode; _logger.LogInformation($"No satellite tiffs found for fieldName '{fieldName}', cropYear {cropYear}");
SaveSettings(settingsfile); }
else
{
if (satelliteItemsCropYear.Count == 0)
{
satelliteItemsAvailable = false;
_logger.LogInformation($"No satellite tiffs found for fieldName '{fieldName}', cropYear {cropYear}");
}
} }
// Now get the tiffs // Sort the list by date
List<Item> satelliteTiffs = await _generalService.FindSatelliteItemsAll(cropfieldItem, _settings.SatelliteTaskCode); if (satelliteItemsAvailable) satelliteItemsCropYear = satelliteItemsCropYear.OrderBy(x => x.DataDate).ToList();
_logger.LogInformation(string.Format($"Downloading {satelliteTiffs.Count} geotiffs for a field in year {input.cropYear} ..."));
foreach (Item satalliteItem in satelliteTiffs) {
// download the geotiff. Returns a zip file with always these three files:
// data.dat.aux.xml
// thumbnail.jpg
// wenr.tif. Contains 5 layers: (1) ndvi, (2) wdvi, (3) Red, (4) Green and (5) Blue
DateTime SatelliteImageDate = (DateTime)satalliteItem.DataDate;
string SatelliteDate = SatelliteImageDate.ToString("yyyyMMdd");
string fileName = string.Format($"{input.cropName}_{input.fieldName}_{SatelliteDate}"); // assuming the fieldName is good enough for an ID. One might add here _{input.fieldID} or _{input.cropName}
string fileNameZip = string.Format($"{fileName}.zip");
string fileNameGeotiff = string.Format($"{fileName}.tif");
await _farmmapsApiService.DownloadItemAsync(satalliteItem.Code, Path.Combine(DownloadFolder, fileNameZip));
// Extract the file "wenr.tif" from zip, rename it, delete "wenr.tif". overwriteFiles = true // Download statistics to a single csv file
ZipFile.ExtractToDirectory(Path.Combine(DownloadFolder, fileNameZip), DownloadFolder, true); if (satelliteItemsAvailable && downloadFolder != null && fileNameStats != null)
File.Delete(Path.Combine(DownloadFolder, fileNameGeotiff)); // Delete the fileNameGeotiff file if exists {
File.Move(Path.Combine(DownloadFolder, "wenr.tif"), Path.Combine(DownloadFolder, fileNameGeotiff)); // Rename the oldFileName into newFileName // Write statistics for all images for all fieldNane and cropYear to a single csv file, fileNameStats
_logger.LogInformation($"Downloading stats for field '{fieldName}' in cropyear {cropYear} to {fileNameStats}");
// Cleanup string downloadedStats = await _generalService.DownloadSatelliteStats(satelliteItemsCropYear, fieldName, satelliteBands, downloadFolder);
string[] filesToDelete = new string[] { fileNameZip, "wenr.tif", "thumbnail.jpg", "data.dat.aux.xml" }; // Add contents of this csv file to thee single large csv file
foreach (string f in filesToDelete) var retainedLines = File.ReadAllLines(downloadedStats).Skip(1);
File.AppendAllLines(fileNameStats, retainedLines);
File.Delete(downloadedStats);
// Optionally, also download the zipped tiffs. This can be a lot of files and increase runtime
if (false)
{ {
File.Delete(Path.Combine(DownloadFolder, f)); foreach (Item selectedSatelliteItem in satelliteItemsCropYear)
{
// download the geotiffs. Returns a zip file with always these three files:
// data.dat.aux.xml
// thumbnail.jpg
// wenr.tif. Contains 5 layers: (1) ndvi, (2) wdvi, (3) Red, (4) Green and (5) Blue
var SatelliteDate = selectedSatelliteItem.DataDate.Value.ToString("yyyyMMdd");
_logger.LogInformation($"Downloading geotiff file for field {fieldName}, date {SatelliteDate}");
string fileName = string.Format($"satelliteGeotiff_{fieldName}_{SatelliteDate}"); // no need to add satelliteBand in the name because the tif contains all bands
string fileNameZip = string.Format($"{fileName}.zip");
string fileNameGeotiff = string.Format($"{fileName}.tif");
await _farmmapsApiService.DownloadItemAsync(selectedSatelliteItem.Code, Path.Combine(downloadFolder, fileNameZip));
if (false)
{
// Extract the file "wenr.tif" from zip, rename it to fileNameGeotiff
ZipFile.ExtractToDirectory(Path.Combine(downloadFolder, fileNameZip), downloadFolder, true);
File.Delete(Path.Combine(downloadFolder, fileNameGeotiff)); // Delete the fileNameGeotiff file if exists
File.Move(Path.Combine(downloadFolder, "wenr.tif"), Path.Combine(downloadFolder, fileNameGeotiff)); // Rename the oldFileName into newFileName
// Cleanup
string[] filesToDelete = new string[] { fileNameZip, "wenr.tif", "thumbnail.jpg", "data.dat.aux.xml" };
foreach (string f in filesToDelete)
{
File.Delete(Path.Combine(downloadFolder, f));
}
}
}
}
}
// Download statistics to database
if (satelliteItemsAvailable && database != null)
{
// Tailormade code for writing to database
// No unnecessary intermediate step here of writing to csv and getting stats for all images in the crop year.
// Efficient is to check if there is any image for which stats are to be added to database and add only add these new stats for not yet archived dates directly to database
// A full check of downloaded dates versus available dates in the database is not made here.
// We assume only new images will be added (i.e. for later dates), assuming no historical images are added in groenmonitor (check!).
// And we assume farmMaps always nicely generates statistics, no hick-ups
// Under this assumptions, we only need to compare with the lastDownloadedSatelliteDate from the database
cntDatesDownloaded = 0;
foreach (Item satelliteItem in satelliteItemsCropYear)
{
dtSatelliteDate = satelliteItem.DataDate.Value;
strSatelliteDate = dtSatelliteDate.ToString("yyyy-MM-dd");
listSatelliteStatistics = await _generalService.ListSatelliteStatistics(satelliteItem, satelliteBands, fieldName);
statsAvailable = true;
if (listSatelliteStatistics == null)
{
statsAvailable = false;
_logger.LogWarning($"No stats found for satellite, fielName '{fieldName}', date '{strSatelliteDate}'");
}
else
{
if (listSatelliteStatistics.Count == 0)
{
statsAvailable = false;
_logger.LogWarning($"No stats found for satellite, fielName '{fieldName}', date '{strSatelliteDate}'");
}
}
if (statsAvailable)
{
if (dtSatelliteDate <= input.lastdownloadedimagedate)
{
_logger.LogInformation($"// Stats for parcelid {input.fieldID}, date '{strSatelliteDate}' already there in {schemaname}.{satelllitetable}");
}
else
{
cntDatesDownloaded++;
// Map satelliteStatistics to groenmonitorTable
satelliteStatistics_wdvi = listSatelliteStatistics.SingleOrDefault(p => p.satelliteBand == "wdvi");
satelliteStatistics_ndvi = listSatelliteStatistics.SingleOrDefault(p => p.satelliteBand == "ndvi");
groenmonitorTable.parcelid = input.fieldID;
groenmonitorTable.date = strSatelliteDate;
groenmonitorTable.source = "akkerwebwenr"; // Like this so SQL in bigdata.groenmonitorlatest_flowerbulbs works properly
groenmonitorTable.wdvi_pixelcount = satelliteStatistics_wdvi.populationCount; //count of pixels with data
groenmonitorTable.wdvi_max = satelliteStatistics_wdvi.max;
groenmonitorTable.wdvi_mean = satelliteStatistics_wdvi.mean;
groenmonitorTable.wdvi_min = satelliteStatistics_wdvi.min;
groenmonitorTable.wdvi_stdev = satelliteStatistics_wdvi.stddev;
groenmonitorTable.wdvi_median = satelliteStatistics_wdvi.median;
groenmonitorTable.wdvi_p90 = -99; // Example of a statistics (90% not included in satelliteStatistics
groenmonitorTable.ndvi_pixelcount = satelliteStatistics_ndvi.populationCount; //count of pixels with data
groenmonitorTable.ndvi_max = satelliteStatistics_ndvi.max;
groenmonitorTable.ndvi_mean = satelliteStatistics_ndvi.mean;
groenmonitorTable.ndvi_min = satelliteStatistics_ndvi.min;
groenmonitorTable.ndvi_stdev = satelliteStatistics_ndvi.stddev;
groenmonitorTable.ndvi_median = satelliteStatistics_ndvi.median;
groenmonitorTable.ndvi_p90 = -99; // Example of a statistics (90% not included in satelliteStatistics
// fill the insertSql query with fieldValues from the groenmonitorTable, then run the query
using (NpgsqlConnection connection = new NpgsqlConnection(database.GetConnectionString()))
{
connection.Open();
NpgsqlCommand insertCmd = connection.CreateCommand();
object[] fieldValues = groenmonitorTable.GetType()
.GetFields()
.Select(field => field.GetValue(groenmonitorTable))
.ToArray();
insertCmd.CommandText = string.Format(insertSql, fieldValues);
//Console.WriteLine(insertCmd.CommandText);
int r = insertCmd.ExecuteNonQuery();
if (r != 1)
throw new Exception("// FarmmapsBulkSatDownload: Insert Failed");
connection.Close();
}
_logger.LogInformation($"// Added stats to {schemaname}.{satelllitetable} for parcelid {input.fieldID}, date '{strSatelliteDate}'. cntDatesDownloaded: {cntDatesDownloaded}");
}
}
} }
} }
} }
// Functions to save previously created cropfields // Functions to save previously created cropfields
private void LoadSettings(string file) private void LoadSettings(string file)
{ {
@ -216,7 +416,9 @@ LIMIT 10;", schemaname, parceltablename, groenmonitorlatestviewname); //LIMIT
} }
else else
{ {
_settings = new Settings(); Settings settings = new Settings();
settings.cropFieldItems = new List<Item>();
_settings = settings;
} }
} }
@ -228,14 +430,67 @@ LIMIT 10;", schemaname, parceltablename, groenmonitorlatestviewname); //LIMIT
var json = JsonConvert.SerializeObject(_settings); var json = JsonConvert.SerializeObject(_settings);
File.WriteAllText(file, json); File.WriteAllText(file, json);
} }
private void SaveInfo(string file) private string strTime(TimeSpan ts)
{ {
if (_settings == null) return String.Format("{0:00}:{1:00}:{2:00}", ts.Hours, ts.Minutes, ts.Seconds);
return; }
private string InsertSQLfromClass(string schemaname, string groenmonitortablename)
var json = JsonConvert.SerializeObject(_settings); {
File.WriteAllText(file, json); // Generates an INSERT query for GroenmonitorTable.
// When writing to a different table structure, just make a new class for the statellite statistics table, similar to GroenmonitorTable.cs
// @"
//INSERT INTO bigdata.groenmonitor (parcelid,date,wdvi_pixelcount,wdvi_max,wdvi_mean,wdvi_min,wdvi_stdev,wdvi_median,wdvi_p90,ndvi_pixelcount,ndvi_max,ndvi_mean,ndvi_min,ndvi_stdev,ndvi_median,ndvi_p90)
//VALUES({0},'{1}',{2},{3},{4},{5},{6},{7},{8},{9},{10},{11},{12},{13},{14},{15})
//ON CONFLICT (parcelid, date)
//DO UPDATE SET parcelid={0},date='{1}',wdvi_pixelcount={2},wdvi_max={3},wdvi_mean={4},wdvi_min={5},wdvi_stdev={6},wdvi_median={7},wdvi_p90={8},ndvi_pixelcount={9},ndvi_max={10},ndvi_mean={11},ndvi_min={12},ndvi_stdev={13},ndvi_median={14},ndvi_p90={15}
//";
string fieldName;
var fieldNames = typeof(GroenmonitorTable).GetFields()
.Select(field => field.Name)
.ToList();
StringBuilder sbInsertSql = new StringBuilder();
string insertSql;
StringBuilder insertSql1 = new StringBuilder();
insertSql1.Append(@"INSERT INTO " + schemaname + "." + groenmonitortablename + " (");
StringBuilder insertSql2 = new StringBuilder();
insertSql2.Append("VALUES(");
StringBuilder insertSql3 = new StringBuilder();
insertSql3.Append("ON CONFLICT(parcelid, date, source)");
StringBuilder insertSql4 = new StringBuilder();
insertSql4.Append("DO UPDATE SET ");
for (int i = 0; i < fieldNames.Count; i++)
{
fieldName = fieldNames[i];
insertSql1.Append(fieldName);
if (fieldName == "date" | fieldName == "source")
{
//add extra quotes ' for fields in the Postgress table for which we know they are date or varchar
insertSql2.Append("'{" + i + "}'");
insertSql4.Append(fieldName + "='{" + i + "}'");
}
else
{
insertSql2.Append("{" + i + "}");
insertSql4.Append(fieldName + "={" + i + "}");
}
if (i < fieldNames.Count - 1)
{
insertSql1.Append(",");
insertSql2.Append(",");
insertSql4.Append(",");
}
else
{
insertSql1.Append(")");
insertSql2.Append(")");
}
}
sbInsertSql.AppendLine(insertSql1.ToString());
sbInsertSql.AppendLine(insertSql2.ToString());
sbInsertSql.AppendLine(insertSql3.ToString());
sbInsertSql.AppendLine(insertSql4.ToString());
insertSql = sbInsertSql.ToString();
return insertSql;
} }
} }

285
FarmmapsBulkSatDownload/BulkSatDownloadInput.json
View File

@ -1,12 +1,63 @@
[ [
{ {
"UseCreatedCropfield": false, "fieldName": "5641", // FarmMaps minimum needs are: fieldName, cropYear & geometryJson
"fieldName": "fld5641", // FarmMaps won't do without a fieldName "cropYear": 1999, //For testing a year for which we know no satellite data available, program shouldn't crash
"DownloadFolder": "C:\\workdir\\groenmonitor\\", //"C:\\workdir\\groenmonitor\\"; // "Downloads" -> if you just put "Downloads" the program will download to somewhere in ..\FarmMapsApiClient_WURtest\FarmmapsDataDownload\bin\Debug\netcoreapp3.1\Downloads\ "fieldID": 5641,
"cropName": "wheat", "SatelliteBands": [ "wdvi", "ndvi" ], // ["ndvi"] or ["wdvi"] or both: [ "wdvi", "ndvi" ]
"lastdownloadedimagedate": "1999-01-01", //downloads images from this date till end of the year
"geometryJson": {
"type": "Polygon",
"coordinates": [
[
[ 3.37837807779104, 51.3231095796538 ],
[ 3.38065689232502, 51.3212527499355 ],
[ 3.38022924592256, 51.3210683536359 ],
[ 3.37980548452565, 51.3208801127141 ],
[ 3.37959556105776, 51.3207540143696 ],
[ 3.3793691292654, 51.3205959677371 ],
[ 3.37822219207335, 51.3215667913007 ],
[ 3.37816999925795, 51.3216109809456 ],
[ 3.37646704574705, 51.3208025481261 ],
[ 3.37646695791282, 51.3208025061493 ],
[ 3.37608401443192, 51.3206231652693 ],
[ 3.37607169507628, 51.3206173959751 ],
[ 3.37606021048754, 51.320612017601 ],
[ 3.37582728410659, 51.3205029306946 ],
[ 3.37580409779263, 51.3206502985963 ],
[ 3.37575872019649, 51.3207993094705 ],
[ 3.37575476634361, 51.3208122883487 ],
[ 3.37571181656268, 51.3208797459348 ],
[ 3.3756624532907, 51.3209415238446 ],
[ 3.37557609963811, 51.3210110142077 ],
[ 3.37541089899821, 51.3211055871218 ],
[ 3.37477516102591, 51.3214102985009 ],
[ 3.37473173914127, 51.3214311108204 ],
[ 3.37455904622072, 51.3215138815012 ],
[ 3.37415098054777, 51.3217199232877 ],
[ 3.37313700916272, 51.3222422862785 ],
[ 3.37748824689601, 51.3242852920348 ],
[ 3.37749760805371, 51.3242713084009 ],
[ 3.37811903757028, 51.3233437635596 ],
[ 3.37818758851947, 51.3232647797363 ],
[ 3.37823803668144, 51.3232236798646 ],
[ 3.37837807779104, 51.3231095796538 ]
]
]
},
"cropfielditemcode": null, //BulkSatDownloadApplication returns a Settings.json file with cropfielditems.Code. After first run, you can copy-paste that here to speed up
"downloadFolder": "C:\\workdir\\groenmonitor\\",
"fileNameStats": "BulkSatDownload.csv",
"database": null,
"schemaname": null,
"cropfieldtable": null,
"satelllitetable": null
},
{
"fieldName": "5641", // FarmMaps minimum needs are: fieldName, cropYear & geometryJson
"cropYear": 2021, "cropYear": 2021,
"fieldID": 5641, "fieldID": 5641,
"lastdownloadedimagedate": "2021-01-01", "SatelliteBands": [ "wdvi", "ndvi" ], // ["ndvi"] or ["wdvi"] or both: [ "wdvi", "ndvi" ]
"lastdownloadedimagedate": "2021-01-01", //downloads images from this date till end of the year
"geometryJson": { "geometryJson": {
"type": "Polygon", "type": "Polygon",
"coordinates": [ "coordinates": [
@ -45,65 +96,22 @@
[ 3.37837807779104, 51.3231095796538 ] [ 3.37837807779104, 51.3231095796538 ]
] ]
] ]
} },
}, "cropfielditemcode": null, //BulkSatDownloadApplication returns a Settings.json file with cropfielditems.Code. After first run, you can copy-paste that here to speed up
"downloadFolder": "C:\\workdir\\groenmonitor\\",
"fileNameStats": "BulkSatDownload.csv",
"database": null,
"schemaname": null,
"cropfieldtable": null,
"satelllitetable": null
}
,
{ {
"UseCreatedCropfield": false, "fieldName": "5641", // FarmMaps minimum needs are: fieldName, cropYear & geometryJson
"fieldName": "fld5641", // FarmMaps won't do without a fieldName
"DownloadFolder": "C:\\workdir\\groenmonitor\\", //"C:\\workdir\\groenmonitor\\"; // "Downloads" -> if you just put "Downloads" the program will download to somewhere in ..\FarmMapsApiClient_WURtest\FarmmapsDataDownload\bin\Debug\netcoreapp3.1\Downloads\
"cropName": "wheat",
"cropYear": 2020, "cropYear": 2020,
"fieldID": 5641, "fieldID": 5641,
"lastdownloadedimagedate": "2020-01-01", "SatelliteBands": [ "wdvi", "ndvi" ], // ["ndvi"] or ["wdvi"] or both: [ "wdvi", "ndvi" ]
"geometryJson": { "lastdownloadedimagedate": "2020-01-01", //downloads images from this date till end of the year
"type": "Polygon",
"coordinates": [
[
[ 3.37837807779104, 51.3231095796538 ],
[ 3.38065689232502, 51.3212527499355 ],
[ 3.38022924592256, 51.3210683536359 ],
[ 3.37980548452565, 51.3208801127141 ],
[ 3.37959556105776, 51.3207540143696 ],
[ 3.3793691292654, 51.3205959677371 ],
[ 3.37822219207335, 51.3215667913007 ],
[ 3.37816999925795, 51.3216109809456 ],
[ 3.37646704574705, 51.3208025481261 ],
[ 3.37646695791282, 51.3208025061493 ],
[ 3.37608401443192, 51.3206231652693 ],
[ 3.37607169507628, 51.3206173959751 ],
[ 3.37606021048754, 51.320612017601 ],
[ 3.37582728410659, 51.3205029306946 ],
[ 3.37580409779263, 51.3206502985963 ],
[ 3.37575872019649, 51.3207993094705 ],
[ 3.37575476634361, 51.3208122883487 ],
[ 3.37571181656268, 51.3208797459348 ],
[ 3.3756624532907, 51.3209415238446 ],
[ 3.37557609963811, 51.3210110142077 ],
[ 3.37541089899821, 51.3211055871218 ],
[ 3.37477516102591, 51.3214102985009 ],
[ 3.37473173914127, 51.3214311108204 ],
[ 3.37455904622072, 51.3215138815012 ],
[ 3.37415098054777, 51.3217199232877 ],
[ 3.37313700916272, 51.3222422862785 ],
[ 3.37748824689601, 51.3242852920348 ],
[ 3.37749760805371, 51.3242713084009 ],
[ 3.37811903757028, 51.3233437635596 ],
[ 3.37818758851947, 51.3232647797363 ],
[ 3.37823803668144, 51.3232236798646 ],
[ 3.37837807779104, 51.3231095796538 ]
]
]
}
},
{
"UseCreatedCropfield": false,
"fieldName": "fld5641", // FarmMaps won't do without a fieldName
"DownloadFolder": "C:\\workdir\\groenmonitor\\", //"C:\\workdir\\groenmonitor\\"; // "Downloads" -> if you just put "Downloads" the program will download to somewhere in ..\FarmMapsApiClient_WURtest\FarmmapsDataDownload\bin\Debug\netcoreapp3.1\Downloads\
"cropName": "wheat",
"cropYear": 2019,
"fieldID": 5641,
"lastdownloadedimagedate": "2019-01-01",
"geometryJson": { "geometryJson": {
"type": "Polygon", "type": "Polygon",
"coordinates": [ "coordinates": [
@ -142,152 +150,13 @@
[ 3.37837807779104, 51.3231095796538 ] [ 3.37837807779104, 51.3231095796538 ]
] ]
] ]
} },
}, "cropfielditemcode": null, //BulkSatDownloadApplication returns a Settings.json file with cropfielditems.Code. After first run, you can copy-paste that here to speed up
{ "downloadFolder": "C:\\workdir\\groenmonitor\\",
"UseCreatedCropfield": false, "fileNameStats": "BulkSatDownload.csv",
"fieldName": "fld5641", // FarmMaps won't do without a fieldName "database": null,
"DownloadFolder": "C:\\workdir\\groenmonitor\\", //"C:\\workdir\\groenmonitor\\"; // "Downloads" -> if you just put "Downloads" the program will download to somewhere in ..\FarmMapsApiClient_WURtest\FarmmapsDataDownload\bin\Debug\netcoreapp3.1\Downloads\ "schemaname": null,
"cropName": "wheat", "cropfieldtable": null,
"cropYear": 2018, "satelllitetable": null
"fieldID": 5641,
"lastdownloadedimagedate": "2018-01-01",
"geometryJson": {
"type": "Polygon",
"coordinates": [
[
[ 3.37837807779104, 51.3231095796538 ],
[ 3.38065689232502, 51.3212527499355 ],
[ 3.38022924592256, 51.3210683536359 ],
[ 3.37980548452565, 51.3208801127141 ],
[ 3.37959556105776, 51.3207540143696 ],
[ 3.3793691292654, 51.3205959677371 ],
[ 3.37822219207335, 51.3215667913007 ],
[ 3.37816999925795, 51.3216109809456 ],
[ 3.37646704574705, 51.3208025481261 ],
[ 3.37646695791282, 51.3208025061493 ],
[ 3.37608401443192, 51.3206231652693 ],
[ 3.37607169507628, 51.3206173959751 ],
[ 3.37606021048754, 51.320612017601 ],
[ 3.37582728410659, 51.3205029306946 ],
[ 3.37580409779263, 51.3206502985963 ],
[ 3.37575872019649, 51.3207993094705 ],
[ 3.37575476634361, 51.3208122883487 ],
[ 3.37571181656268, 51.3208797459348 ],
[ 3.3756624532907, 51.3209415238446 ],
[ 3.37557609963811, 51.3210110142077 ],
[ 3.37541089899821, 51.3211055871218 ],
[ 3.37477516102591, 51.3214102985009 ],
[ 3.37473173914127, 51.3214311108204 ],
[ 3.37455904622072, 51.3215138815012 ],
[ 3.37415098054777, 51.3217199232877 ],
[ 3.37313700916272, 51.3222422862785 ],
[ 3.37748824689601, 51.3242852920348 ],
[ 3.37749760805371, 51.3242713084009 ],
[ 3.37811903757028, 51.3233437635596 ],
[ 3.37818758851947, 51.3232647797363 ],
[ 3.37823803668144, 51.3232236798646 ],
[ 3.37837807779104, 51.3231095796538 ]
]
]
}
},
{
"UseCreatedCropfield": false,
"fieldName": "fld5641", // FarmMaps won't do without a fieldName
"DownloadFolder": "C:\\workdir\\groenmonitor\\", //"C:\\workdir\\groenmonitor\\"; // "Downloads" -> if you just put "Downloads" the program will download to somewhere in ..\FarmMapsApiClient_WURtest\FarmmapsDataDownload\bin\Debug\netcoreapp3.1\Downloads\
"cropName": "wheat",
"cropYear": 2017,
"fieldID": 5641,
"lastdownloadedimagedate": "2017-01-01",
"geometryJson": {
"type": "Polygon",
"coordinates": [
[
[ 3.37837807779104, 51.3231095796538 ],
[ 3.38065689232502, 51.3212527499355 ],
[ 3.38022924592256, 51.3210683536359 ],
[ 3.37980548452565, 51.3208801127141 ],
[ 3.37959556105776, 51.3207540143696 ],
[ 3.3793691292654, 51.3205959677371 ],
[ 3.37822219207335, 51.3215667913007 ],
[ 3.37816999925795, 51.3216109809456 ],
[ 3.37646704574705, 51.3208025481261 ],
[ 3.37646695791282, 51.3208025061493 ],
[ 3.37608401443192, 51.3206231652693 ],
[ 3.37607169507628, 51.3206173959751 ],
[ 3.37606021048754, 51.320612017601 ],
[ 3.37582728410659, 51.3205029306946 ],
[ 3.37580409779263, 51.3206502985963 ],
[ 3.37575872019649, 51.3207993094705 ],
[ 3.37575476634361, 51.3208122883487 ],
[ 3.37571181656268, 51.3208797459348 ],
[ 3.3756624532907, 51.3209415238446 ],
[ 3.37557609963811, 51.3210110142077 ],
[ 3.37541089899821, 51.3211055871218 ],
[ 3.37477516102591, 51.3214102985009 ],
[ 3.37473173914127, 51.3214311108204 ],
[ 3.37455904622072, 51.3215138815012 ],
[ 3.37415098054777, 51.3217199232877 ],
[ 3.37313700916272, 51.3222422862785 ],
[ 3.37748824689601, 51.3242852920348 ],
[ 3.37749760805371, 51.3242713084009 ],
[ 3.37811903757028, 51.3233437635596 ],
[ 3.37818758851947, 51.3232647797363 ],
[ 3.37823803668144, 51.3232236798646 ],
[ 3.37837807779104, 51.3231095796538 ]
]
]
}
},
{
"UseCreatedCropfield": false,
"fieldName": "fld5641", // FarmMaps won't do without a fieldName
"DownloadFolder": "C:\\workdir\\groenmonitor\\", //"C:\\workdir\\groenmonitor\\"; // "Downloads" -> if you just put "Downloads" the program will download to somewhere in ..\FarmMapsApiClient_WURtest\FarmmapsDataDownload\bin\Debug\netcoreapp3.1\Downloads\
"cropName": "wheat",
"cropYear": 2016,
"fieldID": 5641,
"lastdownloadedimagedate": "2016-01-01",
"geometryJson": {
"type": "Polygon",
"coordinates": [
[
[ 3.37837807779104, 51.3231095796538 ],
[ 3.38065689232502, 51.3212527499355 ],
[ 3.38022924592256, 51.3210683536359 ],
[ 3.37980548452565, 51.3208801127141 ],
[ 3.37959556105776, 51.3207540143696 ],
[ 3.3793691292654, 51.3205959677371 ],
[ 3.37822219207335, 51.3215667913007 ],
[ 3.37816999925795, 51.3216109809456 ],
[ 3.37646704574705, 51.3208025481261 ],
[ 3.37646695791282, 51.3208025061493 ],
[ 3.37608401443192, 51.3206231652693 ],
[ 3.37607169507628, 51.3206173959751 ],
[ 3.37606021048754, 51.320612017601 ],
[ 3.37582728410659, 51.3205029306946 ],
[ 3.37580409779263, 51.3206502985963 ],
[ 3.37575872019649, 51.3207993094705 ],
[ 3.37575476634361, 51.3208122883487 ],
[ 3.37571181656268, 51.3208797459348 ],
[ 3.3756624532907, 51.3209415238446 ],
[ 3.37557609963811, 51.3210110142077 ],
[ 3.37541089899821, 51.3211055871218 ],
[ 3.37477516102591, 51.3214102985009 ],
[ 3.37473173914127, 51.3214311108204 ],
[ 3.37455904622072, 51.3215138815012 ],
[ 3.37415098054777, 51.3217199232877 ],
[ 3.37313700916272, 51.3222422862785 ],
[ 3.37748824689601, 51.3242852920348 ],
[ 3.37749760805371, 51.3242713084009 ],
[ 3.37811903757028, 51.3233437635596 ],
[ 3.37818758851947, 51.3232647797363 ],
[ 3.37823803668144, 51.3232236798646 ],
[ 3.37837807779104, 51.3231095796538 ]
]
]
}
} }
] ]

2
FarmmapsBulkSatDownload/FarmmapsBulkSatDownload.csproj
View File

@ -16,7 +16,7 @@
<None Update="BulkSatDownloadInput.json"> <None Update="BulkSatDownloadInput.json">
<CopyToOutputDirectory>Always</CopyToOutputDirectory> <CopyToOutputDirectory>Always</CopyToOutputDirectory>
</None> </None>
<None Update="DBsettings.json"> <None Update="DBsettings.secrets.json">
<CopyToOutputDirectory>Always</CopyToOutputDirectory> <CopyToOutputDirectory>Always</CopyToOutputDirectory>
</None> </None>
</ItemGroup> </ItemGroup>

14
FarmmapsBulkSatDownload/Models/BulkSatDownloadInput.cs
View File

@ -1,18 +1,24 @@
using System; using System;
using System.Collections.Generic;
using Newtonsoft.Json.Linq; using Newtonsoft.Json.Linq;
namespace FarmmapsBulkSatDownload.Models namespace FarmmapsBulkSatDownload.Models
{ {
public class BulkSatDownloadInput public class BulkSatDownloadInput
{ {
public bool UseCreatedCropfield { get; set; }
public string fieldName { get; set; } public string fieldName { get; set; }
public string DownloadFolder { get; set; }
public string cropName { get; set; }
public int cropYear { get; set; } public int cropYear { get; set; }
public int fieldID { get; set; } public int fieldID { get; set; }
public List<string> SatelliteBands { get; set; }
public DateTime lastdownloadedimagedate { get; set; } public DateTime lastdownloadedimagedate { get; set; }
public JObject GeometryJson { get; set; } public JObject GeometryJson { get; set; }
//public string fieldName { get { return string.Format($"{cropName}_fld{fieldID}"); } } public string cropfielditemcode { get; set; }
public string downloadFolder { get; set; }
public string fileNameStats { get; set; }
public DB database { get; set; }
public string schemaname { get; set; }
public string cropfieldtable { get; set; }
public string satelllitetable { get; set; }
} }
} }

2
FarmmapsBulkSatDownload/Models/DB.cs
View File

@ -3,7 +3,7 @@ using System.Collections.Generic;
using System.Text; using System.Text;
using Npgsql; using Npgsql;
namespace FarmmapsApi.Services namespace FarmmapsBulkSatDownload.Models
{ {
public class DB public class DB
{ {

2
FarmmapsBulkSatDownload/Models/GroenmonitorTable.cs
View File

@ -6,7 +6,7 @@ namespace FarmmapsBulkSatDownload.Models
{ {
public class GroenmonitorTable public class GroenmonitorTable
{ {
//public string parceltablename; //PO20190605: parceltablename added as a field (=column) in the GroenmonitorTable. The GroenmonitorTable contains data from multiple parceltables //double? makes a member nullable
public int parcelid; public int parcelid;
public string date; public string date;
public string source; public string source;

11
FarmmapsBulkSatDownload/Models/Settings.cs
View File

@ -1,11 +1,10 @@
namespace FarmmapsBulkSatDownload using System.Collections.Generic;
using FarmmapsApi.Models;
namespace FarmmapsBulkSatDownload
{ {
public class Settings public class Settings
{ {
public string CropfieldItemCode { get; set; } public List<Item> cropFieldItems { get; set; }
public string SatelliteTaskCode { get; set; }
public string VanDerSatTaskCode { get; set; }
public string WatBalTaskCode { get; set; }
} }
} }

66
FarmmapsBulkSatDownload/ShowGeotiff.r
View File

@ -1,15 +1,18 @@
# ShowGeotiff.r # ShowGeotiff.r
# I downloaded and calculated the stats for the polygon defined in C:\git\FarmMapsApiClient_WURtest\FarmmapsDataDownload\DataDownloadInput.json # I downloaded and calculated the stats for the polygon defined in C:\git\FarmMapsApiClient_WURtest\FarmmapsDataDownload\DataDownloadInput.json
# in which I set "SatelliteBand": "wdvi" and in which in the console I requested image 8 for date '2020-06-14' # in which I set "SatelliteBand": "wdvi" and in which in the console I requested the image for date '2020-09-22'
# FarmmapsBulkSatDownload generates many files. Here is what I tried when inputing the same field for a number of years using BulkSatDownloadInput.json
# see list below
library(raster) library(raster)
library(sf) library(sf)
library(rgdal) library(rgdal)
setwd("C:/workdir/groenmonitor/") setwd("C:/workdir/groenmonitor/DataDownload/")
# FarmmapsDataDownload generates two files:
# FarmmapsDataDownload and BulkSatDownload can be used to download zip files with inside two files:
fileGeotiff <- "wenr.tif" fileGeotiff <- "wenr.tif"
# fileJpg <- "thumbnail.jpg" fileJpg <- "thumbnail.jpg"
# FarmmapsBulkSatDownload generates many files. Here is what I tried when inputing the same field for a number of years using BulkSatDownloadInput.json # Here is what I tried when inputing the same field for a number of years:
# fileGeotiff <- "wheat_fld5641_20210224.tif" # 2 files for year 2021. This file is a nice example having in upperleft corner no data # fileGeotiff <- "wheat_fld5641_20210224.tif" # 2 files for year 2021. This file is a nice example having in upperleft corner no data
# fileGeotiff <- "wheat_fld5641_20210331.tif" # 2 files for year 2021 # fileGeotiff <- "wheat_fld5641_20210331.tif" # 2 files for year 2021
# fileGeotiff <- "wheat_fld5641_20200321.tif" # 14 files for year 2020, earliest # fileGeotiff <- "wheat_fld5641_20200321.tif" # 14 files for year 2020, earliest
@ -24,11 +27,11 @@ year <- substr(fileGeotiff,lenfilename-11,lenfilename-8)
imgdate <- substr(fileGeotiff,lenfilename-11,lenfilename-4) imgdate <- substr(fileGeotiff,lenfilename-11,lenfilename-4)
# The thumbnail has the polygon clipped out, has 1 layer, no crs and the mean value is not the mean wdvi we are looking for # The thumbnail has the polygon clipped out, has 1 layer, no crs and the mean value is not the mean wdvi we are looking for
# r.thumbnail <- raster(fileJpg) r.thumbnail <- raster(fileJpg)
# plot(r.thumbnail) plot(r.thumbnail)
# crs(r.thumbnail) crs(r.thumbnail)
# CRS arguments: NA #CRS arguments: NA
# cellStats(r.thumbnail,'mean') #59.91667 cellStats(r.thumbnail,'mean') #87.5128 # nonsense
stk.wenr <- stack(x=fileGeotiff) stk.wenr <- stack(x=fileGeotiff)
# plot(stk.wenr) shows 5 plots (5 bands) # plot(stk.wenr) shows 5 plots (5 bands)
@ -49,11 +52,7 @@ crs(stk.wenr)
r.wenr.rd.wdvi <- subset(stk.wenr,2) r.wenr.rd.wdvi <- subset(stk.wenr,2)
dev.off() dev.off()
plot(r.wenr.rd.wdvi,main=paste("wdvi",imgdate),xlab="RDX",ylab="RDY") plot(r.wenr.rd.wdvi,main=paste("wdvi",imgdate),xlab="RDX",ylab="RDY")
cellStats(r.wenr.rd.wdvi,'mean') #0.1654627 cellStats(r.wenr.rd.wdvi,'mean') #0.1350561
# Same as in downloaded file SatelliteDataStatistics_test_satData_wdvi_2020-06-14.csv
# "mean": 0.16564258790046743
# So FindSatelliteItem in C:\git\FarmMapsApiClient_WURtest\FarmmapsApi\Services\GeneralService.cs returns a a stacked geotiff in
# i.e. the cellstats are calculated from the rectangle
# Furthermore we can see # Furthermore we can see
# shows coordinates in RD # shows coordinates in RD
@ -69,6 +68,15 @@ r.wenr.wgs84.wdvi <- projectRaster(r.wenr.rd.wdvi, crs = CRS('+init=EPSG:4326'))
# plot(p1,add=TRUE, col="transparent",border="black") # plot(p1,add=TRUE, col="transparent",border="black")
# Draw the polygon on top of the raster # Draw the polygon on top of the raster
# Polygon p2 from C:\git\FarmMapsApiClient_WURtest\FarmmapsDataDownload\DataDownloadInput.json # Polygon p2 from C:\git\FarmMapsApiClient_WURtest\FarmmapsDataDownload\DataDownloadInput.json
p2 <- data.frame(id = 1, wkt = gsub("\n","",'POLYGON((
4.960707146896585 52.800583669708487,
4.960645975538824 52.800470217610922,
4.962140695752897 52.799177147194797,
4.967523821195745 52.801502400041208,
4.966336768950911 52.802543735879809,
4.961711880764330 52.801009996856429,
4.960707146896585 52.800583669708487))'))
# Polygon p2 from C:\git\FarmMapsApiClient_WURtest\FarmmapsBulkSatDownload\BulkSatDownloadInput.json
p2 <- data.frame(id = 1, wkt = gsub("\n","",'POLYGON(( p2 <- data.frame(id = 1, wkt = gsub("\n","",'POLYGON((
3.37837807779104 51.3231095796538, 3.37837807779104 51.3231095796538,
3.38065689232502 51.3212527499355, 3.38065689232502 51.3212527499355,
@ -123,11 +131,29 @@ dev.off()
plot(r.wenr.wgs84.wdvi.pol,main=paste("wdvi",imgdate),xlab="LON",ylab="LAT") plot(r.wenr.wgs84.wdvi.pol,main=paste("wdvi",imgdate),xlab="LON",ylab="LAT")
plot(p2.wgs84,add=TRUE, col="transparent",border="red") plot(p2.wgs84,add=TRUE, col="transparent",border="red")
#That's what we want! Now compare the stats #That's what we want! Now compare the stats
cellStats(r.wenr.rd.wdvi,'mean') #0.1654627 # Stats from rectangle, RD cellStats(r.wenr.rd.wdvi,'mean') # [1] 0.1350561 # Stats from rectangle, RD
cellStats(r.wenr.wgs84.wdvi,'mean') #0.1656399 # Stats from rectangle, WGS84 cellStats(r.wenr.wgs84.wdvi,'mean') # [1] 0.1351411 # Stats from rectangle, WGS84
cellStats(r.wenr.rd.wdvi.pol,'mean') #0.1658702 # Stats from raster clipped by polygon, WGS84 cellStats(r.wenr.rd.wdvi.pol,'mean') # [1] 0.05723957 # Stats from raster clipped by polygon, RD
cellStats(r.wenr.wgs84.wdvi.pol,'mean') #0.1658611 # Stats from raster clipped by polygon, WGS84 cellStats(r.wenr.wgs84.wdvi.pol,'mean') # [1] 0.05723607 # Stats from raster clipped by polygon, WGS84
# Conclusion: in this example little difference, but can be quite different! # file SatelliteDataStatistics_test_satData_wdvi_2020-09-22.csv
# "mean": 0.057430520945401985 # SatelliteDataStatistics_test_satData_wdvi_2020.csv returns stats for the clipped raster (.pol). 'mean' almost the same, maybe
# cellStats cannot return median, just a few stats.
cellStats(r.wenr.wgs84.wdvi.pol,'median') # Error in .local(x, stat, ...) : invalid 'stat'. Should be sum, min, max, sd, mean, or 'countNA'
r.wenr.wgs84.wdvi.vals <- values(r.wenr.wgs84.wdvi)
median(r.wenr.wgs84.wdvi.vals) # [1] NA
median(r.wenr.wgs84.wdvi.vals,na.rm=TRUE) # [1] 0.076
r.wenr.wgs84.wdvi.pol.vals <- values(r.wenr.wgs84.wdvi.pol)
median(r.wenr.wgs84.wdvi.pol.vals) # [1] NA
median(r.wenr.wgs84.wdvi.pol.vals,na.rm=TRUE) # [1] 0.048
# "median": 0.04800000041723251 # SatelliteDataStatistics_test_satData_wdvi_2020.csv returns stats for the clipped raster (.pol).
# An image may contain NA values. Check:
cellStats(r.wenr.wgs84.wdvi,'countNA') # [1] 22956
ncell(r.wenr.wgs84.wdvi) # [1] 221696
cellStats(r.wenr.wgs84.wdvi,'countNA') / ncell(r.wenr.wgs84.wdvi) # [1] 0.1035472 # 10% no data? doesn't show in the plot?
cellStats(r.wenr.wgs84.wdvi.pol,'countNA') # [1] 147387
summary(r.wenr.wgs84.wdvi.pol.vals) # shows the same: NA's: 147387
ncell(r.wenr.wgs84.wdvi.pol) # [1] 221696
cellStats(r.wenr.wgs84.wdvi.pol,'countNA') / ncell(r.wenr.wgs84.wdvi.pol) # [1] 0.6648158 # 66% no data? doesn't show in the plot?
# The project FarmmapsNbs can generate a wenr.tif file, application.tif, uptake.tif (in rtest1.uptake.zip)and shape.shp (in rtest1.taskmap.zip) # The project FarmmapsNbs can generate a wenr.tif file, application.tif, uptake.tif (in rtest1.uptake.zip)and shape.shp (in rtest1.taskmap.zip)
r.application <- raster("C:/git/FarmMapsApiClient_WURtest/FarmmapsNbs/bin/Debug/netcoreapp3.1/Downloads/application.tif") r.application <- raster("C:/git/FarmMapsApiClient_WURtest/FarmmapsNbs/bin/Debug/netcoreapp3.1/Downloads/application.tif")

116
FarmmapsDataDownload/DataDownloadApplication.cs
View File

@ -1,6 +1,7 @@
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.IO; using System.IO;
using System.IO.Compression;
using System.Linq; using System.Linq;
using System.Threading.Tasks; using System.Threading.Tasks;
using FarmmapsApi; using FarmmapsApi;
@ -16,7 +17,7 @@ namespace FarmmapsDataDownload
{ {
public class DataDownloadApplication : IApplication public class DataDownloadApplication : IApplication
{ {
private const string DownloadFolder = "Downloads"; //private const string DownloadFolder = "Downloads";
private const string SettingsFile = "settings.json"; private const string SettingsFile = "settings.json";
private readonly ILogger<DataDownloadApplication> _logger; private readonly ILogger<DataDownloadApplication> _logger;
@ -38,10 +39,8 @@ namespace FarmmapsDataDownload
public async Task RunAsync() public async Task RunAsync()
{ {
var fieldsInputJson = File.ReadAllText("DataDownloadInput.json"); var fieldsInputJson = File.ReadAllText("DataDownloadInput.json");
List<DataDownloadInput> fieldsInputs = JsonConvert.DeserializeObject<List<DataDownloadInput>>(fieldsInputJson);
if (!Directory.Exists(DownloadFolder)) List<DataDownloadInput> fieldsInputs = JsonConvert.DeserializeObject<List<DataDownloadInput>>(fieldsInputJson);
Directory.CreateDirectory(DownloadFolder);
// !! this call is needed the first time an api is called with a fresh clientid and secret !! // !! this call is needed the first time an api is called with a fresh clientid and secret !!
await _farmmapsApiService.GetCurrentUserCodeAsync(); await _farmmapsApiService.GetCurrentUserCodeAsync();
@ -62,12 +61,23 @@ namespace FarmmapsDataDownload
private async Task Process(List<UserRoot> roots, DataDownloadInput input) private async Task Process(List<UserRoot> roots, DataDownloadInput input)
{ {
string downloadFolder = input.DownloadFolder;
if (string.IsNullOrEmpty(downloadFolder)) {
downloadFolder = "Downloads";
}
if (!Directory.Exists(downloadFolder))
Directory.CreateDirectory(downloadFolder);
// !!specify if you are using an already created cropfield: // !!specify if you are using an already created cropfield:
bool useCreatedCropfield = input.UseCreatedCropfield; bool useCreatedCropfield = input.UseCreatedCropfield;
var cropYear = input.CropYear; var cropYear = input.CropYear;
var fieldName = input.fieldName; var fieldName = input.fieldName;
bool storeSatelliteStatistics = input.StoreSatelliteStatistics; bool storeSatelliteStatistics = input.StoreSatelliteStatisticsSingleImage;
bool storeSatelliteStatisticsCropYear = input.StoreSatelliteStatisticsCropYear; bool storeSatelliteStatisticsCropYear = input.StoreSatelliteStatisticsCropYear;
List<string> SatelliteBands = new List<string>(1) { input.SatelliteBand };
string headerLineStats = $"FieldName,satelliteDate,satelliteBand,max,min,mean,mode,median,stddev,minPlus,curtosis,maxMinus,skewness,variance,populationCount,variationCoefficient,confidenceIntervalLow, confidenceIntervalHigh,confidenceIntervalErrorMargin" + Environment.NewLine;
string settingsfile = $"Settings_{fieldName}.json"; string settingsfile = $"Settings_{fieldName}.json";
LoadSettings(settingsfile); LoadSettings(settingsfile);
@ -92,7 +102,7 @@ namespace FarmmapsDataDownload
{ {
_logger.LogInformation("Creating cropfield"); _logger.LogInformation("Creating cropfield");
cropfieldItem = await _generalService.CreateCropfieldItemAsync(myDriveRoot.Code, cropfieldItem = await _generalService.CreateCropfieldItemAsync(myDriveRoot.Code,
$"DataCropfield {input.OutputFileName}", cropYear.Year, input.GeometryJson.ToString(Formatting.None)); $"DataCropfield {input.OutputFileName}", cropYear, input.GeometryJson.ToString(Formatting.None));
_settings.CropfieldItemCode = cropfieldItem.Code; _settings.CropfieldItemCode = cropfieldItem.Code;
SaveSettings(settingsfile); SaveSettings(settingsfile);
} }
@ -117,12 +127,10 @@ namespace FarmmapsDataDownload
_logger.LogInformation("Downloading shadow map"); _logger.LogInformation("Downloading shadow map");
await _farmmapsApiService.DownloadItemAsync(shadowItem.Code, await _farmmapsApiService.DownloadItemAsync(shadowItem.Code,
Path.Combine(DownloadFolder, $"{input.OutputFileName}_shadow.zip")); Path.Combine(downloadFolder, $"{input.OutputFileName}_shadow.zip"));
} }
// Get satellite data // Get satellite data
if (input.GetSatelliteData) if (input.GetSatelliteData)
{ {
@ -134,54 +142,72 @@ namespace FarmmapsDataDownload
SaveSettings(settingsfile); SaveSettings(settingsfile);
} }
int selectedLayer= 2;
if (input.SatelliteBand == "ndvi") selectedLayer = 0; // Select all satellite items
if (input.SatelliteBand == "wdvi") selectedLayer = 1; List<Item> satelliteItemsCropYear = await _generalService.FindSatelliteItems(cropfieldItem, _settings.SatelliteTaskCode);
if (input.SatelliteBand == "natural") selectedLayer = 2; satelliteItemsCropYear = satelliteItemsCropYear.OrderBy(x => x.DataDate).ToList();
// Select a particular satellite item from satelliteTask
Item satalliteItem = await _generalService.FindSatelliteItem(cropfieldItem, _settings.SatelliteTaskCode, fieldName, selectedLayer, storeSatelliteStatisticsCropYear, DownloadFolder);
if (input.StoreSatelliteStatisticsSingleImage == true) {
var satelliteBand = satalliteItem.Data["layers"][selectedLayer]["name"]; _logger.LogInformation("Available satellite images:");
var count = 0;
//Store satellite data to csv TimeSpan.FromSeconds(0.5);
if (storeSatelliteStatistics == true && (selectedLayer == 0 || selectedLayer ==1)) foreach (var item in satelliteItemsCropYear)
{
var satelliteStatistics = satalliteItem.Data["layers"][selectedLayer]["renderer"]["band"]["statistics"];
Console.WriteLine($"Satellite image date: {satalliteItem.DataDate}");
var SatelliteStatsFile = $"{DownloadFolder}/SatelliteDataStatistics_{fieldName}_{input.SatelliteBand}_{satalliteItem.DataDate.Value:yyyy-MM-dd}.csv";
using var w = new StreamWriter(SatelliteStatsFile);
{ {
foreach (var item in satelliteStatistics)
Console.WriteLine($"Satellite image #{count}: {item.DataDate}");
count++;
}
_logger.LogInformation("Enter satellite image number");
int element = Int32.Parse(Console.ReadLine());
var selectedSatelliteItem = satelliteItemsCropYear[element];
var SatelliteDate = selectedSatelliteItem.DataDate.Value.ToString("yyyyMMdd");
string fileName = string.Format($"satelliteGeotiff_{fieldName}_{SatelliteDate}"); // no need to add satelliteBand in the name because the tif contains all bands
string fileNameZip = string.Format($"{fileName}.zip");
string fileNameGeotiff = string.Format($"{fileName}.tif");
await _farmmapsApiService.DownloadItemAsync(selectedSatelliteItem.Code, Path.Combine(downloadFolder, fileNameZip));
// Download a csv file with stats
List<Item> selectedSatalliteItems = new List<Item>(1) { selectedSatelliteItem };
string fileNameStats = Path.Combine(downloadFolder, string.Format($"satelliteStats_{fieldName}_{SatelliteDate}.csv"));
string downloadedStats = await _generalService.DownloadSatelliteStats(selectedSatalliteItems, fieldName, SatelliteBands, downloadFolder);
//rename the csv file with stats
File.Delete(fileNameStats);
File.Move(downloadedStats, fileNameStats);
// wenr.tif. Contains 5 layers: (1) ndvi, (2) wdvi, (3) Red, (4) Green and (5) Blue
// download the geotiffs. Returns a zip file with always these three files:
// data.dat.aux.xml
// thumbnail.jpg
// wenr.tif. Contains 5 layers: (1) ndvi, (2) wdvi, (3) Red, (4) Green and (5) Blue
if (true)
{
// Extract the file "wenr.tif" from zip, rename it to fileNameGeotiff
ZipFile.ExtractToDirectory(Path.Combine(downloadFolder, fileNameZip), downloadFolder, true);
File.Delete(Path.Combine(downloadFolder, fileNameGeotiff)); // Delete the fileNameGeotiff file if exists
File.Move(Path.Combine(downloadFolder, "wenr.tif"), Path.Combine(downloadFolder, fileNameGeotiff)); // Rename the oldFileName into newFileName
// Cleanup
string[] filesToDelete = new string[] { fileNameZip, "wenr.tif", "thumbnail.jpg", "data.dat.aux.xml" };
foreach (string f in filesToDelete)
{ {
var line = string.Format("{0}", item); File.Delete(Path.Combine(downloadFolder, f));
w.WriteLine(line);
w.Flush();
} }
} }
_logger.LogInformation($"Downloaded files {fileNameGeotiff} and {fileNameStats} to {downloadFolder}");
var inputType = (satalliteItem.Data["layers"] as JArray)?[selectedLayer]["name"].ToString(); }
if (string.IsNullOrEmpty(inputType)) { if (input.StoreSatelliteStatisticsCropYear == true) {
_logger.LogError("Could not get the input type name from the satellite item"); string fileNameStats = Path.Combine(downloadFolder, string.Format($"satelliteStats_{fieldName}_{cropYear}.csv"));
return; File.Delete(fileNameStats);
} string downloadedStats = await _generalService.DownloadSatelliteStats(satelliteItemsCropYear, fieldName, SatelliteBands, downloadFolder);
File.Move(downloadedStats, fileNameStats);
// download the geotiff of needed inputtype _logger.LogInformation($"Downloaded file {fileNameStats} with stats for field '{fieldName}', cropyear {cropYear}");
var SatelliteImageDate = (DateTime)satalliteItem.DataDate;
var SatelliteDate = SatelliteImageDate.ToString("yyyyMMdd");
_logger.LogInformation("Downloading geotiff file");
await _farmmapsApiService.DownloadItemAsync(satalliteItem.Code,
Path.Combine(DownloadFolder, $"satelliteGeotiff_{input.OutputFileName}_{inputType}_{SatelliteDate}.zip"));
} }
} }
// Get vanDerSat data // Get vanDerSat data
if (input.GetVanDerSatData) if (input.GetVanDerSatData)
{ {

6
FarmmapsDataDownload/DataDownloadInput.json
View File

@ -3,15 +3,15 @@
"UseCreatedCropfield": true, "UseCreatedCropfield": true,
"outputFileName": "testSatData2", "outputFileName": "testSatData2",
"fieldName": "test_satData2", "fieldName": "test_satData2",
"DownloadFolder": "C:\\workdir\\groenmonitor\\", //"C:\\workdir\\groenmonitor\\", // "Downloads", -> if you just put "Downloads" the program will download to somewhere in ..\FarmMapsApiClient_WURtest\FarmmapsDataDownload\bin\Debug\netcoreapp3.1\Downloads\
"GetShadowData": false, "GetShadowData": false,
"GetSatelliteData": true, "GetSatelliteData": true,
"SatelliteBand": "wdvi", // "natural", "ndvi" or "wdvi" "SatelliteBand": "wdvi", // "natural", "ndvi" or "wdvi"
"StoreSatelliteStatistics": false, "StoreSatelliteStatisticsSingleImage": true,
"StoreSatelliteStatisticsCropYear": true, "StoreSatelliteStatisticsCropYear": true,
"GetVanDerSatData": false, "GetVanDerSatData": false,
"StoreVanDerSatStatistics": false, "StoreVanDerSatStatistics": false,
"CropYear": "2020-01-01", "CropYear": 2020,
"geometryJson": { "geometryJson": {
"type": "Polygon", "type": "Polygon",
"coordinates": [ "coordinates": [

5
FarmmapsDataDownload/Models/DataDownloadInput.cs
View File

@ -9,14 +9,15 @@ namespace FarmmapsDataDownload.Models
public string File { get; set; } public string File { get; set; }
public string InputVariable { get; set; } public string InputVariable { get; set; }
public string OutputFileName { get; set; } public string OutputFileName { get; set; }
public DateTime CropYear { get; set; } public string DownloadFolder { get; set; }
public int CropYear { get; set; }
public JObject GeometryJson { get; set; } public JObject GeometryJson { get; set; }
public string InputLayerName { get; set; } public string InputLayerName { get; set; }
public string fieldName { get; set; } public string fieldName { get; set; }
public bool GetSatelliteData { get; set; } public bool GetSatelliteData { get; set; }
public bool GetVanDerSatData { get; set; } public bool GetVanDerSatData { get; set; }
public string SatelliteBand { get; set; } public string SatelliteBand { get; set; }
public bool StoreSatelliteStatistics { get; set; } public bool StoreSatelliteStatisticsSingleImage { get; set; }
public bool StoreSatelliteStatisticsCropYear { get; set; } public bool StoreSatelliteStatisticsCropYear { get; set; }
public bool StoreVanDerSatStatistics { get; set; } public bool StoreVanDerSatStatistics { get; set; }
public bool GetShadowData { get; set; } public bool GetShadowData { get; set; }