pvanoort/FarmMapsApiClient_WUR
1
0
Fork 0
forked from FarmMaps/FarmMapsApiClient
Code Pull Requests Activity

solved issue where sometimes satelliteItem contains only 2 layers

Browse Source 
Normally 3 layers 0: ndvi, 1: wdvi, 2: natural. In some cases only 2 layers: 0: wdvi, 1: natural. Which caused error when requesting statistics.
This happened for example for cropfieldItem.Code ‘e08d71bd92334dbab9e645ad6e72da63’
This commit is contained in:
pepijn van oort
2021-05-28 17:25:31 +02:00
parent 1bc326cfd2
commit 111b4e576f
5 changed files with 232 additions and 194 deletions
Download Patch File Download Diff File Expand all files Collapse all files

210
FarmmapsBulkSatDownload/BulkSatDownloadApplication.cs
View File

@@ -57,16 +57,15 @@ namespace FarmmapsBulkSatDownload
// Crashes if "DBsettings.secrets.json" is absent or empty
DB dbparcels = JsonConvert.DeserializeObject<DB>(File.ReadAllText("DBsettings.secrets.json"));
string schemaname = "bigdata";
string parceltablename = "parcel_disac";//"parcelsijbrandij" "parcel"; "parcel_flowerbulbs"; "parcel_disac"
string groenmonitortablename = "groenmonitor_disac";//"groenmonitorsijbrandij" "groenmonitor" "groenmonitor_flowerbulbs" "groenmonitor_disac"
string parceltablename = "parcel_flowerbulbs";//"parcelsijbrandij" "parcel"; "parcel_flowerbulbs"; "parcel_disac"
string groenmonitortablename = "groenmonitor_flowerbulbs";//"groenmonitorsijbrandij" "groenmonitor" "groenmonitor_flowerbulbs" "groenmonitor_disac"
// The view 'groenmonitorlatestviewname' contains per parcelid (arbid) the year in which it "exists" and the date of the latest image downloaded. It is used to prevent unneccessary downloading of image statistics already in the database
string groenmonitorlatestviewname = "groenmonitorlatest_disac"; //"groenmonitorsijbrandijlatest" "groenmonitorlatest" "groenmonitorlatest_flowerbulbs" "groenmonitorlatest_disac"
//GroenmonitorTable gmtb;
string groenmonitorlatestviewname = "groenmonitorlatest_flowerbulbs"; //"groenmonitorsijbrandijlatest" "groenmonitorlatest" "groenmonitorlatest_flowerbulbs" "groenmonitorlatest_disac"
// Database query and connection. Geometry must be in WGS84 coordinate system, EPSG 4326
// Apparently the FarmmapsApi cannot handle MultiPolygon, so we need to convert to single Polygon
// In case database returns a MultiPolygon use ST_NumGeometries(pt.geom) to count the number of polygons
// If necessary use WHERE T_NumGeometries(pt.geom) = 1 to select only single polygons
// If necessary use WHERE ST_NumGeometries(pt.geom) = 1 to select only single polygons
//
// FarmMaps get's its satellite images from www.groenmonitor.nl through the https://agrodatacube.wur.nl/.
// Many images are available at www.groenmonitor.nl, the https://agrodatacube.wur.nl/ serves only the clean images, 10-30 per year, 2019 onwards. Possibly more images will be added for earlier years
@@ -79,9 +78,11 @@ namespace FarmmapsBulkSatDownload
SELECT pt.arbid, pt.year, gml.lastwenrdate, ST_AsGeoJSON(ST_Transform((ST_DUMP(pt.geom)).geom::geometry(Polygon),4326)) AS geojson_polygon_wgs84, COALESCE(pt.cropfielditemcode,'') AS cropfielditemcode
FROM {0}.{1} pt, {0}.{2} gml
WHERE
pt.arbid = gml.arbid
pt.arbid = gml.arbid AND
pt.crop NOT IN ('Lelie','Tulp') AND
pt.year > 2018 AND pt.arbid >= 8155
ORDER BY pt.arbid
LIMIT 15;", schemaname, parceltablename, groenmonitorlatestviewname); //LIMIT x for testing
LIMIT 2000;", schemaname, parceltablename, groenmonitorlatestviewname); //LIMIT x for testing
using (NpgsqlConnection connection = new NpgsqlConnection(connectionString))
{
@@ -182,12 +183,16 @@ LIMIT 15;", schemaname, parceltablename, groenmonitorlatestviewname); //LIMIT
DateTime dtSatelliteDate;
string strSatelliteDate;
List<Item> satelliteItemsCropYear;
GroenmonitorTable groenmonitorTable = new GroenmonitorTable();
StringBuilder sbInsertSql;
StringBuilder sbInsertSqlInto;
StringBuilder sbInsertSqlUpdate;
StringBuilder sbInsertSqlValues;
List<SatelliteStatistics> listSatelliteStatistics;
SatelliteStatistics satelliteStatistics_wdvi;
SatelliteStatistics satelliteStatistics_ndvi;
int cntDatesDownloaded;
string satelliteSource = "akkerwebwenr"; //same as in groenmonitorlatestviewname SQL code
string fieldName = input.fieldName;
int cropYear = input.cropYear;
List<string> satelliteBands = input.SatelliteBands;
@@ -199,7 +204,6 @@ LIMIT 15;", schemaname, parceltablename, groenmonitorlatestviewname); //LIMIT
string satelllitetable = input.satelllitetable;
DateTime lastDownloadedSatelliteDate = input.lastdownloadedimagedate;
cropfielditemcode = input.cropfielditemcode;
string insertSql = InsertSQLfromClass(schemaname, satelllitetable);
LoadSettings(settingsfile);
@@ -236,7 +240,7 @@ LIMIT 15;", schemaname, parceltablename, groenmonitorlatestviewname); //LIMIT
//Console.WriteLine(insertCmd.CommandText);
int r = updateCmd.ExecuteNonQuery();
if (r != 1)
throw new Exception("// FarmmapsBulkSatDownload: Insert Failed");
throw new Exception("// FarmmapsBulkSatDownload: Update Failed");
connection.Close();
}
_logger.LogInformation($"// FarmmapsBulkSatDownload: Added cropfieldItem.Code '{cropfieldItem.Code}' for parcelid {input.fieldID} to {schemaname}.{cropfieldtable} ");
@@ -260,7 +264,7 @@ LIMIT 15;", schemaname, parceltablename, groenmonitorlatestviewname); //LIMIT
// SaveSettings(settingsfile);
// Getting satellite items
_logger.LogInformation(string.Format($"Running FindSatelliteItemsCropYear for cropfieldItem.Code '{cropfieldItem.Code}', SatelliteTaskCode '{satelliteTaskCode}'"));
_logger.LogInformation(string.Format($"Running FindSatelliteItems for cropfieldItem.Code '{cropfieldItem.Code}', SatelliteTaskCode '{satelliteTaskCode}'"));
satelliteItemsCropYear = await _generalService.FindSatelliteItems(cropfieldItem, satelliteTaskCode);
// Checking if satellite items found
@@ -268,14 +272,14 @@ LIMIT 15;", schemaname, parceltablename, groenmonitorlatestviewname); //LIMIT
if (satelliteItemsCropYear == null)
{
satelliteItemsAvailable = false;
_logger.LogInformation($"No satellite tiffs found for fieldName '{fieldName}', cropYear {cropYear}");
_logger.LogInformation($"No satellite tiffs found for fieldName '{fieldName}', cropYear {cropYear}, cropfielditemcode '{cropfielditemcode}'");
}
else
{
if (satelliteItemsCropYear.Count == 0)
{
satelliteItemsAvailable = false;
_logger.LogInformation($"No satellite tiffs found for fieldName '{fieldName}', cropYear {cropYear}");
_logger.LogInformation($"No satellite tiffs found for fieldName '{fieldName}', cropYear {cropYear}, cropfielditemcode '{cropfielditemcode}'");
}
}
@@ -335,77 +339,94 @@ LIMIT 15;", schemaname, parceltablename, groenmonitorlatestviewname); //LIMIT
// And we assume farmMaps always nicely generates statistics, no hick-ups
// Under this assumptions, we only need to compare with the lastDownloadedSatelliteDate from the database
cntDatesDownloaded = 0;
foreach (Item satelliteItem in satelliteItemsCropYear)
using (NpgsqlConnection connection = new NpgsqlConnection(database.GetConnectionString()))
{
dtSatelliteDate = satelliteItem.DataDate.Value;
strSatelliteDate = dtSatelliteDate.ToString("yyyy-MM-dd");
listSatelliteStatistics = await _generalService.ListSatelliteStatistics(satelliteItem, satelliteBands, fieldName);
statsAvailable = true;
if (listSatelliteStatistics == null)
connection.Open();
foreach (Item satelliteItem in satelliteItemsCropYear)
{
statsAvailable = false;
_logger.LogWarning($"No stats found for satellite, fielName '{fieldName}', date '{strSatelliteDate}'");
}
else
{
if (listSatelliteStatistics.Count == 0)
dtSatelliteDate = satelliteItem.DataDate.Value;
strSatelliteDate = dtSatelliteDate.ToString("yyyy-MM-dd");
listSatelliteStatistics = await _generalService.ListSatelliteStatistics(satelliteItem, satelliteBands, fieldName);
statsAvailable = true;
if (listSatelliteStatistics == null)
{
statsAvailable = false;
_logger.LogWarning($"No stats found for satellite, fielName '{fieldName}', date '{strSatelliteDate}'");
}
}
if (statsAvailable)
{
if (dtSatelliteDate <= input.lastdownloadedimagedate)
{
_logger.LogInformation($"// Stats for parcelid {input.fieldID}, date '{strSatelliteDate}' already there in {schemaname}.{satelllitetable}");
_logger.LogWarning($"No stats found for satellite, fieldName '{fieldName}', date '{strSatelliteDate}'");
}
else
{
cntDatesDownloaded++;
// Map satelliteStatistics to groenmonitorTable
satelliteStatistics_wdvi = listSatelliteStatistics.SingleOrDefault(p => p.satelliteBand == "wdvi");
satelliteStatistics_ndvi = listSatelliteStatistics.SingleOrDefault(p => p.satelliteBand == "ndvi");
groenmonitorTable.parcelid = input.fieldID;
groenmonitorTable.date = strSatelliteDate;
groenmonitorTable.source = "akkerwebwenr"; // Like this so SQL in bigdata.groenmonitorlatest_flowerbulbs works properly
groenmonitorTable.wdvi_pixelcount = satelliteStatistics_wdvi.populationCount; //count of pixels with data
groenmonitorTable.wdvi_max = satelliteStatistics_wdvi.max;
groenmonitorTable.wdvi_mean = satelliteStatistics_wdvi.mean;
groenmonitorTable.wdvi_min = satelliteStatistics_wdvi.min;
groenmonitorTable.wdvi_stdev = satelliteStatistics_wdvi.stddev;
groenmonitorTable.wdvi_median = satelliteStatistics_wdvi.median;
groenmonitorTable.wdvi_p90 = -99; // Example of a statistics (90% not included in satelliteStatistics
groenmonitorTable.ndvi_pixelcount = satelliteStatistics_ndvi.populationCount; //count of pixels with data
groenmonitorTable.ndvi_max = satelliteStatistics_ndvi.max;
groenmonitorTable.ndvi_mean = satelliteStatistics_ndvi.mean;
groenmonitorTable.ndvi_min = satelliteStatistics_ndvi.min;
groenmonitorTable.ndvi_stdev = satelliteStatistics_ndvi.stddev;
groenmonitorTable.ndvi_median = satelliteStatistics_ndvi.median;
groenmonitorTable.ndvi_p90 = -99; // Example of a statistics (90% not included in satelliteStatistics
// fill the insertSql query with fieldValues from the groenmonitorTable, then run the query
using (NpgsqlConnection connection = new NpgsqlConnection(database.GetConnectionString()))
if (listSatelliteStatistics.Count == 0)
{
connection.Open();
statsAvailable = false;
_logger.LogWarning($"No stats found for satellite, fieldName '{fieldName}', date '{strSatelliteDate}'");
}
}
if (statsAvailable)
{
if (dtSatelliteDate <= input.lastdownloadedimagedate)
{
_logger.LogInformation($"// Stats for parcelid {input.fieldID}, date '{strSatelliteDate}' already there in {schemaname}.{satelllitetable}");
}
else
{
cntDatesDownloaded++;
// Map satelliteStatistics to groenmonitorTable
satelliteStatistics_wdvi = listSatelliteStatistics.SingleOrDefault(p => p.satelliteBand == "wdvi");
satelliteStatistics_ndvi = listSatelliteStatistics.SingleOrDefault(p => p.satelliteBand == "ndvi");
sbInsertSql = new StringBuilder();
sbInsertSqlInto = new StringBuilder();
sbInsertSqlUpdate = new StringBuilder();
sbInsertSqlValues = new StringBuilder();
sbInsertSqlInto.Append($"INSERT INTO {schemaname}.{satelllitetable} (parcelid,date,source");
sbInsertSqlValues.Append($"VALUES ({input.fieldID},'{strSatelliteDate}','{satelliteSource}'");
sbInsertSqlUpdate.Append($"ON CONFLICT (parcelid,date,source) DO UPDATE SET parcelid={input.fieldID}, date='{strSatelliteDate}',source='{satelliteSource}'");
if (satelliteBands.Contains("wdvi"))
{
if (satelliteStatistics_wdvi != null)
{
sbInsertSqlInto.Append(",wdvi_pixelcount,wdvi_max,wdvi_mean,wdvi_min,wdvi_stdev,wdvi_median");
sbInsertSqlValues.Append($",{satelliteStatistics_wdvi.populationCount},{satelliteStatistics_wdvi.max},{satelliteStatistics_wdvi.mean},{satelliteStatistics_wdvi.min},{satelliteStatistics_wdvi.stddev},{satelliteStatistics_wdvi.median}");
sbInsertSqlUpdate.Append($",wdvi_pixelcount={satelliteStatistics_wdvi.populationCount},wdvi_max={satelliteStatistics_wdvi.max},wdvi_mean={satelliteStatistics_wdvi.mean},wdvi_min={satelliteStatistics_wdvi.min},wdvi_stdev={satelliteStatistics_wdvi.stddev},wdvi_median={satelliteStatistics_wdvi.median}");
}
else
{
_logger.LogWarning($"wdvi missing for cropfielditemcode {cropfielditemcode}, date '{strSatelliteDate}'");
}
}
if (satelliteBands.Contains("ndvi"))
{
if (satelliteStatistics_ndvi != null)
{
sbInsertSqlInto.Append(",ndvi_pixelcount,ndvi_max,ndvi_mean,ndvi_min,ndvi_stdev,ndvi_median");
sbInsertSqlValues.Append($",{satelliteStatistics_ndvi.populationCount},{satelliteStatistics_ndvi.max},{satelliteStatistics_ndvi.mean},{satelliteStatistics_ndvi.min},{satelliteStatistics_ndvi.stddev},{satelliteStatistics_ndvi.median}");
sbInsertSqlUpdate.Append($",ndvi_pixelcount={satelliteStatistics_ndvi.populationCount},ndvi_max={satelliteStatistics_ndvi.max},ndvi_mean={satelliteStatistics_ndvi.mean},ndvi_min={satelliteStatistics_ndvi.min},ndvi_stdev={satelliteStatistics_ndvi.stddev},ndvi_median={satelliteStatistics_ndvi.median}");
}
else
{
_logger.LogWarning($"ndvi missing for cropfielditemcode {cropfielditemcode}, date '{strSatelliteDate}'");
}
}
sbInsertSqlInto.Append(")");
sbInsertSqlValues.Append(")");
sbInsertSql.AppendLine(sbInsertSqlInto.ToString());
sbInsertSql.AppendLine(sbInsertSqlValues.ToString());
sbInsertSql.AppendLine(sbInsertSqlUpdate.ToString());
//string strInsertSql = sbInsertSql.ToString();
NpgsqlCommand insertCmd = connection.CreateCommand();
object[] fieldValues = groenmonitorTable.GetType()
.GetFields()
.Select(field => field.GetValue(groenmonitorTable))
.ToArray();
insertCmd.CommandText = string.Format(insertSql, fieldValues);
insertCmd.CommandText = sbInsertSql.ToString();
//Console.WriteLine(insertCmd.CommandText);
int r = insertCmd.ExecuteNonQuery();
if (r != 1)
throw new Exception("// FarmmapsBulkSatDownload: Insert Failed");
connection.Close();
_logger.LogInformation($"// Added stats to {schemaname}.{satelllitetable} for parcelid {input.fieldID}, date '{strSatelliteDate}'. cntDatesDownloaded: {cntDatesDownloaded}");
}
_logger.LogInformation($"// Added stats to {schemaname}.{satelllitetable} for parcelid {input.fieldID}, date '{strSatelliteDate}'. cntDatesDownloaded: {cntDatesDownloaded}");
}
}
connection.Close();
}
}
}
// Functions to save previously created cropfields
private void LoadSettings(string file)
{
@@ -434,64 +455,5 @@ LIMIT 15;", schemaname, parceltablename, groenmonitorlatestviewname); //LIMIT
{
return String.Format("{0:00}:{1:00}:{2:00}", ts.Hours, ts.Minutes, ts.Seconds);
}
private string InsertSQLfromClass(string schemaname, string groenmonitortablename)
{
// Generates an INSERT query for GroenmonitorTable.
// When writing to a different table structure, just make a new class for the statellite statistics table, similar to GroenmonitorTable.cs
// @"
//INSERT INTO bigdata.groenmonitor (parcelid,date,wdvi_pixelcount,wdvi_max,wdvi_mean,wdvi_min,wdvi_stdev,wdvi_median,wdvi_p90,ndvi_pixelcount,ndvi_max,ndvi_mean,ndvi_min,ndvi_stdev,ndvi_median,ndvi_p90)
//VALUES({0},'{1}',{2},{3},{4},{5},{6},{7},{8},{9},{10},{11},{12},{13},{14},{15})
//ON CONFLICT (parcelid, date)
//DO UPDATE SET parcelid={0},date='{1}',wdvi_pixelcount={2},wdvi_max={3},wdvi_mean={4},wdvi_min={5},wdvi_stdev={6},wdvi_median={7},wdvi_p90={8},ndvi_pixelcount={9},ndvi_max={10},ndvi_mean={11},ndvi_min={12},ndvi_stdev={13},ndvi_median={14},ndvi_p90={15}
//";
string fieldName;
var fieldNames = typeof(GroenmonitorTable).GetFields()
.Select(field => field.Name)
.ToList();
StringBuilder sbInsertSql = new StringBuilder();
string insertSql;
StringBuilder insertSql1 = new StringBuilder();
insertSql1.Append(@"INSERT INTO " + schemaname + "." + groenmonitortablename + " (");
StringBuilder insertSql2 = new StringBuilder();
insertSql2.Append("VALUES(");
StringBuilder insertSql3 = new StringBuilder();
insertSql3.Append("ON CONFLICT(parcelid, date, source)");
StringBuilder insertSql4 = new StringBuilder();
insertSql4.Append("DO UPDATE SET ");
for (int i = 0; i < fieldNames.Count; i++)
{
fieldName = fieldNames[i];
insertSql1.Append(fieldName);
if (fieldName == "date" | fieldName == "source")
{
//add extra quotes ' for fields in the Postgress table for which we know they are date or varchar
insertSql2.Append("'{" + i + "}'");
insertSql4.Append(fieldName + "='{" + i + "}'");
}
else
{
insertSql2.Append("{" + i + "}");
insertSql4.Append(fieldName + "={" + i + "}");
}
if (i < fieldNames.Count - 1)
{
insertSql1.Append(",");
insertSql2.Append(",");
insertSql4.Append(",");
}
else
{
insertSql1.Append(")");
insertSql2.Append(")");
}
}
sbInsertSql.AppendLine(insertSql1.ToString());
sbInsertSql.AppendLine(insertSql2.ToString());
sbInsertSql.AppendLine(insertSql3.ToString());
sbInsertSql.AppendLine(insertSql4.ToString());
insertSql = sbInsertSql.ToString();
return insertSql;
}
}
}