pvanoort/FarmMapsApiClient_WUR
1
0
Fork 0
forked from FarmMaps/FarmMapsApiClient
Code Pull Requests Activity

also logging satellitetaskcode to cropfieldtable in database

Browse Source
This commit is contained in:
pepijn van oort
2021-05-29 12:50:33 +02:00
parent 111b4e576f
commit d1747427bc
2 changed files with 51 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

66
FarmmapsBulkSatDownload/BulkSatDownloadApplication.cs
View File

@@ -75,12 +75,14 @@ namespace FarmmapsBulkSatDownload
string connectionString = dbparcels.GetConnectionString(); string connectionString = dbparcels.GetConnectionString();
string readSql = string.Format( string readSql = string.Format(
@" @"
SELECT pt.arbid, pt.year, gml.lastwenrdate, ST_AsGeoJSON(ST_Transform((ST_DUMP(pt.geom)).geom::geometry(Polygon),4326)) AS geojson_polygon_wgs84, COALESCE(pt.cropfielditemcode,'') AS cropfielditemcode SELECT pt.arbid, pt.year, gml.lastwenrdate, ST_AsGeoJSON(ST_Transform((ST_DUMP(pt.geom)).geom::geometry(Polygon),4326)) AS geojson_polygon_wgs84,
COALESCE(pt.cropfielditemcode,'') AS cropfielditemcode,
CASE WHEN pt.year >= DATE_PART('year', CURRENT_DATE) THEN '' ELSE COALESCE(pt.satellitetaskcode,'') END AS satellitetaskcode
FROM {0}.{1} pt, {0}.{2} gml FROM {0}.{1} pt, {0}.{2} gml
WHERE WHERE
pt.arbid = gml.arbid AND pt.arbid = gml.arbid AND
pt.crop NOT IN ('Lelie','Tulp') AND pt.crop NOT IN ('Lelie','Tulp') AND
pt.year > 2018 AND pt.arbid >= 8155 pt.year > 2018 AND pt.arbid IN(8276,8314,8315)
ORDER BY pt.arbid ORDER BY pt.arbid
LIMIT 2000;", schemaname, parceltablename, groenmonitorlatestviewname); //LIMIT x for testing LIMIT 2000;", schemaname, parceltablename, groenmonitorlatestviewname); //LIMIT x for testing
@@ -102,6 +104,7 @@ LIMIT 2000;", schemaname, parceltablename, groenmonitorlatestviewname); //LIMI
bulkSatDownloadInput.GeometryJson = JObject.Parse(dr.GetString(3)); bulkSatDownloadInput.GeometryJson = JObject.Parse(dr.GetString(3));
bulkSatDownloadInput.SatelliteBands = satelliteBands; bulkSatDownloadInput.SatelliteBands = satelliteBands;
bulkSatDownloadInput.cropfielditemcode = dr.GetString(4); bulkSatDownloadInput.cropfielditemcode = dr.GetString(4);
bulkSatDownloadInput.satellitetaskcode = dr.GetString(5);
bulkSatDownloadInput.database = dbparcels; bulkSatDownloadInput.database = dbparcels;
bulkSatDownloadInput.schemaname = schemaname; bulkSatDownloadInput.schemaname = schemaname;
bulkSatDownloadInput.cropfieldtable = parceltablename; bulkSatDownloadInput.cropfieldtable = parceltablename;
@@ -177,6 +180,7 @@ LIMIT 2000;", schemaname, parceltablename, groenmonitorlatestviewname); //LIMI
private async Task Process(List<UserRoot> roots, BulkSatDownloadInput input) private async Task Process(List<UserRoot> roots, BulkSatDownloadInput input)
{ {
string cropfielditemcode; string cropfielditemcode;
string satellitetaskcode;
Item cropfieldItem; Item cropfieldItem;
bool satelliteItemsAvailable; bool satelliteItemsAvailable;
bool statsAvailable; bool statsAvailable;
@@ -193,6 +197,7 @@ LIMIT 2000;", schemaname, parceltablename, groenmonitorlatestviewname); //LIMI
int cntDatesDownloaded; int cntDatesDownloaded;
string satelliteSource = "akkerwebwenr"; //same as in groenmonitorlatestviewname SQL code string satelliteSource = "akkerwebwenr"; //same as in groenmonitorlatestviewname SQL code
int fieldID = input.fieldID;
string fieldName = input.fieldName; string fieldName = input.fieldName;
int cropYear = input.cropYear; int cropYear = input.cropYear;
List<string> satelliteBands = input.SatelliteBands; List<string> satelliteBands = input.SatelliteBands;
@@ -204,6 +209,7 @@ LIMIT 2000;", schemaname, parceltablename, groenmonitorlatestviewname); //LIMI
string satelllitetable = input.satelllitetable; string satelllitetable = input.satelllitetable;
DateTime lastDownloadedSatelliteDate = input.lastdownloadedimagedate; DateTime lastDownloadedSatelliteDate = input.lastdownloadedimagedate;
cropfielditemcode = input.cropfielditemcode; cropfielditemcode = input.cropfielditemcode;
satellitetaskcode = input.satellitetaskcode;
LoadSettings(settingsfile); LoadSettings(settingsfile);
@@ -230,20 +236,20 @@ LIMIT 2000;", schemaname, parceltablename, groenmonitorlatestviewname); //LIMI
// If working with a database, add this cropfieldItem.Code to the database so that next case same cropField is requested, will be faster // If working with a database, add this cropfieldItem.Code to the database so that next case same cropField is requested, will be faster
if (database != null) if (database != null)
{ {
// add this CropfieldItemCode to the parceltable // add this cropfielditemcode to the cropfieldtable
using (NpgsqlConnection connection = new NpgsqlConnection(database.GetConnectionString())) using (NpgsqlConnection connection = new NpgsqlConnection(database.GetConnectionString()))
{ {
connection.Open(); connection.Open();
NpgsqlCommand updateCmd = connection.CreateCommand(); NpgsqlCommand updateCmd = connection.CreateCommand();
string updateSql = string.Format($"UPDATE {schemaname}.{cropfieldtable} SET cropfielditemcode = '{cropfieldItem.Code}' WHERE arbid = {input.fieldID};"); string updateSql = string.Format($"UPDATE {schemaname}.{cropfieldtable} SET cropfielditemcode = '{cropfieldItem.Code}' WHERE arbid = {fieldID};");
updateCmd.CommandText = updateSql; updateCmd.CommandText = updateSql;
//Console.WriteLine(insertCmd.CommandText); //Console.WriteLine(insertCmd.CommandText);
int r = updateCmd.ExecuteNonQuery(); int r = updateCmd.ExecuteNonQuery();
if (r != 1) if (r != 1)
throw new Exception("// FarmmapsBulkSatDownload: Update Failed"); throw new Exception("// FarmmapsBulkSatDownload: Update cropfielditemcode Failed");
connection.Close(); connection.Close();
} }
_logger.LogInformation($"// FarmmapsBulkSatDownload: Added cropfieldItem.Code '{cropfieldItem.Code}' for parcelid {input.fieldID} to {schemaname}.{cropfieldtable} "); _logger.LogInformation($"// FarmmapsBulkSatDownload: Added cropfieldItem.Code '{cropfieldItem.Code}' for parcelid {fieldID} to {schemaname}.{cropfieldtable} ");
} }
} }
else else
@@ -252,20 +258,48 @@ LIMIT 2000;", schemaname, parceltablename, groenmonitorlatestviewname); //LIMI
cropfieldItem = new Item(); cropfieldItem = new Item();
cropfieldItem.Code = cropfielditemcode; cropfieldItem.Code = cropfielditemcode;
cropfieldItem.Name = "DataCropfield " + fieldName; cropfieldItem.Name = "DataCropfield " + fieldName;
_logger.LogInformation($"// FarmmapsBulkSatDownload: CropfieldItem.Code for parcelid {input.fieldID} already there in {schemaname}.{cropfieldtable}: '{cropfieldItem.Code}'"); _logger.LogInformation($"// FarmmapsBulkSatDownload: CropfieldItem.Code for parcelid {fieldID} already there in {schemaname}.{cropfieldtable}: '{cropfieldItem.Code}'");
} }
_settings.cropFieldItems.Add(cropfieldItem); _settings.cropFieldItems.Add(cropfieldItem);
SaveSettings(settingsfile); SaveSettings(settingsfile);
//Create satelliteTaskCode & save satelliteTaskCode.Code to settingsfile for retracing last call (can be useful if failed) //Create satelliteTaskCode & save satelliteTaskCode.Code to settingsfile for retracing last call (can be useful if failed)
_logger.LogInformation(string.Format($"Running RunSatelliteTask for cropfieldItem '{cropfielditemcode}' and saving settings to {settingsfile}")); //_logger.LogInformation(string.Format($"Running RunSatelliteTask for cropfieldItem '{cropfielditemcode}' and saving settings to {settingsfile}"));
var satelliteTaskCode = await _generalService.RunSatelliteTask(cropfieldItem); //var satelliteTaskCode = await _generalService.RunSatelliteTask(cropfieldItem);
// POSSIBLE & DESIRABLE TO ALSO LOG satelliteTaskCode? if (string.IsNullOrEmpty(satellitetaskcode))
{
_logger.LogInformation(string.Format($"Running RunSatelliteTask for cropfieldItem '{cropfielditemcode}'"));
satellitetaskcode = await _generalService.RunSatelliteTask(cropfieldItem);
// If working with a database, add this cropfieldItem.Code to the database so that next case same cropField is requested, will be faster
if (database != null)
{
// add this satellitetaskcode to the cropfieldtable
using (NpgsqlConnection connection = new NpgsqlConnection(database.GetConnectionString()))
{
connection.Open();
NpgsqlCommand updateCmd = connection.CreateCommand();
string updateSql = string.Format($"UPDATE {schemaname}.{cropfieldtable} SET satellitetaskcode = '{satellitetaskcode}' WHERE arbid = {fieldID};");
updateCmd.CommandText = updateSql;
//Console.WriteLine(insertCmd.CommandText);
int r = updateCmd.ExecuteNonQuery();
if (r != 1)
throw new Exception("// FarmmapsBulkSatDownload: Update satellitetaskcode Failed");
connection.Close();
}
_logger.LogInformation($"// FarmmapsBulkSatDownload: Added satellitetaskcode '{satellitetaskcode}' for cropfieldItem.Code '{cropfieldItem.Code}' to {schemaname}.{cropfieldtable} ");
}
}
else
{
_logger.LogInformation($"// FarmmapsBulkSatDownload: satellitetaskcode for parcelid {fieldID} already there in {schemaname}.{cropfieldtable}: '{satellitetaskcode}'");
}
// TODO also log satellitetaskcode to settings, how?
// SaveSettings(settingsfile); // SaveSettings(settingsfile);
// Getting satellite items // Getting satellite items
_logger.LogInformation(string.Format($"Running FindSatelliteItems for cropfieldItem.Code '{cropfieldItem.Code}', SatelliteTaskCode '{satelliteTaskCode}'")); _logger.LogInformation(string.Format($"Running FindSatelliteItems for cropfieldItem.Code '{cropfieldItem.Code}', SatelliteTaskCode '{satellitetaskcode}'"));
satelliteItemsCropYear = await _generalService.FindSatelliteItems(cropfieldItem, satelliteTaskCode); satelliteItemsCropYear = await _generalService.FindSatelliteItems(cropfieldItem, satellitetaskcode);
// Checking if satellite items found // Checking if satellite items found
satelliteItemsAvailable = true; satelliteItemsAvailable = true;
@@ -365,7 +399,7 @@ LIMIT 2000;", schemaname, parceltablename, groenmonitorlatestviewname); //LIMI
{ {
if (dtSatelliteDate <= input.lastdownloadedimagedate) if (dtSatelliteDate <= input.lastdownloadedimagedate)
{ {
_logger.LogInformation($"// Stats for parcelid {input.fieldID}, date '{strSatelliteDate}' already there in {schemaname}.{satelllitetable}"); _logger.LogInformation($"// Stats for parcelid {fieldID}, date '{strSatelliteDate}' already there in {schemaname}.{satelllitetable}");
} }
else else
{ {
@@ -378,8 +412,8 @@ LIMIT 2000;", schemaname, parceltablename, groenmonitorlatestviewname); //LIMI
sbInsertSqlUpdate = new StringBuilder(); sbInsertSqlUpdate = new StringBuilder();
sbInsertSqlValues = new StringBuilder(); sbInsertSqlValues = new StringBuilder();
sbInsertSqlInto.Append($"INSERT INTO {schemaname}.{satelllitetable} (parcelid,date,source"); sbInsertSqlInto.Append($"INSERT INTO {schemaname}.{satelllitetable} (parcelid,date,source");
sbInsertSqlValues.Append($"VALUES ({input.fieldID},'{strSatelliteDate}','{satelliteSource}'"); sbInsertSqlValues.Append($"VALUES ({fieldID},'{strSatelliteDate}','{satelliteSource}'");
sbInsertSqlUpdate.Append($"ON CONFLICT (parcelid,date,source) DO UPDATE SET parcelid={input.fieldID}, date='{strSatelliteDate}',source='{satelliteSource}'"); sbInsertSqlUpdate.Append($"ON CONFLICT (parcelid,date,source) DO UPDATE SET parcelid={fieldID}, date='{strSatelliteDate}',source='{satelliteSource}'");
if (satelliteBands.Contains("wdvi")) if (satelliteBands.Contains("wdvi"))
{ {
if (satelliteStatistics_wdvi != null) if (satelliteStatistics_wdvi != null)
@@ -418,7 +452,7 @@ LIMIT 2000;", schemaname, parceltablename, groenmonitorlatestviewname); //LIMI
int r = insertCmd.ExecuteNonQuery(); int r = insertCmd.ExecuteNonQuery();
if (r != 1) if (r != 1)
throw new Exception("// FarmmapsBulkSatDownload: Insert Failed"); throw new Exception("// FarmmapsBulkSatDownload: Insert Failed");
_logger.LogInformation($"// Added stats to {schemaname}.{satelllitetable} for parcelid {input.fieldID}, date '{strSatelliteDate}'. cntDatesDownloaded: {cntDatesDownloaded}"); _logger.LogInformation($"// Added stats to {schemaname}.{satelllitetable} for parcelid {fieldID}, date '{strSatelliteDate}'. cntDatesDownloaded: {cntDatesDownloaded}");
} }
} }
} }

1
FarmmapsBulkSatDownload/Models/BulkSatDownloadInput.cs
View File

@@ -13,6 +13,7 @@ namespace FarmmapsBulkSatDownload.Models
public DateTime lastdownloadedimagedate { get; set; } public DateTime lastdownloadedimagedate { get; set; }
public JObject GeometryJson { get; set; } public JObject GeometryJson { get; set; }
public string cropfielditemcode { get; set; } public string cropfielditemcode { get; set; }
public string satellitetaskcode { get; set; }
public string downloadFolder { get; set; } public string downloadFolder { get; set; }
public string fileNameStats { get; set; } public string fileNameStats { get; set; }
public DB database { get; set; } public DB database { get; set; }