From d1747427bc84bace8d8d1450f65c1ced236ce8aa Mon Sep 17 00:00:00 2001 From: pepijn van oort Date: Sat, 29 May 2021 12:50:33 +0200 Subject: [PATCH] also logging satellitetaskcode to cropfieldtable in database --- .../BulkSatDownloadApplication.cs | 66 ++++++++++++++----- .../Models/BulkSatDownloadInput.cs | 1 + 2 files changed, 51 insertions(+), 16 deletions(-) diff --git a/FarmmapsBulkSatDownload/BulkSatDownloadApplication.cs b/FarmmapsBulkSatDownload/BulkSatDownloadApplication.cs index 35f3bf6..7017f46 100644 --- a/FarmmapsBulkSatDownload/BulkSatDownloadApplication.cs +++ b/FarmmapsBulkSatDownload/BulkSatDownloadApplication.cs @@ -75,12 +75,14 @@ namespace FarmmapsBulkSatDownload string connectionString = dbparcels.GetConnectionString(); string readSql = string.Format( @" -SELECT pt.arbid, pt.year, gml.lastwenrdate, ST_AsGeoJSON(ST_Transform((ST_DUMP(pt.geom)).geom::geometry(Polygon),4326)) AS geojson_polygon_wgs84, COALESCE(pt.cropfielditemcode,'') AS cropfielditemcode +SELECT pt.arbid, pt.year, gml.lastwenrdate, ST_AsGeoJSON(ST_Transform((ST_DUMP(pt.geom)).geom::geometry(Polygon),4326)) AS geojson_polygon_wgs84, + COALESCE(pt.cropfielditemcode,'') AS cropfielditemcode, + CASE WHEN pt.year >= DATE_PART('year', CURRENT_DATE) THEN '' ELSE COALESCE(pt.satellitetaskcode,'') END AS satellitetaskcode FROM {0}.{1} pt, {0}.{2} gml WHERE pt.arbid = gml.arbid AND pt.crop NOT IN ('Lelie','Tulp') AND - pt.year > 2018 AND pt.arbid >= 8155 + pt.year > 2018 AND pt.arbid IN(8276,8314,8315) ORDER BY pt.arbid LIMIT 2000;", schemaname, parceltablename, groenmonitorlatestviewname); //LIMIT x for testing @@ -102,6 +104,7 @@ LIMIT 2000;", schemaname, parceltablename, groenmonitorlatestviewname); //LIMI bulkSatDownloadInput.GeometryJson = JObject.Parse(dr.GetString(3)); bulkSatDownloadInput.SatelliteBands = satelliteBands; bulkSatDownloadInput.cropfielditemcode = dr.GetString(4); + bulkSatDownloadInput.satellitetaskcode = dr.GetString(5); bulkSatDownloadInput.database = dbparcels; bulkSatDownloadInput.schemaname = schemaname; bulkSatDownloadInput.cropfieldtable = parceltablename; @@ -177,6 +180,7 @@ LIMIT 2000;", schemaname, parceltablename, groenmonitorlatestviewname); //LIMI private async Task Process(List roots, BulkSatDownloadInput input) { string cropfielditemcode; + string satellitetaskcode; Item cropfieldItem; bool satelliteItemsAvailable; bool statsAvailable; @@ -193,6 +197,7 @@ LIMIT 2000;", schemaname, parceltablename, groenmonitorlatestviewname); //LIMI int cntDatesDownloaded; string satelliteSource = "akkerwebwenr"; //same as in groenmonitorlatestviewname SQL code + int fieldID = input.fieldID; string fieldName = input.fieldName; int cropYear = input.cropYear; List satelliteBands = input.SatelliteBands; @@ -204,6 +209,7 @@ LIMIT 2000;", schemaname, parceltablename, groenmonitorlatestviewname); //LIMI string satelllitetable = input.satelllitetable; DateTime lastDownloadedSatelliteDate = input.lastdownloadedimagedate; cropfielditemcode = input.cropfielditemcode; + satellitetaskcode = input.satellitetaskcode; LoadSettings(settingsfile); @@ -230,20 +236,20 @@ LIMIT 2000;", schemaname, parceltablename, groenmonitorlatestviewname); //LIMI // If working with a database, add this cropfieldItem.Code to the database so that next case same cropField is requested, will be faster if (database != null) { - // add this CropfieldItemCode to the parceltable + // add this cropfielditemcode to the cropfieldtable using (NpgsqlConnection connection = new NpgsqlConnection(database.GetConnectionString())) { connection.Open(); NpgsqlCommand updateCmd = connection.CreateCommand(); - string updateSql = string.Format($"UPDATE {schemaname}.{cropfieldtable} SET cropfielditemcode = '{cropfieldItem.Code}' WHERE arbid = {input.fieldID};"); + string updateSql = string.Format($"UPDATE {schemaname}.{cropfieldtable} SET cropfielditemcode = '{cropfieldItem.Code}' WHERE arbid = {fieldID};"); updateCmd.CommandText = updateSql; //Console.WriteLine(insertCmd.CommandText); int r = updateCmd.ExecuteNonQuery(); if (r != 1) - throw new Exception("// FarmmapsBulkSatDownload: Update Failed"); + throw new Exception("// FarmmapsBulkSatDownload: Update cropfielditemcode Failed"); connection.Close(); } - _logger.LogInformation($"// FarmmapsBulkSatDownload: Added cropfieldItem.Code '{cropfieldItem.Code}' for parcelid {input.fieldID} to {schemaname}.{cropfieldtable} "); + _logger.LogInformation($"// FarmmapsBulkSatDownload: Added cropfieldItem.Code '{cropfieldItem.Code}' for parcelid {fieldID} to {schemaname}.{cropfieldtable} "); } } else @@ -252,20 +258,48 @@ LIMIT 2000;", schemaname, parceltablename, groenmonitorlatestviewname); //LIMI cropfieldItem = new Item(); cropfieldItem.Code = cropfielditemcode; cropfieldItem.Name = "DataCropfield " + fieldName; - _logger.LogInformation($"// FarmmapsBulkSatDownload: CropfieldItem.Code for parcelid {input.fieldID} already there in {schemaname}.{cropfieldtable}: '{cropfieldItem.Code}'"); + _logger.LogInformation($"// FarmmapsBulkSatDownload: CropfieldItem.Code for parcelid {fieldID} already there in {schemaname}.{cropfieldtable}: '{cropfieldItem.Code}'"); } _settings.cropFieldItems.Add(cropfieldItem); SaveSettings(settingsfile); //Create satelliteTaskCode & save satelliteTaskCode.Code to settingsfile for retracing last call (can be useful if failed) - _logger.LogInformation(string.Format($"Running RunSatelliteTask for cropfieldItem '{cropfielditemcode}' and saving settings to {settingsfile}")); - var satelliteTaskCode = await _generalService.RunSatelliteTask(cropfieldItem); - // POSSIBLE & DESIRABLE TO ALSO LOG satelliteTaskCode? + //_logger.LogInformation(string.Format($"Running RunSatelliteTask for cropfieldItem '{cropfielditemcode}' and saving settings to {settingsfile}")); + //var satelliteTaskCode = await _generalService.RunSatelliteTask(cropfieldItem); + if (string.IsNullOrEmpty(satellitetaskcode)) + { + _logger.LogInformation(string.Format($"Running RunSatelliteTask for cropfieldItem '{cropfielditemcode}'")); + satellitetaskcode = await _generalService.RunSatelliteTask(cropfieldItem); + // If working with a database, add this cropfieldItem.Code to the database so that next case same cropField is requested, will be faster + if (database != null) + { + // add this satellitetaskcode to the cropfieldtable + using (NpgsqlConnection connection = new NpgsqlConnection(database.GetConnectionString())) + { + connection.Open(); + NpgsqlCommand updateCmd = connection.CreateCommand(); + string updateSql = string.Format($"UPDATE {schemaname}.{cropfieldtable} SET satellitetaskcode = '{satellitetaskcode}' WHERE arbid = {fieldID};"); + updateCmd.CommandText = updateSql; + //Console.WriteLine(insertCmd.CommandText); + int r = updateCmd.ExecuteNonQuery(); + if (r != 1) + throw new Exception("// FarmmapsBulkSatDownload: Update satellitetaskcode Failed"); + connection.Close(); + } + _logger.LogInformation($"// FarmmapsBulkSatDownload: Added satellitetaskcode '{satellitetaskcode}' for cropfieldItem.Code '{cropfieldItem.Code}' to {schemaname}.{cropfieldtable} "); + } + } + else + { + _logger.LogInformation($"// FarmmapsBulkSatDownload: satellitetaskcode for parcelid {fieldID} already there in {schemaname}.{cropfieldtable}: '{satellitetaskcode}'"); + } + + // TODO also log satellitetaskcode to settings, how? // SaveSettings(settingsfile); // Getting satellite items - _logger.LogInformation(string.Format($"Running FindSatelliteItems for cropfieldItem.Code '{cropfieldItem.Code}', SatelliteTaskCode '{satelliteTaskCode}'")); - satelliteItemsCropYear = await _generalService.FindSatelliteItems(cropfieldItem, satelliteTaskCode); + _logger.LogInformation(string.Format($"Running FindSatelliteItems for cropfieldItem.Code '{cropfieldItem.Code}', SatelliteTaskCode '{satellitetaskcode}'")); + satelliteItemsCropYear = await _generalService.FindSatelliteItems(cropfieldItem, satellitetaskcode); // Checking if satellite items found satelliteItemsAvailable = true; @@ -365,7 +399,7 @@ LIMIT 2000;", schemaname, parceltablename, groenmonitorlatestviewname); //LIMI { if (dtSatelliteDate <= input.lastdownloadedimagedate) { - _logger.LogInformation($"// Stats for parcelid {input.fieldID}, date '{strSatelliteDate}' already there in {schemaname}.{satelllitetable}"); + _logger.LogInformation($"// Stats for parcelid {fieldID}, date '{strSatelliteDate}' already there in {schemaname}.{satelllitetable}"); } else { @@ -378,8 +412,8 @@ LIMIT 2000;", schemaname, parceltablename, groenmonitorlatestviewname); //LIMI sbInsertSqlUpdate = new StringBuilder(); sbInsertSqlValues = new StringBuilder(); sbInsertSqlInto.Append($"INSERT INTO {schemaname}.{satelllitetable} (parcelid,date,source"); - sbInsertSqlValues.Append($"VALUES ({input.fieldID},'{strSatelliteDate}','{satelliteSource}'"); - sbInsertSqlUpdate.Append($"ON CONFLICT (parcelid,date,source) DO UPDATE SET parcelid={input.fieldID}, date='{strSatelliteDate}',source='{satelliteSource}'"); + sbInsertSqlValues.Append($"VALUES ({fieldID},'{strSatelliteDate}','{satelliteSource}'"); + sbInsertSqlUpdate.Append($"ON CONFLICT (parcelid,date,source) DO UPDATE SET parcelid={fieldID}, date='{strSatelliteDate}',source='{satelliteSource}'"); if (satelliteBands.Contains("wdvi")) { if (satelliteStatistics_wdvi != null) @@ -418,7 +452,7 @@ LIMIT 2000;", schemaname, parceltablename, groenmonitorlatestviewname); //LIMI int r = insertCmd.ExecuteNonQuery(); if (r != 1) throw new Exception("// FarmmapsBulkSatDownload: Insert Failed"); - _logger.LogInformation($"// Added stats to {schemaname}.{satelllitetable} for parcelid {input.fieldID}, date '{strSatelliteDate}'. cntDatesDownloaded: {cntDatesDownloaded}"); + _logger.LogInformation($"// Added stats to {schemaname}.{satelllitetable} for parcelid {fieldID}, date '{strSatelliteDate}'. cntDatesDownloaded: {cntDatesDownloaded}"); } } } diff --git a/FarmmapsBulkSatDownload/Models/BulkSatDownloadInput.cs b/FarmmapsBulkSatDownload/Models/BulkSatDownloadInput.cs index e23b914..cddc01c 100644 --- a/FarmmapsBulkSatDownload/Models/BulkSatDownloadInput.cs +++ b/FarmmapsBulkSatDownload/Models/BulkSatDownloadInput.cs @@ -13,6 +13,7 @@ namespace FarmmapsBulkSatDownload.Models public DateTime lastdownloadedimagedate { get; set; } public JObject GeometryJson { get; set; } public string cropfielditemcode { get; set; } + public string satellitetaskcode { get; set; } public string downloadFolder { get; set; } public string fileNameStats { get; set; } public DB database { get; set; }