solved issue where sometimes satelliteItem contains only 2 layers
Normally 3 layers 0: ndvi, 1: wdvi, 2: natural. In some cases only 2 layers: 0: wdvi, 1: natural. Which caused error when requesting statistics. This happened for example for cropfieldItem.Code ‘e08d71bd92334dbab9e645ad6e72da63’
This commit is contained in:
parent
1bc326cfd2
commit
111b4e576f
@ -423,32 +423,28 @@ namespace FarmmapsApi.Services
|
|||||||
|
|
||||||
string satelliteDataStatisticsFile = Path.Combine(downloadFolder, $"satelliteStats_{fieldName}.csv");
|
string satelliteDataStatisticsFile = Path.Combine(downloadFolder, $"satelliteStats_{fieldName}.csv");
|
||||||
File.Delete(satelliteDataStatisticsFile); // Delete the SatelliteFile file if exists
|
File.Delete(satelliteDataStatisticsFile); // Delete the SatelliteFile file if exists
|
||||||
List<int> selectedLayers = new List<int>();
|
|
||||||
foreach (string satelliteBand in satelliteBands)
|
|
||||||
{
|
|
||||||
if (satelliteBand == "ndvi") selectedLayers.Add(0);
|
|
||||||
if (satelliteBand == "wdvi") selectedLayers.Add(1);
|
|
||||||
};
|
|
||||||
string headerLineStats = $"FieldName,satelliteDate,satelliteBand,max,min,mean,mode,median,stddev,minPlus,curtosis,maxMinus,skewness,variance,populationCount,variationCoefficient,confidenceIntervalLow, confidenceIntervalHigh,confidenceIntervalErrorMargin" + Environment.NewLine;
|
string headerLineStats = $"FieldName,satelliteDate,satelliteBand,max,min,mean,mode,median,stddev,minPlus,curtosis,maxMinus,skewness,variance,populationCount,variationCoefficient,confidenceIntervalLow, confidenceIntervalHigh,confidenceIntervalErrorMargin" + Environment.NewLine;
|
||||||
File.AppendAllText(satelliteDataStatisticsFile, headerLineStats);
|
File.AppendAllText(satelliteDataStatisticsFile, headerLineStats);
|
||||||
foreach (var item in satelliteTiffs)
|
foreach (var satelliteTiff in satelliteTiffs)
|
||||||
foreach (int selectedLayer in selectedLayers)
|
|
||||||
{
|
{
|
||||||
|
List<JToken> layers = satelliteTiff.Data["layers"].Children().ToList();
|
||||||
|
foreach (JToken layer in layers)
|
||||||
{
|
{
|
||||||
var satelliteBand = item.Data["layers"][selectedLayer]["name"];
|
DateTime satelliteImageDate = (DateTime)satelliteTiff.DataDate;
|
||||||
var satelliteImageDate = (DateTime)item.DataDate;
|
string satelliteBand = layer["name"].ToString();
|
||||||
var satelliteStatisticsJtoken = item.Data["layers"][selectedLayer]["renderer"]["band"]["statistics"];
|
if (satelliteBands.Contains(satelliteBand))
|
||||||
|
{
|
||||||
|
JToken satelliteStatisticsJtoken = layer["renderer"]["band"]["statistics"];
|
||||||
if (satelliteStatisticsJtoken == null)
|
if (satelliteStatisticsJtoken == null)
|
||||||
{
|
{
|
||||||
_logger.LogWarning($"{satelliteImageDate.ToString("yyyy-MM-dd")} no statistics found");
|
_logger.LogWarning($"{satelliteImageDate.ToString("yyyy-MM-dd")} no statistics found for satelliteBand '{satelliteBand}'");
|
||||||
//Console.WriteLine($"Available data: {item.Data}");
|
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
SatelliteStatistics satelliteStatistics = satelliteStatisticsJtoken.ToObject<SatelliteStatistics>();
|
SatelliteStatistics satelliteStatistics = satelliteStatisticsJtoken.ToObject<SatelliteStatistics>();
|
||||||
satelliteStatistics.fieldName = fieldName;
|
satelliteStatistics.fieldName = fieldName;
|
||||||
satelliteStatistics.satelliteDate = satelliteImageDate;
|
satelliteStatistics.satelliteDate = satelliteImageDate;
|
||||||
satelliteStatistics.satelliteBand = satelliteBand.ToString();
|
satelliteStatistics.satelliteBand = satelliteBand;
|
||||||
File.AppendAllText(satelliteDataStatisticsFile, $"" +
|
File.AppendAllText(satelliteDataStatisticsFile, $"" +
|
||||||
$"{satelliteStatistics.fieldName}," +
|
$"{satelliteStatistics.fieldName}," +
|
||||||
$"{satelliteStatistics.satelliteDate.ToString("yyyy-MM-dd")}," +
|
$"{satelliteStatistics.satelliteDate.ToString("yyyy-MM-dd")}," +
|
||||||
@ -473,38 +469,42 @@ namespace FarmmapsApi.Services
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
return satelliteDataStatisticsFile;
|
return satelliteDataStatisticsFile;
|
||||||
}
|
}
|
||||||
public async Task<List<SatelliteStatistics>> ListSatelliteStatistics(Item satelliteTiff, List<string> satelliteBands = null, string fieldName = null)
|
public async Task<List<SatelliteStatistics>> ListSatelliteStatistics(Item satelliteTiff, List<string> satelliteBands = null, string fieldName = null)
|
||||||
{
|
{
|
||||||
SatelliteStatistics satelliteStatistics;
|
SatelliteStatistics satelliteStatistics;
|
||||||
List<SatelliteStatistics> listSatelliteStatistics = new List<SatelliteStatistics>();
|
List<SatelliteStatistics> listSatelliteStatistics = new List<SatelliteStatistics>();
|
||||||
List<int> selectedLayers = new List<int>();
|
List<JToken> layers = satelliteTiff.Data["layers"].Children().ToList();
|
||||||
foreach (string satelliteBand in satelliteBands)
|
foreach (JToken layer in layers)
|
||||||
{
|
{
|
||||||
if (satelliteBand == "ndvi") selectedLayers.Add(0);
|
DateTime satelliteImageDate = (DateTime)satelliteTiff.DataDate;
|
||||||
if (satelliteBand == "wdvi") selectedLayers.Add(1);
|
string satelliteBand = layer["name"].ToString();
|
||||||
};
|
//_logger.LogInformation($"Date '{satelliteImageDate.ToString("yyyy-MM-dd")}': satelliteBand: {satelliteBand}");
|
||||||
|
if (satelliteBands.Contains(satelliteBand))
|
||||||
foreach (int selectedLayer in selectedLayers)
|
|
||||||
{
|
{
|
||||||
var satelliteBand = satelliteTiff.Data["layers"][selectedLayer]["name"];
|
JToken satelliteStatisticsJtoken = layer["renderer"]["band"]["statistics"];
|
||||||
var satelliteImageDate = (DateTime)satelliteTiff.DataDate;
|
|
||||||
var satelliteStatisticsJtoken = satelliteTiff.Data["layers"][selectedLayer]["renderer"]["band"]["statistics"];
|
|
||||||
if (satelliteStatisticsJtoken == null)
|
if (satelliteStatisticsJtoken == null)
|
||||||
{
|
{
|
||||||
_logger.LogWarning($"{satelliteImageDate.ToString("yyyy-MM-dd")} no statistics found");
|
_logger.LogWarning($"{satelliteImageDate.ToString("yyyy-MM-dd")} no statistics found for satelliteBand '{satelliteBand}'");
|
||||||
//Console.WriteLine($"Available data: {item.Data}");
|
//Console.WriteLine($"Available data: {item.Data}");
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
|
//_logger.LogInformation($"Adding satelliteStatistics to listSatelliteStatistics");
|
||||||
satelliteStatistics = satelliteStatisticsJtoken.ToObject<SatelliteStatistics>();
|
satelliteStatistics = satelliteStatisticsJtoken.ToObject<SatelliteStatistics>();
|
||||||
satelliteStatistics.fieldName = fieldName.ToString();
|
satelliteStatistics.fieldName = fieldName;
|
||||||
satelliteStatistics.satelliteDate = satelliteImageDate;
|
satelliteStatistics.satelliteDate = satelliteImageDate;
|
||||||
satelliteStatistics.satelliteBand = satelliteBand.ToString();
|
satelliteStatistics.satelliteBand = satelliteBand;
|
||||||
listSatelliteStatistics.Add(satelliteStatistics);
|
listSatelliteStatistics.Add(satelliteStatistics);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
//else
|
||||||
|
//{
|
||||||
|
// _logger.LogInformation($"this satelliteBand is not in your list satelliteBands");
|
||||||
|
//}
|
||||||
|
}
|
||||||
|
|
||||||
return listSatelliteStatistics;
|
return listSatelliteStatistics;
|
||||||
}
|
}
|
||||||
|
@ -57,16 +57,15 @@ namespace FarmmapsBulkSatDownload
|
|||||||
// Crashes if "DBsettings.secrets.json" is absent or empty
|
// Crashes if "DBsettings.secrets.json" is absent or empty
|
||||||
DB dbparcels = JsonConvert.DeserializeObject<DB>(File.ReadAllText("DBsettings.secrets.json"));
|
DB dbparcels = JsonConvert.DeserializeObject<DB>(File.ReadAllText("DBsettings.secrets.json"));
|
||||||
string schemaname = "bigdata";
|
string schemaname = "bigdata";
|
||||||
string parceltablename = "parcel_disac";//"parcelsijbrandij" "parcel"; "parcel_flowerbulbs"; "parcel_disac"
|
string parceltablename = "parcel_flowerbulbs";//"parcelsijbrandij" "parcel"; "parcel_flowerbulbs"; "parcel_disac"
|
||||||
string groenmonitortablename = "groenmonitor_disac";//"groenmonitorsijbrandij" "groenmonitor" "groenmonitor_flowerbulbs" "groenmonitor_disac"
|
string groenmonitortablename = "groenmonitor_flowerbulbs";//"groenmonitorsijbrandij" "groenmonitor" "groenmonitor_flowerbulbs" "groenmonitor_disac"
|
||||||
// The view 'groenmonitorlatestviewname' contains per parcelid (arbid) the year in which it "exists" and the date of the latest image downloaded. It is used to prevent unneccessary downloading of image statistics already in the database
|
// The view 'groenmonitorlatestviewname' contains per parcelid (arbid) the year in which it "exists" and the date of the latest image downloaded. It is used to prevent unneccessary downloading of image statistics already in the database
|
||||||
string groenmonitorlatestviewname = "groenmonitorlatest_disac"; //"groenmonitorsijbrandijlatest" "groenmonitorlatest" "groenmonitorlatest_flowerbulbs" "groenmonitorlatest_disac"
|
string groenmonitorlatestviewname = "groenmonitorlatest_flowerbulbs"; //"groenmonitorsijbrandijlatest" "groenmonitorlatest" "groenmonitorlatest_flowerbulbs" "groenmonitorlatest_disac"
|
||||||
//GroenmonitorTable gmtb;
|
|
||||||
|
|
||||||
// Database query and connection. Geometry must be in WGS84 coordinate system, EPSG 4326
|
// Database query and connection. Geometry must be in WGS84 coordinate system, EPSG 4326
|
||||||
// Apparently the FarmmapsApi cannot handle MultiPolygon, so we need to convert to single Polygon
|
// Apparently the FarmmapsApi cannot handle MultiPolygon, so we need to convert to single Polygon
|
||||||
// In case database returns a MultiPolygon use ST_NumGeometries(pt.geom) to count the number of polygons
|
// In case database returns a MultiPolygon use ST_NumGeometries(pt.geom) to count the number of polygons
|
||||||
// If necessary use WHERE T_NumGeometries(pt.geom) = 1 to select only single polygons
|
// If necessary use WHERE ST_NumGeometries(pt.geom) = 1 to select only single polygons
|
||||||
//
|
//
|
||||||
// FarmMaps get's its satellite images from www.groenmonitor.nl through the https://agrodatacube.wur.nl/.
|
// FarmMaps get's its satellite images from www.groenmonitor.nl through the https://agrodatacube.wur.nl/.
|
||||||
// Many images are available at www.groenmonitor.nl, the https://agrodatacube.wur.nl/ serves only the clean images, 10-30 per year, 2019 onwards. Possibly more images will be added for earlier years
|
// Many images are available at www.groenmonitor.nl, the https://agrodatacube.wur.nl/ serves only the clean images, 10-30 per year, 2019 onwards. Possibly more images will be added for earlier years
|
||||||
@ -79,9 +78,11 @@ namespace FarmmapsBulkSatDownload
|
|||||||
SELECT pt.arbid, pt.year, gml.lastwenrdate, ST_AsGeoJSON(ST_Transform((ST_DUMP(pt.geom)).geom::geometry(Polygon),4326)) AS geojson_polygon_wgs84, COALESCE(pt.cropfielditemcode,'') AS cropfielditemcode
|
SELECT pt.arbid, pt.year, gml.lastwenrdate, ST_AsGeoJSON(ST_Transform((ST_DUMP(pt.geom)).geom::geometry(Polygon),4326)) AS geojson_polygon_wgs84, COALESCE(pt.cropfielditemcode,'') AS cropfielditemcode
|
||||||
FROM {0}.{1} pt, {0}.{2} gml
|
FROM {0}.{1} pt, {0}.{2} gml
|
||||||
WHERE
|
WHERE
|
||||||
pt.arbid = gml.arbid
|
pt.arbid = gml.arbid AND
|
||||||
|
pt.crop NOT IN ('Lelie','Tulp') AND
|
||||||
|
pt.year > 2018 AND pt.arbid >= 8155
|
||||||
ORDER BY pt.arbid
|
ORDER BY pt.arbid
|
||||||
LIMIT 15;", schemaname, parceltablename, groenmonitorlatestviewname); //LIMIT x for testing
|
LIMIT 2000;", schemaname, parceltablename, groenmonitorlatestviewname); //LIMIT x for testing
|
||||||
|
|
||||||
using (NpgsqlConnection connection = new NpgsqlConnection(connectionString))
|
using (NpgsqlConnection connection = new NpgsqlConnection(connectionString))
|
||||||
{
|
{
|
||||||
@ -182,12 +183,16 @@ LIMIT 15;", schemaname, parceltablename, groenmonitorlatestviewname); //LIMIT
|
|||||||
DateTime dtSatelliteDate;
|
DateTime dtSatelliteDate;
|
||||||
string strSatelliteDate;
|
string strSatelliteDate;
|
||||||
List<Item> satelliteItemsCropYear;
|
List<Item> satelliteItemsCropYear;
|
||||||
GroenmonitorTable groenmonitorTable = new GroenmonitorTable();
|
StringBuilder sbInsertSql;
|
||||||
|
StringBuilder sbInsertSqlInto;
|
||||||
|
StringBuilder sbInsertSqlUpdate;
|
||||||
|
StringBuilder sbInsertSqlValues;
|
||||||
List<SatelliteStatistics> listSatelliteStatistics;
|
List<SatelliteStatistics> listSatelliteStatistics;
|
||||||
SatelliteStatistics satelliteStatistics_wdvi;
|
SatelliteStatistics satelliteStatistics_wdvi;
|
||||||
SatelliteStatistics satelliteStatistics_ndvi;
|
SatelliteStatistics satelliteStatistics_ndvi;
|
||||||
int cntDatesDownloaded;
|
int cntDatesDownloaded;
|
||||||
|
|
||||||
|
string satelliteSource = "akkerwebwenr"; //same as in groenmonitorlatestviewname SQL code
|
||||||
string fieldName = input.fieldName;
|
string fieldName = input.fieldName;
|
||||||
int cropYear = input.cropYear;
|
int cropYear = input.cropYear;
|
||||||
List<string> satelliteBands = input.SatelliteBands;
|
List<string> satelliteBands = input.SatelliteBands;
|
||||||
@ -199,7 +204,6 @@ LIMIT 15;", schemaname, parceltablename, groenmonitorlatestviewname); //LIMIT
|
|||||||
string satelllitetable = input.satelllitetable;
|
string satelllitetable = input.satelllitetable;
|
||||||
DateTime lastDownloadedSatelliteDate = input.lastdownloadedimagedate;
|
DateTime lastDownloadedSatelliteDate = input.lastdownloadedimagedate;
|
||||||
cropfielditemcode = input.cropfielditemcode;
|
cropfielditemcode = input.cropfielditemcode;
|
||||||
string insertSql = InsertSQLfromClass(schemaname, satelllitetable);
|
|
||||||
|
|
||||||
LoadSettings(settingsfile);
|
LoadSettings(settingsfile);
|
||||||
|
|
||||||
@ -236,7 +240,7 @@ LIMIT 15;", schemaname, parceltablename, groenmonitorlatestviewname); //LIMIT
|
|||||||
//Console.WriteLine(insertCmd.CommandText);
|
//Console.WriteLine(insertCmd.CommandText);
|
||||||
int r = updateCmd.ExecuteNonQuery();
|
int r = updateCmd.ExecuteNonQuery();
|
||||||
if (r != 1)
|
if (r != 1)
|
||||||
throw new Exception("// FarmmapsBulkSatDownload: Insert Failed");
|
throw new Exception("// FarmmapsBulkSatDownload: Update Failed");
|
||||||
connection.Close();
|
connection.Close();
|
||||||
}
|
}
|
||||||
_logger.LogInformation($"// FarmmapsBulkSatDownload: Added cropfieldItem.Code '{cropfieldItem.Code}' for parcelid {input.fieldID} to {schemaname}.{cropfieldtable} ");
|
_logger.LogInformation($"// FarmmapsBulkSatDownload: Added cropfieldItem.Code '{cropfieldItem.Code}' for parcelid {input.fieldID} to {schemaname}.{cropfieldtable} ");
|
||||||
@ -260,7 +264,7 @@ LIMIT 15;", schemaname, parceltablename, groenmonitorlatestviewname); //LIMIT
|
|||||||
// SaveSettings(settingsfile);
|
// SaveSettings(settingsfile);
|
||||||
|
|
||||||
// Getting satellite items
|
// Getting satellite items
|
||||||
_logger.LogInformation(string.Format($"Running FindSatelliteItemsCropYear for cropfieldItem.Code '{cropfieldItem.Code}', SatelliteTaskCode '{satelliteTaskCode}'"));
|
_logger.LogInformation(string.Format($"Running FindSatelliteItems for cropfieldItem.Code '{cropfieldItem.Code}', SatelliteTaskCode '{satelliteTaskCode}'"));
|
||||||
satelliteItemsCropYear = await _generalService.FindSatelliteItems(cropfieldItem, satelliteTaskCode);
|
satelliteItemsCropYear = await _generalService.FindSatelliteItems(cropfieldItem, satelliteTaskCode);
|
||||||
|
|
||||||
// Checking if satellite items found
|
// Checking if satellite items found
|
||||||
@ -268,14 +272,14 @@ LIMIT 15;", schemaname, parceltablename, groenmonitorlatestviewname); //LIMIT
|
|||||||
if (satelliteItemsCropYear == null)
|
if (satelliteItemsCropYear == null)
|
||||||
{
|
{
|
||||||
satelliteItemsAvailable = false;
|
satelliteItemsAvailable = false;
|
||||||
_logger.LogInformation($"No satellite tiffs found for fieldName '{fieldName}', cropYear {cropYear}");
|
_logger.LogInformation($"No satellite tiffs found for fieldName '{fieldName}', cropYear {cropYear}, cropfielditemcode '{cropfielditemcode}'");
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
if (satelliteItemsCropYear.Count == 0)
|
if (satelliteItemsCropYear.Count == 0)
|
||||||
{
|
{
|
||||||
satelliteItemsAvailable = false;
|
satelliteItemsAvailable = false;
|
||||||
_logger.LogInformation($"No satellite tiffs found for fieldName '{fieldName}', cropYear {cropYear}");
|
_logger.LogInformation($"No satellite tiffs found for fieldName '{fieldName}', cropYear {cropYear}, cropfielditemcode '{cropfielditemcode}'");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -335,6 +339,9 @@ LIMIT 15;", schemaname, parceltablename, groenmonitorlatestviewname); //LIMIT
|
|||||||
// And we assume farmMaps always nicely generates statistics, no hick-ups
|
// And we assume farmMaps always nicely generates statistics, no hick-ups
|
||||||
// Under this assumptions, we only need to compare with the lastDownloadedSatelliteDate from the database
|
// Under this assumptions, we only need to compare with the lastDownloadedSatelliteDate from the database
|
||||||
cntDatesDownloaded = 0;
|
cntDatesDownloaded = 0;
|
||||||
|
using (NpgsqlConnection connection = new NpgsqlConnection(database.GetConnectionString()))
|
||||||
|
{
|
||||||
|
connection.Open();
|
||||||
foreach (Item satelliteItem in satelliteItemsCropYear)
|
foreach (Item satelliteItem in satelliteItemsCropYear)
|
||||||
{
|
{
|
||||||
dtSatelliteDate = satelliteItem.DataDate.Value;
|
dtSatelliteDate = satelliteItem.DataDate.Value;
|
||||||
@ -344,14 +351,14 @@ LIMIT 15;", schemaname, parceltablename, groenmonitorlatestviewname); //LIMIT
|
|||||||
if (listSatelliteStatistics == null)
|
if (listSatelliteStatistics == null)
|
||||||
{
|
{
|
||||||
statsAvailable = false;
|
statsAvailable = false;
|
||||||
_logger.LogWarning($"No stats found for satellite, fielName '{fieldName}', date '{strSatelliteDate}'");
|
_logger.LogWarning($"No stats found for satellite, fieldName '{fieldName}', date '{strSatelliteDate}'");
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
if (listSatelliteStatistics.Count == 0)
|
if (listSatelliteStatistics.Count == 0)
|
||||||
{
|
{
|
||||||
statsAvailable = false;
|
statsAvailable = false;
|
||||||
_logger.LogWarning($"No stats found for satellite, fielName '{fieldName}', date '{strSatelliteDate}'");
|
_logger.LogWarning($"No stats found for satellite, fieldName '{fieldName}', date '{strSatelliteDate}'");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (statsAvailable)
|
if (statsAvailable)
|
||||||
@ -366,46 +373,60 @@ LIMIT 15;", schemaname, parceltablename, groenmonitorlatestviewname); //LIMIT
|
|||||||
// Map satelliteStatistics to groenmonitorTable
|
// Map satelliteStatistics to groenmonitorTable
|
||||||
satelliteStatistics_wdvi = listSatelliteStatistics.SingleOrDefault(p => p.satelliteBand == "wdvi");
|
satelliteStatistics_wdvi = listSatelliteStatistics.SingleOrDefault(p => p.satelliteBand == "wdvi");
|
||||||
satelliteStatistics_ndvi = listSatelliteStatistics.SingleOrDefault(p => p.satelliteBand == "ndvi");
|
satelliteStatistics_ndvi = listSatelliteStatistics.SingleOrDefault(p => p.satelliteBand == "ndvi");
|
||||||
groenmonitorTable.parcelid = input.fieldID;
|
sbInsertSql = new StringBuilder();
|
||||||
groenmonitorTable.date = strSatelliteDate;
|
sbInsertSqlInto = new StringBuilder();
|
||||||
groenmonitorTable.source = "akkerwebwenr"; // Like this so SQL in bigdata.groenmonitorlatest_flowerbulbs works properly
|
sbInsertSqlUpdate = new StringBuilder();
|
||||||
groenmonitorTable.wdvi_pixelcount = satelliteStatistics_wdvi.populationCount; //count of pixels with data
|
sbInsertSqlValues = new StringBuilder();
|
||||||
groenmonitorTable.wdvi_max = satelliteStatistics_wdvi.max;
|
sbInsertSqlInto.Append($"INSERT INTO {schemaname}.{satelllitetable} (parcelid,date,source");
|
||||||
groenmonitorTable.wdvi_mean = satelliteStatistics_wdvi.mean;
|
sbInsertSqlValues.Append($"VALUES ({input.fieldID},'{strSatelliteDate}','{satelliteSource}'");
|
||||||
groenmonitorTable.wdvi_min = satelliteStatistics_wdvi.min;
|
sbInsertSqlUpdate.Append($"ON CONFLICT (parcelid,date,source) DO UPDATE SET parcelid={input.fieldID}, date='{strSatelliteDate}',source='{satelliteSource}'");
|
||||||
groenmonitorTable.wdvi_stdev = satelliteStatistics_wdvi.stddev;
|
if (satelliteBands.Contains("wdvi"))
|
||||||
groenmonitorTable.wdvi_median = satelliteStatistics_wdvi.median;
|
|
||||||
groenmonitorTable.wdvi_p90 = -99; // Example of a statistics (90% not included in satelliteStatistics
|
|
||||||
groenmonitorTable.ndvi_pixelcount = satelliteStatistics_ndvi.populationCount; //count of pixels with data
|
|
||||||
groenmonitorTable.ndvi_max = satelliteStatistics_ndvi.max;
|
|
||||||
groenmonitorTable.ndvi_mean = satelliteStatistics_ndvi.mean;
|
|
||||||
groenmonitorTable.ndvi_min = satelliteStatistics_ndvi.min;
|
|
||||||
groenmonitorTable.ndvi_stdev = satelliteStatistics_ndvi.stddev;
|
|
||||||
groenmonitorTable.ndvi_median = satelliteStatistics_ndvi.median;
|
|
||||||
groenmonitorTable.ndvi_p90 = -99; // Example of a statistics (90% not included in satelliteStatistics
|
|
||||||
|
|
||||||
// fill the insertSql query with fieldValues from the groenmonitorTable, then run the query
|
|
||||||
using (NpgsqlConnection connection = new NpgsqlConnection(database.GetConnectionString()))
|
|
||||||
{
|
{
|
||||||
connection.Open();
|
if (satelliteStatistics_wdvi != null)
|
||||||
|
{
|
||||||
|
sbInsertSqlInto.Append(",wdvi_pixelcount,wdvi_max,wdvi_mean,wdvi_min,wdvi_stdev,wdvi_median");
|
||||||
|
sbInsertSqlValues.Append($",{satelliteStatistics_wdvi.populationCount},{satelliteStatistics_wdvi.max},{satelliteStatistics_wdvi.mean},{satelliteStatistics_wdvi.min},{satelliteStatistics_wdvi.stddev},{satelliteStatistics_wdvi.median}");
|
||||||
|
sbInsertSqlUpdate.Append($",wdvi_pixelcount={satelliteStatistics_wdvi.populationCount},wdvi_max={satelliteStatistics_wdvi.max},wdvi_mean={satelliteStatistics_wdvi.mean},wdvi_min={satelliteStatistics_wdvi.min},wdvi_stdev={satelliteStatistics_wdvi.stddev},wdvi_median={satelliteStatistics_wdvi.median}");
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
_logger.LogWarning($"wdvi missing for cropfielditemcode {cropfielditemcode}, date '{strSatelliteDate}'");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (satelliteBands.Contains("ndvi"))
|
||||||
|
{
|
||||||
|
if (satelliteStatistics_ndvi != null)
|
||||||
|
{
|
||||||
|
sbInsertSqlInto.Append(",ndvi_pixelcount,ndvi_max,ndvi_mean,ndvi_min,ndvi_stdev,ndvi_median");
|
||||||
|
sbInsertSqlValues.Append($",{satelliteStatistics_ndvi.populationCount},{satelliteStatistics_ndvi.max},{satelliteStatistics_ndvi.mean},{satelliteStatistics_ndvi.min},{satelliteStatistics_ndvi.stddev},{satelliteStatistics_ndvi.median}");
|
||||||
|
sbInsertSqlUpdate.Append($",ndvi_pixelcount={satelliteStatistics_ndvi.populationCount},ndvi_max={satelliteStatistics_ndvi.max},ndvi_mean={satelliteStatistics_ndvi.mean},ndvi_min={satelliteStatistics_ndvi.min},ndvi_stdev={satelliteStatistics_ndvi.stddev},ndvi_median={satelliteStatistics_ndvi.median}");
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
_logger.LogWarning($"ndvi missing for cropfielditemcode {cropfielditemcode}, date '{strSatelliteDate}'");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sbInsertSqlInto.Append(")");
|
||||||
|
sbInsertSqlValues.Append(")");
|
||||||
|
sbInsertSql.AppendLine(sbInsertSqlInto.ToString());
|
||||||
|
sbInsertSql.AppendLine(sbInsertSqlValues.ToString());
|
||||||
|
sbInsertSql.AppendLine(sbInsertSqlUpdate.ToString());
|
||||||
|
//string strInsertSql = sbInsertSql.ToString();
|
||||||
NpgsqlCommand insertCmd = connection.CreateCommand();
|
NpgsqlCommand insertCmd = connection.CreateCommand();
|
||||||
object[] fieldValues = groenmonitorTable.GetType()
|
insertCmd.CommandText = sbInsertSql.ToString();
|
||||||
.GetFields()
|
|
||||||
.Select(field => field.GetValue(groenmonitorTable))
|
|
||||||
.ToArray();
|
|
||||||
insertCmd.CommandText = string.Format(insertSql, fieldValues);
|
|
||||||
//Console.WriteLine(insertCmd.CommandText);
|
//Console.WriteLine(insertCmd.CommandText);
|
||||||
int r = insertCmd.ExecuteNonQuery();
|
int r = insertCmd.ExecuteNonQuery();
|
||||||
if (r != 1)
|
if (r != 1)
|
||||||
throw new Exception("// FarmmapsBulkSatDownload: Insert Failed");
|
throw new Exception("// FarmmapsBulkSatDownload: Insert Failed");
|
||||||
connection.Close();
|
|
||||||
}
|
|
||||||
_logger.LogInformation($"// Added stats to {schemaname}.{satelllitetable} for parcelid {input.fieldID}, date '{strSatelliteDate}'. cntDatesDownloaded: {cntDatesDownloaded}");
|
_logger.LogInformation($"// Added stats to {schemaname}.{satelllitetable} for parcelid {input.fieldID}, date '{strSatelliteDate}'. cntDatesDownloaded: {cntDatesDownloaded}");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
connection.Close();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Functions to save previously created cropfields
|
// Functions to save previously created cropfields
|
||||||
private void LoadSettings(string file)
|
private void LoadSettings(string file)
|
||||||
{
|
{
|
||||||
@ -434,64 +455,5 @@ LIMIT 15;", schemaname, parceltablename, groenmonitorlatestviewname); //LIMIT
|
|||||||
{
|
{
|
||||||
return String.Format("{0:00}:{1:00}:{2:00}", ts.Hours, ts.Minutes, ts.Seconds);
|
return String.Format("{0:00}:{1:00}:{2:00}", ts.Hours, ts.Minutes, ts.Seconds);
|
||||||
}
|
}
|
||||||
private string InsertSQLfromClass(string schemaname, string groenmonitortablename)
|
|
||||||
{
|
|
||||||
// Generates an INSERT query for GroenmonitorTable.
|
|
||||||
// When writing to a different table structure, just make a new class for the statellite statistics table, similar to GroenmonitorTable.cs
|
|
||||||
// @"
|
|
||||||
//INSERT INTO bigdata.groenmonitor (parcelid,date,wdvi_pixelcount,wdvi_max,wdvi_mean,wdvi_min,wdvi_stdev,wdvi_median,wdvi_p90,ndvi_pixelcount,ndvi_max,ndvi_mean,ndvi_min,ndvi_stdev,ndvi_median,ndvi_p90)
|
|
||||||
//VALUES({0},'{1}',{2},{3},{4},{5},{6},{7},{8},{9},{10},{11},{12},{13},{14},{15})
|
|
||||||
//ON CONFLICT (parcelid, date)
|
|
||||||
//DO UPDATE SET parcelid={0},date='{1}',wdvi_pixelcount={2},wdvi_max={3},wdvi_mean={4},wdvi_min={5},wdvi_stdev={6},wdvi_median={7},wdvi_p90={8},ndvi_pixelcount={9},ndvi_max={10},ndvi_mean={11},ndvi_min={12},ndvi_stdev={13},ndvi_median={14},ndvi_p90={15}
|
|
||||||
//";
|
|
||||||
string fieldName;
|
|
||||||
var fieldNames = typeof(GroenmonitorTable).GetFields()
|
|
||||||
.Select(field => field.Name)
|
|
||||||
.ToList();
|
|
||||||
StringBuilder sbInsertSql = new StringBuilder();
|
|
||||||
string insertSql;
|
|
||||||
StringBuilder insertSql1 = new StringBuilder();
|
|
||||||
insertSql1.Append(@"INSERT INTO " + schemaname + "." + groenmonitortablename + " (");
|
|
||||||
StringBuilder insertSql2 = new StringBuilder();
|
|
||||||
insertSql2.Append("VALUES(");
|
|
||||||
StringBuilder insertSql3 = new StringBuilder();
|
|
||||||
insertSql3.Append("ON CONFLICT(parcelid, date, source)");
|
|
||||||
StringBuilder insertSql4 = new StringBuilder();
|
|
||||||
insertSql4.Append("DO UPDATE SET ");
|
|
||||||
for (int i = 0; i < fieldNames.Count; i++)
|
|
||||||
{
|
|
||||||
fieldName = fieldNames[i];
|
|
||||||
insertSql1.Append(fieldName);
|
|
||||||
if (fieldName == "date" | fieldName == "source")
|
|
||||||
{
|
|
||||||
//add extra quotes ' for fields in the Postgress table for which we know they are date or varchar
|
|
||||||
insertSql2.Append("'{" + i + "}'");
|
|
||||||
insertSql4.Append(fieldName + "='{" + i + "}'");
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
insertSql2.Append("{" + i + "}");
|
|
||||||
insertSql4.Append(fieldName + "={" + i + "}");
|
|
||||||
}
|
|
||||||
if (i < fieldNames.Count - 1)
|
|
||||||
{
|
|
||||||
insertSql1.Append(",");
|
|
||||||
insertSql2.Append(",");
|
|
||||||
insertSql4.Append(",");
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
insertSql1.Append(")");
|
|
||||||
insertSql2.Append(")");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
sbInsertSql.AppendLine(insertSql1.ToString());
|
|
||||||
sbInsertSql.AppendLine(insertSql2.ToString());
|
|
||||||
sbInsertSql.AppendLine(insertSql3.ToString());
|
|
||||||
sbInsertSql.AppendLine(insertSql4.ToString());
|
|
||||||
insertSql = sbInsertSql.ToString();
|
|
||||||
return insertSql;
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -44,7 +44,6 @@
|
|||||||
]
|
]
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
"cropfielditemcode": null, //BulkSatDownloadApplication returns a Settings.json file with cropfielditems.Code. After first run, you can copy-paste that here to speed up
|
|
||||||
"downloadFolder": "C:\\workdir\\groenmonitor\\",
|
"downloadFolder": "C:\\workdir\\groenmonitor\\",
|
||||||
"fileNameStats": "BulkSatDownload.csv",
|
"fileNameStats": "BulkSatDownload.csv",
|
||||||
"database": null,
|
"database": null,
|
||||||
@ -54,7 +53,7 @@
|
|||||||
},
|
},
|
||||||
{
|
{
|
||||||
"fieldName": "5641", // FarmMaps minimum needs are: fieldName, cropYear & geometryJson
|
"fieldName": "5641", // FarmMaps minimum needs are: fieldName, cropYear & geometryJson
|
||||||
"cropYear": 2021,
|
"cropYear": 2021, //For testing a year for which we know no satellite data available, program shouldn't crash
|
||||||
"fieldID": 5641,
|
"fieldID": 5641,
|
||||||
"SatelliteBands": [ "wdvi", "ndvi" ], // ["ndvi"] or ["wdvi"] or both: [ "wdvi", "ndvi" ]
|
"SatelliteBands": [ "wdvi", "ndvi" ], // ["ndvi"] or ["wdvi"] or both: [ "wdvi", "ndvi" ]
|
||||||
"lastdownloadedimagedate": "2021-01-01", //downloads images from this date till end of the year
|
"lastdownloadedimagedate": "2021-01-01", //downloads images from this date till end of the year
|
||||||
@ -97,15 +96,13 @@
|
|||||||
]
|
]
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
"cropfielditemcode": null, //BulkSatDownloadApplication returns a Settings.json file with cropfielditems.Code. After first run, you can copy-paste that here to speed up
|
|
||||||
"downloadFolder": "C:\\workdir\\groenmonitor\\",
|
"downloadFolder": "C:\\workdir\\groenmonitor\\",
|
||||||
"fileNameStats": "BulkSatDownload.csv",
|
"fileNameStats": "BulkSatDownload.csv",
|
||||||
"database": null,
|
"database": null,
|
||||||
"schemaname": null,
|
"schemaname": null,
|
||||||
"cropfieldtable": null,
|
"cropfieldtable": null,
|
||||||
"satelllitetable": null
|
"satelllitetable": null
|
||||||
}
|
},
|
||||||
,
|
|
||||||
{
|
{
|
||||||
"fieldName": "5641", // FarmMaps minimum needs are: fieldName, cropYear & geometryJson
|
"fieldName": "5641", // FarmMaps minimum needs are: fieldName, cropYear & geometryJson
|
||||||
"cropYear": 2020,
|
"cropYear": 2020,
|
||||||
@ -151,7 +148,110 @@
|
|||||||
]
|
]
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
"cropfielditemcode": null, //BulkSatDownloadApplication returns a Settings.json file with cropfielditems.Code. After first run, you can copy-paste that here to speed up
|
"downloadFolder": "C:\\workdir\\groenmonitor\\",
|
||||||
|
"fileNameStats": "BulkSatDownload.csv",
|
||||||
|
"database": null,
|
||||||
|
"schemaname": null,
|
||||||
|
"cropfieldtable": null,
|
||||||
|
"satelllitetable": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"fieldName": "5641", // FarmMaps minimum needs are: fieldName, cropYear & geometryJson
|
||||||
|
"cropYear": 2019,
|
||||||
|
"fieldID": 5641,
|
||||||
|
"SatelliteBands": [ "wdvi", "ndvi" ], // ["ndvi"] or ["wdvi"] or both: [ "wdvi", "ndvi" ]
|
||||||
|
"lastdownloadedimagedate": "2019-01-01", //downloads images from this date till end of the year
|
||||||
|
"geometryJson": {
|
||||||
|
"type": "Polygon",
|
||||||
|
"coordinates": [
|
||||||
|
[
|
||||||
|
[ 3.37837807779104, 51.3231095796538 ],
|
||||||
|
[ 3.38065689232502, 51.3212527499355 ],
|
||||||
|
[ 3.38022924592256, 51.3210683536359 ],
|
||||||
|
[ 3.37980548452565, 51.3208801127141 ],
|
||||||
|
[ 3.37959556105776, 51.3207540143696 ],
|
||||||
|
[ 3.3793691292654, 51.3205959677371 ],
|
||||||
|
[ 3.37822219207335, 51.3215667913007 ],
|
||||||
|
[ 3.37816999925795, 51.3216109809456 ],
|
||||||
|
[ 3.37646704574705, 51.3208025481261 ],
|
||||||
|
[ 3.37646695791282, 51.3208025061493 ],
|
||||||
|
[ 3.37608401443192, 51.3206231652693 ],
|
||||||
|
[ 3.37607169507628, 51.3206173959751 ],
|
||||||
|
[ 3.37606021048754, 51.320612017601 ],
|
||||||
|
[ 3.37582728410659, 51.3205029306946 ],
|
||||||
|
[ 3.37580409779263, 51.3206502985963 ],
|
||||||
|
[ 3.37575872019649, 51.3207993094705 ],
|
||||||
|
[ 3.37575476634361, 51.3208122883487 ],
|
||||||
|
[ 3.37571181656268, 51.3208797459348 ],
|
||||||
|
[ 3.3756624532907, 51.3209415238446 ],
|
||||||
|
[ 3.37557609963811, 51.3210110142077 ],
|
||||||
|
[ 3.37541089899821, 51.3211055871218 ],
|
||||||
|
[ 3.37477516102591, 51.3214102985009 ],
|
||||||
|
[ 3.37473173914127, 51.3214311108204 ],
|
||||||
|
[ 3.37455904622072, 51.3215138815012 ],
|
||||||
|
[ 3.37415098054777, 51.3217199232877 ],
|
||||||
|
[ 3.37313700916272, 51.3222422862785 ],
|
||||||
|
[ 3.37748824689601, 51.3242852920348 ],
|
||||||
|
[ 3.37749760805371, 51.3242713084009 ],
|
||||||
|
[ 3.37811903757028, 51.3233437635596 ],
|
||||||
|
[ 3.37818758851947, 51.3232647797363 ],
|
||||||
|
[ 3.37823803668144, 51.3232236798646 ],
|
||||||
|
[ 3.37837807779104, 51.3231095796538 ]
|
||||||
|
]
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"downloadFolder": "C:\\workdir\\groenmonitor\\",
|
||||||
|
"fileNameStats": "BulkSatDownload.csv",
|
||||||
|
"database": null,
|
||||||
|
"schemaname": null,
|
||||||
|
"cropfieldtable": null,
|
||||||
|
"satelllitetable": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"fieldName": "5641", // FarmMaps minimum needs are: fieldName, cropYear & geometryJson
|
||||||
|
"cropYear": 2018, //little to no images for 2018
|
||||||
|
"fieldID": 5641,
|
||||||
|
"SatelliteBands": [ "wdvi", "ndvi" ], // ["ndvi"] or ["wdvi"] or both: [ "wdvi", "ndvi" ]
|
||||||
|
"lastdownloadedimagedate": "2018-01-01", //downloads images from this date till end of the year
|
||||||
|
"geometryJson": {
|
||||||
|
"type": "Polygon",
|
||||||
|
"coordinates": [
|
||||||
|
[
|
||||||
|
[ 3.37837807779104, 51.3231095796538 ],
|
||||||
|
[ 3.38065689232502, 51.3212527499355 ],
|
||||||
|
[ 3.38022924592256, 51.3210683536359 ],
|
||||||
|
[ 3.37980548452565, 51.3208801127141 ],
|
||||||
|
[ 3.37959556105776, 51.3207540143696 ],
|
||||||
|
[ 3.3793691292654, 51.3205959677371 ],
|
||||||
|
[ 3.37822219207335, 51.3215667913007 ],
|
||||||
|
[ 3.37816999925795, 51.3216109809456 ],
|
||||||
|
[ 3.37646704574705, 51.3208025481261 ],
|
||||||
|
[ 3.37646695791282, 51.3208025061493 ],
|
||||||
|
[ 3.37608401443192, 51.3206231652693 ],
|
||||||
|
[ 3.37607169507628, 51.3206173959751 ],
|
||||||
|
[ 3.37606021048754, 51.320612017601 ],
|
||||||
|
[ 3.37582728410659, 51.3205029306946 ],
|
||||||
|
[ 3.37580409779263, 51.3206502985963 ],
|
||||||
|
[ 3.37575872019649, 51.3207993094705 ],
|
||||||
|
[ 3.37575476634361, 51.3208122883487 ],
|
||||||
|
[ 3.37571181656268, 51.3208797459348 ],
|
||||||
|
[ 3.3756624532907, 51.3209415238446 ],
|
||||||
|
[ 3.37557609963811, 51.3210110142077 ],
|
||||||
|
[ 3.37541089899821, 51.3211055871218 ],
|
||||||
|
[ 3.37477516102591, 51.3214102985009 ],
|
||||||
|
[ 3.37473173914127, 51.3214311108204 ],
|
||||||
|
[ 3.37455904622072, 51.3215138815012 ],
|
||||||
|
[ 3.37415098054777, 51.3217199232877 ],
|
||||||
|
[ 3.37313700916272, 51.3222422862785 ],
|
||||||
|
[ 3.37748824689601, 51.3242852920348 ],
|
||||||
|
[ 3.37749760805371, 51.3242713084009 ],
|
||||||
|
[ 3.37811903757028, 51.3233437635596 ],
|
||||||
|
[ 3.37818758851947, 51.3232647797363 ],
|
||||||
|
[ 3.37823803668144, 51.3232236798646 ],
|
||||||
|
[ 3.37837807779104, 51.3231095796538 ]
|
||||||
|
]
|
||||||
|
]
|
||||||
|
},
|
||||||
"downloadFolder": "C:\\workdir\\groenmonitor\\",
|
"downloadFolder": "C:\\workdir\\groenmonitor\\",
|
||||||
"fileNameStats": "BulkSatDownload.csv",
|
"fileNameStats": "BulkSatDownload.csv",
|
||||||
"database": null,
|
"database": null,
|
||||||
|
@ -5,6 +5,10 @@
|
|||||||
<TargetFramework>netcoreapp3.1</TargetFramework>
|
<TargetFramework>netcoreapp3.1</TargetFramework>
|
||||||
</PropertyGroup>
|
</PropertyGroup>
|
||||||
|
|
||||||
|
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|AnyCPU'">
|
||||||
|
<WarningLevel>5</WarningLevel>
|
||||||
|
</PropertyGroup>
|
||||||
|
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
<PackageReference Include="Npgsql" Version="5.0.5" />
|
<PackageReference Include="Npgsql" Version="5.0.5" />
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
|
@ -1,28 +0,0 @@
|
|||||||
using System;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
using System.Text;
|
|
||||||
|
|
||||||
namespace FarmmapsBulkSatDownload.Models
|
|
||||||
{
|
|
||||||
public class GroenmonitorTable
|
|
||||||
{
|
|
||||||
//double? makes a member nullable
|
|
||||||
public int parcelid;
|
|
||||||
public string date;
|
|
||||||
public string source;
|
|
||||||
public int wdvi_pixelcount; //count of pixels with data
|
|
||||||
public double wdvi_max;
|
|
||||||
public double wdvi_mean;
|
|
||||||
public double wdvi_min;
|
|
||||||
public double wdvi_stdev;
|
|
||||||
public double wdvi_median;
|
|
||||||
public double wdvi_p90;
|
|
||||||
public int ndvi_pixelcount; //count of pixels with data
|
|
||||||
public double ndvi_max;
|
|
||||||
public double ndvi_mean;
|
|
||||||
public double ndvi_min;
|
|
||||||
public double ndvi_stdev;
|
|
||||||
public double ndvi_median;
|
|
||||||
public double ndvi_p90;
|
|
||||||
}
|
|
||||||
}
|
|
Loading…
Reference in New Issue
Block a user