FarmMaps/FarmMapsApiClient
10
0
Fork 1
Code Issues Pull Requests Releases Activity

new project FarmmapsCleanUp to clean up cropfielditems and satellitetaskcodes used only once

Browse Source
This commit is contained in:
pepijn van oort
2021-06-07 16:38:42 +02:00
parent d1747427bc
commit 915df2ac6b
6 changed files with 260 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

64
FarmmapsBulkSatDownload/BulkSatDownloadApplication.cs
View File

@@ -57,10 +57,10 @@ namespace FarmmapsBulkSatDownload
// Crashes if "DBsettings.secrets.json" is absent or empty
DB dbparcels = JsonConvert.DeserializeObject<DB>(File.ReadAllText("DBsettings.secrets.json"));
string schemaname = "bigdata";
string parceltablename = "parcel_flowerbulbs";//"parcelsijbrandij" "parcel"; "parcel_flowerbulbs"; "parcel_disac"
string groenmonitortablename = "groenmonitor_flowerbulbs";//"groenmonitorsijbrandij" "groenmonitor" "groenmonitor_flowerbulbs" "groenmonitor_disac"
string parceltablename = "parcel_bollenrevolutie_tulips2020"; //"parcelsijbrandij" "parcel"; "parcel_flowerbulbs"; "parcel_disac"; ""parcel_bollenrevolutie_tulips2020""
string groenmonitortablename = "groenmonitor_bollenrevolutie_tulips2020"; //"groenmonitorsijbrandij" "groenmonitor" "groenmonitor_flowerbulbs" "groenmonitor_disac" "groenmonitor_bollenrevolutie_tulips2020"
// The view 'groenmonitorlatestviewname' contains per parcelid (arbid) the year in which it "exists" and the date of the latest image downloaded. It is used to prevent unneccessary downloading of image statistics already in the database
string groenmonitorlatestviewname = "groenmonitorlatest_flowerbulbs"; //"groenmonitorsijbrandijlatest" "groenmonitorlatest" "groenmonitorlatest_flowerbulbs" "groenmonitorlatest_disac"
string groenmonitorlatestviewname = "groenmonitorlatest_bollenrevolutie_tulips2020"; //"groenmonitorsijbrandijlatest" "groenmonitorlatest" "groenmonitorlatest_flowerbulbs" "groenmonitorlatest_disac" "groenmonitorlatest_bollenrevolutie_tulips2020"
// Database query and connection. Geometry must be in WGS84 coordinate system, EPSG 4326
// Apparently the FarmmapsApi cannot handle MultiPolygon, so we need to convert to single Polygon
@@ -80,11 +80,10 @@ SELECT pt.arbid, pt.year, gml.lastwenrdate, ST_AsGeoJSON(ST_Transform((ST_DUMP(p
CASE WHEN pt.year >= DATE_PART('year', CURRENT_DATE) THEN '' ELSE COALESCE(pt.satellitetaskcode,'') END AS satellitetaskcode
FROM {0}.{1} pt, {0}.{2} gml
WHERE
pt.arbid = gml.arbid AND
pt.crop NOT IN ('Lelie','Tulp') AND
pt.year > 2018 AND pt.arbid IN(8276,8314,8315)
pt.arbid = gml.arbid
AND pt.satellitetaskcode IS NULL
ORDER BY pt.arbid
LIMIT 2000;", schemaname, parceltablename, groenmonitorlatestviewname); //LIMIT x for testing
LIMIT 5;", schemaname, parceltablename, groenmonitorlatestviewname); //LIMIT x for testing
using (NpgsqlConnection connection = new NpgsqlConnection(connectionString))
{
@@ -117,30 +116,30 @@ LIMIT 2000;", schemaname, parceltablename, groenmonitorlatestviewname); //LIMI
// Option 2: Example without database. Comment out this part if you want to use database
// Read cropfields "BulkSatDownloadInput.json" and write all stats to a single csv file
// Write all stats for multiple fields will be written to a single csv file
string downloadFolder;
string fileNameStats;
string headerLineStats = $"FieldName,satelliteDate,satelliteBand,max,min,mean,mode,median,stddev,minPlus,curtosis,maxMinus,skewness,variance,populationCount,variationCoefficient,confidenceIntervalLow, confidenceIntervalHigh,confidenceIntervalErrorMargin" + Environment.NewLine;
var fieldsInputJson = File.ReadAllText("BulkSatDownloadInput.json");
bulkSatDownloadInputListCsv = JsonConvert.DeserializeObject<List<BulkSatDownloadInput>>(fieldsInputJson);
for (int i = 0; i < bulkSatDownloadInputListCsv.Count; i++)
{
downloadFolder = bulkSatDownloadInputListCsv[i].downloadFolder;
fileNameStats = Path.Combine(downloadFolder, bulkSatDownloadInputListCsv[i].fileNameStats);
if (!Directory.Exists(downloadFolder))
Directory.CreateDirectory(downloadFolder);
bulkSatDownloadInputListCsv[i].fileNameStats = fileNameStats;
// Header same as in GeneralService.DownloadSatelliteStats
// Delete fileNameStats if existing. Create a new file. Add a header to csv file
File.Delete(fileNameStats);
File.AppendAllText(fileNameStats, headerLineStats);
}
//string downloadFolder;
//string fileNameStats;
//string headerLineStats = $"FieldName,satelliteDate,satelliteBand,max,min,mean,mode,median,stddev,minPlus,curtosis,maxMinus,skewness,variance,populationCount,variationCoefficient,confidenceIntervalLow, confidenceIntervalHigh,confidenceIntervalErrorMargin" + Environment.NewLine;
//var fieldsInputJson = File.ReadAllText("BulkSatDownloadInput.json");
//bulkSatDownloadInputListCsv = JsonConvert.DeserializeObject<List<BulkSatDownloadInput>>(fieldsInputJson);
//for (int i = 0; i < bulkSatDownloadInputListCsv.Count; i++)
//{
// downloadFolder = bulkSatDownloadInputListCsv[i].downloadFolder;
// fileNameStats = Path.Combine(downloadFolder, bulkSatDownloadInputListCsv[i].fileNameStats);
// if (!Directory.Exists(downloadFolder))
// Directory.CreateDirectory(downloadFolder);
// bulkSatDownloadInputListCsv[i].fileNameStats = fileNameStats;
// // Header same as in GeneralService.DownloadSatelliteStats
// // Delete fileNameStats if existing. Create a new file. Add a header to csv file
// File.Delete(fileNameStats);
// File.AppendAllText(fileNameStats, headerLineStats);
//}
// Now choose which list you want to use
bulkSatDownloadInputList = bulkSatDownloadInputListDB; //bulkSatDownloadInputListDB; //bulkSatDownloadInputListCsv;
// Whichever option (database or json/csv), continue here
// Delete the settingsfile
File.Delete(settingsfile);
// File.Delete(settingsfile);
// For each input download all images. Keep track to time, important when doing bulk downloads
var watch = System.Diagnostics.Stopwatch.StartNew();
@@ -157,13 +156,26 @@ LIMIT 2000;", schemaname, parceltablename, groenmonitorlatestviewname); //LIMI
try
{
await Process(roots, bulkSatDownloadInput);
}
catch (Exception ex)
{
_logger.LogError(ex.Message);
}
watch.Stop();
// add this downloadtime to the cropfieldtable.
// Only if cropfieldtable has a field called 'downloadtime' type 'time'!
//using (NpgsqlConnection connection = new NpgsqlConnection(dbparcels.GetConnectionString()))
//{
// connection.Open();
// NpgsqlCommand updateCmd = connection.CreateCommand();
// string updateSql = string.Format($"UPDATE {schemaname}.{parceltablename} SET downloadtime = '{strTime(watch.Elapsed)}' WHERE arbid = {bulkSatDownloadInput.fieldID};");
// updateCmd.CommandText = updateSql;
// int r = updateCmd.ExecuteNonQuery();
// if (r != 1)
// throw new Exception("// FarmmapsBulkSatDownload: Update downloadtime Failed");
// connection.Close();
//}
//_logger.LogInformation($"// FarmmapsBulkSatDownload: Added downloadtime = '{strTime(watch.Elapsed)}' to {schemaname}.{parceltablename} ");
tsSofar = tsSofar + watch.Elapsed;
tsTotalEstimated = tsSofar / (i + 1) * bulkSatDownloadInputList.Count;
tsRemaining = tsTotalEstimated - tsSofar;