with doDB clearer choice on whether to read fields for which to download satellite stats from csv or database

This commit is contained in:
Pepijn van Oort 2022-03-07 13:36:20 +01:00
parent 3da2b71d62
commit f8a412be64
2 changed files with 109 additions and 311 deletions

View File

@ -51,6 +51,13 @@ namespace FarmmapsBulkSatDownload
DateTime lastdownloadedimagedate; DateTime lastdownloadedimagedate;
int cropYear; int cropYear;
//Use doDB to decide if reading from/writing to database (doDB = true) or
//read from file BulkSatDownloadInput.json and write satellite statistics to file(s) specified in BulkSatDownloadInput.json
//note in case of doDB == true you will need to provide a file "DBsettings.secrets.json" with login details for the database
//see empty example "DBsettings.json"
bool doDB = false;
if (doDB == true)
{
// Option 1: When using database need to (1) fill in database data in DBsettings.secrets.json; (2) write tailor made SELECT query for fieldinputs in following lines; // Option 1: When using database need to (1) fill in database data in DBsettings.secrets.json; (2) write tailor made SELECT query for fieldinputs in following lines;
// (3) Write tailor made INSERT INTO query in Task Process() below; // (3) Write tailor made INSERT INTO query in Task Process() below;
// Initialize databases. Username, password etc stored in file "DBsettings.secrets.json". // Initialize databases. Username, password etc stored in file "DBsettings.secrets.json".
@ -74,16 +81,16 @@ namespace FarmmapsBulkSatDownload
List<string> satelliteBands = new List<string> { "wdvi", "ndvi" }; List<string> satelliteBands = new List<string> { "wdvi", "ndvi" };
string connectionString = dbparcels.GetConnectionString(); string connectionString = dbparcels.GetConnectionString();
string readSql = string.Format( string readSql = string.Format(
@" @"
SELECT pt.arbid, pt.year, gml.lastwenrdate, ST_AsGeoJSON(ST_Transform((ST_DUMP(pt.geom)).geom::geometry(Polygon),4326)) AS geojson_polygon_wgs84, SELECT pt.arbid, pt.year, gml.lastwenrdate, ST_AsGeoJSON(ST_Transform((ST_DUMP(pt.geom)).geom::geometry(Polygon),4326)) AS geojson_polygon_wgs84,
COALESCE(pt.cropfielditemcode,'') AS cropfielditemcode, COALESCE(pt.cropfielditemcode,'') AS cropfielditemcode,
CASE WHEN pt.year >= DATE_PART('year', CURRENT_DATE) THEN '' ELSE COALESCE(pt.satellitetaskcode,'') END AS satellitetaskcode CASE WHEN pt.year >= DATE_PART('year', CURRENT_DATE) THEN '' ELSE COALESCE(pt.satellitetaskcode,'') END AS satellitetaskcode
FROM {0}.{1} pt, {0}.{2} gml FROM {0}.{1} pt, {0}.{2} gml
WHERE WHERE
pt.arbid = gml.arbid pt.arbid = gml.arbid
AND pt.satellitetaskcode IS NULL AND pt.satellitetaskcode IS NULL
ORDER BY pt.arbid ORDER BY pt.arbid
LIMIT 5;", schemaname, parceltablename, groenmonitorlatestviewname); //LIMIT x for testing LIMIT 5;", schemaname, parceltablename, groenmonitorlatestviewname); //LIMIT x for testing
using (NpgsqlConnection connection = new NpgsqlConnection(connectionString)) using (NpgsqlConnection connection = new NpgsqlConnection(connectionString))
{ {
@ -112,34 +119,35 @@ LIMIT 5;", schemaname, parceltablename, groenmonitorlatestviewname); //LIMIT x
} }
connection.Close(); connection.Close();
} }
// Now choose which list you want to use
bulkSatDownloadInputList = bulkSatDownloadInputListDB;
}
else
{
// Option 2: Example without database. Comment out this part if you want to use database // Option 2: Example without database. Comment out this part if you want to use database
// Read cropfields "BulkSatDownloadInput.json" and write all stats to a single csv file // Read cropfields "BulkSatDownloadInput.json" and write all stats to a single csv file
// Write all stats for multiple fields will be written to a single csv file // Write all stats for multiple fields will be written to a single csv file
//string downloadFolder; string downloadFolder;
//string fileNameStats; string fileNameStats;
//string headerLineStats = $"FieldName,satelliteDate,satelliteBand,max,min,mean,mode,median,stddev,minPlus,curtosis,maxMinus,skewness,variance,populationCount,variationCoefficient,confidenceIntervalLow, confidenceIntervalHigh,confidenceIntervalErrorMargin" + Environment.NewLine; string headerLineStats = $"FieldName,satelliteDate,satelliteBand,max,min,mean,mode,median,stddev,minPlus,curtosis,maxMinus,skewness,variance,populationCount,variationCoefficient,confidenceIntervalLow, confidenceIntervalHigh,confidenceIntervalErrorMargin" + Environment.NewLine;
//var fieldsInputJson = File.ReadAllText("BulkSatDownloadInput.json"); var fieldsInputJson = File.ReadAllText("BulkSatDownloadInput.json");
//bulkSatDownloadInputListCsv = JsonConvert.DeserializeObject<List<BulkSatDownloadInput>>(fieldsInputJson); bulkSatDownloadInputListCsv = JsonConvert.DeserializeObject<List<BulkSatDownloadInput>>(fieldsInputJson);
//for (int i = 0; i < bulkSatDownloadInputListCsv.Count; i++) for (int i = 0; i < bulkSatDownloadInputListCsv.Count; i++)
//{ {
// downloadFolder = bulkSatDownloadInputListCsv[i].downloadFolder; downloadFolder = bulkSatDownloadInputListCsv[i].downloadFolder;
// fileNameStats = Path.Combine(downloadFolder, bulkSatDownloadInputListCsv[i].fileNameStats); fileNameStats = Path.Combine(downloadFolder, bulkSatDownloadInputListCsv[i].fileNameStats);
// if (!Directory.Exists(downloadFolder)) if (!Directory.Exists(downloadFolder))
// Directory.CreateDirectory(downloadFolder); Directory.CreateDirectory(downloadFolder);
// bulkSatDownloadInputListCsv[i].fileNameStats = fileNameStats; bulkSatDownloadInputListCsv[i].fileNameStats = fileNameStats;
// // Header same as in GeneralService.DownloadSatelliteStats // Header same as in GeneralService.DownloadSatelliteStats
// // Delete fileNameStats if existing. Create a new file. Add a header to csv file // Delete fileNameStats if existing. Create a new file. Add a header to csv file
// File.Delete(fileNameStats); File.Delete(fileNameStats);
// File.AppendAllText(fileNameStats, headerLineStats); File.AppendAllText(fileNameStats, headerLineStats);
//} }
// Now choose which list you want to use // Now choose which list you want to use
bulkSatDownloadInputList = bulkSatDownloadInputListDB; //bulkSatDownloadInputListDB; //bulkSatDownloadInputListCsv; bulkSatDownloadInputList = bulkSatDownloadInputListCsv;
}
// Whichever option (database or json/csv), continue here
// Delete the settingsfile
// File.Delete(settingsfile);
// For each input download all images. Keep track to time, important when doing bulk downloads // For each input download all images. Keep track to time, important when doing bulk downloads
var watch = System.Diagnostics.Stopwatch.StartNew(); var watch = System.Diagnostics.Stopwatch.StartNew();

View File

@ -1,46 +1,19 @@
[ [
{ {
"fieldName": "5641", // FarmMaps minimum needs are: fieldName, cropYear & geometryJson "fieldName": "BvdT FieldlabG92", // FarmMaps minimum needs are: fieldName, cropYear & geometryJson
"cropYear": 1999, //For testing a year for which we know no satellite data available, program shouldn't crash "cropYear": 2022, // for testing same field last year
"fieldID": 5641, "fieldID": 1,
"SatelliteBands": [ "wdvi", "ndvi" ], // ["ndvi"] or ["wdvi"] or both: [ "wdvi", "ndvi" ] "SatelliteBands": [ "wdvi", "ndvi" ], // ["ndvi"] or ["wdvi"] or both: [ "wdvi", "ndvi" ]
"lastdownloadedimagedate": "1999-01-01", //downloads images from this date till end of the year "lastdownloadedimagedate": "2022-01-01", //downloads images from this date till end of the year
"geometryJson": { "geometryJson": {
"type": "Polygon", "type": "Polygon",
"coordinates": [ "coordinates": [
[ [
[ 3.37837807779104, 51.3231095796538 ], [ 5.563472073408009, 52.547554398144172 ],
[ 3.38065689232502, 51.3212527499355 ], [ 5.567425915520115, 52.547725375100377 ],
[ 3.38022924592256, 51.3210683536359 ], [ 5.563878143678981, 52.54048022658143 ],
[ 3.37980548452565, 51.3208801127141 ], [ 5.563878143678981, 52.54048022658143 ],
[ 3.37959556105776, 51.3207540143696 ], [ 5.563472073408009, 52.547554398144172 ]
[ 3.3793691292654, 51.3205959677371 ],
[ 3.37822219207335, 51.3215667913007 ],
[ 3.37816999925795, 51.3216109809456 ],
[ 3.37646704574705, 51.3208025481261 ],
[ 3.37646695791282, 51.3208025061493 ],
[ 3.37608401443192, 51.3206231652693 ],
[ 3.37607169507628, 51.3206173959751 ],
[ 3.37606021048754, 51.320612017601 ],
[ 3.37582728410659, 51.3205029306946 ],
[ 3.37580409779263, 51.3206502985963 ],
[ 3.37575872019649, 51.3207993094705 ],
[ 3.37575476634361, 51.3208122883487 ],
[ 3.37571181656268, 51.3208797459348 ],
[ 3.3756624532907, 51.3209415238446 ],
[ 3.37557609963811, 51.3210110142077 ],
[ 3.37541089899821, 51.3211055871218 ],
[ 3.37477516102591, 51.3214102985009 ],
[ 3.37473173914127, 51.3214311108204 ],
[ 3.37455904622072, 51.3215138815012 ],
[ 3.37415098054777, 51.3217199232877 ],
[ 3.37313700916272, 51.3222422862785 ],
[ 3.37748824689601, 51.3242852920348 ],
[ 3.37749760805371, 51.3242713084009 ],
[ 3.37811903757028, 51.3233437635596 ],
[ 3.37818758851947, 51.3232647797363 ],
[ 3.37823803668144, 51.3232236798646 ],
[ 3.37837807779104, 51.3231095796538 ]
] ]
] ]
}, },
@ -52,203 +25,20 @@
"satelllitetable": null "satelllitetable": null
}, },
{ {
"fieldName": "5641", // FarmMaps minimum needs are: fieldName, cropYear & geometryJson "fieldName": "BvdT FieldlabG92", // FarmMaps minimum needs are: fieldName, cropYear & geometryJson
"cropYear": 2021, //For testing a year for which we know no satellite data available, program shouldn't crash "cropYear": 2021, // for testing same field last year (i.e. 2021)
"fieldID": 5641, "fieldID": 1,
"SatelliteBands": [ "wdvi", "ndvi" ], // ["ndvi"] or ["wdvi"] or both: [ "wdvi", "ndvi" ] "SatelliteBands": [ "wdvi", "ndvi" ], // ["ndvi"] or ["wdvi"] or both: [ "wdvi", "ndvi" ]
"lastdownloadedimagedate": "2021-01-01", //downloads images from this date till end of the year "lastdownloadedimagedate": "2021-01-01", //downloads images from this date till end of the year
"geometryJson": { "geometryJson": {
"type": "Polygon", "type": "Polygon",
"coordinates": [ "coordinates": [
[ [
[ 3.37837807779104, 51.3231095796538 ], [ 5.563472073408009, 52.547554398144172 ],
[ 3.38065689232502, 51.3212527499355 ], [ 5.567425915520115, 52.547725375100377 ],
[ 3.38022924592256, 51.3210683536359 ], [ 5.563878143678981, 52.54048022658143 ],
[ 3.37980548452565, 51.3208801127141 ], [ 5.563878143678981, 52.54048022658143 ],
[ 3.37959556105776, 51.3207540143696 ], [ 5.563472073408009, 52.547554398144172 ]
[ 3.3793691292654, 51.3205959677371 ],
[ 3.37822219207335, 51.3215667913007 ],
[ 3.37816999925795, 51.3216109809456 ],
[ 3.37646704574705, 51.3208025481261 ],
[ 3.37646695791282, 51.3208025061493 ],
[ 3.37608401443192, 51.3206231652693 ],
[ 3.37607169507628, 51.3206173959751 ],
[ 3.37606021048754, 51.320612017601 ],
[ 3.37582728410659, 51.3205029306946 ],
[ 3.37580409779263, 51.3206502985963 ],
[ 3.37575872019649, 51.3207993094705 ],
[ 3.37575476634361, 51.3208122883487 ],
[ 3.37571181656268, 51.3208797459348 ],
[ 3.3756624532907, 51.3209415238446 ],
[ 3.37557609963811, 51.3210110142077 ],
[ 3.37541089899821, 51.3211055871218 ],
[ 3.37477516102591, 51.3214102985009 ],
[ 3.37473173914127, 51.3214311108204 ],
[ 3.37455904622072, 51.3215138815012 ],
[ 3.37415098054777, 51.3217199232877 ],
[ 3.37313700916272, 51.3222422862785 ],
[ 3.37748824689601, 51.3242852920348 ],
[ 3.37749760805371, 51.3242713084009 ],
[ 3.37811903757028, 51.3233437635596 ],
[ 3.37818758851947, 51.3232647797363 ],
[ 3.37823803668144, 51.3232236798646 ],
[ 3.37837807779104, 51.3231095796538 ]
]
]
},
"downloadFolder": "C:\\workdir\\groenmonitor\\",
"fileNameStats": "BulkSatDownload.csv",
"database": null,
"schemaname": null,
"cropfieldtable": null,
"satelllitetable": null
},
{
"fieldName": "5641", // FarmMaps minimum needs are: fieldName, cropYear & geometryJson
"cropYear": 2020,
"fieldID": 5641,
"SatelliteBands": [ "wdvi", "ndvi" ], // ["ndvi"] or ["wdvi"] or both: [ "wdvi", "ndvi" ]
"lastdownloadedimagedate": "2020-01-01", //downloads images from this date till end of the year
"geometryJson": {
"type": "Polygon",
"coordinates": [
[
[ 3.37837807779104, 51.3231095796538 ],
[ 3.38065689232502, 51.3212527499355 ],
[ 3.38022924592256, 51.3210683536359 ],
[ 3.37980548452565, 51.3208801127141 ],
[ 3.37959556105776, 51.3207540143696 ],
[ 3.3793691292654, 51.3205959677371 ],
[ 3.37822219207335, 51.3215667913007 ],
[ 3.37816999925795, 51.3216109809456 ],
[ 3.37646704574705, 51.3208025481261 ],
[ 3.37646695791282, 51.3208025061493 ],
[ 3.37608401443192, 51.3206231652693 ],
[ 3.37607169507628, 51.3206173959751 ],
[ 3.37606021048754, 51.320612017601 ],
[ 3.37582728410659, 51.3205029306946 ],
[ 3.37580409779263, 51.3206502985963 ],
[ 3.37575872019649, 51.3207993094705 ],
[ 3.37575476634361, 51.3208122883487 ],
[ 3.37571181656268, 51.3208797459348 ],
[ 3.3756624532907, 51.3209415238446 ],
[ 3.37557609963811, 51.3210110142077 ],
[ 3.37541089899821, 51.3211055871218 ],
[ 3.37477516102591, 51.3214102985009 ],
[ 3.37473173914127, 51.3214311108204 ],
[ 3.37455904622072, 51.3215138815012 ],
[ 3.37415098054777, 51.3217199232877 ],
[ 3.37313700916272, 51.3222422862785 ],
[ 3.37748824689601, 51.3242852920348 ],
[ 3.37749760805371, 51.3242713084009 ],
[ 3.37811903757028, 51.3233437635596 ],
[ 3.37818758851947, 51.3232647797363 ],
[ 3.37823803668144, 51.3232236798646 ],
[ 3.37837807779104, 51.3231095796538 ]
]
]
},
"downloadFolder": "C:\\workdir\\groenmonitor\\",
"fileNameStats": "BulkSatDownload.csv",
"database": null,
"schemaname": null,
"cropfieldtable": null,
"satelllitetable": null
},
{
"fieldName": "5641", // FarmMaps minimum needs are: fieldName, cropYear & geometryJson
"cropYear": 2019,
"fieldID": 5641,
"SatelliteBands": [ "wdvi", "ndvi" ], // ["ndvi"] or ["wdvi"] or both: [ "wdvi", "ndvi" ]
"lastdownloadedimagedate": "2019-01-01", //downloads images from this date till end of the year
"geometryJson": {
"type": "Polygon",
"coordinates": [
[
[ 3.37837807779104, 51.3231095796538 ],
[ 3.38065689232502, 51.3212527499355 ],
[ 3.38022924592256, 51.3210683536359 ],
[ 3.37980548452565, 51.3208801127141 ],
[ 3.37959556105776, 51.3207540143696 ],
[ 3.3793691292654, 51.3205959677371 ],
[ 3.37822219207335, 51.3215667913007 ],
[ 3.37816999925795, 51.3216109809456 ],
[ 3.37646704574705, 51.3208025481261 ],
[ 3.37646695791282, 51.3208025061493 ],
[ 3.37608401443192, 51.3206231652693 ],
[ 3.37607169507628, 51.3206173959751 ],
[ 3.37606021048754, 51.320612017601 ],
[ 3.37582728410659, 51.3205029306946 ],
[ 3.37580409779263, 51.3206502985963 ],
[ 3.37575872019649, 51.3207993094705 ],
[ 3.37575476634361, 51.3208122883487 ],
[ 3.37571181656268, 51.3208797459348 ],
[ 3.3756624532907, 51.3209415238446 ],
[ 3.37557609963811, 51.3210110142077 ],
[ 3.37541089899821, 51.3211055871218 ],
[ 3.37477516102591, 51.3214102985009 ],
[ 3.37473173914127, 51.3214311108204 ],
[ 3.37455904622072, 51.3215138815012 ],
[ 3.37415098054777, 51.3217199232877 ],
[ 3.37313700916272, 51.3222422862785 ],
[ 3.37748824689601, 51.3242852920348 ],
[ 3.37749760805371, 51.3242713084009 ],
[ 3.37811903757028, 51.3233437635596 ],
[ 3.37818758851947, 51.3232647797363 ],
[ 3.37823803668144, 51.3232236798646 ],
[ 3.37837807779104, 51.3231095796538 ]
]
]
},
"downloadFolder": "C:\\workdir\\groenmonitor\\",
"fileNameStats": "BulkSatDownload.csv",
"database": null,
"schemaname": null,
"cropfieldtable": null,
"satelllitetable": null
},
{
"fieldName": "5641", // FarmMaps minimum needs are: fieldName, cropYear & geometryJson
"cropYear": 2018, //little to no images for 2018
"fieldID": 5641,
"SatelliteBands": [ "wdvi", "ndvi" ], // ["ndvi"] or ["wdvi"] or both: [ "wdvi", "ndvi" ]
"lastdownloadedimagedate": "2018-01-01", //downloads images from this date till end of the year
"geometryJson": {
"type": "Polygon",
"coordinates": [
[
[ 3.37837807779104, 51.3231095796538 ],
[ 3.38065689232502, 51.3212527499355 ],
[ 3.38022924592256, 51.3210683536359 ],
[ 3.37980548452565, 51.3208801127141 ],
[ 3.37959556105776, 51.3207540143696 ],
[ 3.3793691292654, 51.3205959677371 ],
[ 3.37822219207335, 51.3215667913007 ],
[ 3.37816999925795, 51.3216109809456 ],
[ 3.37646704574705, 51.3208025481261 ],
[ 3.37646695791282, 51.3208025061493 ],
[ 3.37608401443192, 51.3206231652693 ],
[ 3.37607169507628, 51.3206173959751 ],
[ 3.37606021048754, 51.320612017601 ],
[ 3.37582728410659, 51.3205029306946 ],
[ 3.37580409779263, 51.3206502985963 ],
[ 3.37575872019649, 51.3207993094705 ],
[ 3.37575476634361, 51.3208122883487 ],
[ 3.37571181656268, 51.3208797459348 ],
[ 3.3756624532907, 51.3209415238446 ],
[ 3.37557609963811, 51.3210110142077 ],
[ 3.37541089899821, 51.3211055871218 ],
[ 3.37477516102591, 51.3214102985009 ],
[ 3.37473173914127, 51.3214311108204 ],
[ 3.37455904622072, 51.3215138815012 ],
[ 3.37415098054777, 51.3217199232877 ],
[ 3.37313700916272, 51.3222422862785 ],
[ 3.37748824689601, 51.3242852920348 ],
[ 3.37749760805371, 51.3242713084009 ],
[ 3.37811903757028, 51.3233437635596 ],
[ 3.37818758851947, 51.3232647797363 ],
[ 3.37823803668144, 51.3232236798646 ],
[ 3.37837807779104, 51.3231095796538 ]
] ]
] ]
}, },