jits
/
FarmMapsApiClient_WURtest
forked from FarmMaps/FarmMapsApiClient
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

with doDB clearer choice on whether to read fields for which to download satellite stats from csv or database

master
Pepijn van Oort 2022-03-07 13:36:20 +01:00
parent 3da2b71d62
commit f8a412be64
2 changed files with 109 additions and 311 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

66
FarmmapsBulkSatDownload/BulkSatDownloadApplication.cs
View File

@ -51,6 +51,13 @@ namespace FarmmapsBulkSatDownload
DateTime lastdownloadedimagedate;
int cropYear;
//Use doDB to decide if reading from/writing to database (doDB = true) or
//read from file BulkSatDownloadInput.json and write satellite statistics to file(s) specified in BulkSatDownloadInput.json
//note in case of doDB == true you will need to provide a file "DBsettings.secrets.json" with login details for the database
//see empty example "DBsettings.json"
bool doDB = false;
if (doDB == true)
{
// Option 1: When using database need to (1) fill in database data in DBsettings.secrets.json; (2) write tailor made SELECT query for fieldinputs in following lines;
// (3) Write tailor made INSERT INTO query in Task Process() below;
// Initialize databases. Username, password etc stored in file "DBsettings.secrets.json".
@ -74,16 +81,16 @@ namespace FarmmapsBulkSatDownload
List<string> satelliteBands = new List<string> { "wdvi", "ndvi" };
string connectionString = dbparcels.GetConnectionString();
string readSql = string.Format(
@"
SELECT pt.arbid, pt.year, gml.lastwenrdate, ST_AsGeoJSON(ST_Transform((ST_DUMP(pt.geom)).geom::geometry(Polygon),4326)) AS geojson_polygon_wgs84,
@"
SELECT pt.arbid, pt.year, gml.lastwenrdate, ST_AsGeoJSON(ST_Transform((ST_DUMP(pt.geom)).geom::geometry(Polygon),4326)) AS geojson_polygon_wgs84,
COALESCE(pt.cropfielditemcode,'') AS cropfielditemcode,
CASE WHEN pt.year >= DATE_PART('year', CURRENT_DATE) THEN '' ELSE COALESCE(pt.satellitetaskcode,'') END AS satellitetaskcode
FROM {0}.{1} pt, {0}.{2} gml
WHERE
FROM {0}.{1} pt, {0}.{2} gml
WHERE
pt.arbid = gml.arbid
AND pt.satellitetaskcode IS NULL
ORDER BY pt.arbid
LIMIT 5;", schemaname, parceltablename, groenmonitorlatestviewname); //LIMIT x for testing
ORDER BY pt.arbid
LIMIT 5;", schemaname, parceltablename, groenmonitorlatestviewname); //LIMIT x for testing
using (NpgsqlConnection connection = new NpgsqlConnection(connectionString))
{
@ -112,34 +119,35 @@ LIMIT 5;", schemaname, parceltablename, groenmonitorlatestviewname); //LIMIT x
}
connection.Close();
}
// Now choose which list you want to use
bulkSatDownloadInputList = bulkSatDownloadInputListDB;
}
else
{
// Option 2: Example without database. Comment out this part if you want to use database
// Read cropfields "BulkSatDownloadInput.json" and write all stats to a single csv file
// Write all stats for multiple fields will be written to a single csv file
//string downloadFolder;
//string fileNameStats;
//string headerLineStats = $"FieldName,satelliteDate,satelliteBand,max,min,mean,mode,median,stddev,minPlus,curtosis,maxMinus,skewness,variance,populationCount,variationCoefficient,confidenceIntervalLow, confidenceIntervalHigh,confidenceIntervalErrorMargin" + Environment.NewLine;
//var fieldsInputJson = File.ReadAllText("BulkSatDownloadInput.json");
//bulkSatDownloadInputListCsv = JsonConvert.DeserializeObject<List<BulkSatDownloadInput>>(fieldsInputJson);
//for (int i = 0; i < bulkSatDownloadInputListCsv.Count; i++)
//{
// downloadFolder = bulkSatDownloadInputListCsv[i].downloadFolder;
// fileNameStats = Path.Combine(downloadFolder, bulkSatDownloadInputListCsv[i].fileNameStats);
// if (!Directory.Exists(downloadFolder))
// Directory.CreateDirectory(downloadFolder);
// bulkSatDownloadInputListCsv[i].fileNameStats = fileNameStats;
// // Header same as in GeneralService.DownloadSatelliteStats
// // Delete fileNameStats if existing. Create a new file. Add a header to csv file
// File.Delete(fileNameStats);
// File.AppendAllText(fileNameStats, headerLineStats);
//}
string downloadFolder;
string fileNameStats;
string headerLineStats = $"FieldName,satelliteDate,satelliteBand,max,min,mean,mode,median,stddev,minPlus,curtosis,maxMinus,skewness,variance,populationCount,variationCoefficient,confidenceIntervalLow, confidenceIntervalHigh,confidenceIntervalErrorMargin" + Environment.NewLine;
var fieldsInputJson = File.ReadAllText("BulkSatDownloadInput.json");
bulkSatDownloadInputListCsv = JsonConvert.DeserializeObject<List<BulkSatDownloadInput>>(fieldsInputJson);
for (int i = 0; i < bulkSatDownloadInputListCsv.Count; i++)
{
downloadFolder = bulkSatDownloadInputListCsv[i].downloadFolder;
fileNameStats = Path.Combine(downloadFolder, bulkSatDownloadInputListCsv[i].fileNameStats);
if (!Directory.Exists(downloadFolder))
Directory.CreateDirectory(downloadFolder);
bulkSatDownloadInputListCsv[i].fileNameStats = fileNameStats;
// Header same as in GeneralService.DownloadSatelliteStats
// Delete fileNameStats if existing. Create a new file. Add a header to csv file
File.Delete(fileNameStats);
File.AppendAllText(fileNameStats, headerLineStats);
}
// Now choose which list you want to use
bulkSatDownloadInputList = bulkSatDownloadInputListDB; //bulkSatDownloadInputListDB; //bulkSatDownloadInputListCsv;
// Whichever option (database or json/csv), continue here
// Delete the settingsfile
// File.Delete(settingsfile);
bulkSatDownloadInputList = bulkSatDownloadInputListCsv;
}
// For each input download all images. Keep track to time, important when doing bulk downloads
var watch = System.Diagnostics.Stopwatch.StartNew();

244
FarmmapsBulkSatDownload/BulkSatDownloadInput.json
View File

@ -1,46 +1,19 @@
[
{
"fieldName": "5641", // FarmMaps minimum needs are: fieldName, cropYear & geometryJson
"cropYear": 1999, //For testing a year for which we know no satellite data available, program shouldn't crash
"fieldID": 5641,
"fieldName": "BvdT FieldlabG92", // FarmMaps minimum needs are: fieldName, cropYear & geometryJson
"cropYear": 2022, // for testing same field last year
"fieldID": 1,
"SatelliteBands": [ "wdvi", "ndvi" ], // ["ndvi"] or ["wdvi"] or both: [ "wdvi", "ndvi" ]
"lastdownloadedimagedate": "1999-01-01", //downloads images from this date till end of the year
"lastdownloadedimagedate": "2022-01-01", //downloads images from this date till end of the year
"geometryJson": {
"type": "Polygon",
"coordinates": [
[
[ 3.37837807779104, 51.3231095796538 ],
[ 3.38065689232502, 51.3212527499355 ],
[ 3.38022924592256, 51.3210683536359 ],
[ 3.37980548452565, 51.3208801127141 ],
[ 3.37959556105776, 51.3207540143696 ],
[ 3.3793691292654, 51.3205959677371 ],
[ 3.37822219207335, 51.3215667913007 ],
[ 3.37816999925795, 51.3216109809456 ],
[ 3.37646704574705, 51.3208025481261 ],
[ 3.37646695791282, 51.3208025061493 ],
[ 3.37608401443192, 51.3206231652693 ],
[ 3.37607169507628, 51.3206173959751 ],
[ 3.37606021048754, 51.320612017601 ],
[ 3.37582728410659, 51.3205029306946 ],
[ 3.37580409779263, 51.3206502985963 ],
[ 3.37575872019649, 51.3207993094705 ],
[ 3.37575476634361, 51.3208122883487 ],
[ 3.37571181656268, 51.3208797459348 ],
[ 3.3756624532907, 51.3209415238446 ],
[ 3.37557609963811, 51.3210110142077 ],
[ 3.37541089899821, 51.3211055871218 ],
[ 3.37477516102591, 51.3214102985009 ],
[ 3.37473173914127, 51.3214311108204 ],
[ 3.37455904622072, 51.3215138815012 ],
[ 3.37415098054777, 51.3217199232877 ],
[ 3.37313700916272, 51.3222422862785 ],
[ 3.37748824689601, 51.3242852920348 ],
[ 3.37749760805371, 51.3242713084009 ],
[ 3.37811903757028, 51.3233437635596 ],
[ 3.37818758851947, 51.3232647797363 ],
[ 3.37823803668144, 51.3232236798646 ],
[ 3.37837807779104, 51.3231095796538 ]
[ 5.563472073408009, 52.547554398144172 ],
[ 5.567425915520115, 52.547725375100377 ],
[ 5.563878143678981, 52.54048022658143 ],
[ 5.563878143678981, 52.54048022658143 ],
[ 5.563472073408009, 52.547554398144172 ]
]
]
},
@ -52,203 +25,20 @@
"satelllitetable": null
},
{
"fieldName": "5641", // FarmMaps minimum needs are: fieldName, cropYear & geometryJson
"cropYear": 2021, //For testing a year for which we know no satellite data available, program shouldn't crash
"fieldID": 5641,
"fieldName": "BvdT FieldlabG92", // FarmMaps minimum needs are: fieldName, cropYear & geometryJson
"cropYear": 2021, // for testing same field last year (i.e. 2021)
"fieldID": 1,
"SatelliteBands": [ "wdvi", "ndvi" ], // ["ndvi"] or ["wdvi"] or both: [ "wdvi", "ndvi" ]
"lastdownloadedimagedate": "2021-01-01", //downloads images from this date till end of the year
"geometryJson": {
"type": "Polygon",
"coordinates": [
[
[ 3.37837807779104, 51.3231095796538 ],
[ 3.38065689232502, 51.3212527499355 ],
[ 3.38022924592256, 51.3210683536359 ],
[ 3.37980548452565, 51.3208801127141 ],
[ 3.37959556105776, 51.3207540143696 ],
[ 3.3793691292654, 51.3205959677371 ],
[ 3.37822219207335, 51.3215667913007 ],
[ 3.37816999925795, 51.3216109809456 ],
[ 3.37646704574705, 51.3208025481261 ],
[ 3.37646695791282, 51.3208025061493 ],
[ 3.37608401443192, 51.3206231652693 ],
[ 3.37607169507628, 51.3206173959751 ],
[ 3.37606021048754, 51.320612017601 ],
[ 3.37582728410659, 51.3205029306946 ],
[ 3.37580409779263, 51.3206502985963 ],
[ 3.37575872019649, 51.3207993094705 ],
[ 3.37575476634361, 51.3208122883487 ],
[ 3.37571181656268, 51.3208797459348 ],
[ 3.3756624532907, 51.3209415238446 ],
[ 3.37557609963811, 51.3210110142077 ],
[ 3.37541089899821, 51.3211055871218 ],
[ 3.37477516102591, 51.3214102985009 ],
[ 3.37473173914127, 51.3214311108204 ],
[ 3.37455904622072, 51.3215138815012 ],
[ 3.37415098054777, 51.3217199232877 ],
[ 3.37313700916272, 51.3222422862785 ],
[ 3.37748824689601, 51.3242852920348 ],
[ 3.37749760805371, 51.3242713084009 ],
[ 3.37811903757028, 51.3233437635596 ],
[ 3.37818758851947, 51.3232647797363 ],
[ 3.37823803668144, 51.3232236798646 ],
[ 3.37837807779104, 51.3231095796538 ]
]
]
},
"downloadFolder": "C:\\workdir\\groenmonitor\\",
"fileNameStats": "BulkSatDownload.csv",
"database": null,
"schemaname": null,
"cropfieldtable": null,
"satelllitetable": null
},
{
"fieldName": "5641", // FarmMaps minimum needs are: fieldName, cropYear & geometryJson
"cropYear": 2020,
"fieldID": 5641,
"SatelliteBands": [ "wdvi", "ndvi" ], // ["ndvi"] or ["wdvi"] or both: [ "wdvi", "ndvi" ]
"lastdownloadedimagedate": "2020-01-01", //downloads images from this date till end of the year
"geometryJson": {
"type": "Polygon",
"coordinates": [
[
[ 3.37837807779104, 51.3231095796538 ],
[ 3.38065689232502, 51.3212527499355 ],
[ 3.38022924592256, 51.3210683536359 ],
[ 3.37980548452565, 51.3208801127141 ],
[ 3.37959556105776, 51.3207540143696 ],
[ 3.3793691292654, 51.3205959677371 ],
[ 3.37822219207335, 51.3215667913007 ],
[ 3.37816999925795, 51.3216109809456 ],
[ 3.37646704574705, 51.3208025481261 ],
[ 3.37646695791282, 51.3208025061493 ],
[ 3.37608401443192, 51.3206231652693 ],
[ 3.37607169507628, 51.3206173959751 ],
[ 3.37606021048754, 51.320612017601 ],
[ 3.37582728410659, 51.3205029306946 ],
[ 3.37580409779263, 51.3206502985963 ],
[ 3.37575872019649, 51.3207993094705 ],
[ 3.37575476634361, 51.3208122883487 ],
[ 3.37571181656268, 51.3208797459348 ],
[ 3.3756624532907, 51.3209415238446 ],
[ 3.37557609963811, 51.3210110142077 ],
[ 3.37541089899821, 51.3211055871218 ],
[ 3.37477516102591, 51.3214102985009 ],
[ 3.37473173914127, 51.3214311108204 ],
[ 3.37455904622072, 51.3215138815012 ],
[ 3.37415098054777, 51.3217199232877 ],
[ 3.37313700916272, 51.3222422862785 ],
[ 3.37748824689601, 51.3242852920348 ],
[ 3.37749760805371, 51.3242713084009 ],
[ 3.37811903757028, 51.3233437635596 ],
[ 3.37818758851947, 51.3232647797363 ],
[ 3.37823803668144, 51.3232236798646 ],
[ 3.37837807779104, 51.3231095796538 ]
]
]
},
"downloadFolder": "C:\\workdir\\groenmonitor\\",
"fileNameStats": "BulkSatDownload.csv",
"database": null,
"schemaname": null,
"cropfieldtable": null,
"satelllitetable": null
},
{
"fieldName": "5641", // FarmMaps minimum needs are: fieldName, cropYear & geometryJson
"cropYear": 2019,
"fieldID": 5641,
"SatelliteBands": [ "wdvi", "ndvi" ], // ["ndvi"] or ["wdvi"] or both: [ "wdvi", "ndvi" ]
"lastdownloadedimagedate": "2019-01-01", //downloads images from this date till end of the year
"geometryJson": {
"type": "Polygon",
"coordinates": [
[
[ 3.37837807779104, 51.3231095796538 ],
[ 3.38065689232502, 51.3212527499355 ],
[ 3.38022924592256, 51.3210683536359 ],
[ 3.37980548452565, 51.3208801127141 ],
[ 3.37959556105776, 51.3207540143696 ],
[ 3.3793691292654, 51.3205959677371 ],
[ 3.37822219207335, 51.3215667913007 ],
[ 3.37816999925795, 51.3216109809456 ],
[ 3.37646704574705, 51.3208025481261 ],
[ 3.37646695791282, 51.3208025061493 ],
[ 3.37608401443192, 51.3206231652693 ],
[ 3.37607169507628, 51.3206173959751 ],
[ 3.37606021048754, 51.320612017601 ],
[ 3.37582728410659, 51.3205029306946 ],
[ 3.37580409779263, 51.3206502985963 ],
[ 3.37575872019649, 51.3207993094705 ],
[ 3.37575476634361, 51.3208122883487 ],
[ 3.37571181656268, 51.3208797459348 ],
[ 3.3756624532907, 51.3209415238446 ],
[ 3.37557609963811, 51.3210110142077 ],
[ 3.37541089899821, 51.3211055871218 ],
[ 3.37477516102591, 51.3214102985009 ],
[ 3.37473173914127, 51.3214311108204 ],
[ 3.37455904622072, 51.3215138815012 ],
[ 3.37415098054777, 51.3217199232877 ],
[ 3.37313700916272, 51.3222422862785 ],
[ 3.37748824689601, 51.3242852920348 ],
[ 3.37749760805371, 51.3242713084009 ],
[ 3.37811903757028, 51.3233437635596 ],
[ 3.37818758851947, 51.3232647797363 ],
[ 3.37823803668144, 51.3232236798646 ],
[ 3.37837807779104, 51.3231095796538 ]
]
]
},
"downloadFolder": "C:\\workdir\\groenmonitor\\",
"fileNameStats": "BulkSatDownload.csv",
"database": null,
"schemaname": null,
"cropfieldtable": null,
"satelllitetable": null
},
{
"fieldName": "5641", // FarmMaps minimum needs are: fieldName, cropYear & geometryJson
"cropYear": 2018, //little to no images for 2018
"fieldID": 5641,
"SatelliteBands": [ "wdvi", "ndvi" ], // ["ndvi"] or ["wdvi"] or both: [ "wdvi", "ndvi" ]
"lastdownloadedimagedate": "2018-01-01", //downloads images from this date till end of the year
"geometryJson": {
"type": "Polygon",
"coordinates": [
[
[ 3.37837807779104, 51.3231095796538 ],
[ 3.38065689232502, 51.3212527499355 ],
[ 3.38022924592256, 51.3210683536359 ],
[ 3.37980548452565, 51.3208801127141 ],
[ 3.37959556105776, 51.3207540143696 ],
[ 3.3793691292654, 51.3205959677371 ],
[ 3.37822219207335, 51.3215667913007 ],
[ 3.37816999925795, 51.3216109809456 ],
[ 3.37646704574705, 51.3208025481261 ],
[ 3.37646695791282, 51.3208025061493 ],
[ 3.37608401443192, 51.3206231652693 ],
[ 3.37607169507628, 51.3206173959751 ],
[ 3.37606021048754, 51.320612017601 ],
[ 3.37582728410659, 51.3205029306946 ],
[ 3.37580409779263, 51.3206502985963 ],
[ 3.37575872019649, 51.3207993094705 ],
[ 3.37575476634361, 51.3208122883487 ],
[ 3.37571181656268, 51.3208797459348 ],
[ 3.3756624532907, 51.3209415238446 ],
[ 3.37557609963811, 51.3210110142077 ],
[ 3.37541089899821, 51.3211055871218 ],
[ 3.37477516102591, 51.3214102985009 ],
[ 3.37473173914127, 51.3214311108204 ],
[ 3.37455904622072, 51.3215138815012 ],
[ 3.37415098054777, 51.3217199232877 ],
[ 3.37313700916272, 51.3222422862785 ],
[ 3.37748824689601, 51.3242852920348 ],
[ 3.37749760805371, 51.3242713084009 ],
[ 3.37811903757028, 51.3233437635596 ],
[ 3.37818758851947, 51.3232647797363 ],
[ 3.37823803668144, 51.3232236798646 ],
[ 3.37837807779104, 51.3231095796538 ]
[ 5.563472073408009, 52.547554398144172 ],
[ 5.567425915520115, 52.547725375100377 ],
[ 5.563878143678981, 52.54048022658143 ],
[ 5.563878143678981, 52.54048022658143 ],
[ 5.563472073408009, 52.547554398144172 ]
]
]
},