FarmMaps/FarmMapsApiClient
11
0
Fork 3
Code Issues Pull Requests Releases Activity

master #15

Merged
mark merged 5 commits from jits/FarmMapsApiClient_WURtest:master into master 2023-11-02 11:01:03 +00:00
Conversation 1 Commits 5 Files Changed 9 +272 -14
2 changed files with 272 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 166fd1f66d - Show all commits

97
FarmmapsBulkSatDownload/BulkSatDownloadApplication.cs
View File

@ -25,6 +25,7 @@ namespace FarmmapsBulkSatDownload
private readonly GeneralService _generalService;
public const string settingsfile = "Settings.json";
public const int firstAvailableYear = 2017;
private Settings _settings;
public BulkSatDownloadApplication(ILogger<BulkSatDownloadApplication> logger, FarmmapsApiService farmmapsApiService,
@ -199,6 +200,27 @@ namespace FarmmapsBulkSatDownload
private async Task Process(List<UserRoot> roots, BulkSatDownloadInput input)
{
//PO20220311: first time a call is made to download satellite images or statistics, an empty list is returned
//If we wait a bit longer, e.g. 10 secs, then e.g. a list of 3 images may be returned
//If we wait still longer, maybe 4 images.
//The solution implemented below is to fire calls as long as the number of images returned keeps increasing
//While in between each call, sleep for sleepSecs
//Continue this until the number no longer increases or the maximum number of calls has been reached
//If you set sleepSecs to a very low value, e.g. 5 secs, then after 1 call you might get images and after 2nd call still zero images.
//to be on the safe side, better bit higher value.
//Just accept this may take a while, have a coffee, we suggest sleepSecs = 30;
int sleepSecs = 30;
int callCntMax = 4 * 60 / sleepSecs; //4*60 = max 4 minutes
//For example we may set: "sleepSecs = 10;" and "callCntMax = 24;" and following result:
//Call no: 1. Giving FarmMaps 10 seconds to get SatelliteItems...
//Call no: 1: Received 2 images
//Call no: 2. Giving FarmMaps 10 seconds to get SatelliteItems...
//Call no: 2: Received 7 images
//Call no: 3. Giving FarmMaps 10 seconds to get SatelliteItems...
//Call no: 3: Received 7 images
//And the firing of calls would stop because the number of images returned is no longer increasing
//In the worst case, this could would lead to a total sleeping period of "sleepSecsSum = sleepSecs * callCntMax" seconds. After that we give up
string cropfielditemcode;
string satellitetaskcode;
Item cropfieldItem;
@ -230,6 +252,10 @@ namespace FarmmapsBulkSatDownload
DateTime lastDownloadedSatelliteDate = input.lastdownloadedimagedate;
cropfielditemcode = input.cropfielditemcode;
satellitetaskcode = input.satellitetaskcode;
int satelliteItemsCropYearCntPrev;
int satelliteItemsCropYearCnt;
int callCnt;
int sleepSecsSum;
LoadSettings(settingsfile);
@ -317,23 +343,80 @@ namespace FarmmapsBulkSatDownload
// TODO also log satellitetaskcode to settings, how?
// SaveSettings(settingsfile);
// Getting satellite items
_logger.LogInformation(string.Format($"Running FindSatelliteItems for cropfieldItem.Code '{cropfieldItem.Code}', SatelliteTaskCode '{satellitetaskcode}'"));
satelliteItemsCropYear = await _generalService.FindSatelliteItems(cropfieldItem, satellitetaskcode);
// Getting satellite items. Only for years for which available
satelliteItemsCropYearCntPrev = 0;
satelliteItemsCropYearCnt = 0;
sleepSecsSum = 0;
satelliteItemsCropYear = null;
if (cropYear >= firstAvailableYear && cropYear <= DateTime.Now.Year)
{
_logger.LogInformation(string.Format($"Running FindSatelliteItems for cropfieldItem.Code '{cropfieldItem.Code}', SatelliteTaskCode '{satellitetaskcode}'"));
//Call first time
callCnt = 1;
//if callCntMax == 0 then don't sleep
//if callCntMax = 1 then sleep first 1x
if (callCntMax > 0)
{
_logger.LogInformation($"Call no: {callCnt}. Giving FarmMaps {sleepSecs} seconds to get SatelliteItems...");
System.Threading.Thread.Sleep(1000 * sleepSecs);
sleepSecsSum = sleepSecsSum + sleepSecs;
}
satelliteItemsCropYear = await _generalService.FindSatelliteItems(cropfieldItem, satellitetaskcode);
satelliteItemsCropYearCntPrev = satelliteItemsCropYear.Count;
_logger.LogInformation($"Call no: {callCnt}. Received {satelliteItemsCropYearCntPrev} images");
callCnt++;
satelliteItemsCropYearCnt = satelliteItemsCropYearCntPrev;
//if callCntMax > 1 then sleep untill (1) no more increase in number of images received OR (2) maximum number of calls reached
if (callCntMax > 1)
{
//Call second time
_logger.LogInformation($"Call no: {callCnt}. Giving FarmMaps another {sleepSecs} seconds to get SatelliteItems...");
System.Threading.Thread.Sleep(1000 * sleepSecs);
satelliteItemsCropYear = await _generalService.FindSatelliteItems(cropfieldItem, satellitetaskcode);
satelliteItemsCropYearCnt = satelliteItemsCropYear.Count;
_logger.LogInformation($"Call no: {callCnt}. Received {satelliteItemsCropYearCnt} images");
sleepSecsSum = sleepSecsSum + sleepSecs;
//As long as there is progress, keep calling
callCnt++;
while (callCnt <= callCntMax && (satelliteItemsCropYearCnt == 0 || satelliteItemsCropYearCnt > satelliteItemsCropYearCntPrev))
{
_logger.LogInformation($"Surprise! The longer we wait, the more images we get. Sleep and call once more");
satelliteItemsCropYearCntPrev = satelliteItemsCropYearCnt;
_logger.LogInformation($"Call no: {callCnt} (max: {callCntMax}). Giving FarmMaps another {sleepSecs} seconds to get SatelliteItems...");
System.Threading.Thread.Sleep(1000 * sleepSecs);
satelliteItemsCropYear = await _generalService.FindSatelliteItems(cropfieldItem, satellitetaskcode);
satelliteItemsCropYearCnt = satelliteItemsCropYear.Count;
_logger.LogInformation($"Call no: {callCnt}. Received {satelliteItemsCropYearCnt} images");
callCnt++;
sleepSecsSum = sleepSecsSum + sleepSecs;
}
}
}
else
{
_logger.LogWarning($"// FarmmapsBulkSatDownload: crop year {cropYear} is out of bounds. No stats will be written!");
}
if (satelliteItemsCropYearCnt == 0)
{
_logger.LogWarning($"// FarmmapsBulkSatDownload: after calling one or more times and " +
$"sleeping in total {sleepSecsSum} seconds, still no images found. " +
$"Please check your settings for parameters callCntMax and sleepSecs in FarmmapsBulkSatDownload.cs or contact FarmMaps");
}
// Checking if satellite items found
satelliteItemsAvailable = true;
if (satelliteItemsCropYear == null)
{
satelliteItemsAvailable = false;
_logger.LogInformation($"No satellite tiffs found for fieldName '{fieldName}', cropYear {cropYear}, cropfielditemcode '{cropfielditemcode}'");
_logger.LogInformation($"// FarmmapsBulkSatDownload: No satellite tiffs found for fieldName '{fieldName}', cropYear {cropYear}, cropfielditemcode '{cropfielditemcode}'");
}
else
{
if (satelliteItemsCropYear.Count == 0)
{
satelliteItemsAvailable = false;
_logger.LogInformation($"No satellite tiffs found for fieldName '{fieldName}', cropYear {cropYear}, cropfielditemcode '{cropfielditemcode}'");
_logger.LogInformation($"// FarmmapsBulkSatDownload: No satellite tiffs found for fieldName '{fieldName}', cropYear {cropYear}, cropfielditemcode '{cropfielditemcode}'");
}
}
@ -343,10 +426,10 @@ namespace FarmmapsBulkSatDownload
// Download statistics to a single csv file
if (satelliteItemsAvailable && downloadFolder != null && fileNameStats != null)
{
// Write statistics for all images for all fieldNane and cropYear to a single csv file, fileNameStats
// Write statistics for all images for all fieldName and cropYear to a single csv file, fileNameStats
_logger.LogInformation($"Downloading stats for field '{fieldName}' in cropyear {cropYear} to {fileNameStats}");
string downloadedStats = await _generalService.DownloadSatelliteStats(satelliteItemsCropYear, fieldName, satelliteBands, downloadFolder);
// Add contents of this csv file to thee single large csv file
// Add contents of this csv file to the single large csv file
var retainedLines = File.ReadAllLines(downloadedStats).Skip(1);
File.AppendAllLines(fileNameStats, retainedLines);
File.Delete(downloadedStats);

189
FarmmapsBulkSatDownload/BulkSatDownloadInput.json
View File

@ -1,9 +1,34 @@
[
{
"fieldName": "BvdT FieldlabG92", // FarmMaps minimum needs are: fieldName, cropYear & geometryJson
"cropYear": 2022, // for testing same field last year
"fieldName": "MyField_1", // FarmMaps minimum needs are: fieldName, cropYear & geometryJson
"cropYear": 2023,
"fieldID": 1,
"SatelliteBands": [ "wdvi", "ndvi" ], // ["ndvi"] or ["wdvi"] or both: [ "wdvi", "ndvi" ]
"SatelliteBands": [ "ci-red", "ndvi", "wdvi" ], // ["ndvi"] or ["wdvi"] or both: [ "wdvi", "ndvi" ]
"lastdownloadedimagedate": "2023-01-01", //downloads images from this date till end of the year
"geometryJson": {
"type": "Polygon",
"coordinates": [
[
[ 5.563472073408009, 52.547554398144172 ],
[ 5.567425915520115, 52.547725375100377 ],
[ 5.563878143678981, 52.54048022658143 ],
[ 5.563878143678981, 52.54048022658143 ],
[ 5.563472073408009, 52.547554398144172 ]
]
]
},
"downloadFolder": "C:\\workdir\\groenmonitor\\", //if not yet existing this folder will be created
"fileNameStats": "BulkSatDownload.csv", //if file exists, probably will be overwritten. Check code in BulkSatDownloadApplication.cs
"database": null, // keep null to work with json and csv. Check code in BulkSatDownloadApplication.cs if reading/writing to/from database
"schemaname": null,
"cropfieldtable": null,
"satelllitetable": null
},
{
"fieldName": "MyField_2", // FarmMaps minimum needs are: fieldName, cropYear & geometryJson
"cropYear": 2022, // for testing same field last year (i.e. 2021)
"fieldID": 2,
"SatelliteBands": [ "ci-red", "ndvi", "wdvi" ], // ["ndvi"] or ["wdvi"] or both: [ "wdvi", "ndvi" ]
"lastdownloadedimagedate": "2022-01-01", //downloads images from this date till end of the year
"geometryJson": {
"type": "Polygon",
@ -25,10 +50,10 @@
"satelllitetable": null
},
{
"fieldName": "BvdT FieldlabG92", // FarmMaps minimum needs are: fieldName, cropYear & geometryJson
"cropYear": 2021, // for testing same field last year (i.e. 2021)
"fieldID": 1,
"SatelliteBands": [ "wdvi", "ndvi" ], // ["ndvi"] or ["wdvi"] or both: [ "wdvi", "ndvi" ]
"fieldName": "MyField_3", // FarmMaps minimum needs are: fieldName, cropYear & geometryJson
"cropYear": 2021,
"fieldID": 3,
"SatelliteBands": [ "ci-red", "ndvi", "wdvi" ], // ["ndvi"] or ["wdvi"] or both: [ "wdvi", "ndvi" ]
"lastdownloadedimagedate": "2021-01-01", //downloads images from this date till end of the year
"geometryJson": {
"type": "Polygon",
@ -48,5 +73,155 @@
"schemaname": null,
"cropfieldtable": null,
"satelllitetable": null
},
{
"fieldName": "MyField_4", // FarmMaps minimum needs are: fieldName, cropYear & geometryJson
"cropYear": 2020,
"fieldID": 4,
"SatelliteBands": [ "ci-red", "ndvi", "wdvi" ], // ["ndvi"] or ["wdvi"] or both: [ "wdvi", "ndvi" ]
"lastdownloadedimagedate": "2020-01-01", //downloads images from this date till end of the year
"geometryJson": {
"type": "Polygon",
"coordinates": [
[
[ 5.563472073408009, 52.547554398144172 ],
[ 5.567425915520115, 52.547725375100377 ],
[ 5.563878143678981, 52.54048022658143 ],
[ 5.563878143678981, 52.54048022658143 ],
[ 5.563472073408009, 52.547554398144172 ]
]
]
},
"downloadFolder": "C:\\workdir\\groenmonitor\\",
"fileNameStats": "BulkSatDownload.csv",
"database": null,
"schemaname": null,
"cropfieldtable": null,
"satelllitetable": null
},
{
"fieldName": "MyField_5", // FarmMaps minimum needs are: fieldName, cropYear & geometryJson
"cropYear": 2019,
"fieldID": 5,
"SatelliteBands": [ "ci-red", "ndvi", "wdvi" ], // ["ndvi"] or ["wdvi"] or both: [ "wdvi", "ndvi" ]
"lastdownloadedimagedate": "2019-01-01", //downloads images from this date till end of the year
"geometryJson": {
"type": "Polygon",
"coordinates": [
[
[ 5.563472073408009, 52.547554398144172 ],
[ 5.567425915520115, 52.547725375100377 ],
[ 5.563878143678981, 52.54048022658143 ],
[ 5.563878143678981, 52.54048022658143 ],
[ 5.563472073408009, 52.547554398144172 ]
]
]
},
"downloadFolder": "C:\\workdir\\groenmonitor\\",
"fileNameStats": "BulkSatDownload.csv",
"database": null,
"schemaname": null,
"cropfieldtable": null,
"satelllitetable": null
},
{
"fieldName": "MyField_6", // FarmMaps minimum needs are: fieldName, cropYear & geometryJson
"cropYear": 2018,
"fieldID": 6,
"SatelliteBands": [ "ci-red", "ndvi", "wdvi" ], // ["ndvi"] or ["wdvi"] or both: [ "wdvi", "ndvi" ]
"lastdownloadedimagedate": "2018-01-01", //downloads images from this date till end of the year
"geometryJson": {
"type": "Polygon",
"coordinates": [
[
[ 5.563472073408009, 52.547554398144172 ],
[ 5.567425915520115, 52.547725375100377 ],
[ 5.563878143678981, 52.54048022658143 ],
[ 5.563878143678981, 52.54048022658143 ],
[ 5.563472073408009, 52.547554398144172 ]
]
]
},
"downloadFolder": "C:\\workdir\\groenmonitor\\",
"fileNameStats": "BulkSatDownload.csv",
"database": null,
"schemaname": null,
"cropfieldtable": null,
"satelllitetable": null
},
{
"fieldName": "MyField_6", // FarmMaps minimum needs are: fieldName, cropYear & geometryJson
"cropYear": 2017,
"fieldID": 7,
"SatelliteBands": [ "ci-red", "ndvi", "wdvi" ], // ["ndvi"] or ["wdvi"] or both: [ "wdvi", "ndvi" ]
"lastdownloadedimagedate": "2017-01-01", //downloads images from this date till end of the year
"geometryJson": {
"type": "Polygon",
"coordinates": [
[
[ 5.563472073408009, 52.547554398144172 ],
[ 5.567425915520115, 52.547725375100377 ],
[ 5.563878143678981, 52.54048022658143 ],
[ 5.563878143678981, 52.54048022658143 ],
[ 5.563472073408009, 52.547554398144172 ]
]
]
},
"downloadFolder": "C:\\workdir\\groenmonitor\\",
"fileNameStats": "BulkSatDownload.csv",
"database": null,
"schemaname": null,
"cropfieldtable": null,
"satelllitetable": null
},
{
"fieldName": "MyField_8", // FarmMaps minimum needs are: fieldName, cropYear & geometryJson. Actually no Satellite data before 2017. This here to illustrate the program does not crash
"cropYear": 2016,
"fieldID": 8,
"SatelliteBands": [ "ci-red", "ndvi", "wdvi" ], // ["ndvi"] or ["wdvi"] or both: [ "wdvi", "ndvi" ]
"lastdownloadedimagedate": "2016-01-01", //downloads images from this date till end of the year
"geometryJson": {
"type": "Polygon",
"coordinates": [
[
[ 5.563472073408009, 52.547554398144172 ],
[ 5.567425915520115, 52.547725375100377 ],
[ 5.563878143678981, 52.54048022658143 ],
[ 5.563878143678981, 52.54048022658143 ],
[ 5.563472073408009, 52.547554398144172 ]
]
]
},
"downloadFolder": "C:\\workdir\\groenmonitor\\",
"fileNameStats": "BulkSatDownload.csv",
"database": null,
"schemaname": null,
"cropfieldtable": null,
"satelllitetable": null
},
{
"fieldName": "MyField_9", // FarmMaps minimum needs are: fieldName, cropYear & geometryJson. Actually no Satellite data before 2017. This here to illustrate the program does not crash
"cropYear": 2015,
"fieldID": 9,
"SatelliteBands": [ "ci-red", "ndvi", "wdvi" ], // ["ndvi"] or ["wdvi"] or both: [ "wdvi", "ndvi" ]
"lastdownloadedimagedate": "2015-01-01", //downloads images from this date till end of the year
"geometryJson": {
"type": "Polygon",
"coordinates": [
[
[ 5.563472073408009, 52.547554398144172 ],
[ 5.567425915520115, 52.547725375100377 ],
[ 5.563878143678981, 52.54048022658143 ],
[ 5.563878143678981, 52.54048022658143 ],
[ 5.563472073408009, 52.547554398144172 ]
]
]
},
"downloadFolder": "C:\\workdir\\groenmonitor\\",
"fileNameStats": "BulkSatDownload.csv",
"database": null,
"schemaname": null,
"cropfieldtable": null,
"satelllitetable": null
}
]