forked from FarmMaps/FarmMapsApiClient
603 lines
39 KiB
C#
603 lines
39 KiB
C#
using System;
|
|
using System.Collections.Generic;
|
|
using System.IO;
|
|
using System.IO.Compression;
|
|
using System.Linq;
|
|
using System.Threading.Tasks;
|
|
using FarmmapsApi;
|
|
using FarmmapsApi.Models;
|
|
using FarmmapsApi.Services;
|
|
using FarmmapsBulkSatDownload.Models;
|
|
using Microsoft.Extensions.Logging;
|
|
using Newtonsoft.Json;
|
|
using Npgsql;
|
|
using Newtonsoft.Json.Linq;
|
|
using static FarmmapsApiSamples.Constants;
|
|
using System.Text;
|
|
|
|
namespace FarmmapsBulkSatDownload
|
|
{
|
|
public class BulkSatDownloadApplication : IApplication
|
|
{
|
|
private readonly ILogger<BulkSatDownloadApplication> _logger;
|
|
private readonly FarmmapsApiService _farmmapsApiService;
|
|
private readonly BulkSatDownloadService _bulkSatDownloadService;
|
|
private readonly GeneralService _generalService;
|
|
|
|
public const string settingsfile = "Settings.json";
|
|
public const int firstAvailableYear = 2017;
|
|
private Settings _settings;
|
|
|
|
public BulkSatDownloadApplication(ILogger<BulkSatDownloadApplication> logger, FarmmapsApiService farmmapsApiService,
|
|
GeneralService generalService, BulkSatDownloadService bulkSatDownloadService)
|
|
{
|
|
_logger = logger;
|
|
_farmmapsApiService = farmmapsApiService;
|
|
_generalService = generalService;
|
|
_bulkSatDownloadService = bulkSatDownloadService;
|
|
}
|
|
public async Task RunAsync()
|
|
{
|
|
// Check if we have permission
|
|
// !! this call is needed the first time an api is called with a fresh clientid and secret !!
|
|
await _farmmapsApiService.GetCurrentUserCodeAsync();
|
|
var roots = await _farmmapsApiService.GetCurrentUserRootsAsync();
|
|
|
|
BulkSatDownloadInput bulkSatDownloadInput;
|
|
List<BulkSatDownloadInput> bulkSatDownloadInputList;
|
|
// Below are two options to for bulk download: (1) from and to database or (2) inputs from json, output to csv
|
|
// For illustration we make two lists bulkSatDownloadInputListDB and bulkSatDownloadInputListCsv and then choose which one we will use
|
|
List<BulkSatDownloadInput> bulkSatDownloadInputListDB;
|
|
List<BulkSatDownloadInput> bulkSatDownloadInputListCsv;
|
|
DateTime lastdownloadedimagedate;
|
|
int cropYear;
|
|
|
|
//Use doDB to decide if reading from/writing to database (doDB = true) or
|
|
//read from file BulkSatDownloadInput.json and write satellite statistics to file(s) specified in BulkSatDownloadInput.json
|
|
//note in case of doDB == true you will need to provide a file "DBsettings.secrets.json" with login details for the database
|
|
//see empty example "DBsettings.json"
|
|
bool doDB = false;
|
|
if (doDB == true)
|
|
{
|
|
// Option 1: When using database need to (1) fill in database data in DBsettings.secrets.json; (2) write tailor made SELECT query for fieldinputs in following lines;
|
|
// (3) Write tailor made INSERT INTO query in Task Process() below;
|
|
// Initialize databases. Username, password etc stored in file "DBsettings.secrets.json".
|
|
// Crashes if "DBsettings.secrets.json" is absent or empty
|
|
DB dbparcels = JsonConvert.DeserializeObject<DB>(File.ReadAllText("DBsettings.secrets.json"));
|
|
string schemaname = "bigdata";
|
|
string parceltablename = "parcel_bollenrevolutie_tulips2020"; //"parcelsijbrandij" "parcel"; "parcel_flowerbulbs"; "parcel_disac"; ""parcel_bollenrevolutie_tulips2020""
|
|
string groenmonitortablename = "groenmonitor_bollenrevolutie_tulips2020"; //"groenmonitorsijbrandij" "groenmonitor" "groenmonitor_flowerbulbs" "groenmonitor_disac" "groenmonitor_bollenrevolutie_tulips2020"
|
|
// The view 'groenmonitorlatestviewname' contains per parcelid (arbid) the year in which it "exists" and the date of the latest image downloaded. It is used to prevent unneccessary downloading of image statistics already in the database
|
|
string groenmonitorlatestviewname = "groenmonitorlatest_bollenrevolutie_tulips2020"; //"groenmonitorsijbrandijlatest" "groenmonitorlatest" "groenmonitorlatest_flowerbulbs" "groenmonitorlatest_disac" "groenmonitorlatest_bollenrevolutie_tulips2020"
|
|
|
|
// Database query and connection. Geometry must be in WGS84 coordinate system, EPSG 4326
|
|
// Apparently the FarmmapsApi cannot handle MultiPolygon, so we need to convert to single Polygon
|
|
// In case database returns a MultiPolygon use ST_NumGeometries(pt.geom) to count the number of polygons
|
|
// If necessary use WHERE ST_NumGeometries(pt.geom) = 1 to select only single polygons
|
|
//
|
|
// FarmMaps get's its satellite images from www.groenmonitor.nl through the https://agrodatacube.wur.nl/.
|
|
// Many images are available at www.groenmonitor.nl, the https://agrodatacube.wur.nl/ serves only the clean images, 10-30 per year, 2019 onwards. Possibly more images will be added for earlier years
|
|
// For other images contact www.groenmonitor.nl, gerbert.roerink@wur.nl
|
|
bulkSatDownloadInputListDB = new List<BulkSatDownloadInput>();
|
|
List<string> satelliteBands = new List<string> { "wdvi", "ndvi" };
|
|
string connectionString = dbparcels.GetConnectionString();
|
|
string readSql = string.Format(
|
|
@"
|
|
SELECT pt.arbid, pt.year, gml.lastwenrdate, ST_AsGeoJSON(ST_Transform((ST_DUMP(pt.geom)).geom::geometry(Polygon),4326)) AS geojson_polygon_wgs84,
|
|
COALESCE(pt.cropfielditemcode,'') AS cropfielditemcode,
|
|
CASE WHEN pt.year >= DATE_PART('year', CURRENT_DATE) THEN '' ELSE COALESCE(pt.satellitetaskcode,'') END AS satellitetaskcode
|
|
FROM {0}.{1} pt, {0}.{2} gml
|
|
WHERE
|
|
pt.arbid = gml.arbid
|
|
AND pt.satellitetaskcode IS NULL
|
|
ORDER BY pt.arbid
|
|
LIMIT 5;", schemaname, parceltablename, groenmonitorlatestviewname); //LIMIT x for testing
|
|
|
|
using (NpgsqlConnection connection = new NpgsqlConnection(connectionString))
|
|
{
|
|
connection.Open();
|
|
|
|
// Read data (run query) = build a list of fields for which to download images
|
|
NpgsqlCommand command = connection.CreateCommand();
|
|
command.CommandText = readSql;
|
|
NpgsqlDataReader dr = command.ExecuteReader();
|
|
while (dr.Read())
|
|
{
|
|
bulkSatDownloadInput = new BulkSatDownloadInput();
|
|
bulkSatDownloadInput.fieldID = dr.GetInt16(0);
|
|
bulkSatDownloadInput.fieldName = string.Format($"{parceltablename}_{bulkSatDownloadInput.fieldID}");
|
|
bulkSatDownloadInput.cropYear = dr.GetInt16(1); ;
|
|
bulkSatDownloadInput.lastdownloadedimagedate = dr.GetDateTime(2);
|
|
bulkSatDownloadInput.GeometryJson = JObject.Parse(dr.GetString(3));
|
|
bulkSatDownloadInput.SatelliteBands = satelliteBands;
|
|
bulkSatDownloadInput.cropfielditemcode = dr.GetString(4);
|
|
bulkSatDownloadInput.satellitetaskcode = dr.GetString(5);
|
|
bulkSatDownloadInput.database = dbparcels;
|
|
bulkSatDownloadInput.schemaname = schemaname;
|
|
bulkSatDownloadInput.cropfieldtable = parceltablename;
|
|
bulkSatDownloadInput.satelllitetable = groenmonitortablename;
|
|
bulkSatDownloadInputListDB.Add(bulkSatDownloadInput);
|
|
}
|
|
connection.Close();
|
|
}
|
|
// Now choose which list you want to use
|
|
bulkSatDownloadInputList = bulkSatDownloadInputListDB;
|
|
}
|
|
else
|
|
{
|
|
// Option 2: Example without database. Comment out this part if you want to use database
|
|
// Read cropfields "BulkSatDownloadInput.json" and write all stats to a single csv file
|
|
// Write all stats for multiple fields will be written to a single csv file
|
|
string downloadFolder;
|
|
string fileNameStats;
|
|
string headerLineStats = $"FieldName,satelliteDate,satelliteBand,max,min,mean,mode,median,stddev,minPlus,curtosis,maxMinus,skewness,variance,populationCount,variationCoefficient,confidenceIntervalLow, confidenceIntervalHigh,confidenceIntervalErrorMargin" + Environment.NewLine;
|
|
var fieldsInputJson = File.ReadAllText("BulkSatDownloadInput.json");
|
|
bulkSatDownloadInputListCsv = JsonConvert.DeserializeObject<List<BulkSatDownloadInput>>(fieldsInputJson);
|
|
for (int i = 0; i < bulkSatDownloadInputListCsv.Count; i++)
|
|
{
|
|
downloadFolder = bulkSatDownloadInputListCsv[i].downloadFolder;
|
|
fileNameStats = Path.Combine(downloadFolder, bulkSatDownloadInputListCsv[i].fileNameStats);
|
|
if (!Directory.Exists(downloadFolder))
|
|
Directory.CreateDirectory(downloadFolder);
|
|
bulkSatDownloadInputListCsv[i].fileNameStats = fileNameStats;
|
|
// Header same as in GeneralService.DownloadSatelliteStats
|
|
// Delete fileNameStats if existing. Create a new file. Add a header to csv file
|
|
File.Delete(fileNameStats);
|
|
File.AppendAllText(fileNameStats, headerLineStats);
|
|
}
|
|
|
|
// Now choose which list you want to use
|
|
bulkSatDownloadInputList = bulkSatDownloadInputListCsv;
|
|
}
|
|
|
|
// For each input download all images. Keep track to time, important when doing bulk downloads
|
|
var watch = System.Diagnostics.Stopwatch.StartNew();
|
|
TimeSpan tsSofar = new TimeSpan();
|
|
TimeSpan tsRemaining;
|
|
TimeSpan tsTotalEstimated;
|
|
|
|
for (int i = 0; i < bulkSatDownloadInputList.Count; i++)
|
|
{
|
|
watch.Restart();
|
|
bulkSatDownloadInput = bulkSatDownloadInputList[i];
|
|
if (string.IsNullOrEmpty(bulkSatDownloadInput.fileNameStats) == false) _logger.LogInformation(string.Format($"// FarmmapsBulkSatDownload: Downloading stats for field {i+1} out of {bulkSatDownloadInputList.Count} to single csv file {bulkSatDownloadInput.fileNameStats}"));
|
|
if (bulkSatDownloadInput.database != null) _logger.LogInformation(string.Format($"// FarmmapsBulkSatDownload: Downloading stats for field {i+1} out of {bulkSatDownloadInputList.Count} to database {bulkSatDownloadInput.schemaname}.{bulkSatDownloadInput.satelllitetable}"));
|
|
try
|
|
{
|
|
await Process(roots, bulkSatDownloadInput);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
_logger.LogError(ex.Message);
|
|
}
|
|
watch.Stop();
|
|
// add this downloadtime to the cropfieldtable.
|
|
// Only if cropfieldtable has a field called 'downloadtime' type 'time'!
|
|
//using (NpgsqlConnection connection = new NpgsqlConnection(dbparcels.GetConnectionString()))
|
|
//{
|
|
// connection.Open();
|
|
// NpgsqlCommand updateCmd = connection.CreateCommand();
|
|
// string updateSql = string.Format($"UPDATE {schemaname}.{parceltablename} SET downloadtime = '{strTime(watch.Elapsed)}' WHERE arbid = {bulkSatDownloadInput.fieldID};");
|
|
// updateCmd.CommandText = updateSql;
|
|
// int r = updateCmd.ExecuteNonQuery();
|
|
// if (r != 1)
|
|
// throw new Exception("// FarmmapsBulkSatDownload: Update downloadtime Failed");
|
|
// connection.Close();
|
|
//}
|
|
//_logger.LogInformation($"// FarmmapsBulkSatDownload: Added downloadtime = '{strTime(watch.Elapsed)}' to {schemaname}.{parceltablename} ");
|
|
tsSofar = tsSofar + watch.Elapsed;
|
|
tsTotalEstimated = tsSofar / (i + 1) * bulkSatDownloadInputList.Count;
|
|
tsRemaining = tsTotalEstimated - tsSofar;
|
|
_logger.LogInformation(string.Format($"// Time (hh:mm:ss): this field: {strTime(watch.Elapsed)}. Sofar: {strTime(tsSofar)}. Total: {strTime(tsTotalEstimated)}. Remaining: {strTime(tsRemaining)}"));
|
|
}
|
|
string strExeFilePath = System.Reflection.Assembly.GetExecutingAssembly().Location;
|
|
string strWorkPath = Path.GetDirectoryName(strExeFilePath);
|
|
_logger.LogInformation(string.Format($"// FarmmapsBulkSatDownload:"));
|
|
_logger.LogInformation(string.Format($"// FarmmapsBulkSatDownload: Done! List of all downloaded cropfieldItems stored in {Path.Combine(strWorkPath,settingsfile)}"));
|
|
_logger.LogInformation(string.Format($"// FarmmapsBulkSatDownload: If you plan to rerun certain fields then adding cropfielditemcode to your input can greatly speed up your application!"));
|
|
|
|
}
|
|
|
|
private async Task Process(List<UserRoot> roots, BulkSatDownloadInput input)
|
|
{
|
|
//PO20220311: first time a call is made to download satellite images or statistics, an empty list is returned
|
|
//If we wait a bit longer, e.g. 10 secs, then e.g. a list of 3 images may be returned
|
|
//If we wait still longer, maybe 4 images.
|
|
//The solution implemented below is to fire calls as long as the number of images returned keeps increasing
|
|
//While in between each call, sleep for sleepSecs
|
|
//Continue this until the number no longer increases or the maximum number of calls has been reached
|
|
//If you set sleepSecs to a very low value, e.g. 5 secs, then after 1 call you might get images and after 2nd call still zero images.
|
|
//to be on the safe side, better bit higher value.
|
|
//Just accept this may take a while, have a coffee, we suggest sleepSecs = 30;
|
|
int sleepSecs = 30;
|
|
int callCntMax = 4 * 60 / sleepSecs; //4*60 = max 4 minutes
|
|
//For example we may set: "sleepSecs = 10;" and "callCntMax = 24;" and following result:
|
|
//Call no: 1. Giving FarmMaps 10 seconds to get SatelliteItems...
|
|
//Call no: 1: Received 2 images
|
|
//Call no: 2. Giving FarmMaps 10 seconds to get SatelliteItems...
|
|
//Call no: 2: Received 7 images
|
|
//Call no: 3. Giving FarmMaps 10 seconds to get SatelliteItems...
|
|
//Call no: 3: Received 7 images
|
|
//And the firing of calls would stop because the number of images returned is no longer increasing
|
|
//In the worst case, this could would lead to a total sleeping period of "sleepSecsSum = sleepSecs * callCntMax" seconds. After that we give up
|
|
|
|
//PO20230801: I also tried creating a cropfieldItem with startDate '2017-01-01' and endDate '2023-08-01'
|
|
//so that only one RunSatelliteTask would need to be executed and statistics would be returned for the e.g. 233 images in this period from start to end date
|
|
//instead of running 7 tasks (one per year) for a single field
|
|
//But code doing that returned much less images than those 233 records which I should be getting.
|
|
//So in the end, stick to downloading stats per year, even if it is for the same location in multiple years
|
|
|
|
string cropfielditemcode;
|
|
string satellitetaskcode;
|
|
Item cropfieldItem;
|
|
bool satelliteItemsAvailable;
|
|
bool statsAvailable;
|
|
DateTime dtSatelliteDate;
|
|
string strSatelliteDate;
|
|
List<Item> satelliteItemsCropYear;
|
|
StringBuilder sbInsertSql;
|
|
StringBuilder sbInsertSqlInto;
|
|
StringBuilder sbInsertSqlUpdate;
|
|
StringBuilder sbInsertSqlValues;
|
|
List<SatelliteStatistics> listSatelliteStatistics;
|
|
SatelliteStatistics satelliteStatistics_wdvi;
|
|
SatelliteStatistics satelliteStatistics_ndvi;
|
|
int cntDatesDownloaded;
|
|
|
|
string satelliteSource = "akkerwebwenr"; //same as in groenmonitorlatestviewname SQL code
|
|
int fieldID = input.fieldID;
|
|
string fieldName = input.fieldName;
|
|
int cropYear = input.cropYear;
|
|
List<string> satelliteBands = input.SatelliteBands;
|
|
string downloadFolder = input.downloadFolder;
|
|
string fileNameStats = input.fileNameStats;
|
|
DB database = input.database;
|
|
string schemaname = input.schemaname;
|
|
string cropfieldtable = input.cropfieldtable;
|
|
string satelllitetable = input.satelllitetable;
|
|
DateTime lastDownloadedSatelliteDate = input.lastdownloadedimagedate;
|
|
cropfielditemcode = input.cropfielditemcode;
|
|
satellitetaskcode = input.satellitetaskcode;
|
|
int satelliteItemsCropYearCntPrev;
|
|
int satelliteItemsCropYearCnt;
|
|
int callCnt;
|
|
int sleepSecsSum;
|
|
|
|
LoadSettings(settingsfile);
|
|
|
|
var uploadedRoot = roots.SingleOrDefault(r => r.Name == "USER_IN");
|
|
if (uploadedRoot == null)
|
|
{
|
|
_logger.LogError("Could not find a needed root item");
|
|
return;
|
|
}
|
|
|
|
var myDriveRoot = roots.SingleOrDefault(r => r.Name == "USER_FILES");
|
|
if (myDriveRoot == null)
|
|
{
|
|
_logger.LogError("Could not find a needed root item");
|
|
return;
|
|
}
|
|
|
|
if (string.IsNullOrEmpty(cropfielditemcode))
|
|
{
|
|
_logger.LogInformation(string.Format($"Creating cropfield '{fieldName}' in the year {cropYear}"));
|
|
cropfieldItem = await _generalService.CreateCropfieldItemAsync(myDriveRoot.Code,
|
|
$"DataCropfield {fieldName}", cropYear, input.GeometryJson.ToString(Formatting.None));
|
|
cropfielditemcode = cropfieldItem.Code;
|
|
// If working with a database, add this cropfieldItem.Code to the database so that next case same cropField is requested, will be faster
|
|
if (database != null)
|
|
{
|
|
// add this cropfielditemcode to the cropfieldtable
|
|
using (NpgsqlConnection connection = new NpgsqlConnection(database.GetConnectionString()))
|
|
{
|
|
connection.Open();
|
|
NpgsqlCommand updateCmd = connection.CreateCommand();
|
|
string updateSql = string.Format($"UPDATE {schemaname}.{cropfieldtable} SET cropfielditemcode = '{cropfieldItem.Code}' WHERE arbid = {fieldID};");
|
|
updateCmd.CommandText = updateSql;
|
|
//Console.WriteLine(insertCmd.CommandText);
|
|
int r = updateCmd.ExecuteNonQuery();
|
|
if (r != 1)
|
|
throw new Exception("// FarmmapsBulkSatDownload: Update cropfielditemcode Failed");
|
|
connection.Close();
|
|
}
|
|
_logger.LogInformation($"// FarmmapsBulkSatDownload: Added cropfieldItem.Code '{cropfieldItem.Code}' for parcelid {fieldID} to {schemaname}.{cropfieldtable} ");
|
|
}
|
|
}
|
|
else
|
|
{
|
|
// WOULD IT BE POSSIBLE TO GET AVAILABLE Item MEMBER VALUES FOR A GIVEN cropfielditemcode?
|
|
cropfieldItem = new Item();
|
|
cropfieldItem.Code = cropfielditemcode;
|
|
cropfieldItem.Name = "DataCropfield " + fieldName;
|
|
_logger.LogInformation($"// FarmmapsBulkSatDownload: CropfieldItem.Code for parcelid {fieldID} already there in {schemaname}.{cropfieldtable}: '{cropfieldItem.Code}'");
|
|
}
|
|
_settings.cropFieldItems.Add(cropfieldItem);
|
|
SaveSettings(settingsfile);
|
|
|
|
//Create satelliteTaskCode & save satelliteTaskCode.Code to settingsfile for retracing last call (can be useful if failed)
|
|
//_logger.LogInformation(string.Format($"Running RunSatelliteTask for cropfieldItem '{cropfielditemcode}' and saving settings to {settingsfile}"));
|
|
//var satelliteTaskCode = await _generalService.RunSatelliteTask(cropfieldItem);
|
|
if (string.IsNullOrEmpty(satellitetaskcode))
|
|
{
|
|
_logger.LogInformation(string.Format($"Running RunSatelliteTask for cropfieldItem '{cropfielditemcode}'"));
|
|
satellitetaskcode = await _generalService.RunSatelliteTask(cropfieldItem);
|
|
// If working with a database, add this cropfieldItem.Code to the database so that next case same cropField is requested, will be faster
|
|
if (database != null)
|
|
{
|
|
// add this satellitetaskcode to the cropfieldtable
|
|
using (NpgsqlConnection connection = new NpgsqlConnection(database.GetConnectionString()))
|
|
{
|
|
connection.Open();
|
|
NpgsqlCommand updateCmd = connection.CreateCommand();
|
|
string updateSql = string.Format($"UPDATE {schemaname}.{cropfieldtable} SET satellitetaskcode = '{satellitetaskcode}' WHERE arbid = {fieldID};");
|
|
updateCmd.CommandText = updateSql;
|
|
//Console.WriteLine(insertCmd.CommandText);
|
|
int r = updateCmd.ExecuteNonQuery();
|
|
if (r != 1)
|
|
throw new Exception("// FarmmapsBulkSatDownload: Update satellitetaskcode Failed");
|
|
connection.Close();
|
|
}
|
|
_logger.LogInformation($"// FarmmapsBulkSatDownload: Added satellitetaskcode '{satellitetaskcode}' for cropfieldItem.Code '{cropfieldItem.Code}' to {schemaname}.{cropfieldtable} ");
|
|
}
|
|
}
|
|
else
|
|
{
|
|
_logger.LogInformation($"// FarmmapsBulkSatDownload: satellitetaskcode for parcelid {fieldID} already there in {schemaname}.{cropfieldtable}: '{satellitetaskcode}'");
|
|
}
|
|
|
|
// TODO also log satellitetaskcode to settings, how?
|
|
// SaveSettings(settingsfile);
|
|
|
|
// Getting satellite items. Only for years for which available
|
|
satelliteItemsCropYearCntPrev = 0;
|
|
satelliteItemsCropYearCnt = 0;
|
|
sleepSecsSum = 0;
|
|
satelliteItemsCropYear = null;
|
|
if (cropYear >= firstAvailableYear && cropYear <= DateTime.Now.Year)
|
|
{
|
|
_logger.LogInformation(string.Format($"Running FindSatelliteItems for cropfieldItem.Code '{cropfieldItem.Code}', SatelliteTaskCode '{satellitetaskcode}'"));
|
|
//Call first time
|
|
callCnt = 1;
|
|
//if callCntMax == 0 then don't sleep
|
|
//if callCntMax = 1 then sleep first 1x
|
|
if (callCntMax > 0)
|
|
{
|
|
_logger.LogInformation($"Call no: {callCnt}. Giving FarmMaps {sleepSecs} seconds to get SatelliteItems...");
|
|
System.Threading.Thread.Sleep(1000 * sleepSecs);
|
|
sleepSecsSum = sleepSecsSum + sleepSecs;
|
|
}
|
|
satelliteItemsCropYear = await _generalService.FindSatelliteItems(cropfieldItem, satellitetaskcode);
|
|
satelliteItemsCropYearCntPrev = satelliteItemsCropYear.Count;
|
|
_logger.LogInformation($"Call no: {callCnt}. Received {satelliteItemsCropYearCntPrev} images");
|
|
callCnt++;
|
|
satelliteItemsCropYearCnt = satelliteItemsCropYearCntPrev;
|
|
//if callCntMax > 1 then sleep untill (1) no more increase in number of images received OR (2) maximum number of calls reached
|
|
if (callCntMax > 1)
|
|
{
|
|
//Call second time
|
|
_logger.LogInformation($"Call no: {callCnt}. Giving FarmMaps another {sleepSecs} seconds to get SatelliteItems...");
|
|
System.Threading.Thread.Sleep(1000 * sleepSecs);
|
|
satelliteItemsCropYear = await _generalService.FindSatelliteItems(cropfieldItem, satellitetaskcode);
|
|
satelliteItemsCropYearCnt = satelliteItemsCropYear.Count;
|
|
_logger.LogInformation($"Call no: {callCnt}. Received {satelliteItemsCropYearCnt} images");
|
|
sleepSecsSum = sleepSecsSum + sleepSecs;
|
|
//As long as there is progress, keep calling
|
|
callCnt++;
|
|
while (callCnt <= callCntMax && (satelliteItemsCropYearCnt == 0 || satelliteItemsCropYearCnt > satelliteItemsCropYearCntPrev))
|
|
{
|
|
_logger.LogInformation($"Surprise! The longer we wait, the more images we get. Sleep and call once more");
|
|
satelliteItemsCropYearCntPrev = satelliteItemsCropYearCnt;
|
|
_logger.LogInformation($"Call no: {callCnt} (max: {callCntMax}). Giving FarmMaps another {sleepSecs} seconds to get SatelliteItems...");
|
|
System.Threading.Thread.Sleep(1000 * sleepSecs);
|
|
satelliteItemsCropYear = await _generalService.FindSatelliteItems(cropfieldItem, satellitetaskcode);
|
|
satelliteItemsCropYearCnt = satelliteItemsCropYear.Count;
|
|
_logger.LogInformation($"Call no: {callCnt}. Received {satelliteItemsCropYearCnt} images");
|
|
callCnt++;
|
|
sleepSecsSum = sleepSecsSum + sleepSecs;
|
|
}
|
|
}
|
|
}
|
|
else
|
|
{
|
|
_logger.LogWarning($"// FarmmapsBulkSatDownload: crop year {cropYear} is out of bounds. No stats will be written!");
|
|
}
|
|
|
|
if (satelliteItemsCropYearCnt == 0)
|
|
{
|
|
_logger.LogWarning($"// FarmmapsBulkSatDownload: after calling one or more times and " +
|
|
$"sleeping in total {sleepSecsSum} seconds, still no images found. " +
|
|
$"Please check your settings for parameters callCntMax and sleepSecs in FarmmapsBulkSatDownload.cs or contact FarmMaps");
|
|
}
|
|
|
|
// Checking if satellite items found
|
|
satelliteItemsAvailable = true;
|
|
if (satelliteItemsCropYear == null)
|
|
{
|
|
satelliteItemsAvailable = false;
|
|
_logger.LogInformation($"// FarmmapsBulkSatDownload: No satellite tiffs found for fieldName '{fieldName}', cropYear {cropYear}, cropfielditemcode '{cropfielditemcode}'");
|
|
}
|
|
else
|
|
{
|
|
if (satelliteItemsCropYear.Count == 0)
|
|
{
|
|
satelliteItemsAvailable = false;
|
|
_logger.LogInformation($"// FarmmapsBulkSatDownload: No satellite tiffs found for fieldName '{fieldName}', cropYear {cropYear}, cropfielditemcode '{cropfielditemcode}'");
|
|
}
|
|
}
|
|
|
|
// Sort the list by date
|
|
if (satelliteItemsAvailable) satelliteItemsCropYear = satelliteItemsCropYear.OrderBy(x => x.DataDate).ToList();
|
|
|
|
// Download statistics to a single csv file
|
|
if (satelliteItemsAvailable && downloadFolder != null && fileNameStats != null)
|
|
{
|
|
// Write statistics for all images for all fieldName and cropYear to a single csv file, fileNameStats
|
|
_logger.LogInformation($"Downloading stats for field '{fieldName}' in cropyear {cropYear} to {fileNameStats}");
|
|
string downloadedStats = await _generalService.DownloadSatelliteStats(satelliteItemsCropYear, fieldName, satelliteBands, downloadFolder);
|
|
// Add contents of this csv file to the single large csv file
|
|
var retainedLines = File.ReadAllLines(downloadedStats).Skip(1);
|
|
File.AppendAllLines(fileNameStats, retainedLines);
|
|
File.Delete(downloadedStats);
|
|
// Optionally, also download the zipped tiffs. This can be a lot of files and increase runtime
|
|
if (false)
|
|
{
|
|
foreach (Item selectedSatelliteItem in satelliteItemsCropYear)
|
|
{
|
|
// download the geotiffs. Returns a zip file with always these three files:
|
|
// data.dat.aux.xml
|
|
// thumbnail.jpg
|
|
// wenr.tif. Contains 5 layers: (1) ndvi, (2) wdvi, (3) Red, (4) Green and (5) Blue
|
|
var SatelliteDate = selectedSatelliteItem.DataDate.Value.ToString("yyyyMMdd");
|
|
_logger.LogInformation($"Downloading geotiff file for field {fieldName}, date {SatelliteDate}");
|
|
string fileName = string.Format($"satelliteGeotiff_{fieldName}_{SatelliteDate}"); // no need to add satelliteBand in the name because the tif contains all bands
|
|
string fileNameZip = string.Format($"{fileName}.zip");
|
|
string fileNameGeotiff = string.Format($"{fileName}.tif");
|
|
await _farmmapsApiService.DownloadItemAsync(selectedSatelliteItem.Code, Path.Combine(downloadFolder, fileNameZip));
|
|
if (false)
|
|
{
|
|
// Extract the file "wenr.tif" from zip, rename it to fileNameGeotiff
|
|
ZipFile.ExtractToDirectory(Path.Combine(downloadFolder, fileNameZip), downloadFolder, true);
|
|
File.Delete(Path.Combine(downloadFolder, fileNameGeotiff)); // Delete the fileNameGeotiff file if exists
|
|
File.Move(Path.Combine(downloadFolder, "wenr.tif"), Path.Combine(downloadFolder, fileNameGeotiff)); // Rename the oldFileName into newFileName
|
|
|
|
// Cleanup
|
|
string[] filesToDelete = new string[] { fileNameZip, "wenr.tif", "thumbnail.jpg", "data.dat.aux.xml" };
|
|
foreach (string f in filesToDelete)
|
|
{
|
|
File.Delete(Path.Combine(downloadFolder, f));
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
// Download statistics to database
|
|
if (satelliteItemsAvailable && database != null)
|
|
{
|
|
// Tailormade code for writing to database
|
|
// No unnecessary intermediate step here of writing to csv and getting stats for all images in the crop year.
|
|
// Efficient is to check if there is any image for which stats are to be added to database and add only add these new stats for not yet archived dates directly to database
|
|
// A full check of downloaded dates versus available dates in the database is not made here.
|
|
// We assume only new images will be added (i.e. for later dates), assuming no historical images are added in groenmonitor (check!).
|
|
// And we assume farmMaps always nicely generates statistics, no hick-ups
|
|
// Under this assumptions, we only need to compare with the lastDownloadedSatelliteDate from the database
|
|
cntDatesDownloaded = 0;
|
|
using (NpgsqlConnection connection = new NpgsqlConnection(database.GetConnectionString()))
|
|
{
|
|
connection.Open();
|
|
foreach (Item satelliteItem in satelliteItemsCropYear)
|
|
{
|
|
dtSatelliteDate = satelliteItem.DataDate.Value;
|
|
strSatelliteDate = dtSatelliteDate.ToString("yyyy-MM-dd");
|
|
listSatelliteStatistics = await _generalService.ListSatelliteStatistics(satelliteItem, satelliteBands, fieldName);
|
|
statsAvailable = true;
|
|
if (listSatelliteStatistics == null)
|
|
{
|
|
statsAvailable = false;
|
|
_logger.LogWarning($"No stats found for satellite, fieldName '{fieldName}', date '{strSatelliteDate}'");
|
|
}
|
|
else
|
|
{
|
|
if (listSatelliteStatistics.Count == 0)
|
|
{
|
|
statsAvailable = false;
|
|
_logger.LogWarning($"No stats found for satellite, fieldName '{fieldName}', date '{strSatelliteDate}'");
|
|
}
|
|
}
|
|
if (statsAvailable)
|
|
{
|
|
if (dtSatelliteDate <= input.lastdownloadedimagedate)
|
|
{
|
|
_logger.LogInformation($"// Stats for parcelid {fieldID}, date '{strSatelliteDate}' already there in {schemaname}.{satelllitetable}");
|
|
}
|
|
else
|
|
{
|
|
cntDatesDownloaded++;
|
|
// Map satelliteStatistics to groenmonitorTable
|
|
satelliteStatistics_wdvi = listSatelliteStatistics.SingleOrDefault(p => p.satelliteBand == "wdvi");
|
|
satelliteStatistics_ndvi = listSatelliteStatistics.SingleOrDefault(p => p.satelliteBand == "ndvi");
|
|
sbInsertSql = new StringBuilder();
|
|
sbInsertSqlInto = new StringBuilder();
|
|
sbInsertSqlUpdate = new StringBuilder();
|
|
sbInsertSqlValues = new StringBuilder();
|
|
sbInsertSqlInto.Append($"INSERT INTO {schemaname}.{satelllitetable} (parcelid,date,source");
|
|
sbInsertSqlValues.Append($"VALUES ({fieldID},'{strSatelliteDate}','{satelliteSource}'");
|
|
sbInsertSqlUpdate.Append($"ON CONFLICT (parcelid,date,source) DO UPDATE SET parcelid={fieldID}, date='{strSatelliteDate}',source='{satelliteSource}'");
|
|
if (satelliteBands.Contains("wdvi"))
|
|
{
|
|
if (satelliteStatistics_wdvi != null)
|
|
{
|
|
sbInsertSqlInto.Append(",wdvi_pixelcount,wdvi_max,wdvi_mean,wdvi_min,wdvi_stdev,wdvi_median");
|
|
sbInsertSqlValues.Append($",{satelliteStatistics_wdvi.populationCount},{satelliteStatistics_wdvi.max},{satelliteStatistics_wdvi.mean},{satelliteStatistics_wdvi.min},{satelliteStatistics_wdvi.stddev},{satelliteStatistics_wdvi.median}");
|
|
sbInsertSqlUpdate.Append($",wdvi_pixelcount={satelliteStatistics_wdvi.populationCount},wdvi_max={satelliteStatistics_wdvi.max},wdvi_mean={satelliteStatistics_wdvi.mean},wdvi_min={satelliteStatistics_wdvi.min},wdvi_stdev={satelliteStatistics_wdvi.stddev},wdvi_median={satelliteStatistics_wdvi.median}");
|
|
}
|
|
else
|
|
{
|
|
_logger.LogWarning($"wdvi missing for cropfielditemcode {cropfielditemcode}, date '{strSatelliteDate}'");
|
|
}
|
|
}
|
|
if (satelliteBands.Contains("ndvi"))
|
|
{
|
|
if (satelliteStatistics_ndvi != null)
|
|
{
|
|
sbInsertSqlInto.Append(",ndvi_pixelcount,ndvi_max,ndvi_mean,ndvi_min,ndvi_stdev,ndvi_median");
|
|
sbInsertSqlValues.Append($",{satelliteStatistics_ndvi.populationCount},{satelliteStatistics_ndvi.max},{satelliteStatistics_ndvi.mean},{satelliteStatistics_ndvi.min},{satelliteStatistics_ndvi.stddev},{satelliteStatistics_ndvi.median}");
|
|
sbInsertSqlUpdate.Append($",ndvi_pixelcount={satelliteStatistics_ndvi.populationCount},ndvi_max={satelliteStatistics_ndvi.max},ndvi_mean={satelliteStatistics_ndvi.mean},ndvi_min={satelliteStatistics_ndvi.min},ndvi_stdev={satelliteStatistics_ndvi.stddev},ndvi_median={satelliteStatistics_ndvi.median}");
|
|
}
|
|
else
|
|
{
|
|
_logger.LogWarning($"ndvi missing for cropfielditemcode {cropfielditemcode}, date '{strSatelliteDate}'");
|
|
}
|
|
}
|
|
sbInsertSqlInto.Append(")");
|
|
sbInsertSqlValues.Append(")");
|
|
sbInsertSql.AppendLine(sbInsertSqlInto.ToString());
|
|
sbInsertSql.AppendLine(sbInsertSqlValues.ToString());
|
|
sbInsertSql.AppendLine(sbInsertSqlUpdate.ToString());
|
|
//string strInsertSql = sbInsertSql.ToString();
|
|
NpgsqlCommand insertCmd = connection.CreateCommand();
|
|
insertCmd.CommandText = sbInsertSql.ToString();
|
|
//Console.WriteLine(insertCmd.CommandText);
|
|
int r = insertCmd.ExecuteNonQuery();
|
|
if (r != 1)
|
|
throw new Exception("// FarmmapsBulkSatDownload: Insert Failed");
|
|
_logger.LogInformation($"// Added stats to {schemaname}.{satelllitetable} for parcelid {fieldID}, date '{strSatelliteDate}'. cntDatesDownloaded: {cntDatesDownloaded}");
|
|
}
|
|
}
|
|
}
|
|
connection.Close();
|
|
}
|
|
}
|
|
}
|
|
|
|
// Functions to save previously created cropfields
|
|
private void LoadSettings(string file)
|
|
{
|
|
if (File.Exists(file))
|
|
{
|
|
var jsonText = File.ReadAllText(file);
|
|
_settings = JsonConvert.DeserializeObject<Settings>(jsonText);
|
|
}
|
|
else
|
|
{
|
|
Settings settings = new Settings();
|
|
settings.cropFieldItems = new List<Item>();
|
|
_settings = settings;
|
|
}
|
|
}
|
|
|
|
private void SaveSettings(string file)
|
|
{
|
|
if (_settings == null)
|
|
return;
|
|
|
|
var json = JsonConvert.SerializeObject(_settings);
|
|
File.WriteAllText(file, json);
|
|
}
|
|
private string strTime(TimeSpan ts)
|
|
{
|
|
return String.Format("{0:00}:{1:00}:{2:00}", ts.Hours, ts.Minutes, ts.Seconds);
|
|
}
|
|
}
|
|
}
|