forked from FarmMaps/FarmMapsApiClient
243 lines
12 KiB
C#
243 lines
12 KiB
C#
|
using System;
|
||
|
|
using System.Collections.Generic;
|
||
|
|
using System.IO;
|
||
|
|
using System.IO.Compression;
|
||
|
|
using System.Linq;
|
||
|
|
using System.Threading.Tasks;
|
||
|
|
using FarmmapsApi;
|
||
|
|
using FarmmapsApi.Models;
|
||
|
|
using FarmmapsApi.Services;
|
||
|
|
using FarmmapsBulkSatDownload.Models;
|
||
|
|
using Microsoft.Extensions.Logging;
|
||
|
|
using Newtonsoft.Json;
|
||
|
|
using Npgsql;
|
||
|
|
using Newtonsoft.Json.Linq;
|
||
|
|
using static FarmmapsApiSamples.Constants;
|
||
|
|
|
||
|
|
namespace FarmmapsBulkSatDownload
|
||
|
|
{
|
||
|
|
public class BulkSatDownloadApplication : IApplication
|
||
|
|
{
|
||
|
|
private const string SettingsFile = "settings.json";
|
||
|
|
|
||
|
|
private readonly ILogger<BulkSatDownloadApplication> _logger;
|
||
|
|
private readonly FarmmapsApiService _farmmapsApiService;
|
||
|
|
private readonly BulkSatDownloadService _dataDownloadService;
|
||
|
|
private readonly GeneralService _generalService;
|
||
|
|
|
||
|
|
private Settings _settings;
|
||
|
|
static DB dbparcels;
|
||
|
|
|
||
|
|
public BulkSatDownloadApplication(ILogger<BulkSatDownloadApplication> logger, FarmmapsApiService farmmapsApiService,
|
||
|
|
GeneralService generalService, BulkSatDownloadService dataDownloadService)
|
||
|
|
{
|
||
|
|
_logger = logger;
|
||
|
|
_farmmapsApiService = farmmapsApiService;
|
||
|
|
_generalService = generalService;
|
||
|
|
_dataDownloadService = dataDownloadService;
|
||
|
|
}
|
||
|
|
public async Task RunAsync()
|
||
|
|
{
|
||
|
|
// Check if we have permission
|
||
|
|
// !! this call is needed the first time an api is called with a fresh clientid and secret !!
|
||
|
|
await _farmmapsApiService.GetCurrentUserCodeAsync();
|
||
|
|
var roots = await _farmmapsApiService.GetCurrentUserRootsAsync();
|
||
|
|
// Initialize databases. Username, password etc stored in file "DBsettings.json"
|
||
|
|
dbparcels = JsonConvert.DeserializeObject<DB>(File.ReadAllText("DBsettings.json"));
|
||
|
|
|
||
|
|
//string date;
|
||
|
|
//string asterices = "*****************************************************************************";
|
||
|
|
//string bboxtiffWithTempPath;
|
||
|
|
//string parcelshp = "parcel.shp";
|
||
|
|
//string parcelshpWithTempPath;
|
||
|
|
//string gwoptions; //= String.Format("-overwrite -s_srs EPSG:28992 -t_srs EPSG:28992 -dstnodata 0 -cutline {0} -crop_to_cutline", parcelshp);
|
||
|
|
//string parceltiff = "parcel.tiff";
|
||
|
|
//string parceltiffWithTempPath;
|
||
|
|
//string[] vegetationindices = { "WDVI", "NDVI" };
|
||
|
|
//string[] sources = { "groenmonitor" }; // { "groenmonitor", "akkerwebwenr", "akkerwebneo" }; { "groenmonitor" };
|
||
|
|
string schemaname = "bigdata";
|
||
|
|
string parceltablename = "parcel_flowerbulbs";//"parcelsijbrandij" "parcel"; "parcel_flowerbulbs"
|
||
|
|
//string groenmonitortablename = "groenmonitor_flowerbulbs";//"groenmonitorsijbrandij" "groenmonitor" "groenmonitor_flowerbulbs" //PO20190605: "groenmonitormpt" contains groenmonitor data from multiple parcel tables (mpt)
|
||
|
|
// The view 'groenmonitorlatestviewname' contains per parcelid (arbid) the year in which it "exists" and the date of the latest image downloaded. It is used to prevent downloading images already downloaded
|
||
|
|
string groenmonitorlatestviewname = "groenmonitorlatest_flowerbulbs"; //"groenmonitorsijbrandijlatest" "groenmonitorlatest" "groenmonitorlatest_flowerbulbs" //PO20190605: "v_groenmonitorlatest" contains latest available dates from groenmonitortablename = "groenmonitormpt"
|
||
|
|
//GroenmonitorTable gmtb;
|
||
|
|
//DateTime dt;
|
||
|
|
//DateTime dtfirst;
|
||
|
|
//DateTime dtlast;
|
||
|
|
//double scalingfactor;
|
||
|
|
BulkSatDownloadInput dataDownloadInput;
|
||
|
|
List<BulkSatDownloadInput> fieldsInputs = new List<BulkSatDownloadInput>();
|
||
|
|
|
||
|
|
// Database query and connection. Geometry must be in WGS84 coordinate system, EPSG 4326
|
||
|
|
// Apparently the FarmmapsApi cannot handle MultiPolygon, so we need to convert to single Polygon
|
||
|
|
// In case database returns a MultiPolygon use ST_NumGeometries(pt.geom) to count the number of polygons
|
||
|
|
// If necessary use WHERE T_NumGeometries(pt.geom) = 1 to use only single polygons
|
||
|
|
string connectionString = dbparcels.GetConnectionString();
|
||
|
|
string readSql = string.Format(
|
||
|
|
@"
|
||
|
|
SELECT pt.arbid, pt.crop, pt.year, gml.lastwenrdate, ST_AsGeoJSON(ST_Transform((ST_DUMP(pt.geom)).geom::geometry(Polygon),4326)) AS geojson_polygon_wgs84
|
||
|
|
FROM {0}.{1} pt, {0}.{2} gml
|
||
|
|
WHERE
|
||
|
|
pt.arbid = gml.arbid AND
|
||
|
|
pt.crop NOT IN('Tulp', 'Lelie') AND
|
||
|
|
pt.year = 2018
|
||
|
|
LIMIT 10;", schemaname, parceltablename, groenmonitorlatestviewname); //LIMIT 10 for testing
|
||
|
|
|
||
|
|
using (NpgsqlConnection connection = new NpgsqlConnection(connectionString))
|
||
|
|
{
|
||
|
|
connection.Open();
|
||
|
|
|
||
|
|
// Read data (run query) = build a list of fields for which to download images
|
||
|
|
NpgsqlCommand command = connection.CreateCommand();
|
||
|
|
command.CommandText = readSql;
|
||
|
|
NpgsqlDataReader dr = command.ExecuteReader();
|
||
|
|
while (dr.Read())
|
||
|
|
{
|
||
|
|
dataDownloadInput = new BulkSatDownloadInput();
|
||
|
|
dataDownloadInput.UseCreatedCropfield = false;
|
||
|
|
dataDownloadInput.DownloadFolder = "C:\\workdir\\groenmonitor\\";
|
||
|
|
dataDownloadInput.fieldID = dr.GetInt16(0);
|
||
|
|
dataDownloadInput.fieldName = string.Format($"{parceltablename}_fld{dataDownloadInput.fieldID}");
|
||
|
|
dataDownloadInput.cropName = dr.GetString(1);
|
||
|
|
dataDownloadInput.cropYear = dr.GetInt16(2);
|
||
|
|
dataDownloadInput.lastdownloadedimagedate = dr.GetDateTime(3);
|
||
|
|
dataDownloadInput.GeometryJson = JObject.Parse(dr.GetString(4));
|
||
|
|
fieldsInputs.Add(dataDownloadInput);
|
||
|
|
}
|
||
|
|
connection.Close();
|
||
|
|
}
|
||
|
|
|
||
|
|
// For testing using json input instead of database input
|
||
|
|
var fieldsInputJson = File.ReadAllText("BulkSatDownloadInput.json");
|
||
|
|
List<BulkSatDownloadInput> fieldsInputs2 = JsonConvert.DeserializeObject<List<BulkSatDownloadInput>>(fieldsInputJson);
|
||
|
|
|
||
|
|
// Now for each input download all images. Use fieldsInputs or fieldsInputs2 for testing
|
||
|
|
foreach (var input in fieldsInputs2)
|
||
|
|
{
|
||
|
|
try
|
||
|
|
{
|
||
|
|
await Process(roots, input);
|
||
|
|
}
|
||
|
|
catch (Exception ex)
|
||
|
|
{
|
||
|
|
_logger.LogError(ex.Message);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
private async Task Process(List<UserRoot> roots, BulkSatDownloadInput input)
|
||
|
|
{
|
||
|
|
string DownloadFolder = input.DownloadFolder;
|
||
|
|
if (!Directory.Exists(DownloadFolder))
|
||
|
|
Directory.CreateDirectory(DownloadFolder);
|
||
|
|
|
||
|
|
// !!specify if you are using an already created cropfield:
|
||
|
|
bool useCreatedCropfield = input.UseCreatedCropfield;
|
||
|
|
string settingsfile = $"Settings_BulkSatDownloadApplication.json";
|
||
|
|
|
||
|
|
LoadSettings(settingsfile);
|
||
|
|
|
||
|
|
var uploadedRoot = roots.SingleOrDefault(r => r.Name == "Uploaded");
|
||
|
|
if (uploadedRoot == null)
|
||
|
|
{
|
||
|
|
_logger.LogError("Could not find a needed root item");
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
|
||
|
|
var myDriveRoot = roots.SingleOrDefault(r => r.Name == "My drive");
|
||
|
|
if (myDriveRoot == null)
|
||
|
|
{
|
||
|
|
_logger.LogError("Could not find a needed root item");
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
|
||
|
|
// Use already created cropfield or create new one
|
||
|
|
Item cropfieldItem;
|
||
|
|
if (useCreatedCropfield == false || string.IsNullOrEmpty(_settings.CropfieldItemCode))
|
||
|
|
{
|
||
|
|
_logger.LogInformation(string.Format($"Creating cropfield for a field in the year {input.cropYear}"));
|
||
|
|
//string geomjson = input.GeometryJson.ToString(Formatting.None); //"{\"type\":\"Polygon\",\"coordinates\":[[[4.960707146896585,52.80058366970849],[4.960645975538824,52.80047021761092],[4.962140695752897,52.7991771471948],[4.967523821195745,52.80150240004121],[4.966336768950911,52.80254373587981],[4.96171188076433,52.80100999685643],[4.960707146896585,52.80058366970849]]]}"
|
||
|
|
string geomjson = input.GeometryJson.ToString(Formatting.None);
|
||
|
|
cropfieldItem = await _generalService.CreateCropfieldItemAsync(myDriveRoot.Code,
|
||
|
|
$"DataCropfield {input.cropName}", input.cropYear, input.GeometryJson.ToString(Formatting.None));
|
||
|
|
_settings.CropfieldItemCode = cropfieldItem.Code;
|
||
|
|
SaveSettings(settingsfile);
|
||
|
|
}
|
||
|
|
else
|
||
|
|
{
|
||
|
|
_logger.LogInformation("Cropfield already exists, trying to get it");
|
||
|
|
cropfieldItem = await _farmmapsApiService.GetItemAsync(_settings.CropfieldItemCode);
|
||
|
|
}
|
||
|
|
|
||
|
|
// check if satellite task not yet done, do here and save taskcode
|
||
|
|
if (useCreatedCropfield == false || string.IsNullOrEmpty(_settings.SatelliteTaskCode))
|
||
|
|
{
|
||
|
|
var satelliteTaskCode = await _generalService.RunSatelliteTask(cropfieldItem);
|
||
|
|
_settings.SatelliteTaskCode = satelliteTaskCode;
|
||
|
|
SaveSettings(settingsfile);
|
||
|
|
}
|
||
|
|
|
||
|
|
// Now get the tiffs
|
||
|
|
List<Item> satelliteTiffs = await _generalService.FindSatelliteItemsAll(cropfieldItem, _settings.SatelliteTaskCode);
|
||
|
|
_logger.LogInformation(string.Format($"Downloading {satelliteTiffs.Count} geotiffs for a field in year {input.cropYear} ..."));
|
||
|
|
foreach (Item satalliteItem in satelliteTiffs) {
|
||
|
|
// download the geotiff. Returns a zip file with always these three files:
|
||
|
|
// data.dat.aux.xml
|
||
|
|
// thumbnail.jpg
|
||
|
|
// wenr.tif. Contains 5 layers: (1) ndvi, (2) wdvi, (3) Red, (4) Green and (5) Blue
|
||
|
|
DateTime SatelliteImageDate = (DateTime)satalliteItem.DataDate;
|
||
|
|
string SatelliteDate = SatelliteImageDate.ToString("yyyyMMdd");
|
||
|
|
string fileName = string.Format($"{input.cropName}_{input.fieldName}_{SatelliteDate}"); // assuming the fieldName is good enough for an ID. One might add here _{input.fieldID} or _{input.cropName}
|
||
|
|
string fileNameZip = string.Format($"{fileName}.zip");
|
||
|
|
string fileNameGeotiff = string.Format($"{fileName}.tif");
|
||
|
|
await _farmmapsApiService.DownloadItemAsync(satalliteItem.Code, Path.Combine(DownloadFolder, fileNameZip));
|
||
|
|
|
||
|
|
// Extract the file "wenr.tif" from zip, rename it, delete "wenr.tif". overwriteFiles = true
|
||
|
|
ZipFile.ExtractToDirectory(Path.Combine(DownloadFolder, fileNameZip), DownloadFolder, true);
|
||
|
|
File.Delete(Path.Combine(DownloadFolder, fileNameGeotiff)); // Delete the fileNameGeotiff file if exists
|
||
|
|
File.Move(Path.Combine(DownloadFolder, "wenr.tif"), Path.Combine(DownloadFolder, fileNameGeotiff)); // Rename the oldFileName into newFileName
|
||
|
|
|
||
|
|
// Cleanup
|
||
|
|
string[] filesToDelete = new string[] { fileNameZip, "wenr.tif", "thumbnail.jpg", "data.dat.aux.xml" };
|
||
|
|
foreach (string f in filesToDelete)
|
||
|
|
{
|
||
|
|
File.Delete(Path.Combine(DownloadFolder, f));
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
// Functions to save previously created cropfields
|
||
|
|
private void LoadSettings(string file)
|
||
|
|
{
|
||
|
|
if (File.Exists(file))
|
||
|
|
{
|
||
|
|
var jsonText = File.ReadAllText(file);
|
||
|
|
_settings = JsonConvert.DeserializeObject<Settings>(jsonText);
|
||
|
|
}
|
||
|
|
else
|
||
|
|
{
|
||
|
|
_settings = new Settings();
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
private void SaveSettings(string file)
|
||
|
|
{
|
||
|
|
if (_settings == null)
|
||
|
|
return;
|
||
|
|
|
||
|
|
var json = JsonConvert.SerializeObject(_settings);
|
||
|
|
File.WriteAllText(file, json);
|
||
|
|
}
|
||
|
|
private void SaveInfo(string file)
|
||
|
|
{
|
||
|
|
if (_settings == null)
|
||
|
|
return;
|
||
|
|
|
||
|
|
var json = JsonConvert.SerializeObject(_settings);
|
||
|
|
File.WriteAllText(file, json);
|
||
|
|
|
||
|
|
}
|
||
|
|
|
||
|
|
}
|
||
|
|
}
|