FarmMapsApiClient/FarmmapsBulkSatDownload/BulkSatDownloadApplication.cs

243 lines
12 KiB
C#
Raw Normal View History

2021-05-18 16:19:28 +00:00
using System;
using System.Collections.Generic;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Threading.Tasks;
using FarmmapsApi;
using FarmmapsApi.Models;
using FarmmapsApi.Services;
using FarmmapsBulkSatDownload.Models;
using Microsoft.Extensions.Logging;
using Newtonsoft.Json;
using Npgsql;
using Newtonsoft.Json.Linq;
using static FarmmapsApiSamples.Constants;
namespace FarmmapsBulkSatDownload
{
public class BulkSatDownloadApplication : IApplication
{
private const string SettingsFile = "settings.json";
private readonly ILogger<BulkSatDownloadApplication> _logger;
private readonly FarmmapsApiService _farmmapsApiService;
private readonly BulkSatDownloadService _dataDownloadService;
private readonly GeneralService _generalService;
private Settings _settings;
static DB dbparcels;
public BulkSatDownloadApplication(ILogger<BulkSatDownloadApplication> logger, FarmmapsApiService farmmapsApiService,
GeneralService generalService, BulkSatDownloadService dataDownloadService)
{
_logger = logger;
_farmmapsApiService = farmmapsApiService;
_generalService = generalService;
_dataDownloadService = dataDownloadService;
}
public async Task RunAsync()
{
// Check if we have permission
// !! this call is needed the first time an api is called with a fresh clientid and secret !!
await _farmmapsApiService.GetCurrentUserCodeAsync();
var roots = await _farmmapsApiService.GetCurrentUserRootsAsync();
// Initialize databases. Username, password etc stored in file "DBsettings.json"
dbparcels = JsonConvert.DeserializeObject<DB>(File.ReadAllText("DBsettings.json"));
//string date;
//string asterices = "*****************************************************************************";
//string bboxtiffWithTempPath;
//string parcelshp = "parcel.shp";
//string parcelshpWithTempPath;
//string gwoptions; //= String.Format("-overwrite -s_srs EPSG:28992 -t_srs EPSG:28992 -dstnodata 0 -cutline {0} -crop_to_cutline", parcelshp);
//string parceltiff = "parcel.tiff";
//string parceltiffWithTempPath;
//string[] vegetationindices = { "WDVI", "NDVI" };
//string[] sources = { "groenmonitor" }; // { "groenmonitor", "akkerwebwenr", "akkerwebneo" }; { "groenmonitor" };
string schemaname = "bigdata";
string parceltablename = "parcel_flowerbulbs";//"parcelsijbrandij" "parcel"; "parcel_flowerbulbs"
//string groenmonitortablename = "groenmonitor_flowerbulbs";//"groenmonitorsijbrandij" "groenmonitor" "groenmonitor_flowerbulbs" //PO20190605: "groenmonitormpt" contains groenmonitor data from multiple parcel tables (mpt)
// The view 'groenmonitorlatestviewname' contains per parcelid (arbid) the year in which it "exists" and the date of the latest image downloaded. It is used to prevent downloading images already downloaded
string groenmonitorlatestviewname = "groenmonitorlatest_flowerbulbs"; //"groenmonitorsijbrandijlatest" "groenmonitorlatest" "groenmonitorlatest_flowerbulbs" //PO20190605: "v_groenmonitorlatest" contains latest available dates from groenmonitortablename = "groenmonitormpt"
//GroenmonitorTable gmtb;
//DateTime dt;
//DateTime dtfirst;
//DateTime dtlast;
//double scalingfactor;
BulkSatDownloadInput dataDownloadInput;
List<BulkSatDownloadInput> fieldsInputs = new List<BulkSatDownloadInput>();
// Database query and connection. Geometry must be in WGS84 coordinate system, EPSG 4326
// Apparently the FarmmapsApi cannot handle MultiPolygon, so we need to convert to single Polygon
// In case database returns a MultiPolygon use ST_NumGeometries(pt.geom) to count the number of polygons
// If necessary use WHERE T_NumGeometries(pt.geom) = 1 to use only single polygons
string connectionString = dbparcels.GetConnectionString();
string readSql = string.Format(
@"
SELECT pt.arbid, pt.crop, pt.year, gml.lastwenrdate, ST_AsGeoJSON(ST_Transform((ST_DUMP(pt.geom)).geom::geometry(Polygon),4326)) AS geojson_polygon_wgs84
FROM {0}.{1} pt, {0}.{2} gml
WHERE
pt.arbid = gml.arbid AND
pt.crop NOT IN('Tulp', 'Lelie') AND
pt.year = 2018
LIMIT 10;", schemaname, parceltablename, groenmonitorlatestviewname); //LIMIT 10 for testing
using (NpgsqlConnection connection = new NpgsqlConnection(connectionString))
{
connection.Open();
// Read data (run query) = build a list of fields for which to download images
NpgsqlCommand command = connection.CreateCommand();
command.CommandText = readSql;
NpgsqlDataReader dr = command.ExecuteReader();
while (dr.Read())
{
dataDownloadInput = new BulkSatDownloadInput();
dataDownloadInput.UseCreatedCropfield = false;
dataDownloadInput.DownloadFolder = "C:\\workdir\\groenmonitor\\";
dataDownloadInput.fieldID = dr.GetInt16(0);
dataDownloadInput.fieldName = string.Format($"{parceltablename}_fld{dataDownloadInput.fieldID}");
dataDownloadInput.cropName = dr.GetString(1);
dataDownloadInput.cropYear = dr.GetInt16(2);
dataDownloadInput.lastdownloadedimagedate = dr.GetDateTime(3);
dataDownloadInput.GeometryJson = JObject.Parse(dr.GetString(4));
fieldsInputs.Add(dataDownloadInput);
}
connection.Close();
}
// For testing using json input instead of database input
var fieldsInputJson = File.ReadAllText("BulkSatDownloadInput.json");
List<BulkSatDownloadInput> fieldsInputs2 = JsonConvert.DeserializeObject<List<BulkSatDownloadInput>>(fieldsInputJson);
// Now for each input download all images. Use fieldsInputs or fieldsInputs2 for testing
foreach (var input in fieldsInputs2)
{
try
{
await Process(roots, input);
}
catch (Exception ex)
{
_logger.LogError(ex.Message);
}
}
}
private async Task Process(List<UserRoot> roots, BulkSatDownloadInput input)
{
string DownloadFolder = input.DownloadFolder;
if (!Directory.Exists(DownloadFolder))
Directory.CreateDirectory(DownloadFolder);
// !!specify if you are using an already created cropfield:
bool useCreatedCropfield = input.UseCreatedCropfield;
string settingsfile = $"Settings_BulkSatDownloadApplication.json";
LoadSettings(settingsfile);
var uploadedRoot = roots.SingleOrDefault(r => r.Name == "Uploaded");
if (uploadedRoot == null)
{
_logger.LogError("Could not find a needed root item");
return;
}
var myDriveRoot = roots.SingleOrDefault(r => r.Name == "My drive");
if (myDriveRoot == null)
{
_logger.LogError("Could not find a needed root item");
return;
}
// Use already created cropfield or create new one
Item cropfieldItem;
if (useCreatedCropfield == false || string.IsNullOrEmpty(_settings.CropfieldItemCode))
{
_logger.LogInformation(string.Format($"Creating cropfield for a field in the year {input.cropYear}"));
//string geomjson = input.GeometryJson.ToString(Formatting.None); //"{\"type\":\"Polygon\",\"coordinates\":[[[4.960707146896585,52.80058366970849],[4.960645975538824,52.80047021761092],[4.962140695752897,52.7991771471948],[4.967523821195745,52.80150240004121],[4.966336768950911,52.80254373587981],[4.96171188076433,52.80100999685643],[4.960707146896585,52.80058366970849]]]}"
string geomjson = input.GeometryJson.ToString(Formatting.None);
cropfieldItem = await _generalService.CreateCropfieldItemAsync(myDriveRoot.Code,
$"DataCropfield {input.cropName}", input.cropYear, input.GeometryJson.ToString(Formatting.None));
_settings.CropfieldItemCode = cropfieldItem.Code;
SaveSettings(settingsfile);
}
else
{
_logger.LogInformation("Cropfield already exists, trying to get it");
cropfieldItem = await _farmmapsApiService.GetItemAsync(_settings.CropfieldItemCode);
}
// check if satellite task not yet done, do here and save taskcode
if (useCreatedCropfield == false || string.IsNullOrEmpty(_settings.SatelliteTaskCode))
{
var satelliteTaskCode = await _generalService.RunSatelliteTask(cropfieldItem);
_settings.SatelliteTaskCode = satelliteTaskCode;
SaveSettings(settingsfile);
}
// Now get the tiffs
List<Item> satelliteTiffs = await _generalService.FindSatelliteItemsAll(cropfieldItem, _settings.SatelliteTaskCode);
_logger.LogInformation(string.Format($"Downloading {satelliteTiffs.Count} geotiffs for a field in year {input.cropYear} ..."));
foreach (Item satalliteItem in satelliteTiffs) {
// download the geotiff. Returns a zip file with always these three files:
// data.dat.aux.xml
// thumbnail.jpg
// wenr.tif. Contains 5 layers: (1) ndvi, (2) wdvi, (3) Red, (4) Green and (5) Blue
DateTime SatelliteImageDate = (DateTime)satalliteItem.DataDate;
string SatelliteDate = SatelliteImageDate.ToString("yyyyMMdd");
string fileName = string.Format($"{input.cropName}_{input.fieldName}_{SatelliteDate}"); // assuming the fieldName is good enough for an ID. One might add here _{input.fieldID} or _{input.cropName}
string fileNameZip = string.Format($"{fileName}.zip");
string fileNameGeotiff = string.Format($"{fileName}.tif");
await _farmmapsApiService.DownloadItemAsync(satalliteItem.Code, Path.Combine(DownloadFolder, fileNameZip));
// Extract the file "wenr.tif" from zip, rename it, delete "wenr.tif". overwriteFiles = true
ZipFile.ExtractToDirectory(Path.Combine(DownloadFolder, fileNameZip), DownloadFolder, true);
File.Delete(Path.Combine(DownloadFolder, fileNameGeotiff)); // Delete the fileNameGeotiff file if exists
File.Move(Path.Combine(DownloadFolder, "wenr.tif"), Path.Combine(DownloadFolder, fileNameGeotiff)); // Rename the oldFileName into newFileName
// Cleanup
string[] filesToDelete = new string[] { fileNameZip, "wenr.tif", "thumbnail.jpg", "data.dat.aux.xml" };
foreach (string f in filesToDelete)
{
File.Delete(Path.Combine(DownloadFolder, f));
}
}
}
// Functions to save previously created cropfields
private void LoadSettings(string file)
{
if (File.Exists(file))
{
var jsonText = File.ReadAllText(file);
_settings = JsonConvert.DeserializeObject<Settings>(jsonText);
}
else
{
_settings = new Settings();
}
}
private void SaveSettings(string file)
{
if (_settings == null)
return;
var json = JsonConvert.SerializeObject(_settings);
File.WriteAllText(file, json);
}
private void SaveInfo(string file)
{
if (_settings == null)
return;
var json = JsonConvert.SerializeObject(_settings);
File.WriteAllText(file, json);
}
}
}