forked from FarmMaps/FarmMapsApiClient
Added project FarmmapsBulkSatDownload
This commit is contained in:
242
FarmmapsBulkSatDownload/BulkSatDownloadApplication.cs
Normal file
242
FarmmapsBulkSatDownload/BulkSatDownloadApplication.cs
Normal file
@@ -0,0 +1,242 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.IO.Compression;
|
||||
using System.Linq;
|
||||
using System.Threading.Tasks;
|
||||
using FarmmapsApi;
|
||||
using FarmmapsApi.Models;
|
||||
using FarmmapsApi.Services;
|
||||
using FarmmapsBulkSatDownload.Models;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Newtonsoft.Json;
|
||||
using Npgsql;
|
||||
using Newtonsoft.Json.Linq;
|
||||
using static FarmmapsApiSamples.Constants;
|
||||
|
||||
namespace FarmmapsBulkSatDownload
|
||||
{
|
||||
public class BulkSatDownloadApplication : IApplication
|
||||
{
|
||||
private const string SettingsFile = "settings.json";
|
||||
|
||||
private readonly ILogger<BulkSatDownloadApplication> _logger;
|
||||
private readonly FarmmapsApiService _farmmapsApiService;
|
||||
private readonly BulkSatDownloadService _dataDownloadService;
|
||||
private readonly GeneralService _generalService;
|
||||
|
||||
private Settings _settings;
|
||||
static DB dbparcels;
|
||||
|
||||
public BulkSatDownloadApplication(ILogger<BulkSatDownloadApplication> logger, FarmmapsApiService farmmapsApiService,
|
||||
GeneralService generalService, BulkSatDownloadService dataDownloadService)
|
||||
{
|
||||
_logger = logger;
|
||||
_farmmapsApiService = farmmapsApiService;
|
||||
_generalService = generalService;
|
||||
_dataDownloadService = dataDownloadService;
|
||||
}
|
||||
public async Task RunAsync()
|
||||
{
|
||||
// Check if we have permission
|
||||
// !! this call is needed the first time an api is called with a fresh clientid and secret !!
|
||||
await _farmmapsApiService.GetCurrentUserCodeAsync();
|
||||
var roots = await _farmmapsApiService.GetCurrentUserRootsAsync();
|
||||
// Initialize databases. Username, password etc stored in file "DBsettings.json"
|
||||
dbparcels = JsonConvert.DeserializeObject<DB>(File.ReadAllText("DBsettings.json"));
|
||||
|
||||
//string date;
|
||||
//string asterices = "*****************************************************************************";
|
||||
//string bboxtiffWithTempPath;
|
||||
//string parcelshp = "parcel.shp";
|
||||
//string parcelshpWithTempPath;
|
||||
//string gwoptions; //= String.Format("-overwrite -s_srs EPSG:28992 -t_srs EPSG:28992 -dstnodata 0 -cutline {0} -crop_to_cutline", parcelshp);
|
||||
//string parceltiff = "parcel.tiff";
|
||||
//string parceltiffWithTempPath;
|
||||
//string[] vegetationindices = { "WDVI", "NDVI" };
|
||||
//string[] sources = { "groenmonitor" }; // { "groenmonitor", "akkerwebwenr", "akkerwebneo" }; { "groenmonitor" };
|
||||
string schemaname = "bigdata";
|
||||
string parceltablename = "parcel_flowerbulbs";//"parcelsijbrandij" "parcel"; "parcel_flowerbulbs"
|
||||
//string groenmonitortablename = "groenmonitor_flowerbulbs";//"groenmonitorsijbrandij" "groenmonitor" "groenmonitor_flowerbulbs" //PO20190605: "groenmonitormpt" contains groenmonitor data from multiple parcel tables (mpt)
|
||||
// The view 'groenmonitorlatestviewname' contains per parcelid (arbid) the year in which it "exists" and the date of the latest image downloaded. It is used to prevent downloading images already downloaded
|
||||
string groenmonitorlatestviewname = "groenmonitorlatest_flowerbulbs"; //"groenmonitorsijbrandijlatest" "groenmonitorlatest" "groenmonitorlatest_flowerbulbs" //PO20190605: "v_groenmonitorlatest" contains latest available dates from groenmonitortablename = "groenmonitormpt"
|
||||
//GroenmonitorTable gmtb;
|
||||
//DateTime dt;
|
||||
//DateTime dtfirst;
|
||||
//DateTime dtlast;
|
||||
//double scalingfactor;
|
||||
BulkSatDownloadInput dataDownloadInput;
|
||||
List<BulkSatDownloadInput> fieldsInputs = new List<BulkSatDownloadInput>();
|
||||
|
||||
// Database query and connection. Geometry must be in WGS84 coordinate system, EPSG 4326
|
||||
// Apparently the FarmmapsApi cannot handle MultiPolygon, so we need to convert to single Polygon
|
||||
// In case database returns a MultiPolygon use ST_NumGeometries(pt.geom) to count the number of polygons
|
||||
// If necessary use WHERE T_NumGeometries(pt.geom) = 1 to use only single polygons
|
||||
string connectionString = dbparcels.GetConnectionString();
|
||||
string readSql = string.Format(
|
||||
@"
|
||||
SELECT pt.arbid, pt.crop, pt.year, gml.lastwenrdate, ST_AsGeoJSON(ST_Transform((ST_DUMP(pt.geom)).geom::geometry(Polygon),4326)) AS geojson_polygon_wgs84
|
||||
FROM {0}.{1} pt, {0}.{2} gml
|
||||
WHERE
|
||||
pt.arbid = gml.arbid AND
|
||||
pt.crop NOT IN('Tulp', 'Lelie') AND
|
||||
pt.year = 2018
|
||||
LIMIT 10;", schemaname, parceltablename, groenmonitorlatestviewname); //LIMIT 10 for testing
|
||||
|
||||
using (NpgsqlConnection connection = new NpgsqlConnection(connectionString))
|
||||
{
|
||||
connection.Open();
|
||||
|
||||
// Read data (run query) = build a list of fields for which to download images
|
||||
NpgsqlCommand command = connection.CreateCommand();
|
||||
command.CommandText = readSql;
|
||||
NpgsqlDataReader dr = command.ExecuteReader();
|
||||
while (dr.Read())
|
||||
{
|
||||
dataDownloadInput = new BulkSatDownloadInput();
|
||||
dataDownloadInput.UseCreatedCropfield = false;
|
||||
dataDownloadInput.DownloadFolder = "C:\\workdir\\groenmonitor\\";
|
||||
dataDownloadInput.fieldID = dr.GetInt16(0);
|
||||
dataDownloadInput.fieldName = string.Format($"{parceltablename}_fld{dataDownloadInput.fieldID}");
|
||||
dataDownloadInput.cropName = dr.GetString(1);
|
||||
dataDownloadInput.cropYear = dr.GetInt16(2);
|
||||
dataDownloadInput.lastdownloadedimagedate = dr.GetDateTime(3);
|
||||
dataDownloadInput.GeometryJson = JObject.Parse(dr.GetString(4));
|
||||
fieldsInputs.Add(dataDownloadInput);
|
||||
}
|
||||
connection.Close();
|
||||
}
|
||||
|
||||
// For testing using json input instead of database input
|
||||
var fieldsInputJson = File.ReadAllText("BulkSatDownloadInput.json");
|
||||
List<BulkSatDownloadInput> fieldsInputs2 = JsonConvert.DeserializeObject<List<BulkSatDownloadInput>>(fieldsInputJson);
|
||||
|
||||
// Now for each input download all images. Use fieldsInputs or fieldsInputs2 for testing
|
||||
foreach (var input in fieldsInputs2)
|
||||
{
|
||||
try
|
||||
{
|
||||
await Process(roots, input);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex.Message);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async Task Process(List<UserRoot> roots, BulkSatDownloadInput input)
|
||||
{
|
||||
string DownloadFolder = input.DownloadFolder;
|
||||
if (!Directory.Exists(DownloadFolder))
|
||||
Directory.CreateDirectory(DownloadFolder);
|
||||
|
||||
// !!specify if you are using an already created cropfield:
|
||||
bool useCreatedCropfield = input.UseCreatedCropfield;
|
||||
string settingsfile = $"Settings_BulkSatDownloadApplication.json";
|
||||
|
||||
LoadSettings(settingsfile);
|
||||
|
||||
var uploadedRoot = roots.SingleOrDefault(r => r.Name == "Uploaded");
|
||||
if (uploadedRoot == null)
|
||||
{
|
||||
_logger.LogError("Could not find a needed root item");
|
||||
return;
|
||||
}
|
||||
|
||||
var myDriveRoot = roots.SingleOrDefault(r => r.Name == "My drive");
|
||||
if (myDriveRoot == null)
|
||||
{
|
||||
_logger.LogError("Could not find a needed root item");
|
||||
return;
|
||||
}
|
||||
|
||||
// Use already created cropfield or create new one
|
||||
Item cropfieldItem;
|
||||
if (useCreatedCropfield == false || string.IsNullOrEmpty(_settings.CropfieldItemCode))
|
||||
{
|
||||
_logger.LogInformation(string.Format($"Creating cropfield for a field in the year {input.cropYear}"));
|
||||
//string geomjson = input.GeometryJson.ToString(Formatting.None); //"{\"type\":\"Polygon\",\"coordinates\":[[[4.960707146896585,52.80058366970849],[4.960645975538824,52.80047021761092],[4.962140695752897,52.7991771471948],[4.967523821195745,52.80150240004121],[4.966336768950911,52.80254373587981],[4.96171188076433,52.80100999685643],[4.960707146896585,52.80058366970849]]]}"
|
||||
string geomjson = input.GeometryJson.ToString(Formatting.None);
|
||||
cropfieldItem = await _generalService.CreateCropfieldItemAsync(myDriveRoot.Code,
|
||||
$"DataCropfield {input.cropName}", input.cropYear, input.GeometryJson.ToString(Formatting.None));
|
||||
_settings.CropfieldItemCode = cropfieldItem.Code;
|
||||
SaveSettings(settingsfile);
|
||||
}
|
||||
else
|
||||
{
|
||||
_logger.LogInformation("Cropfield already exists, trying to get it");
|
||||
cropfieldItem = await _farmmapsApiService.GetItemAsync(_settings.CropfieldItemCode);
|
||||
}
|
||||
|
||||
// check if satellite task not yet done, do here and save taskcode
|
||||
if (useCreatedCropfield == false || string.IsNullOrEmpty(_settings.SatelliteTaskCode))
|
||||
{
|
||||
var satelliteTaskCode = await _generalService.RunSatelliteTask(cropfieldItem);
|
||||
_settings.SatelliteTaskCode = satelliteTaskCode;
|
||||
SaveSettings(settingsfile);
|
||||
}
|
||||
|
||||
// Now get the tiffs
|
||||
List<Item> satelliteTiffs = await _generalService.FindSatelliteItemsAll(cropfieldItem, _settings.SatelliteTaskCode);
|
||||
_logger.LogInformation(string.Format($"Downloading {satelliteTiffs.Count} geotiffs for a field in year {input.cropYear} ..."));
|
||||
foreach (Item satalliteItem in satelliteTiffs) {
|
||||
// download the geotiff. Returns a zip file with always these three files:
|
||||
// data.dat.aux.xml
|
||||
// thumbnail.jpg
|
||||
// wenr.tif. Contains 5 layers: (1) ndvi, (2) wdvi, (3) Red, (4) Green and (5) Blue
|
||||
DateTime SatelliteImageDate = (DateTime)satalliteItem.DataDate;
|
||||
string SatelliteDate = SatelliteImageDate.ToString("yyyyMMdd");
|
||||
string fileName = string.Format($"{input.cropName}_{input.fieldName}_{SatelliteDate}"); // assuming the fieldName is good enough for an ID. One might add here _{input.fieldID} or _{input.cropName}
|
||||
string fileNameZip = string.Format($"{fileName}.zip");
|
||||
string fileNameGeotiff = string.Format($"{fileName}.tif");
|
||||
await _farmmapsApiService.DownloadItemAsync(satalliteItem.Code, Path.Combine(DownloadFolder, fileNameZip));
|
||||
|
||||
// Extract the file "wenr.tif" from zip, rename it, delete "wenr.tif". overwriteFiles = true
|
||||
ZipFile.ExtractToDirectory(Path.Combine(DownloadFolder, fileNameZip), DownloadFolder, true);
|
||||
File.Delete(Path.Combine(DownloadFolder, fileNameGeotiff)); // Delete the fileNameGeotiff file if exists
|
||||
File.Move(Path.Combine(DownloadFolder, "wenr.tif"), Path.Combine(DownloadFolder, fileNameGeotiff)); // Rename the oldFileName into newFileName
|
||||
|
||||
// Cleanup
|
||||
string[] filesToDelete = new string[] { fileNameZip, "wenr.tif", "thumbnail.jpg", "data.dat.aux.xml" };
|
||||
foreach (string f in filesToDelete)
|
||||
{
|
||||
File.Delete(Path.Combine(DownloadFolder, f));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Functions to save previously created cropfields
|
||||
private void LoadSettings(string file)
|
||||
{
|
||||
if (File.Exists(file))
|
||||
{
|
||||
var jsonText = File.ReadAllText(file);
|
||||
_settings = JsonConvert.DeserializeObject<Settings>(jsonText);
|
||||
}
|
||||
else
|
||||
{
|
||||
_settings = new Settings();
|
||||
}
|
||||
}
|
||||
|
||||
private void SaveSettings(string file)
|
||||
{
|
||||
if (_settings == null)
|
||||
return;
|
||||
|
||||
var json = JsonConvert.SerializeObject(_settings);
|
||||
File.WriteAllText(file, json);
|
||||
}
|
||||
private void SaveInfo(string file)
|
||||
{
|
||||
if (_settings == null)
|
||||
return;
|
||||
|
||||
var json = JsonConvert.SerializeObject(_settings);
|
||||
File.WriteAllText(file, json);
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user