using System; using System.Collections.Generic; using System.IO; using System.IO.Compression; using System.Linq; using System.Threading.Tasks; using FarmmapsApi; using FarmmapsApi.Models; using FarmmapsApi.Services; using FarmmapsBulkSatDownload.Models; using Microsoft.Extensions.Logging; using Newtonsoft.Json; using Npgsql; using Newtonsoft.Json.Linq; using static FarmmapsApiSamples.Constants; namespace FarmmapsBulkSatDownload { public class BulkSatDownloadApplication : IApplication { private const string SettingsFile = "settings.json"; private readonly ILogger _logger; private readonly FarmmapsApiService _farmmapsApiService; private readonly BulkSatDownloadService _dataDownloadService; private readonly GeneralService _generalService; private Settings _settings; static DB dbparcels; public BulkSatDownloadApplication(ILogger logger, FarmmapsApiService farmmapsApiService, GeneralService generalService, BulkSatDownloadService dataDownloadService) { _logger = logger; _farmmapsApiService = farmmapsApiService; _generalService = generalService; _dataDownloadService = dataDownloadService; } public async Task RunAsync() { // Check if we have permission // !! this call is needed the first time an api is called with a fresh clientid and secret !! await _farmmapsApiService.GetCurrentUserCodeAsync(); var roots = await _farmmapsApiService.GetCurrentUserRootsAsync(); // Initialize databases. Username, password etc stored in file "DBsettings.json" dbparcels = JsonConvert.DeserializeObject(File.ReadAllText("DBsettings.json")); //string date; //string asterices = "*****************************************************************************"; //string bboxtiffWithTempPath; //string parcelshp = "parcel.shp"; //string parcelshpWithTempPath; //string gwoptions; //= String.Format("-overwrite -s_srs EPSG:28992 -t_srs EPSG:28992 -dstnodata 0 -cutline {0} -crop_to_cutline", parcelshp); //string parceltiff = "parcel.tiff"; //string parceltiffWithTempPath; //string[] vegetationindices = { "WDVI", "NDVI" }; //string[] sources = { "groenmonitor" }; // { "groenmonitor", "akkerwebwenr", "akkerwebneo" }; { "groenmonitor" }; string schemaname = "bigdata"; string parceltablename = "parcel_flowerbulbs";//"parcelsijbrandij" "parcel"; "parcel_flowerbulbs" //string groenmonitortablename = "groenmonitor_flowerbulbs";//"groenmonitorsijbrandij" "groenmonitor" "groenmonitor_flowerbulbs" //PO20190605: "groenmonitormpt" contains groenmonitor data from multiple parcel tables (mpt) // The view 'groenmonitorlatestviewname' contains per parcelid (arbid) the year in which it "exists" and the date of the latest image downloaded. It is used to prevent downloading images already downloaded string groenmonitorlatestviewname = "groenmonitorlatest_flowerbulbs"; //"groenmonitorsijbrandijlatest" "groenmonitorlatest" "groenmonitorlatest_flowerbulbs" //PO20190605: "v_groenmonitorlatest" contains latest available dates from groenmonitortablename = "groenmonitormpt" //GroenmonitorTable gmtb; //DateTime dt; //DateTime dtfirst; //DateTime dtlast; //double scalingfactor; BulkSatDownloadInput dataDownloadInput; List fieldsInputs = new List(); // Database query and connection. Geometry must be in WGS84 coordinate system, EPSG 4326 // Apparently the FarmmapsApi cannot handle MultiPolygon, so we need to convert to single Polygon // In case database returns a MultiPolygon use ST_NumGeometries(pt.geom) to count the number of polygons // If necessary use WHERE T_NumGeometries(pt.geom) = 1 to use only single polygons string connectionString = dbparcels.GetConnectionString(); string readSql = string.Format( @" SELECT pt.arbid, pt.crop, pt.year, gml.lastwenrdate, ST_AsGeoJSON(ST_Transform((ST_DUMP(pt.geom)).geom::geometry(Polygon),4326)) AS geojson_polygon_wgs84 FROM {0}.{1} pt, {0}.{2} gml WHERE pt.arbid = gml.arbid AND pt.crop NOT IN('Tulp', 'Lelie') AND pt.year = 2018 LIMIT 10;", schemaname, parceltablename, groenmonitorlatestviewname); //LIMIT 10 for testing using (NpgsqlConnection connection = new NpgsqlConnection(connectionString)) { connection.Open(); // Read data (run query) = build a list of fields for which to download images NpgsqlCommand command = connection.CreateCommand(); command.CommandText = readSql; NpgsqlDataReader dr = command.ExecuteReader(); while (dr.Read()) { dataDownloadInput = new BulkSatDownloadInput(); dataDownloadInput.UseCreatedCropfield = false; dataDownloadInput.DownloadFolder = "C:\\workdir\\groenmonitor\\"; dataDownloadInput.fieldID = dr.GetInt16(0); dataDownloadInput.fieldName = string.Format($"{parceltablename}_fld{dataDownloadInput.fieldID}"); dataDownloadInput.cropName = dr.GetString(1); dataDownloadInput.cropYear = dr.GetInt16(2); dataDownloadInput.lastdownloadedimagedate = dr.GetDateTime(3); dataDownloadInput.GeometryJson = JObject.Parse(dr.GetString(4)); fieldsInputs.Add(dataDownloadInput); } connection.Close(); } // For testing using json input instead of database input var fieldsInputJson = File.ReadAllText("BulkSatDownloadInput.json"); List fieldsInputs2 = JsonConvert.DeserializeObject>(fieldsInputJson); // Now for each input download all images. Use fieldsInputs or fieldsInputs2 for testing foreach (var input in fieldsInputs2) { try { await Process(roots, input); } catch (Exception ex) { _logger.LogError(ex.Message); } } } private async Task Process(List roots, BulkSatDownloadInput input) { string DownloadFolder = input.DownloadFolder; if (!Directory.Exists(DownloadFolder)) Directory.CreateDirectory(DownloadFolder); // !!specify if you are using an already created cropfield: bool useCreatedCropfield = input.UseCreatedCropfield; string settingsfile = $"Settings_BulkSatDownloadApplication.json"; LoadSettings(settingsfile); var uploadedRoot = roots.SingleOrDefault(r => r.Name == "Uploaded"); if (uploadedRoot == null) { _logger.LogError("Could not find a needed root item"); return; } var myDriveRoot = roots.SingleOrDefault(r => r.Name == "My drive"); if (myDriveRoot == null) { _logger.LogError("Could not find a needed root item"); return; } // Use already created cropfield or create new one Item cropfieldItem; if (useCreatedCropfield == false || string.IsNullOrEmpty(_settings.CropfieldItemCode)) { _logger.LogInformation(string.Format($"Creating cropfield for a field in the year {input.cropYear}")); //string geomjson = input.GeometryJson.ToString(Formatting.None); //"{\"type\":\"Polygon\",\"coordinates\":[[[4.960707146896585,52.80058366970849],[4.960645975538824,52.80047021761092],[4.962140695752897,52.7991771471948],[4.967523821195745,52.80150240004121],[4.966336768950911,52.80254373587981],[4.96171188076433,52.80100999685643],[4.960707146896585,52.80058366970849]]]}" string geomjson = input.GeometryJson.ToString(Formatting.None); cropfieldItem = await _generalService.CreateCropfieldItemAsync(myDriveRoot.Code, $"DataCropfield {input.cropName}", input.cropYear, input.GeometryJson.ToString(Formatting.None)); _settings.CropfieldItemCode = cropfieldItem.Code; SaveSettings(settingsfile); } else { _logger.LogInformation("Cropfield already exists, trying to get it"); cropfieldItem = await _farmmapsApiService.GetItemAsync(_settings.CropfieldItemCode); } // check if satellite task not yet done, do here and save taskcode if (useCreatedCropfield == false || string.IsNullOrEmpty(_settings.SatelliteTaskCode)) { var satelliteTaskCode = await _generalService.RunSatelliteTask(cropfieldItem); _settings.SatelliteTaskCode = satelliteTaskCode; SaveSettings(settingsfile); } // Now get the tiffs List satelliteTiffs = await _generalService.FindSatelliteItemsAll(cropfieldItem, _settings.SatelliteTaskCode); _logger.LogInformation(string.Format($"Downloading {satelliteTiffs.Count} geotiffs for a field in year {input.cropYear} ...")); foreach (Item satalliteItem in satelliteTiffs) { // download the geotiff. Returns a zip file with always these three files: // data.dat.aux.xml // thumbnail.jpg // wenr.tif. Contains 5 layers: (1) ndvi, (2) wdvi, (3) Red, (4) Green and (5) Blue DateTime SatelliteImageDate = (DateTime)satalliteItem.DataDate; string SatelliteDate = SatelliteImageDate.ToString("yyyyMMdd"); string fileName = string.Format($"{input.cropName}_{input.fieldName}_{SatelliteDate}"); // assuming the fieldName is good enough for an ID. One might add here _{input.fieldID} or _{input.cropName} string fileNameZip = string.Format($"{fileName}.zip"); string fileNameGeotiff = string.Format($"{fileName}.tif"); await _farmmapsApiService.DownloadItemAsync(satalliteItem.Code, Path.Combine(DownloadFolder, fileNameZip)); // Extract the file "wenr.tif" from zip, rename it, delete "wenr.tif". overwriteFiles = true ZipFile.ExtractToDirectory(Path.Combine(DownloadFolder, fileNameZip), DownloadFolder, true); File.Delete(Path.Combine(DownloadFolder, fileNameGeotiff)); // Delete the fileNameGeotiff file if exists File.Move(Path.Combine(DownloadFolder, "wenr.tif"), Path.Combine(DownloadFolder, fileNameGeotiff)); // Rename the oldFileName into newFileName // Cleanup string[] filesToDelete = new string[] { fileNameZip, "wenr.tif", "thumbnail.jpg", "data.dat.aux.xml" }; foreach (string f in filesToDelete) { File.Delete(Path.Combine(DownloadFolder, f)); } } } // Functions to save previously created cropfields private void LoadSettings(string file) { if (File.Exists(file)) { var jsonText = File.ReadAllText(file); _settings = JsonConvert.DeserializeObject(jsonText); } else { _settings = new Settings(); } } private void SaveSettings(string file) { if (_settings == null) return; var json = JsonConvert.SerializeObject(_settings); File.WriteAllText(file, json); } private void SaveInfo(string file) { if (_settings == null) return; var json = JsonConvert.SerializeObject(_settings); File.WriteAllText(file, json); } } }