From bea4d207d04ddab0949fc299e7a334841869f349 Mon Sep 17 00:00:00 2001 From: Riepma Date: Tue, 11 May 2021 15:03:08 +0200 Subject: [PATCH] added project to only download some data --- .../DataDownloadApplication.cs | 235 ++++++++++++++++++ FarmmapsDataDownload/DataDownloadInput.json | 31 +++ FarmmapsDataDownload/DataDownloadService.cs | 29 +++ .../FarmmapsDataDownload.csproj | 27 ++ .../Models/DataDownloadInput.cs | 26 ++ FarmmapsDataDownload/Models/Settings.cs | 11 + FarmmapsDataDownload/Models/TargetNData.cs | 9 + FarmmapsDataDownload/Program.cs | 21 ++ FarmmapsPoten/PotenApplication.cs | 11 +- FarmmapsPoten/PotenInput.json | 6 +- 10 files changed, 401 insertions(+), 5 deletions(-) create mode 100644 FarmmapsDataDownload/DataDownloadApplication.cs create mode 100644 FarmmapsDataDownload/DataDownloadInput.json create mode 100644 FarmmapsDataDownload/DataDownloadService.cs create mode 100644 FarmmapsDataDownload/FarmmapsDataDownload.csproj create mode 100644 FarmmapsDataDownload/Models/DataDownloadInput.cs create mode 100644 FarmmapsDataDownload/Models/Settings.cs create mode 100644 FarmmapsDataDownload/Models/TargetNData.cs create mode 100644 FarmmapsDataDownload/Program.cs diff --git a/FarmmapsDataDownload/DataDownloadApplication.cs b/FarmmapsDataDownload/DataDownloadApplication.cs new file mode 100644 index 0000000..9aec1d2 --- /dev/null +++ b/FarmmapsDataDownload/DataDownloadApplication.cs @@ -0,0 +1,235 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using FarmmapsApi; +using FarmmapsApi.Models; +using FarmmapsApi.Services; +using FarmmapsDataDownload.Models; +using Microsoft.Extensions.Logging; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using static FarmmapsApiSamples.Constants; + +namespace FarmmapsDataDownload +{ + public class DataDownloadApplication : IApplication + { + private const string DownloadFolder = "Downloads"; + private const string SettingsFile = "settings.json"; + + private readonly ILogger _logger; + private readonly FarmmapsApiService _farmmapsApiService; + private readonly DataDownloadService _dataDownloadService; + private readonly GeneralService _generalService; + + private Settings _settings; + + public DataDownloadApplication(ILogger logger, FarmmapsApiService farmmapsApiService, + GeneralService generalService, DataDownloadService dataDownloadService) + { + _logger = logger; + _farmmapsApiService = farmmapsApiService; + _generalService = generalService; + _dataDownloadService = dataDownloadService; + } + + public async Task RunAsync() + { + var fieldsInputJson = File.ReadAllText("DataDownloadInput.json"); + List fieldsInputs = JsonConvert.DeserializeObject>(fieldsInputJson); + + if (!Directory.Exists(DownloadFolder)) + Directory.CreateDirectory(DownloadFolder); + + // !! this call is needed the first time an api is called with a fresh clientid and secret !! + await _farmmapsApiService.GetCurrentUserCodeAsync(); + var roots = await _farmmapsApiService.GetCurrentUserRootsAsync(); + + foreach (var input in fieldsInputs) + { + try + { + await Process(roots, input); + } + catch (Exception ex) + { + _logger.LogError(ex.Message); + } + } + } + + private async Task Process(List roots, DataDownloadInput input) + { + // !!specify if you are using an already created cropfield: + bool useCreatedCropfield = input.UseCreatedCropfield; + var cropYear = input.CropYear; + var fieldName = input.fieldName; + bool storeSatelliteStatistics = input.StoreSatelliteStatistics; + string settingsfile = $"Settings_{fieldName}.json"; + + LoadSettings(settingsfile); + + var uploadedRoot = roots.SingleOrDefault(r => r.Name == "Uploaded"); + if (uploadedRoot == null) + { + _logger.LogError("Could not find a needed root item"); + return; + } + + var myDriveRoot = roots.SingleOrDefault(r => r.Name == "My drive"); + if (myDriveRoot == null) + { + _logger.LogError("Could not find a needed root item"); + return; + } + + // Use already created cropfield or create new one + Item cropfieldItem; + if (useCreatedCropfield == false || string.IsNullOrEmpty(_settings.CropfieldItemCode)) + { + _logger.LogInformation("Creating cropfield"); + cropfieldItem = await _generalService.CreateCropfieldItemAsync(myDriveRoot.Code, + $"DataCropfield {input.OutputFileName}", cropYear.Year, input.GeometryJson.ToString(Formatting.None)); + _settings.CropfieldItemCode = cropfieldItem.Code; + SaveSettings(settingsfile); + } + else + { + _logger.LogInformation("Cropfield already exists, trying to get it"); + cropfieldItem = await _farmmapsApiService.GetItemAsync(_settings.CropfieldItemCode); + } + + + // Get shadow data + + if (input.GetShadowData) + { + _logger.LogInformation("Calculate shadow map for field"); + var shadowItem = await _generalService.RunShadowTask(cropfieldItem); + if (shadowItem == null) + { + _logger.LogError("Something went wrong while obtaining the shadow map"); + return; + } + + _logger.LogInformation("Downloading shadow map"); + await _farmmapsApiService.DownloadItemAsync(shadowItem.Code, + Path.Combine(DownloadFolder, $"{input.OutputFileName}_shadow.zip")); + } + + + + + // Get satellite data + if (input.GetSatelliteData) + { + // check if satellite task not yet done, do here and save taskcode + if (useCreatedCropfield == false || string.IsNullOrEmpty(_settings.SatelliteTaskCode)) + { + var satelliteTaskCode = await _generalService.RunSatelliteTask(cropfieldItem); + _settings.SatelliteTaskCode = satelliteTaskCode; + SaveSettings(settingsfile); + } + + // Select a particular satellite item from satelliteTask + Item satalliteItem = await _generalService.FindSatelliteItem(cropfieldItem, _settings.SatelliteTaskCode); + int selectedLayer= 2; + + if (input.SatelliteBand == "ndvi") selectedLayer = 0; + if (input.SatelliteBand == "wdvi") selectedLayer = 1; + if (input.SatelliteBand == "natural") selectedLayer = 2; + + var satelliteBand = satalliteItem.Data["layers"][selectedLayer]["name"]; + + //Store satellite data to csv + if (storeSatelliteStatistics == true && (selectedLayer == 0 || selectedLayer ==1)) + { + var satelliteStatistics = satalliteItem.Data["layers"][selectedLayer]["renderer"]["band"]["statistics"]; + Console.WriteLine($"Satellite image date: {satalliteItem.DataDate}"); + + var SatelliteStatsFile = $"{DownloadFolder}/SatelliteDataStatistics_{fieldName}_{input.SatelliteBand}_{satalliteItem.DataDate.Value:yyyy-MM-dd}.csv"; + using var w = new StreamWriter(SatelliteStatsFile); + { + foreach (var item in satelliteStatistics) + { + var line = string.Format("{0}", item); + w.WriteLine(line); + w.Flush(); + } + } + + } + + var inputType = (satalliteItem.Data["layers"] as JArray)?[selectedLayer]["name"].ToString(); + if (string.IsNullOrEmpty(inputType)) + { + _logger.LogError("Could not get the input type name from the satellite item"); + return; + } + + // download the geotiff of needed inputtype + var SatelliteImageDate = (DateTime)satalliteItem.DataDate; + var SatelliteDate = SatelliteImageDate.ToString("yyyyMMdd"); + _logger.LogInformation("Downloading geotiff file"); + await _farmmapsApiService.DownloadItemAsync(satalliteItem.Code, + Path.Combine(DownloadFolder, $"satelliteGeotiff_{input.OutputFileName}_{inputType}_{SatelliteDate}.zip")); + } + + + // Get vanDerSat data + if (input.GetVanDerSatData) + { + // check if satellite task not yet done, do here and save taskcode + if (useCreatedCropfield == false || string.IsNullOrEmpty(_settings.VanDerSatTaskCode)) + { + var vanDerSatTaskCode = await _generalService.RunVanDerSatTask(cropfieldItem); + _settings.VanDerSatTaskCode = vanDerSatTaskCode; + SaveSettings(settingsfile); + } + + // Select a particular satellite item from satelliteTask + Item vanDerSatItem = await _generalService.FindVanDerSatItem(cropfieldItem, _settings.VanDerSatTaskCode, fieldName, input.StoreVanDerSatStatistics); + + + + } + + + } + + // Functions to save previously created cropfields + private void LoadSettings(string file) + { + if (File.Exists(file)) + { + var jsonText = File.ReadAllText(file); + _settings = JsonConvert.DeserializeObject(jsonText); + } + else + { + _settings = new Settings(); + } + } + + private void SaveSettings(string file) + { + if (_settings == null) + return; + + var json = JsonConvert.SerializeObject(_settings); + File.WriteAllText(file, json); + } + private void SaveInfo(string file) + { + if (_settings == null) + return; + + var json = JsonConvert.SerializeObject(_settings); + File.WriteAllText(file, json); + + } + + } +} diff --git a/FarmmapsDataDownload/DataDownloadInput.json b/FarmmapsDataDownload/DataDownloadInput.json new file mode 100644 index 0000000..15678ae --- /dev/null +++ b/FarmmapsDataDownload/DataDownloadInput.json @@ -0,0 +1,31 @@ +[ + { + "UseCreatedCropfield": true, + "outputFileName": "testSatData", + "fieldName": "test_satData", + "GetShadowData": true, + "GetSatelliteData": true, + "SatelliteBand": "natural", // "ndvi" or "wdvi" + "StoreSatelliteStatistics": true, + + "GetVanDerSatData": false, + "StoreVanDerSatStatistics": false, + "CropYear": "2020-01-01", + "geometryJson": { + "type": "Polygon", + "coordinates": [ + [ + [ 4.960707146896585, 52.800583669708487 ], + [ 4.960645975538824, 52.800470217610922 ], + [ 4.962140695752897, 52.799177147194797 ], + [ 4.967523821195745, 52.801502400041208 ], + [ 4.966336768950911, 52.802543735879809 ], + [ 4.961711880764330, 52.801009996856429 ], + [ 4.960707146896585, 52.800583669708487 ] + ] + ] + } + } + + +] \ No newline at end of file diff --git a/FarmmapsDataDownload/DataDownloadService.cs b/FarmmapsDataDownload/DataDownloadService.cs new file mode 100644 index 0000000..81904bf --- /dev/null +++ b/FarmmapsDataDownload/DataDownloadService.cs @@ -0,0 +1,29 @@ +using System; +using System.Globalization; +using System.Threading.Tasks; +using FarmmapsApi.Models; +using FarmmapsApi.Services; +using FarmmapsDataDownload.Models; +using Microsoft.Extensions.Logging; +using static FarmmapsApi.Extensions; +using static FarmmapsApiSamples.Constants; + +namespace FarmmapsDataDownload +{ + public class DataDownloadService + { + private readonly ILogger _logger; + private readonly FarmmapsApiService _farmmapsApiService; + private readonly GeneralService _generalService; + + public DataDownloadService(ILogger logger, FarmmapsApiService farmmapsApiService, + GeneralService generalService) + { + _logger = logger; + _farmmapsApiService = farmmapsApiService; + _generalService = generalService; + } + + + } +} \ No newline at end of file diff --git a/FarmmapsDataDownload/FarmmapsDataDownload.csproj b/FarmmapsDataDownload/FarmmapsDataDownload.csproj new file mode 100644 index 0000000..f1a5343 --- /dev/null +++ b/FarmmapsDataDownload/FarmmapsDataDownload.csproj @@ -0,0 +1,27 @@ + + + + Exe + netcoreapp3.1 + + + + + Always + + + Always + + + Always + + + Always + + + + + + + + diff --git a/FarmmapsDataDownload/Models/DataDownloadInput.cs b/FarmmapsDataDownload/Models/DataDownloadInput.cs new file mode 100644 index 0000000..57cc6e8 --- /dev/null +++ b/FarmmapsDataDownload/Models/DataDownloadInput.cs @@ -0,0 +1,26 @@ +using System; +using Newtonsoft.Json.Linq; + +namespace FarmmapsDataDownload.Models +{ + public class DataDownloadInput + { + public bool UseCreatedCropfield { get; set; } + public string File { get; set; } + public string InputVariable { get; set; } + public string OutputFileName { get; set; } + public DateTime CropYear { get; set; } + public JObject GeometryJson { get; set; } + public string InputLayerName { get; set; } + public string fieldName { get; set; } + public bool GetSatelliteData { get; set; } + public bool GetVanDerSatData { get; set; } + public string SatelliteBand { get; set; } + public bool StoreSatelliteStatistics { get; set; } + public bool StoreVanDerSatStatistics { get; set; } + public bool GetShadowData { get; set; } + + + + } +} \ No newline at end of file diff --git a/FarmmapsDataDownload/Models/Settings.cs b/FarmmapsDataDownload/Models/Settings.cs new file mode 100644 index 0000000..21506b4 --- /dev/null +++ b/FarmmapsDataDownload/Models/Settings.cs @@ -0,0 +1,11 @@ +namespace FarmmapsDataDownload +{ + public class Settings + { + public string CropfieldItemCode { get; set; } + public string SatelliteTaskCode { get; set; } + public string VanDerSatTaskCode { get; set; } + public string WatBalTaskCode { get; set; } + + } +} \ No newline at end of file diff --git a/FarmmapsDataDownload/Models/TargetNData.cs b/FarmmapsDataDownload/Models/TargetNData.cs new file mode 100644 index 0000000..b829a1a --- /dev/null +++ b/FarmmapsDataDownload/Models/TargetNData.cs @@ -0,0 +1,9 @@ +namespace FarmmapsDataDownload.Models +{ + public class TargetNData + { + public double TSum { get; set; } + public int TargetYield { get; set; } + public double TargetN { get; set; } + } +} \ No newline at end of file diff --git a/FarmmapsDataDownload/Program.cs b/FarmmapsDataDownload/Program.cs new file mode 100644 index 0000000..0c1ad78 --- /dev/null +++ b/FarmmapsDataDownload/Program.cs @@ -0,0 +1,21 @@ +using System.Threading.Tasks; +using FarmmapsApi; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; + +namespace FarmmapsDataDownload +{ + class Program : FarmmapsProgram + { + private static async Task Main(string[] args) + { + await new Program().Start(args); + } + + protected override void Configure(IServiceCollection serviceCollection) + { + serviceCollection.AddLogging() + .AddTransient(); + } + } +} \ No newline at end of file diff --git a/FarmmapsPoten/PotenApplication.cs b/FarmmapsPoten/PotenApplication.cs index 8982a9c..7479bfb 100644 --- a/FarmmapsPoten/PotenApplication.cs +++ b/FarmmapsPoten/PotenApplication.cs @@ -81,6 +81,7 @@ namespace FarmmapsVRApoten _logger.LogInformation($"CropfielditemCode: {cropfieldItem.Code}"); + //Downloading shadowMap for own interpretation if (useShadow) { _logger.LogInformation("Calculate shadow map for field"); @@ -142,6 +143,9 @@ namespace FarmmapsVRApoten _logger.LogInformation($"Converting shape to geotiff"); geotiffItem = await _generalService.ShapeToGeotiff(shapeItem); + _logger.LogInformation($"ShapeToGeotiff_GeotiffItemcode: {geotiffItem.Code}"); + + if (geotiffItem == null) { _logger.LogError("Something went wrong with shape to geotiff transformation"); return; @@ -193,7 +197,7 @@ namespace FarmmapsVRApoten taskmap = await _generalService.CreateTaskmap(cropfieldItem: cropfieldItem, tiffItem: applianceMapItem, outputType: input.OutputType, cellWidth: input.CellWidth, cellHeight: input.CellHeight, startPoint: input.StartPoint.ToString(Formatting.None), ddiCode: input.DdiCode, centered: input.Centered, endPoint: input.EndPoint.ToString(Formatting.None), angle: input.Angle, precision: input.Precision, - cropTypeName: null, costumerName: null, ProductGroupName: null, productName : null, resolution: null, unitScale: null, maximumClasses: input.MaximumClasses); + cropTypeName: null, costumerName: null, ProductGroupName: null, productName : null, resolution: "3", unitScale: null, maximumClasses: input.MaximumClasses); } else { @@ -212,7 +216,10 @@ namespace FarmmapsVRApoten _logger.LogInformation("Downloading taskmap"); await _farmmapsApiService.DownloadItemAsync(taskmap.Code, - Path.Combine(DownloadFolder, $"VRApoten_taskmap_{input.OutputFileName}.zip")); + Path.Combine(DownloadFolder, $"VRApoten_taskmap_{input.OutputFileName}_isoxml.zip")); + + + } } } diff --git a/FarmmapsPoten/PotenInput.json b/FarmmapsPoten/PotenInput.json index 3b7a3c1..6753abd 100644 --- a/FarmmapsPoten/PotenInput.json +++ b/FarmmapsPoten/PotenInput.json @@ -3,13 +3,13 @@ { "File": "PlantingSampleDataLutumANDec.zip", "InputLayerName": "EC0-60", - "OutputFileName": "2021.04.14_vraPoten_SampleDataMultipleLayers", + "OutputFileName": "2021.05.11_vraPoten_SampleDataMultipleLayers", "FieldName": "lutum", "PlantingYear": 2021, "MeanDensity": "30", "Variation": "20", "UseShadow": false, - "CountPerArea": false, // don't forget to change ddi if isoxml is created + "CountPerArea": true, // don't forget to change ddi if isoxml is created "geometryJson": { "type": "Polygon", "coordinates": [ @@ -24,7 +24,7 @@ }, "GenerateTaskmap": true, - "OutputType": "shape", // "shape" or "isoxml" if isoxml also add ddiCode + "OutputType": "isoxml", // "shape" or "isoxml" if isoxml also add ddiCode "Precision": "2", "MaximumClasses": "5", "DdiCode": "0016",