Projects FarmmapsBulkSatDownload and FarmmapsDataDownload fully updated and tested
Added some extra Tasks to /FarmmapsApi/Services/GeneralService.cs Added class SatelliteStatistics.cs to /FarmmapsApi/Models
This commit is contained in:
@@ -1,6 +1,7 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.IO.Compression;
|
||||
using System.Linq;
|
||||
using System.Threading.Tasks;
|
||||
using FarmmapsApi;
|
||||
@@ -16,7 +17,7 @@ namespace FarmmapsDataDownload
|
||||
{
|
||||
public class DataDownloadApplication : IApplication
|
||||
{
|
||||
private const string DownloadFolder = "Downloads";
|
||||
//private const string DownloadFolder = "Downloads";
|
||||
private const string SettingsFile = "settings.json";
|
||||
|
||||
private readonly ILogger<DataDownloadApplication> _logger;
|
||||
@@ -38,10 +39,8 @@ namespace FarmmapsDataDownload
|
||||
public async Task RunAsync()
|
||||
{
|
||||
var fieldsInputJson = File.ReadAllText("DataDownloadInput.json");
|
||||
List<DataDownloadInput> fieldsInputs = JsonConvert.DeserializeObject<List<DataDownloadInput>>(fieldsInputJson);
|
||||
|
||||
if (!Directory.Exists(DownloadFolder))
|
||||
Directory.CreateDirectory(DownloadFolder);
|
||||
List<DataDownloadInput> fieldsInputs = JsonConvert.DeserializeObject<List<DataDownloadInput>>(fieldsInputJson);
|
||||
|
||||
// !! this call is needed the first time an api is called with a fresh clientid and secret !!
|
||||
await _farmmapsApiService.GetCurrentUserCodeAsync();
|
||||
@@ -62,12 +61,23 @@ namespace FarmmapsDataDownload
|
||||
|
||||
private async Task Process(List<UserRoot> roots, DataDownloadInput input)
|
||||
{
|
||||
string downloadFolder = input.DownloadFolder;
|
||||
if (string.IsNullOrEmpty(downloadFolder)) {
|
||||
downloadFolder = "Downloads";
|
||||
}
|
||||
if (!Directory.Exists(downloadFolder))
|
||||
Directory.CreateDirectory(downloadFolder);
|
||||
|
||||
// !!specify if you are using an already created cropfield:
|
||||
bool useCreatedCropfield = input.UseCreatedCropfield;
|
||||
var cropYear = input.CropYear;
|
||||
var fieldName = input.fieldName;
|
||||
bool storeSatelliteStatistics = input.StoreSatelliteStatistics;
|
||||
bool storeSatelliteStatistics = input.StoreSatelliteStatisticsSingleImage;
|
||||
bool storeSatelliteStatisticsCropYear = input.StoreSatelliteStatisticsCropYear;
|
||||
List<string> SatelliteBands = new List<string>(1) { input.SatelliteBand };
|
||||
string headerLineStats = $"FieldName,satelliteDate,satelliteBand,max,min,mean,mode,median,stddev,minPlus,curtosis,maxMinus,skewness,variance,populationCount,variationCoefficient,confidenceIntervalLow, confidenceIntervalHigh,confidenceIntervalErrorMargin" + Environment.NewLine;
|
||||
|
||||
|
||||
string settingsfile = $"Settings_{fieldName}.json";
|
||||
|
||||
LoadSettings(settingsfile);
|
||||
@@ -92,7 +102,7 @@ namespace FarmmapsDataDownload
|
||||
{
|
||||
_logger.LogInformation("Creating cropfield");
|
||||
cropfieldItem = await _generalService.CreateCropfieldItemAsync(myDriveRoot.Code,
|
||||
$"DataCropfield {input.OutputFileName}", cropYear.Year, input.GeometryJson.ToString(Formatting.None));
|
||||
$"DataCropfield {input.OutputFileName}", cropYear, input.GeometryJson.ToString(Formatting.None));
|
||||
_settings.CropfieldItemCode = cropfieldItem.Code;
|
||||
SaveSettings(settingsfile);
|
||||
}
|
||||
@@ -117,12 +127,10 @@ namespace FarmmapsDataDownload
|
||||
|
||||
_logger.LogInformation("Downloading shadow map");
|
||||
await _farmmapsApiService.DownloadItemAsync(shadowItem.Code,
|
||||
Path.Combine(DownloadFolder, $"{input.OutputFileName}_shadow.zip"));
|
||||
Path.Combine(downloadFolder, $"{input.OutputFileName}_shadow.zip"));
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
// Get satellite data
|
||||
if (input.GetSatelliteData)
|
||||
{
|
||||
@@ -134,54 +142,72 @@ namespace FarmmapsDataDownload
|
||||
SaveSettings(settingsfile);
|
||||
}
|
||||
|
||||
int selectedLayer= 2;
|
||||
|
||||
if (input.SatelliteBand == "ndvi") selectedLayer = 0;
|
||||
if (input.SatelliteBand == "wdvi") selectedLayer = 1;
|
||||
if (input.SatelliteBand == "natural") selectedLayer = 2;
|
||||
// Select a particular satellite item from satelliteTask
|
||||
Item satalliteItem = await _generalService.FindSatelliteItem(cropfieldItem, _settings.SatelliteTaskCode, fieldName, selectedLayer, storeSatelliteStatisticsCropYear, DownloadFolder);
|
||||
// Select all satellite items
|
||||
List<Item> satelliteItemsCropYear = await _generalService.FindSatelliteItems(cropfieldItem, _settings.SatelliteTaskCode);
|
||||
satelliteItemsCropYear = satelliteItemsCropYear.OrderBy(x => x.DataDate).ToList();
|
||||
|
||||
|
||||
var satelliteBand = satalliteItem.Data["layers"][selectedLayer]["name"];
|
||||
|
||||
//Store satellite data to csv
|
||||
if (storeSatelliteStatistics == true && (selectedLayer == 0 || selectedLayer ==1))
|
||||
{
|
||||
|
||||
|
||||
var satelliteStatistics = satalliteItem.Data["layers"][selectedLayer]["renderer"]["band"]["statistics"];
|
||||
Console.WriteLine($"Satellite image date: {satalliteItem.DataDate}");
|
||||
|
||||
var SatelliteStatsFile = $"{DownloadFolder}/SatelliteDataStatistics_{fieldName}_{input.SatelliteBand}_{satalliteItem.DataDate.Value:yyyy-MM-dd}.csv";
|
||||
using var w = new StreamWriter(SatelliteStatsFile);
|
||||
if (input.StoreSatelliteStatisticsSingleImage == true) {
|
||||
_logger.LogInformation("Available satellite images:");
|
||||
var count = 0;
|
||||
TimeSpan.FromSeconds(0.5);
|
||||
foreach (var item in satelliteItemsCropYear)
|
||||
{
|
||||
foreach (var item in satelliteStatistics)
|
||||
|
||||
Console.WriteLine($"Satellite image #{count}: {item.DataDate}");
|
||||
count++;
|
||||
}
|
||||
|
||||
_logger.LogInformation("Enter satellite image number");
|
||||
int element = Int32.Parse(Console.ReadLine());
|
||||
var selectedSatelliteItem = satelliteItemsCropYear[element];
|
||||
|
||||
var SatelliteDate = selectedSatelliteItem.DataDate.Value.ToString("yyyyMMdd");
|
||||
string fileName = string.Format($"satelliteGeotiff_{fieldName}_{SatelliteDate}"); // no need to add satelliteBand in the name because the tif contains all bands
|
||||
string fileNameZip = string.Format($"{fileName}.zip");
|
||||
string fileNameGeotiff = string.Format($"{fileName}.tif");
|
||||
await _farmmapsApiService.DownloadItemAsync(selectedSatelliteItem.Code, Path.Combine(downloadFolder, fileNameZip));
|
||||
|
||||
// Download a csv file with stats
|
||||
List<Item> selectedSatalliteItems = new List<Item>(1) { selectedSatelliteItem };
|
||||
string fileNameStats = Path.Combine(downloadFolder, string.Format($"satelliteStats_{fieldName}_{SatelliteDate}.csv"));
|
||||
string downloadedStats = await _generalService.DownloadSatelliteStats(selectedSatalliteItems, fieldName, SatelliteBands, downloadFolder);
|
||||
//rename the csv file with stats
|
||||
File.Delete(fileNameStats);
|
||||
File.Move(downloadedStats, fileNameStats);
|
||||
// wenr.tif. Contains 5 layers: (1) ndvi, (2) wdvi, (3) Red, (4) Green and (5) Blue
|
||||
// download the geotiffs. Returns a zip file with always these three files:
|
||||
// data.dat.aux.xml
|
||||
// thumbnail.jpg
|
||||
// wenr.tif. Contains 5 layers: (1) ndvi, (2) wdvi, (3) Red, (4) Green and (5) Blue
|
||||
if (true)
|
||||
{
|
||||
// Extract the file "wenr.tif" from zip, rename it to fileNameGeotiff
|
||||
ZipFile.ExtractToDirectory(Path.Combine(downloadFolder, fileNameZip), downloadFolder, true);
|
||||
File.Delete(Path.Combine(downloadFolder, fileNameGeotiff)); // Delete the fileNameGeotiff file if exists
|
||||
File.Move(Path.Combine(downloadFolder, "wenr.tif"), Path.Combine(downloadFolder, fileNameGeotiff)); // Rename the oldFileName into newFileName
|
||||
|
||||
// Cleanup
|
||||
string[] filesToDelete = new string[] { fileNameZip, "wenr.tif", "thumbnail.jpg", "data.dat.aux.xml" };
|
||||
foreach (string f in filesToDelete)
|
||||
{
|
||||
var line = string.Format("{0}", item);
|
||||
w.WriteLine(line);
|
||||
w.Flush();
|
||||
File.Delete(Path.Combine(downloadFolder, f));
|
||||
}
|
||||
}
|
||||
_logger.LogInformation($"Downloaded files {fileNameGeotiff} and {fileNameStats} to {downloadFolder}");
|
||||
|
||||
var inputType = (satalliteItem.Data["layers"] as JArray)?[selectedLayer]["name"].ToString();
|
||||
if (string.IsNullOrEmpty(inputType)) {
|
||||
_logger.LogError("Could not get the input type name from the satellite item");
|
||||
return;
|
||||
}
|
||||
|
||||
// download the geotiff of needed inputtype
|
||||
var SatelliteImageDate = (DateTime)satalliteItem.DataDate;
|
||||
var SatelliteDate = SatelliteImageDate.ToString("yyyyMMdd");
|
||||
_logger.LogInformation("Downloading geotiff file");
|
||||
await _farmmapsApiService.DownloadItemAsync(satalliteItem.Code,
|
||||
Path.Combine(DownloadFolder, $"satelliteGeotiff_{input.OutputFileName}_{inputType}_{SatelliteDate}.zip"));
|
||||
}
|
||||
if (input.StoreSatelliteStatisticsCropYear == true) {
|
||||
string fileNameStats = Path.Combine(downloadFolder, string.Format($"satelliteStats_{fieldName}_{cropYear}.csv"));
|
||||
File.Delete(fileNameStats);
|
||||
string downloadedStats = await _generalService.DownloadSatelliteStats(satelliteItemsCropYear, fieldName, SatelliteBands, downloadFolder);
|
||||
File.Move(downloadedStats, fileNameStats);
|
||||
_logger.LogInformation($"Downloaded file {fileNameStats} with stats for field '{fieldName}', cropyear {cropYear}");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
// Get vanDerSat data
|
||||
if (input.GetVanDerSatData)
|
||||
{
|
||||
|
||||
@@ -3,15 +3,15 @@
|
||||
"UseCreatedCropfield": true,
|
||||
"outputFileName": "testSatData2",
|
||||
"fieldName": "test_satData2",
|
||||
"DownloadFolder": "C:\\workdir\\groenmonitor\\", //"C:\\workdir\\groenmonitor\\", // "Downloads", -> if you just put "Downloads" the program will download to somewhere in ..\FarmMapsApiClient_WURtest\FarmmapsDataDownload\bin\Debug\netcoreapp3.1\Downloads\
|
||||
"GetShadowData": false,
|
||||
"GetSatelliteData": true,
|
||||
"SatelliteBand": "wdvi", // "natural", "ndvi" or "wdvi"
|
||||
"StoreSatelliteStatistics": false,
|
||||
"StoreSatelliteStatisticsSingleImage": true,
|
||||
"StoreSatelliteStatisticsCropYear": true,
|
||||
|
||||
"GetVanDerSatData": false,
|
||||
"StoreVanDerSatStatistics": false,
|
||||
"CropYear": "2020-01-01",
|
||||
"CropYear": 2020,
|
||||
"geometryJson": {
|
||||
"type": "Polygon",
|
||||
"coordinates": [
|
||||
|
||||
@@ -9,14 +9,15 @@ namespace FarmmapsDataDownload.Models
|
||||
public string File { get; set; }
|
||||
public string InputVariable { get; set; }
|
||||
public string OutputFileName { get; set; }
|
||||
public DateTime CropYear { get; set; }
|
||||
public string DownloadFolder { get; set; }
|
||||
public int CropYear { get; set; }
|
||||
public JObject GeometryJson { get; set; }
|
||||
public string InputLayerName { get; set; }
|
||||
public string fieldName { get; set; }
|
||||
public bool GetSatelliteData { get; set; }
|
||||
public bool GetVanDerSatData { get; set; }
|
||||
public string SatelliteBand { get; set; }
|
||||
public bool StoreSatelliteStatistics { get; set; }
|
||||
public bool StoreSatelliteStatisticsSingleImage { get; set; }
|
||||
public bool StoreSatelliteStatisticsCropYear { get; set; }
|
||||
public bool StoreVanDerSatStatistics { get; set; }
|
||||
public bool GetShadowData { get; set; }
|
||||
|
||||
Reference in New Issue
Block a user