FarmMaps/FarmMapsApiClient
11
0
Fork 2
Code Issues Pull Requests Releases Activity

new project FarmmapsCleanUp to clean up cropfielditems and satellitetaskcodes used only once

Browse Source
This commit is contained in:
pepijn van oort 2021-06-07 16:38:42 +02:00
parent d1747427bc
commit 915df2ac6b
6 changed files with 260 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
FarmmapsApiSamples.sln
View File

@ -27,7 +27,9 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FarmmapsDataDownload", "Far
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FarmmapsBulkSatDownload", "FarmmapsBulkSatDownload\FarmmapsBulkSatDownload.csproj", "{772DBDCD-9FAA-40A7-8551-2C1620C4AB67}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Secrets", "Secrets\Secrets.csproj", "{C4EE5ECA-253A-4B71-9F67-D231AC4517D6}"
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Secrets", "Secrets\Secrets.csproj", "{C4EE5ECA-253A-4B71-9F67-D231AC4517D6}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FarmmapsCleanUp", "FarmmapsCleanUp\FarmmapsCleanUp.csproj", "{5E4387F9-5953-4A9B-BCA5-DF3964EED3CB}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
@ -75,6 +77,10 @@ Global
{C4EE5ECA-253A-4B71-9F67-D231AC4517D6}.Debug|Any CPU.Build.0 = Debug|Any CPU
{C4EE5ECA-253A-4B71-9F67-D231AC4517D6}.Release|Any CPU.ActiveCfg = Release|Any CPU
{C4EE5ECA-253A-4B71-9F67-D231AC4517D6}.Release|Any CPU.Build.0 = Release|Any CPU
{5E4387F9-5953-4A9B-BCA5-DF3964EED3CB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{5E4387F9-5953-4A9B-BCA5-DF3964EED3CB}.Debug|Any CPU.Build.0 = Debug|Any CPU
{5E4387F9-5953-4A9B-BCA5-DF3964EED3CB}.Release|Any CPU.ActiveCfg = Release|Any CPU
{5E4387F9-5953-4A9B-BCA5-DF3964EED3CB}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE

64
FarmmapsBulkSatDownload/BulkSatDownloadApplication.cs
View File

@ -57,10 +57,10 @@ namespace FarmmapsBulkSatDownload
// Crashes if "DBsettings.secrets.json" is absent or empty
DB dbparcels = JsonConvert.DeserializeObject<DB>(File.ReadAllText("DBsettings.secrets.json"));
string schemaname = "bigdata";
string parceltablename = "parcel_flowerbulbs";//"parcelsijbrandij" "parcel"; "parcel_flowerbulbs"; "parcel_disac"
string groenmonitortablename = "groenmonitor_flowerbulbs";//"groenmonitorsijbrandij" "groenmonitor" "groenmonitor_flowerbulbs" "groenmonitor_disac"
string parceltablename = "parcel_bollenrevolutie_tulips2020"; //"parcelsijbrandij" "parcel"; "parcel_flowerbulbs"; "parcel_disac"; ""parcel_bollenrevolutie_tulips2020""
string groenmonitortablename = "groenmonitor_bollenrevolutie_tulips2020"; //"groenmonitorsijbrandij" "groenmonitor" "groenmonitor_flowerbulbs" "groenmonitor_disac" "groenmonitor_bollenrevolutie_tulips2020"
// The view 'groenmonitorlatestviewname' contains per parcelid (arbid) the year in which it "exists" and the date of the latest image downloaded. It is used to prevent unneccessary downloading of image statistics already in the database
string groenmonitorlatestviewname = "groenmonitorlatest_flowerbulbs"; //"groenmonitorsijbrandijlatest" "groenmonitorlatest" "groenmonitorlatest_flowerbulbs" "groenmonitorlatest_disac"
string groenmonitorlatestviewname = "groenmonitorlatest_bollenrevolutie_tulips2020"; //"groenmonitorsijbrandijlatest" "groenmonitorlatest" "groenmonitorlatest_flowerbulbs" "groenmonitorlatest_disac" "groenmonitorlatest_bollenrevolutie_tulips2020"
// Database query and connection. Geometry must be in WGS84 coordinate system, EPSG 4326
// Apparently the FarmmapsApi cannot handle MultiPolygon, so we need to convert to single Polygon
@ -80,11 +80,10 @@ SELECT pt.arbid, pt.year, gml.lastwenrdate, ST_AsGeoJSON(ST_Transform((ST_DUMP(p
CASE WHEN pt.year >= DATE_PART('year', CURRENT_DATE) THEN '' ELSE COALESCE(pt.satellitetaskcode,'') END AS satellitetaskcode
FROM {0}.{1} pt, {0}.{2} gml
WHERE
pt.arbid = gml.arbid AND
pt.crop NOT IN ('Lelie','Tulp') AND
pt.year > 2018 AND pt.arbid IN(8276,8314,8315)
pt.arbid = gml.arbid
AND pt.satellitetaskcode IS NULL
ORDER BY pt.arbid
LIMIT 2000;", schemaname, parceltablename, groenmonitorlatestviewname); //LIMIT x for testing
LIMIT 5;", schemaname, parceltablename, groenmonitorlatestviewname); //LIMIT x for testing
using (NpgsqlConnection connection = new NpgsqlConnection(connectionString))
{
@ -117,30 +116,30 @@ LIMIT 2000;", schemaname, parceltablename, groenmonitorlatestviewname); //LIMI
// Option 2: Example without database. Comment out this part if you want to use database
// Read cropfields "BulkSatDownloadInput.json" and write all stats to a single csv file
// Write all stats for multiple fields will be written to a single csv file
string downloadFolder;
string fileNameStats;
string headerLineStats = $"FieldName,satelliteDate,satelliteBand,max,min,mean,mode,median,stddev,minPlus,curtosis,maxMinus,skewness,variance,populationCount,variationCoefficient,confidenceIntervalLow, confidenceIntervalHigh,confidenceIntervalErrorMargin" + Environment.NewLine;
var fieldsInputJson = File.ReadAllText("BulkSatDownloadInput.json");
bulkSatDownloadInputListCsv = JsonConvert.DeserializeObject<List<BulkSatDownloadInput>>(fieldsInputJson);
for (int i = 0; i < bulkSatDownloadInputListCsv.Count; i++)
{
downloadFolder = bulkSatDownloadInputListCsv[i].downloadFolder;
fileNameStats = Path.Combine(downloadFolder, bulkSatDownloadInputListCsv[i].fileNameStats);
if (!Directory.Exists(downloadFolder))
Directory.CreateDirectory(downloadFolder);
bulkSatDownloadInputListCsv[i].fileNameStats = fileNameStats;
// Header same as in GeneralService.DownloadSatelliteStats
// Delete fileNameStats if existing. Create a new file. Add a header to csv file
File.Delete(fileNameStats);
File.AppendAllText(fileNameStats, headerLineStats);
}
//string downloadFolder;
//string fileNameStats;
//string headerLineStats = $"FieldName,satelliteDate,satelliteBand,max,min,mean,mode,median,stddev,minPlus,curtosis,maxMinus,skewness,variance,populationCount,variationCoefficient,confidenceIntervalLow, confidenceIntervalHigh,confidenceIntervalErrorMargin" + Environment.NewLine;
//var fieldsInputJson = File.ReadAllText("BulkSatDownloadInput.json");
//bulkSatDownloadInputListCsv = JsonConvert.DeserializeObject<List<BulkSatDownloadInput>>(fieldsInputJson);
//for (int i = 0; i < bulkSatDownloadInputListCsv.Count; i++)
//{
// downloadFolder = bulkSatDownloadInputListCsv[i].downloadFolder;
// fileNameStats = Path.Combine(downloadFolder, bulkSatDownloadInputListCsv[i].fileNameStats);
// if (!Directory.Exists(downloadFolder))
// Directory.CreateDirectory(downloadFolder);
// bulkSatDownloadInputListCsv[i].fileNameStats = fileNameStats;
// // Header same as in GeneralService.DownloadSatelliteStats
// // Delete fileNameStats if existing. Create a new file. Add a header to csv file
// File.Delete(fileNameStats);
// File.AppendAllText(fileNameStats, headerLineStats);
//}
// Now choose which list you want to use
bulkSatDownloadInputList = bulkSatDownloadInputListDB; //bulkSatDownloadInputListDB; //bulkSatDownloadInputListCsv;
// Whichever option (database or json/csv), continue here
// Delete the settingsfile
File.Delete(settingsfile);
// File.Delete(settingsfile);
// For each input download all images. Keep track to time, important when doing bulk downloads
var watch = System.Diagnostics.Stopwatch.StartNew();
@ -157,13 +156,26 @@ LIMIT 2000;", schemaname, parceltablename, groenmonitorlatestviewname); //LIMI
try
{
await Process(roots, bulkSatDownloadInput);
}
catch (Exception ex)
{
_logger.LogError(ex.Message);
}
watch.Stop();
// add this downloadtime to the cropfieldtable.
// Only if cropfieldtable has a field called 'downloadtime' type 'time'!
//using (NpgsqlConnection connection = new NpgsqlConnection(dbparcels.GetConnectionString()))
//{
// connection.Open();
// NpgsqlCommand updateCmd = connection.CreateCommand();
// string updateSql = string.Format($"UPDATE {schemaname}.{parceltablename} SET downloadtime = '{strTime(watch.Elapsed)}' WHERE arbid = {bulkSatDownloadInput.fieldID};");
// updateCmd.CommandText = updateSql;
// int r = updateCmd.ExecuteNonQuery();
// if (r != 1)
// throw new Exception("// FarmmapsBulkSatDownload: Update downloadtime Failed");
// connection.Close();
//}
//_logger.LogInformation($"// FarmmapsBulkSatDownload: Added downloadtime = '{strTime(watch.Elapsed)}' to {schemaname}.{parceltablename} ");
tsSofar = tsSofar + watch.Elapsed;
tsTotalEstimated = tsSofar / (i + 1) * bulkSatDownloadInputList.Count;
tsRemaining = tsTotalEstimated - tsSofar;

143
FarmmapsCleanUp/CleanUpApplication.cs Normal file
View File

@ -0,0 +1,143 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Threading.Tasks;
using FarmmapsApi;
using FarmmapsApi.Models;
using FarmmapsApi.Services;
using FarmmapsBulkSatDownload.Models;
using Microsoft.Extensions.Logging;
using Newtonsoft.Json;
using Npgsql;
using Newtonsoft.Json.Linq;
using static FarmmapsApiSamples.Constants;
namespace FarmmapsCleanup
{
public class CleanupApplication : IApplication
{
private readonly ILogger<CleanupApplication> _logger;
private readonly FarmmapsApiService _farmmapsApiService;
private readonly CleanupService _cleanupService;
private readonly GeneralService _generalService;
public CleanupApplication(ILogger<CleanupApplication> logger, FarmmapsApiService farmmapsApiService,
GeneralService generalService, CleanupService cleanupService)
{
_logger = logger;
_farmmapsApiService = farmmapsApiService;
_generalService = generalService;
_cleanupService = cleanupService;
}
public async Task RunAsync()
{
// !! this call is needed the first time an api is called with a fresh clientid and secret !!
await _farmmapsApiService.GetCurrentUserCodeAsync();
var roots = await _farmmapsApiService.GetCurrentUserRootsAsync();
//Establish a database connection. Expecting you are using same database server as in project FarmmapsBulkSatDownload
DB dbparcels = JsonConvert.DeserializeObject<DB>(File.ReadAllText("DBsettings.secrets.json"));
string schemaname = "bigdata";
string parceltablename = "parcel_bollenrevolutie_tulips2020"; //"parcelsijbrandij" "parcel"; "parcel_flowerbulbs"; "parcel_disac"; ""parcel_bollenrevolutie_tulips2020""
//Query to get lists of items to delete from FarmmapsDatabase and from parceltablename
string cropfielditemcode;
string satellitetaskcode;
List<string> cropfieldItemCodes = new List<string>();
List<string> satellitetaskCodes = new List<string>();
string connectionString = dbparcels.GetConnectionString();
string readSql = string.Format(
@"
SELECT pt.cropfielditemcode, pt.satellitetaskcode
FROM {0}.{1} pt
WHERE
pt.arbid IN(1,2)
ORDER BY pt.arbid
;", schemaname, parceltablename);
string updateCropfieldItemCodesSql = string.Format(
@"
UPDATE {0}.{1}
SET cropfielditemcode=NULL
WHERE arbid IN(1,2)
;", schemaname, parceltablename); //Same WHERE AS above
string updateSatellitetaskCodesSql = string.Format(
@"
UPDATE {0}.{1}
SET satellitetaskcode=NULL
WHERE arbid IN(1,2)
;", schemaname, parceltablename); //Same WHERE AS above
using (NpgsqlConnection connection = new NpgsqlConnection(connectionString))
{
connection.Open();
// Read data (run query) = build a list of fields for which to download images
NpgsqlCommand command = connection.CreateCommand();
command.CommandText = readSql;
NpgsqlDataReader dr = command.ExecuteReader();
while (dr.Read())
{
cropfielditemcode = dr.GetString(0);
satellitetaskcode = dr.GetString(1);
if(!string.IsNullOrEmpty(cropfielditemcode))
cropfieldItemCodes.Add(cropfielditemcode);
if (!string.IsNullOrEmpty(satellitetaskcode))
satellitetaskCodes.Add(satellitetaskcode);
}
connection.Close();
}
_logger.LogWarning($"// FarmmapsCleanUp: WARNING: you are about to delete {cropfieldItemCodes.Count} cropfieldItemCodes and {satellitetaskCodes.Count} satellitetaskCodes from the FarmMaps database and your own table {schemaname}.{parceltablename}");
_logger.LogInformation($"// Nice of you to clean up after the work is done. You would typically do this for cropfieldItemCodes used only once.");
_logger.LogInformation($"// You do NOT want to do this if you think you may later on still want to use these items.");
_logger.LogInformation($"// Please carefully check the SQL queries 'readSql' 'updateSql' in CleanUpApplication.cs before proceeding.");
_logger.LogInformation($"// FarmmapsCleanUp: delete selected cropfieldItemCodes from FarmMaps database and table {schemaname}.{parceltablename}? 0 = no, 1 = yes");
int i;
i = Int32.Parse(Console.ReadLine());
if (i == 1)
{
await _farmmapsApiService.DeleteItemsAsync(cropfieldItemCodes);
//TODO _farmmapsApiService.DeleteItemsAsync throws an error: {StatusCode: 415, ReasonPhrase: 'Unsupported Media Type', Version: 1.1, Content: System.Net.Http.HttpConnectionResponseContent, Headers: {...}}
//what is wrong with cropfieldItemCodes?
//and shouldn't we be telling _farmmapsApiService.DeleteItemsAsync what item type to delete?
using (NpgsqlConnection connection = new NpgsqlConnection(connectionString))
{
connection.Open();
NpgsqlCommand updateCmd = connection.CreateCommand();
updateCmd.CommandText = updateCropfieldItemCodesSql;
int r = updateCmd.ExecuteNonQuery();
if (r == -1)
throw new Exception("// FarmmapsCleanUp: Update cropfielditemcode Failed");
connection.Close();
}
}
_logger.LogInformation($"// FarmmapsCleanUp: delete selected satellitetaskCodes from FarmMaps database and table {schemaname}.{parceltablename}? 0 = no, 1 = yes");
i = Int32.Parse(Console.ReadLine());
if (i == 1)
{
await _farmmapsApiService.DeleteItemsAsync(satellitetaskCodes);
//TODO _farmmapsApiService.DeleteItemsAsync throws an error: {StatusCode: 415, ReasonPhrase: 'Unsupported Media Type', Version: 1.1, Content: System.Net.Http.HttpConnectionResponseContent, Headers: {...}}
//what is wrong with satellitetaskCodes?
//and shouldn't we be telling _farmmapsApiService.DeleteItemsAsync what item type to delete?
using (NpgsqlConnection connection = new NpgsqlConnection(connectionString))
{
connection.Open();
NpgsqlCommand updateCmd = connection.CreateCommand();
updateCmd.CommandText = updateSatellitetaskCodesSql;
int r = updateCmd.ExecuteNonQuery();
if (r == -1)
throw new Exception("// FarmmapsCleanUp: Update cropfielditemcode Failed");
connection.Close();
}
}
_logger.LogInformation($"// FarmmapsCleanUp: done! Hit any key to exit ...");
Console.ReadKey();
}
}
}

28
FarmmapsCleanUp/CleanUpService.cs Normal file
View File

@ -0,0 +1,28 @@
using System;
using System.Globalization;
using System.Threading.Tasks;
using FarmmapsApi.Models;
using FarmmapsApi.Services;
using Microsoft.Extensions.Logging;
using static FarmmapsApi.Extensions;
using static FarmmapsApiSamples.Constants;
namespace FarmmapsCleanup
{
public class CleanupService
{
private readonly ILogger<CleanupService> _logger;
private readonly FarmmapsApiService _farmmapsApiService;
private readonly GeneralService _generalService;
public CleanupService(ILogger<CleanupService> logger, FarmmapsApiService farmmapsApiService,
GeneralService generalService)
{
_logger = logger;
_farmmapsApiService = farmmapsApiService;
_generalService = generalService;
}
}
}

23
FarmmapsCleanUp/FarmmapsCleanUp.csproj Normal file
View File

@ -0,0 +1,23 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>netcoreapp3.1</TargetFramework>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Npgsql" Version="5.0.5" />
</ItemGroup>
<ItemGroup>
<None Update="Data\**\*">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</None>
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\FarmmapsApi\FarmmapsApi.csproj" />
<ProjectReference Include="..\FarmmapsBulkSatDownload\FarmmapsBulkSatDownload.csproj" />
</ItemGroup>
</Project>

21
FarmmapsCleanUp/Program.cs Normal file
View File

@ -0,0 +1,21 @@
using System.Threading.Tasks;
using FarmmapsApi;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
namespace FarmmapsCleanup
{
class Program : FarmmapsProgram<CleanupApplication>
{
private static async Task Main(string[] args)
{
await new Program().Start(args);
}
protected override void Configure(IServiceCollection serviceCollection)
{
serviceCollection.AddLogging()
.AddTransient<CleanupService>();
}
}
}