jits/FarmMapsApiClient_WURtest
1
0
Fork 0
forked from FarmMaps/FarmMapsApiClient
Code Issues Pull Requests Releases Wiki Activity

new project FarmmapsCleanUp to clean up cropfielditems and satellitetaskcodes used only once

Browse Source
This commit is contained in:
pepijn van oort 2021-06-07 16:38:42 +02:00
parent d1747427bc
commit 915df2ac6b
6 changed files with 260 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
FarmmapsApiSamples.sln
View File

@ -27,7 +27,9 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FarmmapsDataDownload", "Far
EndProject EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FarmmapsBulkSatDownload", "FarmmapsBulkSatDownload\FarmmapsBulkSatDownload.csproj", "{772DBDCD-9FAA-40A7-8551-2C1620C4AB67}" Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FarmmapsBulkSatDownload", "FarmmapsBulkSatDownload\FarmmapsBulkSatDownload.csproj", "{772DBDCD-9FAA-40A7-8551-2C1620C4AB67}"
EndProject EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Secrets", "Secrets\Secrets.csproj", "{C4EE5ECA-253A-4B71-9F67-D231AC4517D6}" Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Secrets", "Secrets\Secrets.csproj", "{C4EE5ECA-253A-4B71-9F67-D231AC4517D6}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FarmmapsCleanUp", "FarmmapsCleanUp\FarmmapsCleanUp.csproj", "{5E4387F9-5953-4A9B-BCA5-DF3964EED3CB}"
EndProject EndProject
Global Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution GlobalSection(SolutionConfigurationPlatforms) = preSolution
@ -75,6 +77,10 @@ Global
{C4EE5ECA-253A-4B71-9F67-D231AC4517D6}.Debug|Any CPU.Build.0 = Debug|Any CPU {C4EE5ECA-253A-4B71-9F67-D231AC4517D6}.Debug|Any CPU.Build.0 = Debug|Any CPU
{C4EE5ECA-253A-4B71-9F67-D231AC4517D6}.Release|Any CPU.ActiveCfg = Release|Any CPU {C4EE5ECA-253A-4B71-9F67-D231AC4517D6}.Release|Any CPU.ActiveCfg = Release|Any CPU
{C4EE5ECA-253A-4B71-9F67-D231AC4517D6}.Release|Any CPU.Build.0 = Release|Any CPU {C4EE5ECA-253A-4B71-9F67-D231AC4517D6}.Release|Any CPU.Build.0 = Release|Any CPU
{5E4387F9-5953-4A9B-BCA5-DF3964EED3CB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{5E4387F9-5953-4A9B-BCA5-DF3964EED3CB}.Debug|Any CPU.Build.0 = Debug|Any CPU
{5E4387F9-5953-4A9B-BCA5-DF3964EED3CB}.Release|Any CPU.ActiveCfg = Release|Any CPU
{5E4387F9-5953-4A9B-BCA5-DF3964EED3CB}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection EndGlobalSection
GlobalSection(SolutionProperties) = preSolution GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE HideSolutionNode = FALSE

64
FarmmapsBulkSatDownload/BulkSatDownloadApplication.cs
View File

@ -57,10 +57,10 @@ namespace FarmmapsBulkSatDownload
// Crashes if "DBsettings.secrets.json" is absent or empty // Crashes if "DBsettings.secrets.json" is absent or empty
DB dbparcels = JsonConvert.DeserializeObject<DB>(File.ReadAllText("DBsettings.secrets.json")); DB dbparcels = JsonConvert.DeserializeObject<DB>(File.ReadAllText("DBsettings.secrets.json"));
string schemaname = "bigdata"; string schemaname = "bigdata";
string parceltablename = "parcel_flowerbulbs";//"parcelsijbrandij" "parcel"; "parcel_flowerbulbs"; "parcel_disac" string parceltablename = "parcel_bollenrevolutie_tulips2020"; //"parcelsijbrandij" "parcel"; "parcel_flowerbulbs"; "parcel_disac"; ""parcel_bollenrevolutie_tulips2020""
string groenmonitortablename = "groenmonitor_flowerbulbs";//"groenmonitorsijbrandij" "groenmonitor" "groenmonitor_flowerbulbs" "groenmonitor_disac" string groenmonitortablename = "groenmonitor_bollenrevolutie_tulips2020"; //"groenmonitorsijbrandij" "groenmonitor" "groenmonitor_flowerbulbs" "groenmonitor_disac" "groenmonitor_bollenrevolutie_tulips2020"
// The view 'groenmonitorlatestviewname' contains per parcelid (arbid) the year in which it "exists" and the date of the latest image downloaded. It is used to prevent unneccessary downloading of image statistics already in the database // The view 'groenmonitorlatestviewname' contains per parcelid (arbid) the year in which it "exists" and the date of the latest image downloaded. It is used to prevent unneccessary downloading of image statistics already in the database
string groenmonitorlatestviewname = "groenmonitorlatest_flowerbulbs"; //"groenmonitorsijbrandijlatest" "groenmonitorlatest" "groenmonitorlatest_flowerbulbs" "groenmonitorlatest_disac" string groenmonitorlatestviewname = "groenmonitorlatest_bollenrevolutie_tulips2020"; //"groenmonitorsijbrandijlatest" "groenmonitorlatest" "groenmonitorlatest_flowerbulbs" "groenmonitorlatest_disac" "groenmonitorlatest_bollenrevolutie_tulips2020"
// Database query and connection. Geometry must be in WGS84 coordinate system, EPSG 4326 // Database query and connection. Geometry must be in WGS84 coordinate system, EPSG 4326
// Apparently the FarmmapsApi cannot handle MultiPolygon, so we need to convert to single Polygon // Apparently the FarmmapsApi cannot handle MultiPolygon, so we need to convert to single Polygon
@ -80,11 +80,10 @@ SELECT pt.arbid, pt.year, gml.lastwenrdate, ST_AsGeoJSON(ST_Transform((ST_DUMP(p
CASE WHEN pt.year >= DATE_PART('year', CURRENT_DATE) THEN '' ELSE COALESCE(pt.satellitetaskcode,'') END AS satellitetaskcode CASE WHEN pt.year >= DATE_PART('year', CURRENT_DATE) THEN '' ELSE COALESCE(pt.satellitetaskcode,'') END AS satellitetaskcode
FROM {0}.{1} pt, {0}.{2} gml FROM {0}.{1} pt, {0}.{2} gml
WHERE WHERE
pt.arbid = gml.arbid AND pt.arbid = gml.arbid
pt.crop NOT IN ('Lelie','Tulp') AND AND pt.satellitetaskcode IS NULL
pt.year > 2018 AND pt.arbid IN(8276,8314,8315)
ORDER BY pt.arbid ORDER BY pt.arbid
LIMIT 2000;", schemaname, parceltablename, groenmonitorlatestviewname); //LIMIT x for testing LIMIT 5;", schemaname, parceltablename, groenmonitorlatestviewname); //LIMIT x for testing
using (NpgsqlConnection connection = new NpgsqlConnection(connectionString)) using (NpgsqlConnection connection = new NpgsqlConnection(connectionString))
{ {
@ -117,30 +116,30 @@ LIMIT 2000;", schemaname, parceltablename, groenmonitorlatestviewname); //LIMI
// Option 2: Example without database. Comment out this part if you want to use database // Option 2: Example without database. Comment out this part if you want to use database
// Read cropfields "BulkSatDownloadInput.json" and write all stats to a single csv file // Read cropfields "BulkSatDownloadInput.json" and write all stats to a single csv file
// Write all stats for multiple fields will be written to a single csv file // Write all stats for multiple fields will be written to a single csv file
string downloadFolder; //string downloadFolder;
string fileNameStats; //string fileNameStats;
string headerLineStats = $"FieldName,satelliteDate,satelliteBand,max,min,mean,mode,median,stddev,minPlus,curtosis,maxMinus,skewness,variance,populationCount,variationCoefficient,confidenceIntervalLow, confidenceIntervalHigh,confidenceIntervalErrorMargin" + Environment.NewLine; //string headerLineStats = $"FieldName,satelliteDate,satelliteBand,max,min,mean,mode,median,stddev,minPlus,curtosis,maxMinus,skewness,variance,populationCount,variationCoefficient,confidenceIntervalLow, confidenceIntervalHigh,confidenceIntervalErrorMargin" + Environment.NewLine;
var fieldsInputJson = File.ReadAllText("BulkSatDownloadInput.json"); //var fieldsInputJson = File.ReadAllText("BulkSatDownloadInput.json");
bulkSatDownloadInputListCsv = JsonConvert.DeserializeObject<List<BulkSatDownloadInput>>(fieldsInputJson); //bulkSatDownloadInputListCsv = JsonConvert.DeserializeObject<List<BulkSatDownloadInput>>(fieldsInputJson);
for (int i = 0; i < bulkSatDownloadInputListCsv.Count; i++) //for (int i = 0; i < bulkSatDownloadInputListCsv.Count; i++)
{ //{
downloadFolder = bulkSatDownloadInputListCsv[i].downloadFolder; // downloadFolder = bulkSatDownloadInputListCsv[i].downloadFolder;
fileNameStats = Path.Combine(downloadFolder, bulkSatDownloadInputListCsv[i].fileNameStats); // fileNameStats = Path.Combine(downloadFolder, bulkSatDownloadInputListCsv[i].fileNameStats);
if (!Directory.Exists(downloadFolder)) // if (!Directory.Exists(downloadFolder))
Directory.CreateDirectory(downloadFolder); // Directory.CreateDirectory(downloadFolder);
bulkSatDownloadInputListCsv[i].fileNameStats = fileNameStats; // bulkSatDownloadInputListCsv[i].fileNameStats = fileNameStats;
// Header same as in GeneralService.DownloadSatelliteStats // // Header same as in GeneralService.DownloadSatelliteStats
// Delete fileNameStats if existing. Create a new file. Add a header to csv file // // Delete fileNameStats if existing. Create a new file. Add a header to csv file
File.Delete(fileNameStats); // File.Delete(fileNameStats);
File.AppendAllText(fileNameStats, headerLineStats); // File.AppendAllText(fileNameStats, headerLineStats);
} //}
// Now choose which list you want to use // Now choose which list you want to use
bulkSatDownloadInputList = bulkSatDownloadInputListDB; //bulkSatDownloadInputListDB; //bulkSatDownloadInputListCsv; bulkSatDownloadInputList = bulkSatDownloadInputListDB; //bulkSatDownloadInputListDB; //bulkSatDownloadInputListCsv;
// Whichever option (database or json/csv), continue here // Whichever option (database or json/csv), continue here
// Delete the settingsfile // Delete the settingsfile
File.Delete(settingsfile); // File.Delete(settingsfile);
// For each input download all images. Keep track to time, important when doing bulk downloads // For each input download all images. Keep track to time, important when doing bulk downloads
var watch = System.Diagnostics.Stopwatch.StartNew(); var watch = System.Diagnostics.Stopwatch.StartNew();
@ -157,13 +156,26 @@ LIMIT 2000;", schemaname, parceltablename, groenmonitorlatestviewname); //LIMI
try try
{ {
await Process(roots, bulkSatDownloadInput); await Process(roots, bulkSatDownloadInput);
} }
catch (Exception ex) catch (Exception ex)
{ {
_logger.LogError(ex.Message); _logger.LogError(ex.Message);
} }
watch.Stop(); watch.Stop();
// add this downloadtime to the cropfieldtable.
// Only if cropfieldtable has a field called 'downloadtime' type 'time'!
//using (NpgsqlConnection connection = new NpgsqlConnection(dbparcels.GetConnectionString()))
//{
// connection.Open();
// NpgsqlCommand updateCmd = connection.CreateCommand();
// string updateSql = string.Format($"UPDATE {schemaname}.{parceltablename} SET downloadtime = '{strTime(watch.Elapsed)}' WHERE arbid = {bulkSatDownloadInput.fieldID};");
// updateCmd.CommandText = updateSql;
// int r = updateCmd.ExecuteNonQuery();
// if (r != 1)
// throw new Exception("// FarmmapsBulkSatDownload: Update downloadtime Failed");
// connection.Close();
//}
//_logger.LogInformation($"// FarmmapsBulkSatDownload: Added downloadtime = '{strTime(watch.Elapsed)}' to {schemaname}.{parceltablename} ");
tsSofar = tsSofar + watch.Elapsed; tsSofar = tsSofar + watch.Elapsed;
tsTotalEstimated = tsSofar / (i + 1) * bulkSatDownloadInputList.Count; tsTotalEstimated = tsSofar / (i + 1) * bulkSatDownloadInputList.Count;
tsRemaining = tsTotalEstimated - tsSofar; tsRemaining = tsTotalEstimated - tsSofar;

143
FarmmapsCleanUp/CleanUpApplication.cs Normal file
View File

@ -0,0 +1,143 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Threading.Tasks;
using FarmmapsApi;
using FarmmapsApi.Models;
using FarmmapsApi.Services;
using FarmmapsBulkSatDownload.Models;
using Microsoft.Extensions.Logging;
using Newtonsoft.Json;
using Npgsql;
using Newtonsoft.Json.Linq;
using static FarmmapsApiSamples.Constants;
namespace FarmmapsCleanup
{
public class CleanupApplication : IApplication
{
private readonly ILogger<CleanupApplication> _logger;
private readonly FarmmapsApiService _farmmapsApiService;
private readonly CleanupService _cleanupService;
private readonly GeneralService _generalService;
public CleanupApplication(ILogger<CleanupApplication> logger, FarmmapsApiService farmmapsApiService,
GeneralService generalService, CleanupService cleanupService)
{
_logger = logger;
_farmmapsApiService = farmmapsApiService;
_generalService = generalService;
_cleanupService = cleanupService;
}
public async Task RunAsync()
{
// !! this call is needed the first time an api is called with a fresh clientid and secret !!
await _farmmapsApiService.GetCurrentUserCodeAsync();
var roots = await _farmmapsApiService.GetCurrentUserRootsAsync();
//Establish a database connection. Expecting you are using same database server as in project FarmmapsBulkSatDownload
DB dbparcels = JsonConvert.DeserializeObject<DB>(File.ReadAllText("DBsettings.secrets.json"));
string schemaname = "bigdata";
string parceltablename = "parcel_bollenrevolutie_tulips2020"; //"parcelsijbrandij" "parcel"; "parcel_flowerbulbs"; "parcel_disac"; ""parcel_bollenrevolutie_tulips2020""
//Query to get lists of items to delete from FarmmapsDatabase and from parceltablename
string cropfielditemcode;
string satellitetaskcode;
List<string> cropfieldItemCodes = new List<string>();
List<string> satellitetaskCodes = new List<string>();
string connectionString = dbparcels.GetConnectionString();
string readSql = string.Format(
@"
SELECT pt.cropfielditemcode, pt.satellitetaskcode
FROM {0}.{1} pt
WHERE
pt.arbid IN(1,2)
ORDER BY pt.arbid
;", schemaname, parceltablename);
string updateCropfieldItemCodesSql = string.Format(
@"
UPDATE {0}.{1}
SET cropfielditemcode=NULL
WHERE arbid IN(1,2)
;", schemaname, parceltablename); //Same WHERE AS above
string updateSatellitetaskCodesSql = string.Format(
@"
UPDATE {0}.{1}
SET satellitetaskcode=NULL
WHERE arbid IN(1,2)
;", schemaname, parceltablename); //Same WHERE AS above
using (NpgsqlConnection connection = new NpgsqlConnection(connectionString))
{
connection.Open();
// Read data (run query) = build a list of fields for which to download images
NpgsqlCommand command = connection.CreateCommand();
command.CommandText = readSql;
NpgsqlDataReader dr = command.ExecuteReader();
while (dr.Read())
{
cropfielditemcode = dr.GetString(0);
satellitetaskcode = dr.GetString(1);
if(!string.IsNullOrEmpty(cropfielditemcode))
cropfieldItemCodes.Add(cropfielditemcode);
if (!string.IsNullOrEmpty(satellitetaskcode))
satellitetaskCodes.Add(satellitetaskcode);
}
connection.Close();
}
_logger.LogWarning($"// FarmmapsCleanUp: WARNING: you are about to delete {cropfieldItemCodes.Count} cropfieldItemCodes and {satellitetaskCodes.Count} satellitetaskCodes from the FarmMaps database and your own table {schemaname}.{parceltablename}");
_logger.LogInformation($"// Nice of you to clean up after the work is done. You would typically do this for cropfieldItemCodes used only once.");
_logger.LogInformation($"// You do NOT want to do this if you think you may later on still want to use these items.");
_logger.LogInformation($"// Please carefully check the SQL queries 'readSql' 'updateSql' in CleanUpApplication.cs before proceeding.");
_logger.LogInformation($"// FarmmapsCleanUp: delete selected cropfieldItemCodes from FarmMaps database and table {schemaname}.{parceltablename}? 0 = no, 1 = yes");
int i;
i = Int32.Parse(Console.ReadLine());
if (i == 1)
{
await _farmmapsApiService.DeleteItemsAsync(cropfieldItemCodes);
//TODO _farmmapsApiService.DeleteItemsAsync throws an error: {StatusCode: 415, ReasonPhrase: 'Unsupported Media Type', Version: 1.1, Content: System.Net.Http.HttpConnectionResponseContent, Headers: {...}}
//what is wrong with cropfieldItemCodes?
//and shouldn't we be telling _farmmapsApiService.DeleteItemsAsync what item type to delete?
using (NpgsqlConnection connection = new NpgsqlConnection(connectionString))
{
connection.Open();
NpgsqlCommand updateCmd = connection.CreateCommand();
updateCmd.CommandText = updateCropfieldItemCodesSql;
int r = updateCmd.ExecuteNonQuery();
if (r == -1)
throw new Exception("// FarmmapsCleanUp: Update cropfielditemcode Failed");
connection.Close();
}
}
_logger.LogInformation($"// FarmmapsCleanUp: delete selected satellitetaskCodes from FarmMaps database and table {schemaname}.{parceltablename}? 0 = no, 1 = yes");
i = Int32.Parse(Console.ReadLine());
if (i == 1)
{
await _farmmapsApiService.DeleteItemsAsync(satellitetaskCodes);
//TODO _farmmapsApiService.DeleteItemsAsync throws an error: {StatusCode: 415, ReasonPhrase: 'Unsupported Media Type', Version: 1.1, Content: System.Net.Http.HttpConnectionResponseContent, Headers: {...}}
//what is wrong with satellitetaskCodes?
//and shouldn't we be telling _farmmapsApiService.DeleteItemsAsync what item type to delete?
using (NpgsqlConnection connection = new NpgsqlConnection(connectionString))
{
connection.Open();
NpgsqlCommand updateCmd = connection.CreateCommand();
updateCmd.CommandText = updateSatellitetaskCodesSql;
int r = updateCmd.ExecuteNonQuery();
if (r == -1)
throw new Exception("// FarmmapsCleanUp: Update cropfielditemcode Failed");
connection.Close();
}
}
_logger.LogInformation($"// FarmmapsCleanUp: done! Hit any key to exit ...");
Console.ReadKey();
}
}
}

28
FarmmapsCleanUp/CleanUpService.cs Normal file
View File

@ -0,0 +1,28 @@
using System;
using System.Globalization;
using System.Threading.Tasks;
using FarmmapsApi.Models;
using FarmmapsApi.Services;
using Microsoft.Extensions.Logging;
using static FarmmapsApi.Extensions;
using static FarmmapsApiSamples.Constants;
namespace FarmmapsCleanup
{
public class CleanupService
{
private readonly ILogger<CleanupService> _logger;
private readonly FarmmapsApiService _farmmapsApiService;
private readonly GeneralService _generalService;
public CleanupService(ILogger<CleanupService> logger, FarmmapsApiService farmmapsApiService,
GeneralService generalService)
{
_logger = logger;
_farmmapsApiService = farmmapsApiService;
_generalService = generalService;
}
}
}

23
FarmmapsCleanUp/FarmmapsCleanUp.csproj Normal file
View File

@ -0,0 +1,23 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>netcoreapp3.1</TargetFramework>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Npgsql" Version="5.0.5" />
</ItemGroup>
<ItemGroup>
<None Update="Data\**\*">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</None>
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\FarmmapsApi\FarmmapsApi.csproj" />
<ProjectReference Include="..\FarmmapsBulkSatDownload\FarmmapsBulkSatDownload.csproj" />
</ItemGroup>
</Project>

21
FarmmapsCleanUp/Program.cs Normal file
View File

@ -0,0 +1,21 @@
using System.Threading.Tasks;
using FarmmapsApi;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
namespace FarmmapsCleanup
{
class Program : FarmmapsProgram<CleanupApplication>
{
private static async Task Main(string[] args)
{
await new Program().Start(args);
}
protected override void Configure(IServiceCollection serviceCollection)
{
serviceCollection.AddLogging()
.AddTransient<CleanupService>();
}
}
}