diff --git a/FarmmapsApiSamples.sln b/FarmmapsApiSamples.sln index f7bf4b2..e44d74d 100644 --- a/FarmmapsApiSamples.sln +++ b/FarmmapsApiSamples.sln @@ -27,7 +27,9 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FarmmapsDataDownload", "Far EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FarmmapsBulkSatDownload", "FarmmapsBulkSatDownload\FarmmapsBulkSatDownload.csproj", "{772DBDCD-9FAA-40A7-8551-2C1620C4AB67}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Secrets", "Secrets\Secrets.csproj", "{C4EE5ECA-253A-4B71-9F67-D231AC4517D6}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Secrets", "Secrets\Secrets.csproj", "{C4EE5ECA-253A-4B71-9F67-D231AC4517D6}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FarmmapsCleanUp", "FarmmapsCleanUp\FarmmapsCleanUp.csproj", "{5E4387F9-5953-4A9B-BCA5-DF3964EED3CB}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -75,6 +77,10 @@ Global {C4EE5ECA-253A-4B71-9F67-D231AC4517D6}.Debug|Any CPU.Build.0 = Debug|Any CPU {C4EE5ECA-253A-4B71-9F67-D231AC4517D6}.Release|Any CPU.ActiveCfg = Release|Any CPU {C4EE5ECA-253A-4B71-9F67-D231AC4517D6}.Release|Any CPU.Build.0 = Release|Any CPU + {5E4387F9-5953-4A9B-BCA5-DF3964EED3CB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {5E4387F9-5953-4A9B-BCA5-DF3964EED3CB}.Debug|Any CPU.Build.0 = Debug|Any CPU + {5E4387F9-5953-4A9B-BCA5-DF3964EED3CB}.Release|Any CPU.ActiveCfg = Release|Any CPU + {5E4387F9-5953-4A9B-BCA5-DF3964EED3CB}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/FarmmapsBulkSatDownload/BulkSatDownloadApplication.cs b/FarmmapsBulkSatDownload/BulkSatDownloadApplication.cs index 7017f46..52ed864 100644 --- a/FarmmapsBulkSatDownload/BulkSatDownloadApplication.cs +++ b/FarmmapsBulkSatDownload/BulkSatDownloadApplication.cs @@ -57,10 +57,10 @@ namespace FarmmapsBulkSatDownload // Crashes if "DBsettings.secrets.json" is absent or empty DB dbparcels = JsonConvert.DeserializeObject(File.ReadAllText("DBsettings.secrets.json")); string schemaname = "bigdata"; - string parceltablename = "parcel_flowerbulbs";//"parcelsijbrandij" "parcel"; "parcel_flowerbulbs"; "parcel_disac" - string groenmonitortablename = "groenmonitor_flowerbulbs";//"groenmonitorsijbrandij" "groenmonitor" "groenmonitor_flowerbulbs" "groenmonitor_disac" + string parceltablename = "parcel_bollenrevolutie_tulips2020"; //"parcelsijbrandij" "parcel"; "parcel_flowerbulbs"; "parcel_disac"; ""parcel_bollenrevolutie_tulips2020"" + string groenmonitortablename = "groenmonitor_bollenrevolutie_tulips2020"; //"groenmonitorsijbrandij" "groenmonitor" "groenmonitor_flowerbulbs" "groenmonitor_disac" "groenmonitor_bollenrevolutie_tulips2020" // The view 'groenmonitorlatestviewname' contains per parcelid (arbid) the year in which it "exists" and the date of the latest image downloaded. It is used to prevent unneccessary downloading of image statistics already in the database - string groenmonitorlatestviewname = "groenmonitorlatest_flowerbulbs"; //"groenmonitorsijbrandijlatest" "groenmonitorlatest" "groenmonitorlatest_flowerbulbs" "groenmonitorlatest_disac" + string groenmonitorlatestviewname = "groenmonitorlatest_bollenrevolutie_tulips2020"; //"groenmonitorsijbrandijlatest" "groenmonitorlatest" "groenmonitorlatest_flowerbulbs" "groenmonitorlatest_disac" "groenmonitorlatest_bollenrevolutie_tulips2020" // Database query and connection. Geometry must be in WGS84 coordinate system, EPSG 4326 // Apparently the FarmmapsApi cannot handle MultiPolygon, so we need to convert to single Polygon @@ -80,11 +80,10 @@ SELECT pt.arbid, pt.year, gml.lastwenrdate, ST_AsGeoJSON(ST_Transform((ST_DUMP(p CASE WHEN pt.year >= DATE_PART('year', CURRENT_DATE) THEN '' ELSE COALESCE(pt.satellitetaskcode,'') END AS satellitetaskcode FROM {0}.{1} pt, {0}.{2} gml WHERE - pt.arbid = gml.arbid AND - pt.crop NOT IN ('Lelie','Tulp') AND - pt.year > 2018 AND pt.arbid IN(8276,8314,8315) + pt.arbid = gml.arbid + AND pt.satellitetaskcode IS NULL ORDER BY pt.arbid -LIMIT 2000;", schemaname, parceltablename, groenmonitorlatestviewname); //LIMIT x for testing +LIMIT 5;", schemaname, parceltablename, groenmonitorlatestviewname); //LIMIT x for testing using (NpgsqlConnection connection = new NpgsqlConnection(connectionString)) { @@ -117,30 +116,30 @@ LIMIT 2000;", schemaname, parceltablename, groenmonitorlatestviewname); //LIMI // Option 2: Example without database. Comment out this part if you want to use database // Read cropfields "BulkSatDownloadInput.json" and write all stats to a single csv file // Write all stats for multiple fields will be written to a single csv file - string downloadFolder; - string fileNameStats; - string headerLineStats = $"FieldName,satelliteDate,satelliteBand,max,min,mean,mode,median,stddev,minPlus,curtosis,maxMinus,skewness,variance,populationCount,variationCoefficient,confidenceIntervalLow, confidenceIntervalHigh,confidenceIntervalErrorMargin" + Environment.NewLine; - var fieldsInputJson = File.ReadAllText("BulkSatDownloadInput.json"); - bulkSatDownloadInputListCsv = JsonConvert.DeserializeObject>(fieldsInputJson); - for (int i = 0; i < bulkSatDownloadInputListCsv.Count; i++) - { - downloadFolder = bulkSatDownloadInputListCsv[i].downloadFolder; - fileNameStats = Path.Combine(downloadFolder, bulkSatDownloadInputListCsv[i].fileNameStats); - if (!Directory.Exists(downloadFolder)) - Directory.CreateDirectory(downloadFolder); - bulkSatDownloadInputListCsv[i].fileNameStats = fileNameStats; - // Header same as in GeneralService.DownloadSatelliteStats - // Delete fileNameStats if existing. Create a new file. Add a header to csv file - File.Delete(fileNameStats); - File.AppendAllText(fileNameStats, headerLineStats); - } + //string downloadFolder; + //string fileNameStats; + //string headerLineStats = $"FieldName,satelliteDate,satelliteBand,max,min,mean,mode,median,stddev,minPlus,curtosis,maxMinus,skewness,variance,populationCount,variationCoefficient,confidenceIntervalLow, confidenceIntervalHigh,confidenceIntervalErrorMargin" + Environment.NewLine; + //var fieldsInputJson = File.ReadAllText("BulkSatDownloadInput.json"); + //bulkSatDownloadInputListCsv = JsonConvert.DeserializeObject>(fieldsInputJson); + //for (int i = 0; i < bulkSatDownloadInputListCsv.Count; i++) + //{ + // downloadFolder = bulkSatDownloadInputListCsv[i].downloadFolder; + // fileNameStats = Path.Combine(downloadFolder, bulkSatDownloadInputListCsv[i].fileNameStats); + // if (!Directory.Exists(downloadFolder)) + // Directory.CreateDirectory(downloadFolder); + // bulkSatDownloadInputListCsv[i].fileNameStats = fileNameStats; + // // Header same as in GeneralService.DownloadSatelliteStats + // // Delete fileNameStats if existing. Create a new file. Add a header to csv file + // File.Delete(fileNameStats); + // File.AppendAllText(fileNameStats, headerLineStats); + //} // Now choose which list you want to use bulkSatDownloadInputList = bulkSatDownloadInputListDB; //bulkSatDownloadInputListDB; //bulkSatDownloadInputListCsv; // Whichever option (database or json/csv), continue here // Delete the settingsfile - File.Delete(settingsfile); + // File.Delete(settingsfile); // For each input download all images. Keep track to time, important when doing bulk downloads var watch = System.Diagnostics.Stopwatch.StartNew(); @@ -157,13 +156,26 @@ LIMIT 2000;", schemaname, parceltablename, groenmonitorlatestviewname); //LIMI try { await Process(roots, bulkSatDownloadInput); - } catch (Exception ex) { _logger.LogError(ex.Message); } watch.Stop(); + // add this downloadtime to the cropfieldtable. + // Only if cropfieldtable has a field called 'downloadtime' type 'time'! + //using (NpgsqlConnection connection = new NpgsqlConnection(dbparcels.GetConnectionString())) + //{ + // connection.Open(); + // NpgsqlCommand updateCmd = connection.CreateCommand(); + // string updateSql = string.Format($"UPDATE {schemaname}.{parceltablename} SET downloadtime = '{strTime(watch.Elapsed)}' WHERE arbid = {bulkSatDownloadInput.fieldID};"); + // updateCmd.CommandText = updateSql; + // int r = updateCmd.ExecuteNonQuery(); + // if (r != 1) + // throw new Exception("// FarmmapsBulkSatDownload: Update downloadtime Failed"); + // connection.Close(); + //} + //_logger.LogInformation($"// FarmmapsBulkSatDownload: Added downloadtime = '{strTime(watch.Elapsed)}' to {schemaname}.{parceltablename} "); tsSofar = tsSofar + watch.Elapsed; tsTotalEstimated = tsSofar / (i + 1) * bulkSatDownloadInputList.Count; tsRemaining = tsTotalEstimated - tsSofar; diff --git a/FarmmapsCleanUp/CleanUpApplication.cs b/FarmmapsCleanUp/CleanUpApplication.cs new file mode 100644 index 0000000..ab95e24 --- /dev/null +++ b/FarmmapsCleanUp/CleanUpApplication.cs @@ -0,0 +1,143 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Threading.Tasks; +using FarmmapsApi; +using FarmmapsApi.Models; +using FarmmapsApi.Services; +using FarmmapsBulkSatDownload.Models; +using Microsoft.Extensions.Logging; +using Newtonsoft.Json; +using Npgsql; +using Newtonsoft.Json.Linq; +using static FarmmapsApiSamples.Constants; + +namespace FarmmapsCleanup +{ + public class CleanupApplication : IApplication + { + private readonly ILogger _logger; + private readonly FarmmapsApiService _farmmapsApiService; + private readonly CleanupService _cleanupService; + private readonly GeneralService _generalService; + + public CleanupApplication(ILogger logger, FarmmapsApiService farmmapsApiService, + GeneralService generalService, CleanupService cleanupService) + { + _logger = logger; + _farmmapsApiService = farmmapsApiService; + _generalService = generalService; + _cleanupService = cleanupService; + } + + public async Task RunAsync() + { + // !! this call is needed the first time an api is called with a fresh clientid and secret !! + await _farmmapsApiService.GetCurrentUserCodeAsync(); + var roots = await _farmmapsApiService.GetCurrentUserRootsAsync(); + + //Establish a database connection. Expecting you are using same database server as in project FarmmapsBulkSatDownload + DB dbparcels = JsonConvert.DeserializeObject(File.ReadAllText("DBsettings.secrets.json")); + string schemaname = "bigdata"; + string parceltablename = "parcel_bollenrevolutie_tulips2020"; //"parcelsijbrandij" "parcel"; "parcel_flowerbulbs"; "parcel_disac"; ""parcel_bollenrevolutie_tulips2020"" + + //Query to get lists of items to delete from FarmmapsDatabase and from parceltablename + string cropfielditemcode; + string satellitetaskcode; + List cropfieldItemCodes = new List(); + List satellitetaskCodes = new List(); + string connectionString = dbparcels.GetConnectionString(); + string readSql = string.Format( +@" +SELECT pt.cropfielditemcode, pt.satellitetaskcode +FROM {0}.{1} pt +WHERE + pt.arbid IN(1,2) +ORDER BY pt.arbid +;", schemaname, parceltablename); + + string updateCropfieldItemCodesSql = string.Format( +@" +UPDATE {0}.{1} + SET cropfielditemcode=NULL + WHERE arbid IN(1,2) +;", schemaname, parceltablename); //Same WHERE AS above + string updateSatellitetaskCodesSql = string.Format( +@" +UPDATE {0}.{1} + SET satellitetaskcode=NULL + WHERE arbid IN(1,2) +;", schemaname, parceltablename); //Same WHERE AS above + + using (NpgsqlConnection connection = new NpgsqlConnection(connectionString)) + { + connection.Open(); + + // Read data (run query) = build a list of fields for which to download images + NpgsqlCommand command = connection.CreateCommand(); + command.CommandText = readSql; + NpgsqlDataReader dr = command.ExecuteReader(); + while (dr.Read()) + { + cropfielditemcode = dr.GetString(0); + satellitetaskcode = dr.GetString(1); + if(!string.IsNullOrEmpty(cropfielditemcode)) + cropfieldItemCodes.Add(cropfielditemcode); + if (!string.IsNullOrEmpty(satellitetaskcode)) + satellitetaskCodes.Add(satellitetaskcode); + } + connection.Close(); + } + _logger.LogWarning($"// FarmmapsCleanUp: WARNING: you are about to delete {cropfieldItemCodes.Count} cropfieldItemCodes and {satellitetaskCodes.Count} satellitetaskCodes from the FarmMaps database and your own table {schemaname}.{parceltablename}"); + _logger.LogInformation($"// Nice of you to clean up after the work is done. You would typically do this for cropfieldItemCodes used only once."); + _logger.LogInformation($"// You do NOT want to do this if you think you may later on still want to use these items."); + _logger.LogInformation($"// Please carefully check the SQL queries 'readSql' 'updateSql' in CleanUpApplication.cs before proceeding."); + + _logger.LogInformation($"// FarmmapsCleanUp: delete selected cropfieldItemCodes from FarmMaps database and table {schemaname}.{parceltablename}? 0 = no, 1 = yes"); + int i; + i = Int32.Parse(Console.ReadLine()); + if (i == 1) + { + await _farmmapsApiService.DeleteItemsAsync(cropfieldItemCodes); + //TODO _farmmapsApiService.DeleteItemsAsync throws an error: {StatusCode: 415, ReasonPhrase: 'Unsupported Media Type', Version: 1.1, Content: System.Net.Http.HttpConnectionResponseContent, Headers: {...}} + //what is wrong with cropfieldItemCodes? + //and shouldn't we be telling _farmmapsApiService.DeleteItemsAsync what item type to delete? + using (NpgsqlConnection connection = new NpgsqlConnection(connectionString)) + { + connection.Open(); + NpgsqlCommand updateCmd = connection.CreateCommand(); + updateCmd.CommandText = updateCropfieldItemCodesSql; + int r = updateCmd.ExecuteNonQuery(); + if (r == -1) + throw new Exception("// FarmmapsCleanUp: Update cropfielditemcode Failed"); + connection.Close(); + } + } + _logger.LogInformation($"// FarmmapsCleanUp: delete selected satellitetaskCodes from FarmMaps database and table {schemaname}.{parceltablename}? 0 = no, 1 = yes"); + i = Int32.Parse(Console.ReadLine()); + if (i == 1) + { + await _farmmapsApiService.DeleteItemsAsync(satellitetaskCodes); + //TODO _farmmapsApiService.DeleteItemsAsync throws an error: {StatusCode: 415, ReasonPhrase: 'Unsupported Media Type', Version: 1.1, Content: System.Net.Http.HttpConnectionResponseContent, Headers: {...}} + //what is wrong with satellitetaskCodes? + //and shouldn't we be telling _farmmapsApiService.DeleteItemsAsync what item type to delete? + using (NpgsqlConnection connection = new NpgsqlConnection(connectionString)) + { + connection.Open(); + NpgsqlCommand updateCmd = connection.CreateCommand(); + updateCmd.CommandText = updateSatellitetaskCodesSql; + int r = updateCmd.ExecuteNonQuery(); + if (r == -1) + throw new Exception("// FarmmapsCleanUp: Update cropfielditemcode Failed"); + connection.Close(); + } + } + _logger.LogInformation($"// FarmmapsCleanUp: done! Hit any key to exit ..."); + Console.ReadKey(); + + + } + } +} diff --git a/FarmmapsCleanUp/CleanUpService.cs b/FarmmapsCleanUp/CleanUpService.cs new file mode 100644 index 0000000..b569e19 --- /dev/null +++ b/FarmmapsCleanUp/CleanUpService.cs @@ -0,0 +1,28 @@ +using System; +using System.Globalization; +using System.Threading.Tasks; +using FarmmapsApi.Models; +using FarmmapsApi.Services; +using Microsoft.Extensions.Logging; +using static FarmmapsApi.Extensions; +using static FarmmapsApiSamples.Constants; + +namespace FarmmapsCleanup +{ + public class CleanupService + { + private readonly ILogger _logger; + private readonly FarmmapsApiService _farmmapsApiService; + private readonly GeneralService _generalService; + + public CleanupService(ILogger logger, FarmmapsApiService farmmapsApiService, + GeneralService generalService) + { + _logger = logger; + _farmmapsApiService = farmmapsApiService; + _generalService = generalService; + } + + + } +} \ No newline at end of file diff --git a/FarmmapsCleanUp/FarmmapsCleanUp.csproj b/FarmmapsCleanUp/FarmmapsCleanUp.csproj new file mode 100644 index 0000000..74cf8ea --- /dev/null +++ b/FarmmapsCleanUp/FarmmapsCleanUp.csproj @@ -0,0 +1,23 @@ + + + + Exe + netcoreapp3.1 + + + + + + + + + Always + + + + + + + + + diff --git a/FarmmapsCleanUp/Program.cs b/FarmmapsCleanUp/Program.cs new file mode 100644 index 0000000..182d69e --- /dev/null +++ b/FarmmapsCleanUp/Program.cs @@ -0,0 +1,21 @@ +using System.Threading.Tasks; +using FarmmapsApi; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; + +namespace FarmmapsCleanup +{ + class Program : FarmmapsProgram + { + private static async Task Main(string[] args) + { + await new Program().Start(args); + } + + protected override void Configure(IServiceCollection serviceCollection) + { + serviceCollection.AddLogging() + .AddTransient(); + } + } +} \ No newline at end of file