new project FarmmapsCleanUp to clean up cropfielditems and satellitetaskcodes used only once

FarmmapsApiSamples.sln

@@ -27,7 +27,9 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FarmmapsDataDownload", "Far
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FarmmapsBulkSatDownload", "FarmmapsBulkSatDownload\FarmmapsBulkSatDownload.csproj", "{772DBDCD-9FAA-40A7-8551-2C1620C4AB67}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Secrets", "Secrets\Secrets.csproj", "{C4EE5ECA-253A-4B71-9F67-D231AC4517D6}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Secrets", "Secrets\Secrets.csproj", "{C4EE5ECA-253A-4B71-9F67-D231AC4517D6}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FarmmapsCleanUp", "FarmmapsCleanUp\FarmmapsCleanUp.csproj", "{5E4387F9-5953-4A9B-BCA5-DF3964EED3CB}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
@@ -75,6 +77,10 @@ Global
 		{C4EE5ECA-253A-4B71-9F67-D231AC4517D6}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{C4EE5ECA-253A-4B71-9F67-D231AC4517D6}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{C4EE5ECA-253A-4B71-9F67-D231AC4517D6}.Release|Any CPU.Build.0 = Release|Any CPU
+		{5E4387F9-5953-4A9B-BCA5-DF3964EED3CB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{5E4387F9-5953-4A9B-BCA5-DF3964EED3CB}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{5E4387F9-5953-4A9B-BCA5-DF3964EED3CB}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{5E4387F9-5953-4A9B-BCA5-DF3964EED3CB}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

FarmmapsBulkSatDownload/BulkSatDownloadApplication.cs

@@ -57,10 +57,10 @@ namespace FarmmapsBulkSatDownload
             // Crashes if "DBsettings.secrets.json" is absent or empty
             DB dbparcels = JsonConvert.DeserializeObject<DB>(File.ReadAllText("DBsettings.secrets.json"));
             string schemaname = "bigdata";
-            string parceltablename = "parcel_flowerbulbs";//"parcelsijbrandij" "parcel"; "parcel_flowerbulbs"; "parcel_disac"
+            string parceltablename = "parcel_bollenrevolutie_tulips2020"; //"parcelsijbrandij" "parcel"; "parcel_flowerbulbs"; "parcel_disac"; ""parcel_bollenrevolutie_tulips2020""
-            string groenmonitortablename = "groenmonitor_flowerbulbs";//"groenmonitorsijbrandij" "groenmonitor" "groenmonitor_flowerbulbs" "groenmonitor_disac"
+            string groenmonitortablename = "groenmonitor_bollenrevolutie_tulips2020";   //"groenmonitorsijbrandij" "groenmonitor" "groenmonitor_flowerbulbs" "groenmonitor_disac" "groenmonitor_bollenrevolutie_tulips2020"
             // The  view 'groenmonitorlatestviewname' contains per parcelid (arbid) the year in which it "exists" and the date of the latest image downloaded. It is used to prevent unneccessary downloading of image statistics already in the database
-            string groenmonitorlatestviewname = "groenmonitorlatest_flowerbulbs"; //"groenmonitorsijbrandijlatest" "groenmonitorlatest" "groenmonitorlatest_flowerbulbs" "groenmonitorlatest_disac"
+            string groenmonitorlatestviewname = "groenmonitorlatest_bollenrevolutie_tulips2020"; //"groenmonitorsijbrandijlatest" "groenmonitorlatest" "groenmonitorlatest_flowerbulbs" "groenmonitorlatest_disac" "groenmonitorlatest_bollenrevolutie_tulips2020"
 
             // Database query and connection. Geometry must be in WGS84 coordinate system, EPSG 4326
             // Apparently the FarmmapsApi cannot handle MultiPolygon, so we need to convert to single Polygon
@@ -80,11 +80,10 @@ SELECT pt.arbid, pt.year, gml.lastwenrdate, ST_AsGeoJSON(ST_Transform((ST_DUMP(p
     CASE WHEN pt.year >= DATE_PART('year', CURRENT_DATE) THEN '' ELSE COALESCE(pt.satellitetaskcode,'') END AS satellitetaskcode
 FROM {0}.{1} pt, {0}.{2} gml
 WHERE
-    pt.arbid = gml.arbid AND
+    pt.arbid = gml.arbid
-    pt.crop NOT IN ('Lelie','Tulp') AND
+    AND pt.satellitetaskcode IS NULL
-    pt.year > 2018 AND pt.arbid IN(8276,8314,8315)
 ORDER BY pt.arbid
-LIMIT 2000;", schemaname, parceltablename, groenmonitorlatestviewname);   //LIMIT x for testing
+LIMIT 5;", schemaname, parceltablename, groenmonitorlatestviewname);   //LIMIT x for testing
 
             using (NpgsqlConnection connection = new NpgsqlConnection(connectionString))
             {
@@ -117,30 +116,30 @@ LIMIT 2000;", schemaname, parceltablename, groenmonitorlatestviewname);   //LIMI
             // Option 2: Example without database. Comment out this part if you want to use database
             // Read cropfields "BulkSatDownloadInput.json" and write all stats to a single csv file
             // Write all stats for multiple fields will be written to a single csv file
-            string downloadFolder;
+            //string downloadFolder;
-            string fileNameStats;
+            //string fileNameStats;
-            string headerLineStats = $"FieldName,satelliteDate,satelliteBand,max,min,mean,mode,median,stddev,minPlus,curtosis,maxMinus,skewness,variance,populationCount,variationCoefficient,confidenceIntervalLow, confidenceIntervalHigh,confidenceIntervalErrorMargin" + Environment.NewLine;
+            //string headerLineStats = $"FieldName,satelliteDate,satelliteBand,max,min,mean,mode,median,stddev,minPlus,curtosis,maxMinus,skewness,variance,populationCount,variationCoefficient,confidenceIntervalLow, confidenceIntervalHigh,confidenceIntervalErrorMargin" + Environment.NewLine;
-            var fieldsInputJson = File.ReadAllText("BulkSatDownloadInput.json");
+            //var fieldsInputJson = File.ReadAllText("BulkSatDownloadInput.json");
-            bulkSatDownloadInputListCsv = JsonConvert.DeserializeObject<List<BulkSatDownloadInput>>(fieldsInputJson);
+            //bulkSatDownloadInputListCsv = JsonConvert.DeserializeObject<List<BulkSatDownloadInput>>(fieldsInputJson);
-            for (int i = 0; i < bulkSatDownloadInputListCsv.Count; i++)
+            //for (int i = 0; i < bulkSatDownloadInputListCsv.Count; i++)
-            {
+            //{
-                downloadFolder = bulkSatDownloadInputListCsv[i].downloadFolder;
+            //    downloadFolder = bulkSatDownloadInputListCsv[i].downloadFolder;
-                fileNameStats = Path.Combine(downloadFolder, bulkSatDownloadInputListCsv[i].fileNameStats);
+            //    fileNameStats = Path.Combine(downloadFolder, bulkSatDownloadInputListCsv[i].fileNameStats);
-                if (!Directory.Exists(downloadFolder))
+            //    if (!Directory.Exists(downloadFolder))
-                    Directory.CreateDirectory(downloadFolder);
+            //        Directory.CreateDirectory(downloadFolder);
-                bulkSatDownloadInputListCsv[i].fileNameStats = fileNameStats;
+            //    bulkSatDownloadInputListCsv[i].fileNameStats = fileNameStats;
-                // Header same as in GeneralService.DownloadSatelliteStats
+            //    // Header same as in GeneralService.DownloadSatelliteStats
-                // Delete fileNameStats if existing. Create a new file. Add a header to csv file
+            //    // Delete fileNameStats if existing. Create a new file. Add a header to csv file
-                File.Delete(fileNameStats);
+            //    File.Delete(fileNameStats);
-                File.AppendAllText(fileNameStats, headerLineStats);
+            //    File.AppendAllText(fileNameStats, headerLineStats);
-            }
+            //}
 
             // Now choose which list you want to use
             bulkSatDownloadInputList = bulkSatDownloadInputListDB;  //bulkSatDownloadInputListDB;  //bulkSatDownloadInputListCsv;
 
             // Whichever option (database or json/csv), continue here
             // Delete the settingsfile
-            File.Delete(settingsfile);
+            // File.Delete(settingsfile);
             
             // For each input download all images. Keep track to time, important when doing bulk downloads
             var watch = System.Diagnostics.Stopwatch.StartNew();
@@ -157,13 +156,26 @@ LIMIT 2000;", schemaname, parceltablename, groenmonitorlatestviewname);   //LIMI
                 try
                 {
                     await Process(roots, bulkSatDownloadInput);
-
                 }
                 catch (Exception ex)
                 {
                     _logger.LogError(ex.Message);
                 }
                 watch.Stop();
+                // add this downloadtime to the cropfieldtable. 
+                // Only if cropfieldtable has a field called 'downloadtime' type 'time'!
+                //using (NpgsqlConnection connection = new NpgsqlConnection(dbparcels.GetConnectionString()))
+                //{
+                //    connection.Open();
+                //    NpgsqlCommand updateCmd = connection.CreateCommand();
+                //    string updateSql = string.Format($"UPDATE {schemaname}.{parceltablename} SET downloadtime = '{strTime(watch.Elapsed)}' WHERE arbid = {bulkSatDownloadInput.fieldID};");
+                //    updateCmd.CommandText = updateSql;
+                //    int r = updateCmd.ExecuteNonQuery();
+                //    if (r != 1)
+                //        throw new Exception("// FarmmapsBulkSatDownload: Update downloadtime Failed");
+                //    connection.Close();
+                //}
+                //_logger.LogInformation($"// FarmmapsBulkSatDownload: Added downloadtime = '{strTime(watch.Elapsed)}' to {schemaname}.{parceltablename} ");
                 tsSofar = tsSofar + watch.Elapsed;
                 tsTotalEstimated = tsSofar / (i + 1) * bulkSatDownloadInputList.Count;
                 tsRemaining = tsTotalEstimated - tsSofar;

FarmmapsCleanUp/CleanUpApplication.cs

@@ -0,0 +1,143 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.IO.Compression;
+using System.Linq;
+using System.Threading.Tasks;
+using FarmmapsApi;
+using FarmmapsApi.Models;
+using FarmmapsApi.Services;
+using FarmmapsBulkSatDownload.Models;
+using Microsoft.Extensions.Logging;
+using Newtonsoft.Json;
+using Npgsql;
+using Newtonsoft.Json.Linq;
+using static FarmmapsApiSamples.Constants;
+
+namespace FarmmapsCleanup
+{
+    public class CleanupApplication : IApplication
+    {
+        private readonly ILogger<CleanupApplication> _logger;
+        private readonly FarmmapsApiService _farmmapsApiService;
+        private readonly CleanupService _cleanupService;
+        private readonly GeneralService _generalService;
+
+        public CleanupApplication(ILogger<CleanupApplication> logger, FarmmapsApiService farmmapsApiService,
+            GeneralService generalService, CleanupService cleanupService)
+        {
+            _logger = logger;
+            _farmmapsApiService = farmmapsApiService;
+            _generalService = generalService;
+            _cleanupService = cleanupService;
+        }
+
+        public async Task RunAsync()
+        {
+            // !! this call is needed the first time an api is called with a fresh clientid and secret !!
+            await _farmmapsApiService.GetCurrentUserCodeAsync();
+            var roots = await _farmmapsApiService.GetCurrentUserRootsAsync();
+
+            //Establish a database connection. Expecting you are using same database server as in project FarmmapsBulkSatDownload
+            DB dbparcels = JsonConvert.DeserializeObject<DB>(File.ReadAllText("DBsettings.secrets.json"));
+            string schemaname = "bigdata";
+            string parceltablename = "parcel_bollenrevolutie_tulips2020"; //"parcelsijbrandij" "parcel"; "parcel_flowerbulbs"; "parcel_disac"; ""parcel_bollenrevolutie_tulips2020""
+
+            //Query to get lists of items to delete from FarmmapsDatabase and from parceltablename
+            string cropfielditemcode;
+            string satellitetaskcode;
+            List<string> cropfieldItemCodes = new List<string>();
+            List<string> satellitetaskCodes = new List<string>();
+            string connectionString = dbparcels.GetConnectionString();
+            string readSql = string.Format(
+@"
+SELECT pt.cropfielditemcode, pt.satellitetaskcode
+FROM {0}.{1} pt
+WHERE
+    pt.arbid IN(1,2)
+ORDER BY pt.arbid
+;", schemaname, parceltablename);
+
+            string updateCropfieldItemCodesSql = string.Format(
+@"
+UPDATE {0}.{1}
+	SET cropfielditemcode=NULL
+	WHERE arbid IN(1,2)
+;", schemaname, parceltablename);   //Same WHERE AS above
+            string updateSatellitetaskCodesSql = string.Format(
+@"
+UPDATE {0}.{1}
+	SET satellitetaskcode=NULL
+	WHERE arbid IN(1,2)
+;", schemaname, parceltablename);   //Same WHERE AS above
+
+            using (NpgsqlConnection connection = new NpgsqlConnection(connectionString))
+            {
+                connection.Open();
+
+                // Read data (run query) = build a list of fields for which to download images
+                NpgsqlCommand command = connection.CreateCommand();
+                command.CommandText = readSql;
+                NpgsqlDataReader dr = command.ExecuteReader();
+                while (dr.Read())
+                {
+                    cropfielditemcode = dr.GetString(0);
+                    satellitetaskcode = dr.GetString(1);
+                    if(!string.IsNullOrEmpty(cropfielditemcode))
+                        cropfieldItemCodes.Add(cropfielditemcode);
+                    if (!string.IsNullOrEmpty(satellitetaskcode))
+                        satellitetaskCodes.Add(satellitetaskcode);
+                }
+                connection.Close();
+            }
+            _logger.LogWarning($"// FarmmapsCleanUp: WARNING: you are about to delete {cropfieldItemCodes.Count} cropfieldItemCodes and {satellitetaskCodes.Count} satellitetaskCodes from the FarmMaps database and your own table {schemaname}.{parceltablename}");
+            _logger.LogInformation($"// Nice of you to clean up after the work is done. You would typically do this for cropfieldItemCodes used only once.");
+            _logger.LogInformation($"// You do NOT want to do this if you think you may later on still want to use these items.");
+            _logger.LogInformation($"// Please carefully check the SQL queries 'readSql' 'updateSql' in CleanUpApplication.cs before proceeding.");
+            
+            _logger.LogInformation($"// FarmmapsCleanUp: delete selected cropfieldItemCodes from FarmMaps database and table {schemaname}.{parceltablename}? 0 = no, 1 = yes");
+            int i;
+            i = Int32.Parse(Console.ReadLine());
+            if (i == 1)
+            {
+                await _farmmapsApiService.DeleteItemsAsync(cropfieldItemCodes);
+                //TODO _farmmapsApiService.DeleteItemsAsync throws an error: {StatusCode: 415, ReasonPhrase: 'Unsupported Media Type', Version: 1.1, Content: System.Net.Http.HttpConnectionResponseContent, Headers: {...}}
+                //what is wrong with cropfieldItemCodes?
+                //and shouldn't we be telling _farmmapsApiService.DeleteItemsAsync what item type to delete?
+                using (NpgsqlConnection connection = new NpgsqlConnection(connectionString))
+                {
+                    connection.Open();
+                    NpgsqlCommand updateCmd = connection.CreateCommand();
+                    updateCmd.CommandText = updateCropfieldItemCodesSql;
+                    int r = updateCmd.ExecuteNonQuery();
+                    if (r == -1)
+                        throw new Exception("// FarmmapsCleanUp: Update cropfielditemcode Failed");
+                    connection.Close();
+                }
+            }
+            _logger.LogInformation($"// FarmmapsCleanUp: delete selected satellitetaskCodes from FarmMaps database and table {schemaname}.{parceltablename}? 0 = no, 1 = yes");
+            i = Int32.Parse(Console.ReadLine());
+            if (i == 1)
+            {
+                await _farmmapsApiService.DeleteItemsAsync(satellitetaskCodes);
+                //TODO _farmmapsApiService.DeleteItemsAsync throws an error: {StatusCode: 415, ReasonPhrase: 'Unsupported Media Type', Version: 1.1, Content: System.Net.Http.HttpConnectionResponseContent, Headers: {...}}
+                //what is wrong with satellitetaskCodes?
+                //and shouldn't we be telling _farmmapsApiService.DeleteItemsAsync what item type to delete?
+                using (NpgsqlConnection connection = new NpgsqlConnection(connectionString))
+                {
+                    connection.Open();
+                    NpgsqlCommand updateCmd = connection.CreateCommand();
+                    updateCmd.CommandText = updateSatellitetaskCodesSql;
+                    int r = updateCmd.ExecuteNonQuery();
+                    if (r == -1)
+                        throw new Exception("// FarmmapsCleanUp: Update cropfielditemcode Failed");
+                    connection.Close();
+                }
+            }
+            _logger.LogInformation($"// FarmmapsCleanUp: done! Hit any key to exit ...");
+            Console.ReadKey();
+
+
+        }
+    }
+}

FarmmapsCleanUp/CleanUpService.cs

@@ -0,0 +1,28 @@
+using System;
+using System.Globalization;
+using System.Threading.Tasks;
+using FarmmapsApi.Models;
+using FarmmapsApi.Services;
+using Microsoft.Extensions.Logging;
+using static FarmmapsApi.Extensions;
+using static FarmmapsApiSamples.Constants;
+
+namespace FarmmapsCleanup
+{
+    public class CleanupService
+    {
+        private readonly ILogger<CleanupService> _logger;
+        private readonly FarmmapsApiService _farmmapsApiService;
+        private readonly GeneralService _generalService;
+
+        public CleanupService(ILogger<CleanupService> logger, FarmmapsApiService farmmapsApiService,
+            GeneralService generalService)
+        {
+            _logger = logger;
+            _farmmapsApiService = farmmapsApiService;
+            _generalService = generalService;
+        }
+
+       
+    }
+}

FarmmapsCleanUp/FarmmapsCleanUp.csproj

@@ -0,0 +1,23 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+    <PropertyGroup>
+        <OutputType>Exe</OutputType>
+        <TargetFramework>netcoreapp3.1</TargetFramework>
+    </PropertyGroup>
+
+    <ItemGroup>
+      <PackageReference Include="Npgsql" Version="5.0.5" />
+    </ItemGroup>
+
+    <ItemGroup>
+      <None Update="Data\**\*">
+        <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+      </None>
+    </ItemGroup>
+
+    <ItemGroup>
+      <ProjectReference Include="..\FarmmapsApi\FarmmapsApi.csproj" />
+      <ProjectReference Include="..\FarmmapsBulkSatDownload\FarmmapsBulkSatDownload.csproj" />
+    </ItemGroup>
+
+</Project>

FarmmapsCleanUp/Program.cs

@@ -0,0 +1,21 @@
+using System.Threading.Tasks;
+using FarmmapsApi;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging;
+
+namespace FarmmapsCleanup
+{
+    class Program : FarmmapsProgram<CleanupApplication>
+    {
+        private static async Task Main(string[] args)
+        {
+            await new Program().Start(args);
+        }
+
+        protected override void Configure(IServiceCollection serviceCollection)
+        {
+            serviceCollection.AddLogging()
+                .AddTransient<CleanupService>();
+        }
+    }
+}