forked from FarmMaps/FarmMapsApiClient
new project to download latest version of editeelt codelists
parent
2d588026e2
commit
d8c2ecb88b
|
|
@ -33,6 +33,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FarmmapsCleanUp", "Farmmaps
|
|||
EndProject
|
||||
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FarmmapsKPI", "FarmmapsKPI\FarmmapsKPI.csproj", "{14575235-9867-4CE5-A22F-3F9FE002FF42}"
|
||||
EndProject
|
||||
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FarmmapsDownloadCL", "FarmmapsDownloadCL\FarmmapsDownloadCL.csproj", "{63E69101-D804-4DBC-B2E4-A33771CD5C5F}"
|
||||
EndProject
|
||||
Global
|
||||
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
||||
Debug|Any CPU = Debug|Any CPU
|
||||
|
|
@ -137,6 +139,14 @@ Global
|
|||
{14575235-9867-4CE5-A22F-3F9FE002FF42}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{14575235-9867-4CE5-A22F-3F9FE002FF42}.Release|x64.ActiveCfg = Release|x64
|
||||
{14575235-9867-4CE5-A22F-3F9FE002FF42}.Release|x64.Build.0 = Release|x64
|
||||
{63E69101-D804-4DBC-B2E4-A33771CD5C5F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{63E69101-D804-4DBC-B2E4-A33771CD5C5F}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{63E69101-D804-4DBC-B2E4-A33771CD5C5F}.Debug|x64.ActiveCfg = Debug|x64
|
||||
{63E69101-D804-4DBC-B2E4-A33771CD5C5F}.Debug|x64.Build.0 = Debug|x64
|
||||
{63E69101-D804-4DBC-B2E4-A33771CD5C5F}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{63E69101-D804-4DBC-B2E4-A33771CD5C5F}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{63E69101-D804-4DBC-B2E4-A33771CD5C5F}.Release|x64.ActiveCfg = Release|x64
|
||||
{63E69101-D804-4DBC-B2E4-A33771CD5C5F}.Release|x64.Build.0 = Release|x64
|
||||
EndGlobalSection
|
||||
GlobalSection(SolutionProperties) = preSolution
|
||||
HideSolutionNode = FALSE
|
||||
|
|
|
|||
|
|
@ -0,0 +1,178 @@
|
|||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.IO.Compression;
|
||||
using System.Linq;
|
||||
using System.Reflection;
|
||||
using System.Threading.Tasks;
|
||||
using FarmmapsApi;
|
||||
using FarmmapsApi.Models;
|
||||
using FarmmapsApi.Services;
|
||||
using FarmmapsDownloadCL.Models;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Newtonsoft.Json;
|
||||
using Newtonsoft.Json.Linq;
|
||||
using static FarmmapsApiSamples.Constants;
|
||||
|
||||
namespace FarmmapsDownloadCL
|
||||
{
|
||||
public class DownloadCLApplication : IApplication
|
||||
{
|
||||
|
||||
private readonly ILogger<DownloadCLApplication> _logger;
|
||||
private readonly FarmmapsApiService _farmmapsApiService;
|
||||
private readonly DownloadCLService _DownloadCLService;
|
||||
private readonly GeneralService _generalService;
|
||||
public readonly Dictionary<string, List<string>> _dictCl;
|
||||
string _itemcode;
|
||||
public DownloadCLApplication(ILogger<DownloadCLApplication> logger, FarmmapsApiService farmmapsApiService,
|
||||
GeneralService generalService, DownloadCLService DownloadCLService)
|
||||
{
|
||||
_logger = logger;
|
||||
_farmmapsApiService = farmmapsApiService;
|
||||
_generalService = generalService;
|
||||
_DownloadCLService = DownloadCLService;
|
||||
CodelistsClasses clc = new CodelistsClasses();
|
||||
_itemcode = clc.itemcode;
|
||||
_dictCl = clc.dictCl;
|
||||
}
|
||||
|
||||
public async Task RunAsync()
|
||||
{
|
||||
var fieldsInputJson = File.ReadAllText("DownloadCLInput.json");
|
||||
|
||||
|
||||
DownloadCLInput clInput = JsonConvert.DeserializeObject<DownloadCLInput>(fieldsInputJson);
|
||||
|
||||
// !! this call is needed the first time an api is called with a fresh clientid and secret !!
|
||||
await _farmmapsApiService.GetCurrentUserCodeAsync();
|
||||
var roots = await _farmmapsApiService.GetCurrentUserRootsAsync();
|
||||
|
||||
//Where to write the output
|
||||
string downloadFolder = clInput.DownloadFolder;
|
||||
if (string.IsNullOrEmpty(downloadFolder))
|
||||
{
|
||||
downloadFolder = "Downloads";
|
||||
}
|
||||
if (!Directory.Exists(downloadFolder))
|
||||
Directory.CreateDirectory(downloadFolder);
|
||||
|
||||
//Get the most recent codelists
|
||||
foreach (string codelist in clInput.codelists)
|
||||
{
|
||||
try
|
||||
{
|
||||
await Process(roots, codelist, downloadFolder);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex.Message);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async Task Process(List<UserRoot> roots, string codelistname, string downloadFolder)
|
||||
{
|
||||
List<Item> codelist;
|
||||
string itemtype;
|
||||
string className;
|
||||
string body;
|
||||
string header;
|
||||
string value;
|
||||
string[] aboutArray = null;
|
||||
string[] headerArray = null;
|
||||
PropertyInfo[] props;
|
||||
PropertyInfo prop;
|
||||
List<string> dataList;
|
||||
string clJson = Path.Combine(downloadFolder, $"{codelistname}.json");
|
||||
string clCsv = Path.Combine(downloadFolder, $"{codelistname}.csv");
|
||||
|
||||
try
|
||||
{
|
||||
itemtype = _dictCl[codelistname][0];
|
||||
className = _dictCl[codelistname][1];
|
||||
codelist = await _farmmapsApiService.GetItemChildrenAsync(_itemcode, itemtype);
|
||||
|
||||
//Write full codelist in json format to clJson.
|
||||
body = JsonConvert.SerializeObject(codelist);
|
||||
File.WriteAllText(clJson, body);
|
||||
_logger.LogInformation($"Downloaded file {clJson}");
|
||||
|
||||
//Write full list in csv format to clCsv.
|
||||
StreamWriter sw = new StreamWriter(clCsv);
|
||||
//Write metadata in top: when downloaded
|
||||
sw.WriteLine($"Editeelt codelist {codelistname} downloaded on {DateTime.Now} with the FarmmapsDownloadCL application by Pepijn van Oort (WPR)");
|
||||
|
||||
//Generic, for any Codelist as long as it is also defined in FarmmapsDownloadCL.Models in
|
||||
string typerequest = "FarmmapsDownloadCL.Models." + className + ", FarmmapsDownloadCL, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null";
|
||||
Type objType = Type.GetType(typerequest);
|
||||
object clitem = Activator.CreateInstance(objType);
|
||||
props = clitem.GetType().GetProperties();
|
||||
|
||||
//Lookup about info and write to top
|
||||
foreach (PropertyInfo pi in props)
|
||||
{
|
||||
if (pi.Name == "about")
|
||||
{
|
||||
aboutArray = (string[])pi.GetValue(clitem);
|
||||
foreach (string about in aboutArray)
|
||||
{
|
||||
//Add quotes
|
||||
value = "\"" + about + "\"";
|
||||
//Write each about to a separate line in the csv file
|
||||
sw.WriteLine(value);
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
//Write an empty line
|
||||
sw.WriteLine();
|
||||
|
||||
//Look up the headerArray and write the header comma separated
|
||||
int i = 0;
|
||||
while (headerArray == null)
|
||||
{
|
||||
prop = props[i];
|
||||
if (prop.Name == "headers")
|
||||
{
|
||||
headerArray = (string[])prop.GetValue(clitem);
|
||||
}
|
||||
i++;
|
||||
}
|
||||
header = string.Join(",", headerArray);
|
||||
sw.WriteLine(header);
|
||||
|
||||
//Add quotes because a value for field 'description' could be '21% Cl, 16.5%CaO, 1330 g/l' and without the quotes that
|
||||
//would show up as separate columns. So per definition we will add quotes, except if we have a fieldname in the listNotToQuote, then value for that fieldname will not be given quotes
|
||||
//e.g. not code "3" but code 3.
|
||||
List<string> listNotToQuote = new List<string>() { "code", "n", "p", "k", "quantity", "cropGroupCode", "cultivationGroupCode" };
|
||||
//Loop through all items in the codelist
|
||||
foreach (Item item in codelist)
|
||||
{
|
||||
dataList = new List<string> { };
|
||||
clitem = JsonConvert.DeserializeObject(item.Data.ToString(), objType);
|
||||
|
||||
//Add values of the cLitem to the dataList in the same order as the header
|
||||
foreach (string h in headerArray)
|
||||
{
|
||||
value = (string)clitem.GetType().GetProperty(h).GetValue(clitem, null);
|
||||
if(listNotToQuote.Contains(h) == false)
|
||||
value = "\"" +value + "\"";
|
||||
|
||||
dataList.Add(value);
|
||||
}
|
||||
string dataLine = string.Join(",", dataList);
|
||||
sw.WriteLine(string.Join(",", dataList));
|
||||
}
|
||||
|
||||
sw.Close();
|
||||
_logger.LogInformation($"Downloaded file {clCsv}");
|
||||
|
||||
}
|
||||
catch
|
||||
{
|
||||
_logger.LogWarning($"Missing lookup information on codelist {codelistname}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
{
|
||||
"codeLists": [ "CL022", "CL127", "CL263", "CL042" ]
|
||||
//"DownloadFolder": "Downloads", //"C:\\hugoschrererdir\\kpidir\\", // "Downloads", -> if you just put "Downloads" the program will download to somewhere in ..\FarmMapsApiClient_WURtest\FarmmapsDataDownload\bin\Debug\netcoreapp3.1\Downloads\
|
||||
}
|
||||
|
|
@ -0,0 +1,24 @@
|
|||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<OutputType>Exe</OutputType>
|
||||
<TargetFramework>netcoreapp3.1</TargetFramework>
|
||||
<PlatformTarget>x64</PlatformTarget>
|
||||
<Platforms>AnyCPU;x64</Platforms>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="System.Collections" Version="4.3.0" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\FarmmapsApi\FarmmapsApi.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<None Update="DownloadCLInput.json">
|
||||
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
|
||||
</None>
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
|
|
@ -0,0 +1,87 @@
|
|||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Text;
|
||||
|
||||
namespace FarmmapsDownloadCL.Models
|
||||
{
|
||||
public class CodelistsClasses
|
||||
{
|
||||
public CodelistsClasses()
|
||||
{
|
||||
this.itemcode = "938c0aec4efc46e88783fd029f17309e%3ASYSTEM_DATA";
|
||||
this.dictCl = new Dictionary<string, List<string>>();
|
||||
//Add here once you have defined a new CLxxxitem below
|
||||
//To add a new item, look it up in https://test.farmmaps.eu/swagger/index.html. Download the item you are interested in (like "vnd.farmmaps.itemtype.codelist.cl263")
|
||||
//then below create a new CLxxxitem with fieldnames as you see them in the just downloaded new codelist
|
||||
this.dictCl.Add("CL022", new List<string> { "vnd.farmmaps.itemtype.codelist.cl022", "CL022item" });
|
||||
this.dictCl.Add("CL127", new List<string> { "vnd.farmmaps.itemtype.codelist.cl127", "CL127item" });
|
||||
this.dictCl.Add("CL263", new List<string> { "vnd.farmmaps.itemtype.codelist.cl263", "CL263item" });
|
||||
}
|
||||
public string itemcode;
|
||||
//Define here the codelist names, their item type and the class defining the contents of the data type (and defined in the Models.CodelistsClasses)
|
||||
public Dictionary<string, List<string>> dictCl;
|
||||
}
|
||||
|
||||
public class CL022item
|
||||
{
|
||||
public CL022item()
|
||||
{
|
||||
this.headers = new string[] { "codelist","code", "description", "type","k", "n","p","composition" };
|
||||
this.codelist = "CL022";
|
||||
this.about = new string[] { "EDI-Crop, coderingslijst meststoffen", "type MOR = organische meststof; MAN = anorganische meststof" };
|
||||
}
|
||||
|
||||
public string[] headers { get; }
|
||||
public string codelist { get; }
|
||||
public string[] about { get; }
|
||||
public string k { get; set; }
|
||||
public string n { get; set; }
|
||||
public string p { get; set; }
|
||||
public string code { get; set; }
|
||||
public string type { get; set; }
|
||||
public string composition { get; set; }
|
||||
public string description { get; set; }
|
||||
|
||||
}
|
||||
public class CL127item
|
||||
{
|
||||
public CL127item()
|
||||
{
|
||||
this.headers = new string[] { "codelist", "code", "description", "description_nl", "culturalPracticeLevel1", "culturalPracticeLevel2" };
|
||||
this.codelist = "CL127";
|
||||
}
|
||||
|
||||
public string[] headers { get; }
|
||||
public string codelist { get; }
|
||||
public string code { get; set; }
|
||||
public string description { get; set; }
|
||||
public string description_nl { get; set; }
|
||||
public string culturalPracticeLevel1 { get; set; }
|
||||
public string culturalPracticeLevel2 { get; set; }
|
||||
}
|
||||
public class CL263item
|
||||
{
|
||||
public CL263item()
|
||||
{
|
||||
this.headers = new string[] { "codelist", "code", "eppoCode", "botanicName", "description", "description_fr", "description_nl", "cropGroupCode", "cropGroupName", "cultivationGroupCode", "cultivationGroupName" };
|
||||
this.codelist = "CL263";
|
||||
this.about = new string[] { "EDI-Crop, coderingslijst gewassoorten" };
|
||||
}
|
||||
|
||||
public string[] headers { get; }
|
||||
public string codelist { get; }
|
||||
public string[] about { get; }
|
||||
public string code { get; set; }
|
||||
public string eppoCode { get; set; }
|
||||
public string botanicName { get; set; }
|
||||
public string description { get; set; }
|
||||
public string description_nl { get; set; }
|
||||
public string description_fr { get; set; }
|
||||
public string cropGroupCode { get; set; }
|
||||
public string cropGroupName { get; set; }
|
||||
public string cultivationGroupCode { get; set; }
|
||||
public string cultivationGroupName { get; set; }
|
||||
public string composition { get; set; }
|
||||
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,12 @@
|
|||
using System;
|
||||
using Newtonsoft.Json.Linq;
|
||||
|
||||
namespace FarmmapsDownloadCL.Models
|
||||
{
|
||||
public class DownloadCLInput
|
||||
{
|
||||
public string[] codelists { get; set; }
|
||||
public string DownloadFolder { get; set; }
|
||||
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
using System.Threading.Tasks;
|
||||
using FarmmapsApi;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace FarmmapsDownloadCL
|
||||
{
|
||||
class Program : FarmmapsProgram<DownloadCLApplication>
|
||||
{
|
||||
private static async Task Main(string[] args)
|
||||
{
|
||||
await new Program().Start(args);
|
||||
}
|
||||
|
||||
protected override void Configure(IServiceCollection serviceCollection)
|
||||
{
|
||||
serviceCollection.AddLogging()
|
||||
.AddTransient<DownloadCLService>();
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue