From 691be2185ea64bc748f3c8ad3a0041dd5de557de Mon Sep 17 00:00:00 2001 From: Mark van der Wal Date: Wed, 25 Mar 2020 21:32:28 +0100 Subject: [PATCH] Seperated eventhub service from api service. wip nbs flow. Changed uploading a bit. --- FarmmapsApi/Extensions.cs | 3 +- .../FarmmapsAuthenticationHandler.cs | 6 +- FarmmapsApi/Models/HttpClientSettings.cs | 7 ++ FarmmapsApi/Models/UploadResults.cs | 18 ++++ FarmmapsApi/ResourceEndpoints.cs | 1 + FarmmapsApi/Services/FarmmapsApiService.cs | 86 ++++++++----------- FarmmapsApi/Services/FarmmapsEventHub.cs | 45 ++++++++++ FarmmapsApi/Services/FarmmapsUploader.cs | 7 +- FarmmapsApiSamples/Constants.cs | 1 + FarmmapsApiSamples/NbsApp.cs | 43 ++++++---- FarmmapsApiSamples/Program.cs | 15 ++-- 11 files changed, 151 insertions(+), 81 deletions(-) create mode 100644 FarmmapsApi/Models/HttpClientSettings.cs create mode 100644 FarmmapsApi/Models/UploadResults.cs create mode 100644 FarmmapsApi/Services/FarmmapsEventHub.cs diff --git a/FarmmapsApi/Extensions.cs b/FarmmapsApi/Extensions.cs index 0c63b4d..2ef39ea 100644 --- a/FarmmapsApi/Extensions.cs +++ b/FarmmapsApi/Extensions.cs @@ -23,12 +23,13 @@ namespace FarmmapsApi return new DiscoveryCache(configuration.DiscoveryEndpointUrl, () => httpFactory.CreateClient()); }) + .AddSingleton() + .AddSingleton() .AddTransient() .AddTransient() .AddHttpClient() .AddHttpMessageHandler() .Services; - ; } public static async Task PollTask(TimeSpan retryTime, Func callback) diff --git a/FarmmapsApi/HttpMessageHandlers/FarmmapsAuthenticationHandler.cs b/FarmmapsApi/HttpMessageHandlers/FarmmapsAuthenticationHandler.cs index b727903..5ec3ad0 100644 --- a/FarmmapsApi/HttpMessageHandlers/FarmmapsAuthenticationHandler.cs +++ b/FarmmapsApi/HttpMessageHandlers/FarmmapsAuthenticationHandler.cs @@ -10,12 +10,12 @@ namespace FarmmapsApi.HttpMessageHandlers { protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) { - if (!request.Headers.Authorization.Scheme.Equals(OidcConstants.AuthenticationSchemes.AuthorizationHeaderBearer) || - string.IsNullOrEmpty(request.Headers.Authorization.Parameter)) + if (!OidcConstants.AuthenticationSchemes.AuthorizationHeaderBearer.Equals(request.Headers?.Authorization.Scheme) || + string.IsNullOrEmpty(request.Headers?.Authorization.Parameter)) { return new HttpResponseMessage(HttpStatusCode.Unauthorized) { - Content = new StringContent("You must authenticate before using the API") + Content = new StringContent("You must authenticated before using the API") }; } return await base.SendAsync(request, cancellationToken); diff --git a/FarmmapsApi/Models/HttpClientSettings.cs b/FarmmapsApi/Models/HttpClientSettings.cs new file mode 100644 index 0000000..74c334b --- /dev/null +++ b/FarmmapsApi/Models/HttpClientSettings.cs @@ -0,0 +1,7 @@ +namespace FarmmapsApi.Models +{ + public class HttpClientSettings + { + public string BearerToken { get; set; } + } +} \ No newline at end of file diff --git a/FarmmapsApi/Models/UploadResults.cs b/FarmmapsApi/Models/UploadResults.cs new file mode 100644 index 0000000..437589b --- /dev/null +++ b/FarmmapsApi/Models/UploadResults.cs @@ -0,0 +1,18 @@ +using System; +using Google.Apis.Upload; + +namespace FarmmapsApi.Models +{ + public class UploadResults + { + public IUploadProgress Progress { get; } + + public Uri Location { get; } + + public UploadResults(IUploadProgress progress, Uri location) + { + Progress = progress; + Location = location; + } + } +} \ No newline at end of file diff --git a/FarmmapsApi/ResourceEndpoints.cs b/FarmmapsApi/ResourceEndpoints.cs index 8c29321..6e65681 100644 --- a/FarmmapsApi/ResourceEndpoints.cs +++ b/FarmmapsApi/ResourceEndpoints.cs @@ -8,6 +8,7 @@ namespace FarmmapsApi public const string ITEMS_RESOURCE = "items/{0}"; public const string ITEMS_CREATE_RESOURCE = "items"; public const string ITEMS_DOWNLOAD_RESOURCE = "items/{0}/download"; + public const string ITEMS_UPLOAD_RESOURCE = "api/v1/file"; public const string ITEMS_CHILDREN_RESOURCE = "items/{0}/children"; public const string ITEMS_DELETE_RESOURCE = "items/delete"; diff --git a/FarmmapsApi/Services/FarmmapsApiService.cs b/FarmmapsApi/Services/FarmmapsApiService.cs index 6d33957..f931e62 100644 --- a/FarmmapsApi/Services/FarmmapsApiService.cs +++ b/FarmmapsApi/Services/FarmmapsApiService.cs @@ -13,9 +13,6 @@ using FarmmapsApi.Models; using Google.Apis.Http; using Google.Apis.Upload; using IdentityModel; -using Microsoft.AspNetCore.Http.Connections; -using Microsoft.AspNetCore.SignalR.Client; -using Microsoft.Extensions.Logging; using Newtonsoft.Json; using Newtonsoft.Json.Linq; using Winista.Mime; @@ -24,23 +21,28 @@ namespace FarmmapsApi.Services { public class FarmmapsApiService { + private readonly Configuration _configuration; + private readonly HttpClientSettings _httpClientSettings; private readonly HttpClient _httpClient; private readonly OpenIdConnectService _openIdConnectService; - private readonly Configuration _configuration; - private HubConnection _hubConnection; - - public event Action EventCallback; - - public FarmmapsApiService(HttpClient httpClient, + public FarmmapsApiService(HttpClientSettings httpClientSettings, HttpClient httpClient, OpenIdConnectService openIdConnectService, Configuration configuration) { + _configuration = configuration; + _httpClientSettings = httpClientSettings; _httpClient = httpClient; _openIdConnectService = openIdConnectService; - _configuration = configuration; _httpClient.BaseAddress = new Uri(configuration.Endpoint); _httpClient.DefaultRequestHeaders.Add("Accept", MediaTypeNames.Application.Json); + + if (!string.IsNullOrEmpty(_httpClientSettings.BearerToken)) + { + _httpClient.DefaultRequestHeaders.Authorization = + new AuthenticationHeaderValue(OidcConstants.AuthenticationSchemes.AuthorizationHeaderBearer, + _httpClientSettings.BearerToken); + } } public async Task AuthenticateAsync() @@ -57,33 +59,10 @@ namespace FarmmapsApi.Services if (token.IsError) throw new AuthenticationException(token.Error); + _httpClientSettings.BearerToken = token.AccessToken; _httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue(OidcConstants.AuthenticationSchemes.AuthorizationHeaderBearer, token.AccessToken); - - await StartEventHub(token.AccessToken); - } - - private async Task StartEventHub(string accessToken) - { - var eventEndpoint = $"{_configuration.Endpoint}/EventHub"; - _hubConnection = new HubConnectionBuilder() - .WithUrl(eventEndpoint, HttpTransportType.WebSockets, - options => options.SkipNegotiation = true) - .ConfigureLogging(log => log.AddConsole()) - .WithAutomaticReconnect() - .Build(); - - await _hubConnection.StartAsync(); - await AuthenticateEventHub(accessToken); - - _hubConnection.Reconnected += async s => await AuthenticateEventHub(accessToken); - _hubConnection.On("event", (EventMessage eventMessage) => EventCallback?.Invoke(eventMessage)); - } - - private async Task AuthenticateEventHub(string accessToken) - { - await _hubConnection.SendAsync("authenticate", accessToken); } public async Task GetCurrentUserCodeAsync() @@ -301,19 +280,20 @@ namespace FarmmapsApi.Services /// /// /// - /// + /// /// /// - public async Task UploadFile(string filePath, string parentItemCode, Action uploadCallback) + public async Task UploadFile(string filePath, string parentItemCode, + Action progressCallback = null) { if (!File.Exists(filePath)) throw new FileNotFoundException($"File not found {filePath}"); - + var mimeTypes = new MimeTypes(); var mimeType = mimeTypes.GetMimeTypeFromFile(filePath); - + await using var uploadStream = new FileStream(filePath, FileMode.OpenOrCreate); - + var request = new FileRequest() { Name = Path.GetFileName(filePath), @@ -321,16 +301,17 @@ namespace FarmmapsApi.Services Size = uploadStream.Length }; - Uri uploadIdentifierUri = null; using var httpClient = CreateConfigurableHttpClient(_httpClient); - var farmmapsUploader = new FarmmapsUploader(httpClient, uploadStream, request, mimeType.ToString()); - - farmmapsUploader.ProgressChanged += uploadCallback; - farmmapsUploader.UploadSessionData += data => uploadIdentifierUri = data.UploadUri; - - await farmmapsUploader.UploadAsync(); + var farmmapsUploader = new FarmmapsUploader(httpClient, uploadStream, request, + mimeType.ToString(), ResourceEndpoints.ITEMS_UPLOAD_RESOURCE); - return uploadIdentifierUri; + Uri location = null; + farmmapsUploader.ProgressChanged += progressCallback; + farmmapsUploader.UploadSessionData += data => location = data.UploadUri; + + var progress = await farmmapsUploader.UploadAsync(); + + return new UploadResults(progress, location); } /// @@ -347,7 +328,7 @@ namespace FarmmapsApi.Services throw new FileNotFoundException($"File not found {filePath}"); await using var uploadStream = new FileStream(filePath, FileMode.OpenOrCreate); - + var request = new FileRequest() { Name = Path.GetFileName(filePath), @@ -356,7 +337,8 @@ namespace FarmmapsApi.Services }; using var httpClient = CreateConfigurableHttpClient(_httpClient); - var farmmapsUploader = new FarmmapsUploader(httpClient, uploadStream, request, string.Empty); + var farmmapsUploader = new FarmmapsUploader(httpClient, uploadStream, + request, string.Empty, ResourceEndpoints.ITEMS_UPLOAD_RESOURCE); farmmapsUploader.ProgressChanged += uploadCallback; await farmmapsUploader.ResumeAsync(location); @@ -366,12 +348,14 @@ namespace FarmmapsApi.Services private ConfigurableHttpClient CreateConfigurableHttpClient(HttpClient parent) { - var googleHttpClient = new HttpClientFactory().CreateHttpClient(new CreateHttpClientArgs {GZipEnabled = true, ApplicationName = "FarmMaps"}); + var googleHttpClient = new HttpClientFactory().CreateHttpClient(new CreateHttpClientArgs + {GZipEnabled = true, ApplicationName = "FarmMaps"}); googleHttpClient.BaseAddress = parent.BaseAddress; googleHttpClient.DefaultRequestHeaders.Add("Accept", MediaTypeNames.Application.Json); var authHeader = parent.DefaultRequestHeaders.Authorization; - googleHttpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue(authHeader.Scheme, authHeader.Parameter); + googleHttpClient.DefaultRequestHeaders.Authorization = + new AuthenticationHeaderValue(authHeader.Scheme, authHeader.Parameter); return googleHttpClient; } diff --git a/FarmmapsApi/Services/FarmmapsEventHub.cs b/FarmmapsApi/Services/FarmmapsEventHub.cs new file mode 100644 index 0000000..382c292 --- /dev/null +++ b/FarmmapsApi/Services/FarmmapsEventHub.cs @@ -0,0 +1,45 @@ +using System; +using System.Threading.Tasks; +using FarmmapsApi.Models; +using Microsoft.AspNetCore.Http.Connections; +using Microsoft.AspNetCore.SignalR.Client; +using Microsoft.Extensions.Logging; + +namespace FarmmapsApi.Services +{ + public class FarmmapsEventHub + { + private readonly Configuration _configuration; + private readonly HttpClientSettings _httpClientSettings; + + private HubConnection _hubConnection; + + public event Action EventCallback; + + public FarmmapsEventHub(HttpClientSettings httpClientSettings, Configuration configuration) + { + _httpClientSettings = httpClientSettings; + _configuration = configuration; + } + + public async Task StartEventHub() + { + if(_hubConnection != null) + throw new Exception("EventHub already started"); + + var eventEndpoint = $"{_configuration.Endpoint}/EventHub"; + _hubConnection = new HubConnectionBuilder() + .WithUrl(eventEndpoint, HttpTransportType.WebSockets, + options => options.SkipNegotiation = true) + .ConfigureLogging(log => log.AddConsole()) + .WithAutomaticReconnect() + .Build(); + + await _hubConnection.StartAsync(); + await _hubConnection.SendAsync("authenticate", _httpClientSettings.BearerToken); + + _hubConnection.Reconnected += async s => await _hubConnection.SendAsync("authenticate", _httpClientSettings.BearerToken); + _hubConnection.On("Event", (EventMessage eventMessage) => EventCallback?.Invoke(eventMessage)); + } + } +} \ No newline at end of file diff --git a/FarmmapsApi/Services/FarmmapsUploader.cs b/FarmmapsApi/Services/FarmmapsUploader.cs index 02842dd..1cae5fe 100644 --- a/FarmmapsApi/Services/FarmmapsUploader.cs +++ b/FarmmapsApi/Services/FarmmapsUploader.cs @@ -55,7 +55,7 @@ namespace FarmmapsApi.Services /// completed. /// Caller is responsible for closing the . /// - public FarmmapsUploader(ConfigurableHttpClient httpClient, Stream contentStream, FileRequest body, string contentType) + public FarmmapsUploader(ConfigurableHttpClient httpClient, Stream contentStream, FileRequest body, string contentType, string path) : base(contentStream, new ResumableUploadOptions { @@ -69,9 +69,12 @@ namespace FarmmapsApi.Services _streamLength = ContentStream.CanSeek ? ContentStream.Length : UnknownSize; + var twoMB = 2 * 0x100000; + ChunkSize = twoMB; body.ChunkSize = ChunkSize; Body = body; - Path = "api/v1/file"; + + Path = path; HttpClient = httpClient; HttpMethod = HttpConsts.Post; ContentType = contentType; diff --git a/FarmmapsApiSamples/Constants.cs b/FarmmapsApiSamples/Constants.cs index 852f44b..1d458b3 100644 --- a/FarmmapsApiSamples/Constants.cs +++ b/FarmmapsApiSamples/Constants.cs @@ -5,6 +5,7 @@ namespace FarmmapsApiSamples public const string USERINPUT_ITEMTYPE = "vnd.farmmaps.itemtype.user.input"; public const string GEOTIFF_ITEMTYPE = "vnd.farmmaps.itemtype.geotiff"; public const string CROPFIELD_ITEMTYPE = "vnd.farmmaps.itemtype.cropfield"; + public const string SHAPE_PROCESSED_ITEMTYPE = "vnd.farmmaps.itemtype.shape.processed"; public const string VRANBS_TASK = "vnd.farmmaps.task.vranbs"; } } \ No newline at end of file diff --git a/FarmmapsApiSamples/NbsApp.cs b/FarmmapsApiSamples/NbsApp.cs index 6dd7ee9..7f8d7a6 100644 --- a/FarmmapsApiSamples/NbsApp.cs +++ b/FarmmapsApiSamples/NbsApp.cs @@ -15,48 +15,59 @@ namespace FarmmapsApiSamples { private readonly ILogger _logger; private readonly FarmmapsApiService _farmmapsApiService; + private readonly FarmmapsEventHub _farmmapsEventHub; private readonly NitrogenService _nitrogenService; public NbsApp(ILogger logger, FarmmapsApiService farmmapsApiService, - NitrogenService nitrogenService) + FarmmapsEventHub farmmapsEventHub, NitrogenService nitrogenService) { _logger = logger; _farmmapsApiService = farmmapsApiService; + _farmmapsEventHub = farmmapsEventHub; _nitrogenService = nitrogenService; - farmmapsApiService.EventCallback += OnEvent; + _farmmapsEventHub.EventCallback += OnEvent; } private void OnEvent(EventMessage @event) { - _logger.LogInformation(@event.EventType); +// _logger.LogInformation(@event.EventType); } public async Task RunAsync() { try { - _logger.LogInformation("NBS sample app started"); - - await _farmmapsApiService.AuthenticateAsync(); - - _logger.LogInformation("Authenticated client credentials"); - var roots = await _farmmapsApiService.GetCurrentUserRootsAsync(); // upload data to Uploaded var uploadedRoot = roots.SingleOrDefault(r => r.Name == "Uploaded"); if (uploadedRoot != null) { - await _farmmapsApiService.UploadFile(Path.Combine("Data", "Scan_1_20190605.zip"), uploadedRoot.Code, - progress => - { - _logger.LogInformation($"Status: {progress.Status} - BytesSent: {progress.BytesSent}"); - if (progress.Status == UploadStatus.Failed) - _logger.LogError($"Uploading failed {progress.Exception.Message}"); - }); + var dataPath = Path.Combine("Data", "Scan_1_20190605.zip"); + var result = await _farmmapsApiService.UploadFile(dataPath, uploadedRoot.Code, + progress => _logger.LogInformation($"Status: {progress.Status} - BytesSent: {progress.BytesSent}")); + + if (result.Progress.Status == UploadStatus.Failed) + { + _logger.LogError($"Uploading failed {result.Progress.Exception.Message}"); + return; + } + + // find file we need to use, we use delay as hack until events work correctly +// await Task.Delay(10000); + +// var isariaDataFilter = JObject.Parse(@""); +// var uploadedFilesChildren = await +// _farmmapsApiService.GetItemChildrenAsync(uploadedRoot.Code, SHAPE_PROCESSED_ITEMTYPE); // need to transform shape data to geotiff +// var shapeToGeotiffRequest = new TaskRequest() +// { +// TaskType = "vnd.farmmaps.task.shapetogeotiff" +// }; +// var taskCode = await _farmmapsApiService.QueueTaskAsync(, shapeToGeotiffRequest); + var myDriveRoot = roots.SingleOrDefault(r => r.Name == "My drive"); if (myDriveRoot != null) diff --git a/FarmmapsApiSamples/Program.cs b/FarmmapsApiSamples/Program.cs index 51cb081..744e9b0 100644 --- a/FarmmapsApiSamples/Program.cs +++ b/FarmmapsApiSamples/Program.cs @@ -1,10 +1,11 @@ -using System.Threading.Tasks; +using System; +using System.Threading.Tasks; using FarmmapsApi; using FarmmapsApi.Models; +using FarmmapsApi.Services; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Logging.Console; namespace FarmmapsApiSamples { @@ -20,17 +21,15 @@ namespace FarmmapsApiSamples var serviceProvider = new ServiceCollection() .AddLogging(opts => opts - .AddConsole(c => - { - c.IncludeScopes = false; - c.Format = ConsoleLoggerFormat.Default; - }) + .AddConsole() .AddFilter("System.Net.Http", LogLevel.Warning)) .AddFarmmapsServices(configuration) .AddTransient() .AddSingleton() .BuildServiceProvider(); - + + await serviceProvider.GetService().AuthenticateAsync(); + await serviceProvider.GetService().StartEventHub(); await serviceProvider.GetService().RunAsync(); } }