From cb892e919292175ae55dcb7eeabd2495e466fb4f Mon Sep 17 00:00:00 2001 From: Mark van der Wal Date: Wed, 8 Apr 2020 12:18:17 +0200 Subject: [PATCH] Fixed uploading internal server error caused by timeout caused by too many request. --- FarmmapsApi/Services/GeneralService.cs | 38 +++++------ FarmmapsApiSamples/NitrogenService.cs | 88 ++++++-------------------- 2 files changed, 38 insertions(+), 88 deletions(-) diff --git a/FarmmapsApi/Services/GeneralService.cs b/FarmmapsApi/Services/GeneralService.cs index 377cc4e..d0ce9eb 100644 --- a/FarmmapsApi/Services/GeneralService.cs +++ b/FarmmapsApi/Services/GeneralService.cs @@ -40,6 +40,7 @@ namespace FarmmapsApi.Services public async Task UploadZipWithShapeAsync(UserRoot root, string filePath, string itemName) { + var startUpload = DateTime.UtcNow; var result = await _farmmapsApiService.UploadFile(filePath, root.Code, progress => _logger.LogInformation($"Status: {progress.Status} - BytesSent: {progress.BytesSent}")); @@ -47,28 +48,29 @@ namespace FarmmapsApi.Services if (result.Progress.Status == UploadStatus.Failed) return null; - var zipName = Path.GetFileNameWithoutExtension(filePath); + var zipName = Path.GetFileName(filePath); Item shapeItem = null; await PollTask(TimeSpan.FromSeconds(3), async source => { var uploadedFilesChildren = await _farmmapsApiService.GetItemChildrenAsync(root.Code); var zipItems = uploadedFilesChildren.Where(i => i.Name.Contains(zipName)); - var childrenTasks = new List>>(); + foreach (var zipItem in zipItems) { - childrenTasks.Add(_farmmapsApiService.GetItemChildrenAsync(zipItem.Code, - SHAPE_PROCESSED_ITEMTYPE)); - } - - List[] items = await Task.WhenAll(childrenTasks); - if (items.Length > 0) - { - var flatItems = items.SelectMany(i => i).Where(i => i.Name.Contains(itemName)).ToList(); - if (flatItems.Count > 0) + List items = await _farmmapsApiService.GetItemChildrenAsync(zipItem.Code, + SHAPE_PROCESSED_ITEMTYPE); + + if (items.Count(i => i.Created >= startUpload) > 0) { - shapeItem = flatItems.Where(i => i.Created >= startUpload).OrderByDescending(i => i.Created) + shapeItem = items.Where(i => i.Name.Contains(itemName)) + .OrderByDescending(i => i.Created) .First(); - source.Cancel(); + + if(shapeItem != null) + { + source.Cancel(); + break; + } } } }); @@ -76,18 +78,18 @@ namespace FarmmapsApi.Services return shapeItem; } - public async Task ShapeToGeotiff(Item shapeItemCode) + public async Task ShapeToGeotiff(Item shapeItem) { var shapeToGeotiffRequest = new TaskRequest() { TaskType = "vnd.farmmaps.task.shapetogeotiff" }; - var taskCode = await _farmmapsApiService.QueueTaskAsync(shapeItemCode.Code, shapeToGeotiffRequest); + var taskCode = await _farmmapsApiService.QueueTaskAsync(shapeItem.Code, shapeToGeotiffRequest); await PollTask(TimeSpan.FromSeconds(3), async (tokenSource) => { _logger.LogInformation("Checking shapetogeotiff task status"); - var itemTaskStatus = await _farmmapsApiService.GetTaskStatusAsync(shapeItemCode.Code, taskCode); + var itemTaskStatus = await _farmmapsApiService.GetTaskStatusAsync(shapeItem.Code, taskCode); if (itemTaskStatus.IsFinished) tokenSource.Cancel(); }); @@ -95,8 +97,8 @@ namespace FarmmapsApi.Services _logger.LogInformation("Data shape converted to geotiff"); // the parent of the shape item is now the tiff item - shapeItemCode = await _farmmapsApiService.GetItemAsync(shapeItemCode.Code); - return await _farmmapsApiService.GetItemAsync(shapeItemCode.ParentCode); + shapeItem = await _farmmapsApiService.GetItemAsync(shapeItem.Code); + return await _farmmapsApiService.GetItemAsync(shapeItem.ParentCode); } public async Task FindChildItemAsync(string parentCode, string itemType, string containsName, diff --git a/FarmmapsApiSamples/NitrogenService.cs b/FarmmapsApiSamples/NitrogenService.cs index 9e351f5..721e3ee 100644 --- a/FarmmapsApiSamples/NitrogenService.cs +++ b/FarmmapsApiSamples/NitrogenService.cs @@ -6,7 +6,6 @@ using System.Linq; using System.Threading.Tasks; using FarmmapsApi.Models; using FarmmapsApi.Services; -using Google.Apis.Upload; using Microsoft.Extensions.Logging; using Newtonsoft.Json.Linq; using static FarmmapsApi.Extensions; @@ -18,11 +17,14 @@ namespace FarmmapsApiSamples { private readonly ILogger _logger; private readonly FarmmapsApiService _farmmapsApiService; + private readonly GeneralService _generalService; - public NitrogenService(ILogger logger, FarmmapsApiService farmmapsApiService) + public NitrogenService(ILogger logger, FarmmapsApiService farmmapsApiService, + GeneralService generalService) { _logger = logger; _farmmapsApiService = farmmapsApiService; + _generalService = generalService; } public async Task TestFlow(List roots) @@ -51,16 +53,19 @@ namespace FarmmapsApiSamples var cropfieldItem = await GetOrCreateCropfieldItemAsync(myDriveRoot.Code); var dataPath = Path.Combine("Data", "Scan_1_20190605.zip"); - var result = await _farmmapsApiService.UploadFile(dataPath, uploadedRoot.Code, - progress => _logger.LogInformation($"Status: {progress.Status} - BytesSent: {progress.BytesSent}")); - - if (result.Progress.Status == UploadStatus.Failed) + var isariaShapeItem = await _generalService.UploadZipWithShapeAsync(uploadedRoot, dataPath, "Scan_1_20190605"); + if (isariaShapeItem == null) { - _logger.LogError($"Uploading failed {result.Progress.Exception.Message}"); + _logger.LogError("Could not find isaria shape item"); + return; + } + + var isariaGeotiffItem = await _generalService.ShapeToGeotiff(isariaShapeItem); + if (isariaGeotiffItem == null) + { + _logger.LogError("Something went wrong with isaria shape to geotiff transformation"); return; } - - var isariaGeotiffItem = await ProcessIsaria(uploadedRoot.Code,"Scan_1_20190605.zip"); _logger.LogInformation($"Calculating targetN with targetYield: {60}"); var targetN = await CalculateTargetN(cropfieldItem, plantingDate, measurementDate, 60); @@ -104,64 +109,7 @@ namespace FarmmapsApiSamples return await _farmmapsApiService.CreateItemAsync(cropfieldItemRequest); } - - public async Task ProcessIsaria(string parentCode, string zipName) - { - // get processed isaria data (this can end in an infinite loop...) - // need better way of getting new items processed in a context... - Item isariaShapeItem = null; - await PollTask(TimeSpan.FromSeconds(3), async source => - { - _logger.LogInformation("Trying to get isaria data"); - var uploadedFilesChildren = await _farmmapsApiService.GetItemChildrenAsync(parentCode); - - var zipItems = uploadedFilesChildren.Where(i => i.Name == zipName); - var isariaTasks = new List>>(); - foreach (var zipItem in zipItems) - { - isariaTasks.Add(_farmmapsApiService.GetItemChildrenAsync(zipItem.Code, - SHAPE_PROCESSED_ITEMTYPE)); - } - - List[] items = await Task.WhenAll(isariaTasks); - if (items.Length > 0) - { - var nameWithoutExtension = Path.GetFileNameWithoutExtension(zipName); - var flatItems = items.SelectMany(i => i).Where(i => i.Name.Contains(nameWithoutExtension)).ToList(); - if (flatItems.Count > 0) - { - isariaShapeItem = flatItems.OrderByDescending(i => i.Created).First(); - source.Cancel(); - } - } - }); - - _logger.LogInformation("Found isaria data"); - _logger.LogInformation("Converting shape to geotiff"); - - // need to transform shape data to geotiff - var shapeToGeotiffRequest = new TaskRequest() - { - TaskType = "vnd.farmmaps.task.shapetogeotiff" - }; - var taskCode = await _farmmapsApiService.QueueTaskAsync(isariaShapeItem.Code, shapeToGeotiffRequest); - - await PollTask(TimeSpan.FromSeconds(3), async (tokenSource) => - { - _logger.LogInformation("Checking shapetogeotiff task status"); - var itemTaskStatus = await _farmmapsApiService.GetTaskStatusAsync(isariaShapeItem.Code, taskCode); - if (itemTaskStatus.IsFinished) - tokenSource.Cancel(); - }); - - _logger.LogInformation("Isaria shape converted to geotiff"); - - // the parent of the shape item is now the tiff isaria item - isariaShapeItem = await _farmmapsApiService.GetItemAsync(isariaShapeItem.Code); - - return await _farmmapsApiService.GetItemAsync(isariaShapeItem.ParentCode); - } - + /// /// Calculates TargetN, makes the assumption the cropfield and user.input(targetn) item have the same parent /// @@ -201,7 +149,7 @@ namespace FarmmapsApiSamples await PollTask(TimeSpan.FromSeconds(3), async (tokenSource) => { var itemTaskStatus = await _farmmapsApiService.GetTaskStatusAsync(cropfieldItem.Code, itemTaskCode); - if (itemTaskStatus.State == ItemTaskState.Error || itemTaskStatus.State == ItemTaskState.Ok) + if (itemTaskStatus.IsFinished) tokenSource.Cancel(); }); @@ -242,7 +190,7 @@ namespace FarmmapsApiSamples await PollTask(TimeSpan.FromSeconds(5), async (tokenSource) => { var itemTaskStatus = await _farmmapsApiService.GetTaskStatusAsync(cropfieldItem.Code, itemTaskCode); - if (itemTaskStatus.State == ItemTaskState.Error || itemTaskStatus.State == ItemTaskState.Ok) + if (itemTaskStatus.IsFinished) tokenSource.Cancel(); }); @@ -296,7 +244,7 @@ namespace FarmmapsApiSamples await PollTask(TimeSpan.FromSeconds(5), async (tokenSource) => { var itemTaskStatus = await _farmmapsApiService.GetTaskStatusAsync(cropfieldItem.Code, itemTaskCode); - if (itemTaskStatus.State == ItemTaskState.Error || itemTaskStatus.State == ItemTaskState.Ok) + if (itemTaskStatus.IsFinished) tokenSource.Cancel(); });