FarmMaps/FarmMapsApiClient
11
0
Fork 3
Code Issues Pull Requests Releases Activity

Fixed uploading internal server error caused by timeout caused by too many request.

Browse Source
This commit is contained in:
Mark van der Wal 2020-04-08 12:18:17 +02:00
parent 418446e9d2
commit cb892e9192
2 changed files with 38 additions and 88 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
FarmmapsApi/Services/GeneralService.cs
View File

@ -40,6 +40,7 @@ namespace FarmmapsApi.Services
public async Task<Item> UploadZipWithShapeAsync(UserRoot root, string filePath, string itemName) public async Task<Item> UploadZipWithShapeAsync(UserRoot root, string filePath, string itemName)
{ {
var startUpload = DateTime.UtcNow; var startUpload = DateTime.UtcNow;
var result = await _farmmapsApiService.UploadFile(filePath, root.Code, var result = await _farmmapsApiService.UploadFile(filePath, root.Code,
progress => _logger.LogInformation($"Status: {progress.Status} - BytesSent: {progress.BytesSent}")); progress => _logger.LogInformation($"Status: {progress.Status} - BytesSent: {progress.BytesSent}"));
@ -47,28 +48,29 @@ namespace FarmmapsApi.Services
if (result.Progress.Status == UploadStatus.Failed) if (result.Progress.Status == UploadStatus.Failed)
return null; return null;
var zipName = Path.GetFileNameWithoutExtension(filePath); var zipName = Path.GetFileName(filePath);
Item shapeItem = null; Item shapeItem = null;
await PollTask(TimeSpan.FromSeconds(3), async source => await PollTask(TimeSpan.FromSeconds(3), async source =>
{ {
var uploadedFilesChildren = await _farmmapsApiService.GetItemChildrenAsync(root.Code); var uploadedFilesChildren = await _farmmapsApiService.GetItemChildrenAsync(root.Code);
var zipItems = uploadedFilesChildren.Where(i => i.Name.Contains(zipName)); var zipItems = uploadedFilesChildren.Where(i => i.Name.Contains(zipName));
var childrenTasks = new List<Task<List<Item>>>();
foreach (var zipItem in zipItems) foreach (var zipItem in zipItems)
{ {
childrenTasks.Add(_farmmapsApiService.GetItemChildrenAsync(zipItem.Code, List<Item> items = await _farmmapsApiService.GetItemChildrenAsync(zipItem.Code,
SHAPE_PROCESSED_ITEMTYPE)); SHAPE_PROCESSED_ITEMTYPE);
}
List<Item>[] items = await Task.WhenAll(childrenTasks); if (items.Count(i => i.Created >= startUpload) > 0)
if (items.Length > 0)
{ {
var flatItems = items.SelectMany(i => i).Where(i => i.Name.Contains(itemName)).ToList(); shapeItem = items.Where(i => i.Name.Contains(itemName))
if (flatItems.Count > 0) .OrderByDescending(i => i.Created)
{
shapeItem = flatItems.Where(i => i.Created >= startUpload).OrderByDescending(i => i.Created)
.First(); .First();
if(shapeItem != null)
{
source.Cancel(); source.Cancel();
break;
}
} }
} }
}); });
@ -76,18 +78,18 @@ namespace FarmmapsApi.Services
return shapeItem; return shapeItem;
} }
public async Task<Item> ShapeToGeotiff(Item shapeItemCode) public async Task<Item> ShapeToGeotiff(Item shapeItem)
{ {
var shapeToGeotiffRequest = new TaskRequest() var shapeToGeotiffRequest = new TaskRequest()
{ {
TaskType = "vnd.farmmaps.task.shapetogeotiff" TaskType = "vnd.farmmaps.task.shapetogeotiff"
}; };
var taskCode = await _farmmapsApiService.QueueTaskAsync(shapeItemCode.Code, shapeToGeotiffRequest); var taskCode = await _farmmapsApiService.QueueTaskAsync(shapeItem.Code, shapeToGeotiffRequest);
await PollTask(TimeSpan.FromSeconds(3), async (tokenSource) => await PollTask(TimeSpan.FromSeconds(3), async (tokenSource) =>
{ {
_logger.LogInformation("Checking shapetogeotiff task status"); _logger.LogInformation("Checking shapetogeotiff task status");
var itemTaskStatus = await _farmmapsApiService.GetTaskStatusAsync(shapeItemCode.Code, taskCode); var itemTaskStatus = await _farmmapsApiService.GetTaskStatusAsync(shapeItem.Code, taskCode);
if (itemTaskStatus.IsFinished) if (itemTaskStatus.IsFinished)
tokenSource.Cancel(); tokenSource.Cancel();
}); });
@ -95,8 +97,8 @@ namespace FarmmapsApi.Services
_logger.LogInformation("Data shape converted to geotiff"); _logger.LogInformation("Data shape converted to geotiff");
// the parent of the shape item is now the tiff item // the parent of the shape item is now the tiff item
shapeItemCode = await _farmmapsApiService.GetItemAsync(shapeItemCode.Code); shapeItem = await _farmmapsApiService.GetItemAsync(shapeItem.Code);
return await _farmmapsApiService.GetItemAsync(shapeItemCode.ParentCode); return await _farmmapsApiService.GetItemAsync(shapeItem.ParentCode);
} }
public async Task<Item> FindChildItemAsync(string parentCode, string itemType, string containsName, public async Task<Item> FindChildItemAsync(string parentCode, string itemType, string containsName,

84
FarmmapsApiSamples/NitrogenService.cs
View File

@ -6,7 +6,6 @@ using System.Linq;
using System.Threading.Tasks; using System.Threading.Tasks;
using FarmmapsApi.Models; using FarmmapsApi.Models;
using FarmmapsApi.Services; using FarmmapsApi.Services;
using Google.Apis.Upload;
using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging;
using Newtonsoft.Json.Linq; using Newtonsoft.Json.Linq;
using static FarmmapsApi.Extensions; using static FarmmapsApi.Extensions;
@ -18,11 +17,14 @@ namespace FarmmapsApiSamples
{ {
private readonly ILogger<NitrogenService> _logger; private readonly ILogger<NitrogenService> _logger;
private readonly FarmmapsApiService _farmmapsApiService; private readonly FarmmapsApiService _farmmapsApiService;
private readonly GeneralService _generalService;
public NitrogenService(ILogger<NitrogenService> logger, FarmmapsApiService farmmapsApiService) public NitrogenService(ILogger<NitrogenService> logger, FarmmapsApiService farmmapsApiService,
GeneralService generalService)
{ {
_logger = logger; _logger = logger;
_farmmapsApiService = farmmapsApiService; _farmmapsApiService = farmmapsApiService;
_generalService = generalService;
} }
public async Task TestFlow(List<UserRoot> roots) public async Task TestFlow(List<UserRoot> roots)
@ -51,16 +53,19 @@ namespace FarmmapsApiSamples
var cropfieldItem = await GetOrCreateCropfieldItemAsync(myDriveRoot.Code); var cropfieldItem = await GetOrCreateCropfieldItemAsync(myDriveRoot.Code);
var dataPath = Path.Combine("Data", "Scan_1_20190605.zip"); var dataPath = Path.Combine("Data", "Scan_1_20190605.zip");
var result = await _farmmapsApiService.UploadFile(dataPath, uploadedRoot.Code, var isariaShapeItem = await _generalService.UploadZipWithShapeAsync(uploadedRoot, dataPath, "Scan_1_20190605");
progress => _logger.LogInformation($"Status: {progress.Status} - BytesSent: {progress.BytesSent}")); if (isariaShapeItem == null)
if (result.Progress.Status == UploadStatus.Failed)
{ {
_logger.LogError($"Uploading failed {result.Progress.Exception.Message}"); _logger.LogError("Could not find isaria shape item");
return; return;
} }
var isariaGeotiffItem = await ProcessIsaria(uploadedRoot.Code,"Scan_1_20190605.zip"); var isariaGeotiffItem = await _generalService.ShapeToGeotiff(isariaShapeItem);
if (isariaGeotiffItem == null)
{
_logger.LogError("Something went wrong with isaria shape to geotiff transformation");
return;
}
_logger.LogInformation($"Calculating targetN with targetYield: {60}"); _logger.LogInformation($"Calculating targetN with targetYield: {60}");
var targetN = await CalculateTargetN(cropfieldItem, plantingDate, measurementDate, 60); var targetN = await CalculateTargetN(cropfieldItem, plantingDate, measurementDate, 60);
@ -105,63 +110,6 @@ namespace FarmmapsApiSamples
return await _farmmapsApiService.CreateItemAsync(cropfieldItemRequest); return await _farmmapsApiService.CreateItemAsync(cropfieldItemRequest);
} }
public async Task<Item> ProcessIsaria(string parentCode, string zipName)
{
// get processed isaria data (this can end in an infinite loop...)
// need better way of getting new items processed in a context...
Item isariaShapeItem = null;
await PollTask(TimeSpan.FromSeconds(3), async source =>
{
_logger.LogInformation("Trying to get isaria data");
var uploadedFilesChildren = await _farmmapsApiService.GetItemChildrenAsync(parentCode);
var zipItems = uploadedFilesChildren.Where(i => i.Name == zipName);
var isariaTasks = new List<Task<List<Item>>>();
foreach (var zipItem in zipItems)
{
isariaTasks.Add(_farmmapsApiService.GetItemChildrenAsync(zipItem.Code,
SHAPE_PROCESSED_ITEMTYPE));
}
List<Item>[] items = await Task.WhenAll(isariaTasks);
if (items.Length > 0)
{
var nameWithoutExtension = Path.GetFileNameWithoutExtension(zipName);
var flatItems = items.SelectMany(i => i).Where(i => i.Name.Contains(nameWithoutExtension)).ToList();
if (flatItems.Count > 0)
{
isariaShapeItem = flatItems.OrderByDescending(i => i.Created).First();
source.Cancel();
}
}
});
_logger.LogInformation("Found isaria data");
_logger.LogInformation("Converting shape to geotiff");
// need to transform shape data to geotiff
var shapeToGeotiffRequest = new TaskRequest()
{
TaskType = "vnd.farmmaps.task.shapetogeotiff"
};
var taskCode = await _farmmapsApiService.QueueTaskAsync(isariaShapeItem.Code, shapeToGeotiffRequest);
await PollTask(TimeSpan.FromSeconds(3), async (tokenSource) =>
{
_logger.LogInformation("Checking shapetogeotiff task status");
var itemTaskStatus = await _farmmapsApiService.GetTaskStatusAsync(isariaShapeItem.Code, taskCode);
if (itemTaskStatus.IsFinished)
tokenSource.Cancel();
});
_logger.LogInformation("Isaria shape converted to geotiff");
// the parent of the shape item is now the tiff isaria item
isariaShapeItem = await _farmmapsApiService.GetItemAsync(isariaShapeItem.Code);
return await _farmmapsApiService.GetItemAsync(isariaShapeItem.ParentCode);
}
/// <summary> /// <summary>
/// Calculates TargetN, makes the assumption the cropfield and user.input(targetn) item have the same parent /// Calculates TargetN, makes the assumption the cropfield and user.input(targetn) item have the same parent
/// </summary> /// </summary>
@ -201,7 +149,7 @@ namespace FarmmapsApiSamples
await PollTask(TimeSpan.FromSeconds(3), async (tokenSource) => await PollTask(TimeSpan.FromSeconds(3), async (tokenSource) =>
{ {
var itemTaskStatus = await _farmmapsApiService.GetTaskStatusAsync(cropfieldItem.Code, itemTaskCode); var itemTaskStatus = await _farmmapsApiService.GetTaskStatusAsync(cropfieldItem.Code, itemTaskCode);
if (itemTaskStatus.State == ItemTaskState.Error || itemTaskStatus.State == ItemTaskState.Ok) if (itemTaskStatus.IsFinished)
tokenSource.Cancel(); tokenSource.Cancel();
}); });
@ -242,7 +190,7 @@ namespace FarmmapsApiSamples
await PollTask(TimeSpan.FromSeconds(5), async (tokenSource) => await PollTask(TimeSpan.FromSeconds(5), async (tokenSource) =>
{ {
var itemTaskStatus = await _farmmapsApiService.GetTaskStatusAsync(cropfieldItem.Code, itemTaskCode); var itemTaskStatus = await _farmmapsApiService.GetTaskStatusAsync(cropfieldItem.Code, itemTaskCode);
if (itemTaskStatus.State == ItemTaskState.Error || itemTaskStatus.State == ItemTaskState.Ok) if (itemTaskStatus.IsFinished)
tokenSource.Cancel(); tokenSource.Cancel();
}); });
@ -296,7 +244,7 @@ namespace FarmmapsApiSamples
await PollTask(TimeSpan.FromSeconds(5), async (tokenSource) => await PollTask(TimeSpan.FromSeconds(5), async (tokenSource) =>
{ {
var itemTaskStatus = await _farmmapsApiService.GetTaskStatusAsync(cropfieldItem.Code, itemTaskCode); var itemTaskStatus = await _farmmapsApiService.GetTaskStatusAsync(cropfieldItem.Code, itemTaskCode);
if (itemTaskStatus.State == ItemTaskState.Error || itemTaskStatus.State == ItemTaskState.Ok) if (itemTaskStatus.IsFinished)
tokenSource.Cancel(); tokenSource.Cancel();
}); });