FarmMaps/FarmMapsApiClient
11
0
Fork 3
Code Issues Pull Requests Releases Activity

Fixed uploading internal server error caused by timeout caused by too many request.

Browse Source
This commit is contained in:
Mark van der Wal 2020-04-08 12:18:17 +02:00
parent 418446e9d2
commit cb892e9192
2 changed files with 38 additions and 88 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
FarmmapsApi/Services/GeneralService.cs
View File

@ -40,6 +40,7 @@ namespace FarmmapsApi.Services
public async Task<Item> UploadZipWithShapeAsync(UserRoot root, string filePath, string itemName)
{
var startUpload = DateTime.UtcNow;
var result = await _farmmapsApiService.UploadFile(filePath, root.Code,
progress => _logger.LogInformation($"Status: {progress.Status} - BytesSent: {progress.BytesSent}"));
@ -47,28 +48,29 @@ namespace FarmmapsApi.Services
if (result.Progress.Status == UploadStatus.Failed)
return null;
var zipName = Path.GetFileNameWithoutExtension(filePath);
var zipName = Path.GetFileName(filePath);
Item shapeItem = null;
await PollTask(TimeSpan.FromSeconds(3), async source =>
{
var uploadedFilesChildren = await _farmmapsApiService.GetItemChildrenAsync(root.Code);
var zipItems = uploadedFilesChildren.Where(i => i.Name.Contains(zipName));
var childrenTasks = new List<Task<List<Item>>>();
foreach (var zipItem in zipItems)
{
childrenTasks.Add(_farmmapsApiService.GetItemChildrenAsync(zipItem.Code,
SHAPE_PROCESSED_ITEMTYPE));
}
List<Item> items = await _farmmapsApiService.GetItemChildrenAsync(zipItem.Code,
SHAPE_PROCESSED_ITEMTYPE);
List<Item>[] items = await Task.WhenAll(childrenTasks);
if (items.Length > 0)
if (items.Count(i => i.Created >= startUpload) > 0)
{
var flatItems = items.SelectMany(i => i).Where(i => i.Name.Contains(itemName)).ToList();
if (flatItems.Count > 0)
{
shapeItem = flatItems.Where(i => i.Created >= startUpload).OrderByDescending(i => i.Created)
shapeItem = items.Where(i => i.Name.Contains(itemName))
.OrderByDescending(i => i.Created)
.First();
if(shapeItem != null)
{
source.Cancel();
break;
}
}
}
});
@ -76,18 +78,18 @@ namespace FarmmapsApi.Services
return shapeItem;
}
public async Task<Item> ShapeToGeotiff(Item shapeItemCode)
public async Task<Item> ShapeToGeotiff(Item shapeItem)
{
var shapeToGeotiffRequest = new TaskRequest()
{
TaskType = "vnd.farmmaps.task.shapetogeotiff"
};
var taskCode = await _farmmapsApiService.QueueTaskAsync(shapeItemCode.Code, shapeToGeotiffRequest);
var taskCode = await _farmmapsApiService.QueueTaskAsync(shapeItem.Code, shapeToGeotiffRequest);
await PollTask(TimeSpan.FromSeconds(3), async (tokenSource) =>
{
_logger.LogInformation("Checking shapetogeotiff task status");
var itemTaskStatus = await _farmmapsApiService.GetTaskStatusAsync(shapeItemCode.Code, taskCode);
var itemTaskStatus = await _farmmapsApiService.GetTaskStatusAsync(shapeItem.Code, taskCode);
if (itemTaskStatus.IsFinished)
tokenSource.Cancel();
});
@ -95,8 +97,8 @@ namespace FarmmapsApi.Services
_logger.LogInformation("Data shape converted to geotiff");
// the parent of the shape item is now the tiff item
shapeItemCode = await _farmmapsApiService.GetItemAsync(shapeItemCode.Code);
return await _farmmapsApiService.GetItemAsync(shapeItemCode.ParentCode);
shapeItem = await _farmmapsApiService.GetItemAsync(shapeItem.Code);
return await _farmmapsApiService.GetItemAsync(shapeItem.ParentCode);
}
public async Task<Item> FindChildItemAsync(string parentCode, string itemType, string containsName,

84
FarmmapsApiSamples/NitrogenService.cs
View File

@ -6,7 +6,6 @@ using System.Linq;
using System.Threading.Tasks;
using FarmmapsApi.Models;
using FarmmapsApi.Services;
using Google.Apis.Upload;
using Microsoft.Extensions.Logging;
using Newtonsoft.Json.Linq;
using static FarmmapsApi.Extensions;
@ -18,11 +17,14 @@ namespace FarmmapsApiSamples
{
private readonly ILogger<NitrogenService> _logger;
private readonly FarmmapsApiService _farmmapsApiService;
private readonly GeneralService _generalService;
public NitrogenService(ILogger<NitrogenService> logger, FarmmapsApiService farmmapsApiService)
public NitrogenService(ILogger<NitrogenService> logger, FarmmapsApiService farmmapsApiService,
GeneralService generalService)
{
_logger = logger;
_farmmapsApiService = farmmapsApiService;
_generalService = generalService;
}
public async Task TestFlow(List<UserRoot> roots)
@ -51,16 +53,19 @@ namespace FarmmapsApiSamples
var cropfieldItem = await GetOrCreateCropfieldItemAsync(myDriveRoot.Code);
var dataPath = Path.Combine("Data", "Scan_1_20190605.zip");
var result = await _farmmapsApiService.UploadFile(dataPath, uploadedRoot.Code,
progress => _logger.LogInformation($"Status: {progress.Status} - BytesSent: {progress.BytesSent}"));
if (result.Progress.Status == UploadStatus.Failed)
var isariaShapeItem = await _generalService.UploadZipWithShapeAsync(uploadedRoot, dataPath, "Scan_1_20190605");
if (isariaShapeItem == null)
{
_logger.LogError($"Uploading failed {result.Progress.Exception.Message}");
_logger.LogError("Could not find isaria shape item");
return;
}
var isariaGeotiffItem = await ProcessIsaria(uploadedRoot.Code,"Scan_1_20190605.zip");
var isariaGeotiffItem = await _generalService.ShapeToGeotiff(isariaShapeItem);
if (isariaGeotiffItem == null)
{
_logger.LogError("Something went wrong with isaria shape to geotiff transformation");
return;
}
_logger.LogInformation($"Calculating targetN with targetYield: {60}");
var targetN = await CalculateTargetN(cropfieldItem, plantingDate, measurementDate, 60);
@ -105,63 +110,6 @@ namespace FarmmapsApiSamples
return await _farmmapsApiService.CreateItemAsync(cropfieldItemRequest);
}
public async Task<Item> ProcessIsaria(string parentCode, string zipName)
{
// get processed isaria data (this can end in an infinite loop...)
// need better way of getting new items processed in a context...
Item isariaShapeItem = null;
await PollTask(TimeSpan.FromSeconds(3), async source =>
{
_logger.LogInformation("Trying to get isaria data");
var uploadedFilesChildren = await _farmmapsApiService.GetItemChildrenAsync(parentCode);
var zipItems = uploadedFilesChildren.Where(i => i.Name == zipName);
var isariaTasks = new List<Task<List<Item>>>();
foreach (var zipItem in zipItems)
{
isariaTasks.Add(_farmmapsApiService.GetItemChildrenAsync(zipItem.Code,
SHAPE_PROCESSED_ITEMTYPE));
}
List<Item>[] items = await Task.WhenAll(isariaTasks);
if (items.Length > 0)
{
var nameWithoutExtension = Path.GetFileNameWithoutExtension(zipName);
var flatItems = items.SelectMany(i => i).Where(i => i.Name.Contains(nameWithoutExtension)).ToList();
if (flatItems.Count > 0)
{
isariaShapeItem = flatItems.OrderByDescending(i => i.Created).First();
source.Cancel();
}
}
});
_logger.LogInformation("Found isaria data");
_logger.LogInformation("Converting shape to geotiff");
// need to transform shape data to geotiff
var shapeToGeotiffRequest = new TaskRequest()
{
TaskType = "vnd.farmmaps.task.shapetogeotiff"
};
var taskCode = await _farmmapsApiService.QueueTaskAsync(isariaShapeItem.Code, shapeToGeotiffRequest);
await PollTask(TimeSpan.FromSeconds(3), async (tokenSource) =>
{
_logger.LogInformation("Checking shapetogeotiff task status");
var itemTaskStatus = await _farmmapsApiService.GetTaskStatusAsync(isariaShapeItem.Code, taskCode);
if (itemTaskStatus.IsFinished)
tokenSource.Cancel();
});
_logger.LogInformation("Isaria shape converted to geotiff");
// the parent of the shape item is now the tiff isaria item
isariaShapeItem = await _farmmapsApiService.GetItemAsync(isariaShapeItem.Code);
return await _farmmapsApiService.GetItemAsync(isariaShapeItem.ParentCode);
}
/// <summary>
/// Calculates TargetN, makes the assumption the cropfield and user.input(targetn) item have the same parent
/// </summary>
@ -201,7 +149,7 @@ namespace FarmmapsApiSamples
await PollTask(TimeSpan.FromSeconds(3), async (tokenSource) =>
{
var itemTaskStatus = await _farmmapsApiService.GetTaskStatusAsync(cropfieldItem.Code, itemTaskCode);
if (itemTaskStatus.State == ItemTaskState.Error || itemTaskStatus.State == ItemTaskState.Ok)
if (itemTaskStatus.IsFinished)
tokenSource.Cancel();
});
@ -242,7 +190,7 @@ namespace FarmmapsApiSamples
await PollTask(TimeSpan.FromSeconds(5), async (tokenSource) =>
{
var itemTaskStatus = await _farmmapsApiService.GetTaskStatusAsync(cropfieldItem.Code, itemTaskCode);
if (itemTaskStatus.State == ItemTaskState.Error || itemTaskStatus.State == ItemTaskState.Ok)
if (itemTaskStatus.IsFinished)
tokenSource.Cancel();
});
@ -296,7 +244,7 @@ namespace FarmmapsApiSamples
await PollTask(TimeSpan.FromSeconds(5), async (tokenSource) =>
{
var itemTaskStatus = await _farmmapsApiService.GetTaskStatusAsync(cropfieldItem.Code, itemTaskCode);
if (itemTaskStatus.State == ItemTaskState.Error || itemTaskStatus.State == ItemTaskState.Ok)
if (itemTaskStatus.IsFinished)
tokenSource.Cancel();
});