Fixed uploading internal server error caused by timeout caused by too many request.

FarmmapsApi/Services/GeneralService.cs

@@ -40,6 +40,7 @@ namespace FarmmapsApi.Services
 
         public async Task<Item> UploadZipWithShapeAsync(UserRoot root, string filePath, string itemName)
         {
+            
             var startUpload = DateTime.UtcNow;
             var result = await _farmmapsApiService.UploadFile(filePath, root.Code,
                 progress => _logger.LogInformation($"Status: {progress.Status} - BytesSent: {progress.BytesSent}"));
@@ -47,28 +48,29 @@ namespace FarmmapsApi.Services
             if (result.Progress.Status == UploadStatus.Failed)
                 return null;
 
-            var zipName = Path.GetFileNameWithoutExtension(filePath);
+            var zipName = Path.GetFileName(filePath);
             Item shapeItem = null;
             await PollTask(TimeSpan.FromSeconds(3), async source =>
             {
                 var uploadedFilesChildren = await _farmmapsApiService.GetItemChildrenAsync(root.Code);
                 var zipItems = uploadedFilesChildren.Where(i => i.Name.Contains(zipName));
-                var childrenTasks = new List<Task<List<Item>>>();
+
                 foreach (var zipItem in zipItems)
                 {
-                    childrenTasks.Add(_farmmapsApiService.GetItemChildrenAsync(zipItem.Code,
+                    List<Item> items = await _farmmapsApiService.GetItemChildrenAsync(zipItem.Code,
-                        SHAPE_PROCESSED_ITEMTYPE));
+                        SHAPE_PROCESSED_ITEMTYPE);
-                }
                     
-                List<Item>[] items = await Task.WhenAll(childrenTasks);
+                    if (items.Count(i => i.Created >= startUpload) > 0)
-                if (items.Length > 0)
-                {
-                    var flatItems = items.SelectMany(i => i).Where(i => i.Name.Contains(itemName)).ToList();
-                    if (flatItems.Count > 0)
                     {
-                        shapeItem = flatItems.Where(i => i.Created >= startUpload).OrderByDescending(i => i.Created)
+                        shapeItem = items.Where(i => i.Name.Contains(itemName))
+                            .OrderByDescending(i => i.Created)
                             .First();
-                        source.Cancel();
+                        
+                        if(shapeItem != null)
+                        {
+                            source.Cancel();
+                            break;
+                        }
                     }
                 }
             });
@@ -76,18 +78,18 @@ namespace FarmmapsApi.Services
             return shapeItem;
         }
 
-        public async Task<Item> ShapeToGeotiff(Item shapeItemCode)
+        public async Task<Item> ShapeToGeotiff(Item shapeItem)
         {
             var shapeToGeotiffRequest = new TaskRequest()
             {
                 TaskType = "vnd.farmmaps.task.shapetogeotiff"
             };
-            var taskCode = await _farmmapsApiService.QueueTaskAsync(shapeItemCode.Code, shapeToGeotiffRequest);
+            var taskCode = await _farmmapsApiService.QueueTaskAsync(shapeItem.Code, shapeToGeotiffRequest);
 
             await PollTask(TimeSpan.FromSeconds(3), async (tokenSource) =>
             {
                 _logger.LogInformation("Checking shapetogeotiff task status");
-                var itemTaskStatus = await _farmmapsApiService.GetTaskStatusAsync(shapeItemCode.Code, taskCode);
+                var itemTaskStatus = await _farmmapsApiService.GetTaskStatusAsync(shapeItem.Code, taskCode);
                 if (itemTaskStatus.IsFinished)
                     tokenSource.Cancel();
             });
@@ -95,8 +97,8 @@ namespace FarmmapsApi.Services
             _logger.LogInformation("Data shape converted to geotiff");
 
             // the parent of the shape item is now the tiff item
-            shapeItemCode = await _farmmapsApiService.GetItemAsync(shapeItemCode.Code);
+            shapeItem = await _farmmapsApiService.GetItemAsync(shapeItem.Code);
-            return await _farmmapsApiService.GetItemAsync(shapeItemCode.ParentCode);
+            return await _farmmapsApiService.GetItemAsync(shapeItem.ParentCode);
         }
 
         public async Task<Item> FindChildItemAsync(string parentCode, string itemType, string containsName,

FarmmapsApiSamples/NitrogenService.cs

@@ -6,7 +6,6 @@ using System.Linq;
 using System.Threading.Tasks;
 using FarmmapsApi.Models;
 using FarmmapsApi.Services;
-using Google.Apis.Upload;
 using Microsoft.Extensions.Logging;
 using Newtonsoft.Json.Linq;
 using static FarmmapsApi.Extensions;
@@ -18,11 +17,14 @@ namespace FarmmapsApiSamples
     {
         private readonly ILogger<NitrogenService> _logger;
         private readonly FarmmapsApiService _farmmapsApiService;
+        private readonly GeneralService _generalService;
 
-        public NitrogenService(ILogger<NitrogenService> logger, FarmmapsApiService farmmapsApiService)
+        public NitrogenService(ILogger<NitrogenService> logger, FarmmapsApiService farmmapsApiService,
+            GeneralService generalService)
         {
             _logger = logger;
             _farmmapsApiService = farmmapsApiService;
+            _generalService = generalService;
         }
 
         public async Task TestFlow(List<UserRoot> roots)
@@ -51,16 +53,19 @@ namespace FarmmapsApiSamples
             var cropfieldItem = await GetOrCreateCropfieldItemAsync(myDriveRoot.Code);
             
             var dataPath = Path.Combine("Data", "Scan_1_20190605.zip");
-            var result = await _farmmapsApiService.UploadFile(dataPath, uploadedRoot.Code,
+            var isariaShapeItem = await _generalService.UploadZipWithShapeAsync(uploadedRoot, dataPath, "Scan_1_20190605");
-                progress => _logger.LogInformation($"Status: {progress.Status} - BytesSent: {progress.BytesSent}"));
+            if (isariaShapeItem == null)
-
-            if (result.Progress.Status == UploadStatus.Failed)
             {
-                _logger.LogError($"Uploading failed {result.Progress.Exception.Message}");
+                _logger.LogError("Could not find isaria shape item");
                 return;
             }
             
-            var isariaGeotiffItem = await ProcessIsaria(uploadedRoot.Code,"Scan_1_20190605.zip");
+            var isariaGeotiffItem = await _generalService.ShapeToGeotiff(isariaShapeItem);
+            if (isariaGeotiffItem == null)
+            {
+                _logger.LogError("Something went wrong with isaria shape to geotiff transformation");
+                return;
+            }
 
             _logger.LogInformation($"Calculating targetN with targetYield: {60}");
             var targetN = await CalculateTargetN(cropfieldItem, plantingDate, measurementDate, 60);
@@ -105,63 +110,6 @@ namespace FarmmapsApiSamples
             return await _farmmapsApiService.CreateItemAsync(cropfieldItemRequest);
         }
         
-        public async Task<Item> ProcessIsaria(string parentCode, string zipName)
-        {
-            // get processed isaria data (this can end in an infinite loop...)
-            // need better way of getting new items processed in a context...
-            Item isariaShapeItem = null;
-            await PollTask(TimeSpan.FromSeconds(3), async source =>
-            {
-                _logger.LogInformation("Trying to get isaria data");
-                var uploadedFilesChildren = await _farmmapsApiService.GetItemChildrenAsync(parentCode);
-
-                var zipItems = uploadedFilesChildren.Where(i => i.Name == zipName);
-                var isariaTasks = new List<Task<List<Item>>>();
-                foreach (var zipItem in zipItems)
-                {
-                    isariaTasks.Add(_farmmapsApiService.GetItemChildrenAsync(zipItem.Code,
-                        SHAPE_PROCESSED_ITEMTYPE));
-                }
-
-                List<Item>[] items = await Task.WhenAll(isariaTasks);
-                if (items.Length > 0)
-                {
-                    var nameWithoutExtension = Path.GetFileNameWithoutExtension(zipName);
-                    var flatItems = items.SelectMany(i => i).Where(i => i.Name.Contains(nameWithoutExtension)).ToList();
-                    if (flatItems.Count > 0)
-                    {
-                        isariaShapeItem = flatItems.OrderByDescending(i => i.Created).First();
-                        source.Cancel();
-                    }
-                }
-            });
-            
-            _logger.LogInformation("Found isaria data");
-            _logger.LogInformation("Converting shape to geotiff");
-
-            // need to transform shape data to geotiff
-            var shapeToGeotiffRequest = new TaskRequest()
-            {
-                TaskType = "vnd.farmmaps.task.shapetogeotiff"
-            };
-            var taskCode = await _farmmapsApiService.QueueTaskAsync(isariaShapeItem.Code, shapeToGeotiffRequest);
-                    
-            await PollTask(TimeSpan.FromSeconds(3), async (tokenSource) =>
-            {
-                _logger.LogInformation("Checking shapetogeotiff task status");
-                var itemTaskStatus = await _farmmapsApiService.GetTaskStatusAsync(isariaShapeItem.Code, taskCode);
-                if (itemTaskStatus.IsFinished)
-                    tokenSource.Cancel();
-            });
-            
-            _logger.LogInformation("Isaria shape converted to geotiff");
-            
-            // the parent of the shape item is now the tiff isaria item
-            isariaShapeItem = await _farmmapsApiService.GetItemAsync(isariaShapeItem.Code);
-
-            return await _farmmapsApiService.GetItemAsync(isariaShapeItem.ParentCode);
-        }
-
         /// <summary>
         /// Calculates TargetN, makes the assumption the cropfield and user.input(targetn) item have the same parent
         /// </summary>
@@ -201,7 +149,7 @@ namespace FarmmapsApiSamples
             await PollTask(TimeSpan.FromSeconds(3), async (tokenSource) =>
             {
                 var itemTaskStatus = await _farmmapsApiService.GetTaskStatusAsync(cropfieldItem.Code, itemTaskCode);
-                if (itemTaskStatus.State == ItemTaskState.Error || itemTaskStatus.State == ItemTaskState.Ok)
+                if (itemTaskStatus.IsFinished)
                     tokenSource.Cancel();
             });
 
@@ -242,7 +190,7 @@ namespace FarmmapsApiSamples
             await PollTask(TimeSpan.FromSeconds(5), async (tokenSource) =>
             {
                 var itemTaskStatus = await _farmmapsApiService.GetTaskStatusAsync(cropfieldItem.Code, itemTaskCode);
-                if (itemTaskStatus.State == ItemTaskState.Error || itemTaskStatus.State == ItemTaskState.Ok)
+                if (itemTaskStatus.IsFinished)
                     tokenSource.Cancel();
             });
             
@@ -296,7 +244,7 @@ namespace FarmmapsApiSamples
             await PollTask(TimeSpan.FromSeconds(5), async (tokenSource) =>
             {
                 var itemTaskStatus = await _farmmapsApiService.GetTaskStatusAsync(cropfieldItem.Code, itemTaskCode);
-                if (itemTaskStatus.State == ItemTaskState.Error || itemTaskStatus.State == ItemTaskState.Ok)
+                if (itemTaskStatus.IsFinished)
                     tokenSource.Cancel();
             });