Add more diagnostics to BackgroundWorker

AsbCloudInfrastructure/Background/BackgroundWorker.cs

@@ -1,6 +1,7 @@
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Hosting;
 using System;
+using System.Diagnostics;
 using System.Threading;
 using System.Threading.Tasks;
 
@@ -18,6 +19,8 @@ public class BackgroundWorker : BackgroundService
     public WorkStore WorkStore { get; } = new WorkStore();
     public Work? CurrentWork;
 
+    public Exception? MainLoopLastException { get; private set; } = null;
+
     public BackgroundWorker(IServiceProvider serviceProvider)
     {
         this.serviceProvider = serviceProvider;
@@ -27,23 +30,39 @@ public class BackgroundWorker : BackgroundService
     {
         while (!token.IsCancellationRequested)
         {
-            var work = WorkStore.GetNext();
+            try
-            if (work is null)
             {
-                await Task.Delay(executePeriod, token);
+                var work = WorkStore.GetNext();
-                continue;
+                if (work is null)
+                {
+                    await Task.Delay(executePeriod, token);
+                    continue;
+                }
+
+                CurrentWork = work;
+                using var scope = serviceProvider.CreateScope();
+
+                var result = await work.Start(scope.ServiceProvider, token);
+
+                if (!result)
+                    WorkStore.Felled.Add(work);
+                else
+                    WorkStore.Done.Add(work);
+
+                CurrentWork = null;
+                await Task.Delay(minDelay, token);
+            }
+            catch (Exception ex)
+            {
+                MainLoopLastException = ex;
+                var mainLoopLastException = $"BackgroundWorker " +
+                    $"MainLoopLastException: \r\n" +
+                    $"date: {DateTime.Now:O}\r\n" +
+                    $"message: {ex.Message}\r\n" +
+                    $"inner: {ex.InnerException?.Message}\r\n" +
+                    $"stackTrace: {ex.StackTrace}";
+                Trace.TraceError(mainLoopLastException);
             }
-
-            CurrentWork = work;
-            using var scope = serviceProvider.CreateScope();
-
-            var result = await work.Start(scope.ServiceProvider, token);
-
-            if (!result)
-                WorkStore.Felled.Add(work);
-
-            CurrentWork = null;
-            await Task.Delay(minDelay, token);
         }
     }
 }

AsbCloudInfrastructure/Background/Work.cs

@@ -1,5 +1,6 @@
 using AsbCloudApp.Data;
 using System;
+using System.Diagnostics;
 using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
@@ -79,10 +80,18 @@ public abstract class Work : BackgroundWorkDto
             SetLastError(message);
             if (OnErrorAsync is not null)
             {
-                var task = Task.Run(
+                try
-                    async () => await OnErrorAsync(Id, exception, token),
+                {
-                    token);
+                    var task = Task.Run(
-                await task.WaitAsync(OnErrorHandlerTimeout, token);
+                        async () => await OnErrorAsync(Id, exception, token),
+                        token);
+                    await task.WaitAsync(OnErrorHandlerTimeout, token);
+                }
+                catch (Exception onErrorAsyncException)
+                {
+                    var message2 = FormatExceptionMessage(onErrorAsyncException);
+                    Trace.TraceError($"Backgroud work:\"{Id}\" OnError handler throws exception: {message2}");
+                }
             }
         }
         return false;

AsbCloudInfrastructure/Background/WorkStore.cs

@@ -23,12 +23,17 @@ public class WorkStore
     /// Работы выполняемые один раз
     /// </summary>
     public Queue<Work> RunOnceQueue { get; private set; } = new(8);
-            
+
     /// <summary>
-    /// Завершившиеся с ошибкой
+    /// последние 16 завершившиеся с ошибкой
     /// </summary>
     public CyclycArray<Work> Felled { get; } = new(16);
 
+    /// <summary>
+    /// последние 16 успешно завершенных
+    /// </summary>
+    public CyclycArray<Work> Done { get; } = new(16);
+
     /// <summary>
     /// Добавить фоновую работу выполняющуюся с заданным периодом
     /// </summary>

AsbCloudInfrastructure/Services/Subsystems/WorkSubsystemOperationTimeCalc.cs

@@ -26,7 +26,7 @@ public class WorkSubsystemOperationTimeCalc : Work
     public WorkSubsystemOperationTimeCalc()
         : base("Subsystem operation time calc")
     {
-        Timeout = TimeSpan.FromMinutes(20);
+        Timeout = TimeSpan.FromMinutes(30);
     }
 
     protected override async Task Action(string id, IServiceProvider services, Action<string, double?> onProgressCallback, CancellationToken token)

AsbCloudWebApi/Controllers/BackgroundWorkController.cs

@@ -2,7 +2,10 @@
 using AsbCloudInfrastructure.Background;
 using Microsoft.AspNetCore.Authorization;
 using Microsoft.AspNetCore.Mvc;
+using System;
 using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
 
 namespace AsbCloudWebApi.Controllers
 {
@@ -23,14 +26,16 @@ namespace AsbCloudWebApi.Controllers
         {
             var result = new {
                 CurrentWork = (BackgroundWorkDto?)backgroundWorker.CurrentWork,
+                backgroundWorker.MainLoopLastException,
                 RunOnceQueue = backgroundWorker.WorkStore.RunOnceQueue.Select(work => (BackgroundWorkDto)work),
                 Periodics = backgroundWorker.WorkStore.Periodics.Select(work => (BackgroundWorkDto)work.Work),
+                Done = backgroundWorker.WorkStore.Done.Select(work => (BackgroundWorkDto)work),
                 Felled = backgroundWorker.WorkStore.Felled.Select(work => (BackgroundWorkDto)work),
             };
             return Ok(result);
         }
 
-        [HttpGet("Current")]
+        [HttpGet("current")]
         public IActionResult GetCurrent()
         {
             var work = backgroundWorker.CurrentWork;
@@ -40,11 +45,26 @@ namespace AsbCloudWebApi.Controllers
             return Ok(work);
         }
 
-        [HttpGet("Failed")]
+        [HttpGet("failed")]
         public IActionResult GetFelled()
         {
             var result = backgroundWorker.WorkStore.Felled.Select(work => (BackgroundWorkDto)work);
             return Ok(result);
         }
+
+        [HttpGet("done")]
+        public IActionResult GetDone()
+        {
+            var result = backgroundWorker.WorkStore.Done.Select(work => (BackgroundWorkDto)work);
+            return Ok(result);
+        }
+
+        [HttpPost("restart"), Obsolete("temporary method")]
+        public async Task<IActionResult> RestartAsync(CancellationToken token)
+        {
+            await backgroundWorker.StopAsync(token);
+            await backgroundWorker.StartAsync(token);
+            return Ok();
+        }
     }
 }