Add more diagnostics to BackgroundWorker

AsbCloudInfrastructure/Background/BackgroundWorker.cs

@@ -1,6 +1,7 @@
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Hosting;
 using System;
+using System.Diagnostics;
 using System.Threading;
 using System.Threading.Tasks;
 
@@ -18,6 +19,8 @@ public class BackgroundWorker : BackgroundService
     public WorkStore WorkStore { get; } = new WorkStore();
     public Work? CurrentWork;
 
+    public Exception? MainLoopLastException { get; private set; } = null;
+
     public BackgroundWorker(IServiceProvider serviceProvider)
     {
         this.serviceProvider = serviceProvider;
@@ -26,6 +29,8 @@ public class BackgroundWorker : BackgroundService
     protected override async Task ExecuteAsync(CancellationToken token)
     {
         while (!token.IsCancellationRequested)
+        {
+            try
             {
                 var work = WorkStore.GetNext();
                 if (work is null)
@@ -41,9 +46,23 @@ public class BackgroundWorker : BackgroundService
 
                 if (!result)
                     WorkStore.Felled.Add(work);
+                else
+                    WorkStore.Done.Add(work);
 
                 CurrentWork = null;
                 await Task.Delay(minDelay, token);
             }
+            catch (Exception ex)
+            {
+                MainLoopLastException = ex;
+                var mainLoopLastException = $"BackgroundWorker " +
+                    $"MainLoopLastException: \r\n" +
+                    $"date: {DateTime.Now:O}\r\n" +
+                    $"message: {ex.Message}\r\n" +
+                    $"inner: {ex.InnerException?.Message}\r\n" +
+                    $"stackTrace: {ex.StackTrace}";
+                Trace.TraceError(mainLoopLastException);
+            }
+        }
     }
 }

AsbCloudInfrastructure/Background/Work.cs

@@ -1,5 +1,6 @@
 using AsbCloudApp.Data;
 using System;
+using System.Diagnostics;
 using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
@@ -78,12 +79,20 @@ public abstract class Work : BackgroundWorkDto
             var message = FormatExceptionMessage(exception);
             SetLastError(message);
             if (OnErrorAsync is not null)
+            {
+                try
                 {
                     var task = Task.Run(
                         async () => await OnErrorAsync(Id, exception, token),
                         token);
                     await task.WaitAsync(OnErrorHandlerTimeout, token);
                 }
+                catch (Exception onErrorAsyncException)
+                {
+                    var message2 = FormatExceptionMessage(onErrorAsyncException);
+                    Trace.TraceError($"Backgroud work:\"{Id}\" OnError handler throws exception: {message2}");
+                }
+            }
         }
         return false;
     }

AsbCloudInfrastructure/Background/WorkStore.cs

@@ -25,10 +25,15 @@ public class WorkStore
     public Queue<Work> RunOnceQueue { get; private set; } = new(8);
 
     /// <summary>
-    /// Завершившиеся с ошибкой
+    /// последние 16 завершившиеся с ошибкой
     /// </summary>
     public CyclycArray<Work> Felled { get; } = new(16);
 
+    /// <summary>
+    /// последние 16 успешно завершенных
+    /// </summary>
+    public CyclycArray<Work> Done { get; } = new(16);
+
     /// <summary>
     /// Добавить фоновую работу выполняющуюся с заданным периодом
     /// </summary>

AsbCloudInfrastructure/Services/Subsystems/WorkSubsystemOperationTimeCalc.cs

@@ -26,7 +26,7 @@ public class WorkSubsystemOperationTimeCalc : Work
     public WorkSubsystemOperationTimeCalc()
         : base("Subsystem operation time calc")
     {
-        Timeout = TimeSpan.FromMinutes(20);
+        Timeout = TimeSpan.FromMinutes(30);
     }
 
     protected override async Task Action(string id, IServiceProvider services, Action<string, double?> onProgressCallback, CancellationToken token)

AsbCloudWebApi/Controllers/BackgroundWorkController.cs

@@ -2,7 +2,10 @@
 using AsbCloudInfrastructure.Background;
 using Microsoft.AspNetCore.Authorization;
 using Microsoft.AspNetCore.Mvc;
+using System;
 using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
 
 namespace AsbCloudWebApi.Controllers
 {
@@ -23,14 +26,16 @@ namespace AsbCloudWebApi.Controllers
         {
             var result = new {
                 CurrentWork = (BackgroundWorkDto?)backgroundWorker.CurrentWork,
+                backgroundWorker.MainLoopLastException,
                 RunOnceQueue = backgroundWorker.WorkStore.RunOnceQueue.Select(work => (BackgroundWorkDto)work),
                 Periodics = backgroundWorker.WorkStore.Periodics.Select(work => (BackgroundWorkDto)work.Work),
+                Done = backgroundWorker.WorkStore.Done.Select(work => (BackgroundWorkDto)work),
                 Felled = backgroundWorker.WorkStore.Felled.Select(work => (BackgroundWorkDto)work),
             };
             return Ok(result);
         }
 
-        [HttpGet("Current")]
+        [HttpGet("current")]
         public IActionResult GetCurrent()
         {
             var work = backgroundWorker.CurrentWork;
@@ -40,11 +45,26 @@ namespace AsbCloudWebApi.Controllers
             return Ok(work);
         }
 
-        [HttpGet("Failed")]
+        [HttpGet("failed")]
         public IActionResult GetFelled()
         {
             var result = backgroundWorker.WorkStore.Felled.Select(work => (BackgroundWorkDto)work);
             return Ok(result);
         }
+
+        [HttpGet("done")]
+        public IActionResult GetDone()
+        {
+            var result = backgroundWorker.WorkStore.Done.Select(work => (BackgroundWorkDto)work);
+            return Ok(result);
+        }
+
+        [HttpPost("restart"), Obsolete("temporary method")]
+        public async Task<IActionResult> RestartAsync(CancellationToken token)
+        {
+            await backgroundWorker.StopAsync(token);
+            await backgroundWorker.StartAsync(token);
+            return Ok();
+        }
     }
 }