diff --git a/AsbCloudInfrastructure/Background/BackgroundWorker.cs b/AsbCloudInfrastructure/Background/BackgroundWorker.cs index cacb5a60..0b563ae7 100644 --- a/AsbCloudInfrastructure/Background/BackgroundWorker.cs +++ b/AsbCloudInfrastructure/Background/BackgroundWorker.cs @@ -1,6 +1,7 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using System; +using System.Diagnostics; using System.Threading; using System.Threading.Tasks; @@ -18,6 +19,8 @@ public class BackgroundWorker : BackgroundService public WorkStore WorkStore { get; } = new WorkStore(); public Work? CurrentWork; + public Exception? MainLoopLastException { get; private set; } = null; + public BackgroundWorker(IServiceProvider serviceProvider) { this.serviceProvider = serviceProvider; @@ -27,23 +30,39 @@ public class BackgroundWorker : BackgroundService { while (!token.IsCancellationRequested) { - var work = WorkStore.GetNext(); - if (work is null) + try { - await Task.Delay(executePeriod, token); - continue; + var work = WorkStore.GetNext(); + if (work is null) + { + await Task.Delay(executePeriod, token); + continue; + } + + CurrentWork = work; + using var scope = serviceProvider.CreateScope(); + + var result = await work.Start(scope.ServiceProvider, token); + + if (!result) + WorkStore.Felled.Add(work); + else + WorkStore.Done.Add(work); + + CurrentWork = null; + await Task.Delay(minDelay, token); + } + catch (Exception ex) + { + MainLoopLastException = ex; + var mainLoopLastException = $"BackgroundWorker " + + $"MainLoopLastException: \r\n" + + $"date: {DateTime.Now:O}\r\n" + + $"message: {ex.Message}\r\n" + + $"inner: {ex.InnerException?.Message}\r\n" + + $"stackTrace: {ex.StackTrace}"; + Trace.TraceError(mainLoopLastException); } - - CurrentWork = work; - using var scope = serviceProvider.CreateScope(); - - var result = await work.Start(scope.ServiceProvider, token); - - if (!result) - WorkStore.Felled.Add(work); - - CurrentWork = null; - await Task.Delay(minDelay, token); } } } diff --git a/AsbCloudInfrastructure/Background/Work.cs b/AsbCloudInfrastructure/Background/Work.cs index eba7df66..4820aa26 100644 --- a/AsbCloudInfrastructure/Background/Work.cs +++ b/AsbCloudInfrastructure/Background/Work.cs @@ -1,5 +1,6 @@ using AsbCloudApp.Data; using System; +using System.Diagnostics; using System.Text; using System.Threading; using System.Threading.Tasks; @@ -79,10 +80,18 @@ public abstract class Work : BackgroundWorkDto SetLastError(message); if (OnErrorAsync is not null) { - var task = Task.Run( - async () => await OnErrorAsync(Id, exception, token), - token); - await task.WaitAsync(OnErrorHandlerTimeout, token); + try + { + var task = Task.Run( + async () => await OnErrorAsync(Id, exception, token), + token); + await task.WaitAsync(OnErrorHandlerTimeout, token); + } + catch (Exception onErrorAsyncException) + { + var message2 = FormatExceptionMessage(onErrorAsyncException); + Trace.TraceError($"Backgroud work:\"{Id}\" OnError handler throws exception: {message2}"); + } } } return false; diff --git a/AsbCloudInfrastructure/Background/WorkStore.cs b/AsbCloudInfrastructure/Background/WorkStore.cs index b57fc104..be7386e6 100644 --- a/AsbCloudInfrastructure/Background/WorkStore.cs +++ b/AsbCloudInfrastructure/Background/WorkStore.cs @@ -23,12 +23,17 @@ public class WorkStore /// Работы выполняемые один раз /// public Queue RunOnceQueue { get; private set; } = new(8); - + /// - /// Завершившиеся с ошибкой + /// последние 16 завершившиеся с ошибкой /// public CyclycArray Felled { get; } = new(16); + /// + /// последние 16 успешно завершенных + /// + public CyclycArray Done { get; } = new(16); + /// /// Добавить фоновую работу выполняющуюся с заданным периодом /// diff --git a/AsbCloudInfrastructure/Services/Subsystems/WorkSubsystemOperationTimeCalc.cs b/AsbCloudInfrastructure/Services/Subsystems/WorkSubsystemOperationTimeCalc.cs index 5b9d00f8..7da8051d 100644 --- a/AsbCloudInfrastructure/Services/Subsystems/WorkSubsystemOperationTimeCalc.cs +++ b/AsbCloudInfrastructure/Services/Subsystems/WorkSubsystemOperationTimeCalc.cs @@ -26,7 +26,7 @@ public class WorkSubsystemOperationTimeCalc : Work public WorkSubsystemOperationTimeCalc() : base("Subsystem operation time calc") { - Timeout = TimeSpan.FromMinutes(20); + Timeout = TimeSpan.FromMinutes(30); } protected override async Task Action(string id, IServiceProvider services, Action onProgressCallback, CancellationToken token) diff --git a/AsbCloudWebApi/Controllers/BackgroundWorkController.cs b/AsbCloudWebApi/Controllers/BackgroundWorkController.cs index a0bf6560..0a8c02c0 100644 --- a/AsbCloudWebApi/Controllers/BackgroundWorkController.cs +++ b/AsbCloudWebApi/Controllers/BackgroundWorkController.cs @@ -2,7 +2,10 @@ using AsbCloudInfrastructure.Background; using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; +using System; using System.Linq; +using System.Threading; +using System.Threading.Tasks; namespace AsbCloudWebApi.Controllers { @@ -23,14 +26,16 @@ namespace AsbCloudWebApi.Controllers { var result = new { CurrentWork = (BackgroundWorkDto?)backgroundWorker.CurrentWork, + backgroundWorker.MainLoopLastException, RunOnceQueue = backgroundWorker.WorkStore.RunOnceQueue.Select(work => (BackgroundWorkDto)work), Periodics = backgroundWorker.WorkStore.Periodics.Select(work => (BackgroundWorkDto)work.Work), + Done = backgroundWorker.WorkStore.Done.Select(work => (BackgroundWorkDto)work), Felled = backgroundWorker.WorkStore.Felled.Select(work => (BackgroundWorkDto)work), }; return Ok(result); } - [HttpGet("Current")] + [HttpGet("current")] public IActionResult GetCurrent() { var work = backgroundWorker.CurrentWork; @@ -40,11 +45,26 @@ namespace AsbCloudWebApi.Controllers return Ok(work); } - [HttpGet("Failed")] + [HttpGet("failed")] public IActionResult GetFelled() { var result = backgroundWorker.WorkStore.Felled.Select(work => (BackgroundWorkDto)work); return Ok(result); } + + [HttpGet("done")] + public IActionResult GetDone() + { + var result = backgroundWorker.WorkStore.Done.Select(work => (BackgroundWorkDto)work); + return Ok(result); + } + + [HttpPost("restart"), Obsolete("temporary method")] + public async Task RestartAsync(CancellationToken token) + { + await backgroundWorker.StopAsync(token); + await backgroundWorker.StartAsync(token); + return Ok(); + } } }