BackgroudWork Add onprogres callback

AsbCloudInfrastructure/Background/BackgroudWorkDto.cs

@@ -0,0 +1,170 @@
+using System;
+using System.Diagnostics;
+
+namespace AsbCloudInfrastructure.Background
+{
+    public class BackgroudWorkDto
+    {
+        /// <summary>
+        /// Идентификатор работы. Должен быть уникальным. Используется в логах и передается в колбэки.
+        /// </summary>
+        public string Id { get; init; } = null!;
+
+        public class CurrentStateInfo
+        {
+            private string state = "start";
+
+            /// <summary>
+            /// Время последнего запуска
+            /// </summary>
+            public DateTime Start { get; } = DateTime.Now;
+
+            /// <summary>
+            /// Текущее время выполнения
+            /// </summary>
+            public TimeSpan ExecutionTime => DateTime.Now - Start;
+
+            /// <summary>
+            /// Текстовое описание того, что происходит в задаче.
+            /// </summary>
+            public string State
+            {
+                get => state;
+                internal set
+                {
+                    state = value;
+                    StateUpdate = DateTime.Now;
+                }
+            }
+
+            public double Progress { get; internal set; } = 0;
+
+            /// <summary>
+            /// Время последнего запуска
+            /// </summary>
+            public DateTime StateUpdate { get; private set; } = DateTime.Now;
+        }
+
+        public class LastErrorInfo: LastCompleteInfo
+        {
+            public LastErrorInfo(CurrentStateInfo state, string errorText) 
+                : base(state)
+            {
+                ErrorText = errorText;
+            }
+
+            /// <summary>
+            /// Последняя ошибка
+            /// </summary>
+            public string ErrorText { get; init; } = null!;
+        }
+
+        public class LastCompleteInfo
+        {
+            /// <summary>
+            /// Дата запуска
+            /// </summary>
+            public DateTime Start {get; init;}
+
+            /// <summary>
+            /// Дата завершения
+            /// </summary>
+            public DateTime End { get; init; }
+
+            /// <summary>
+            /// Продолжительность последнего выполнения
+            /// </summary>
+            public TimeSpan ExecutionTime => End - Start;
+
+            /// <summary>
+            /// Состояние на момент завершения
+            /// </summary>
+            public string State { get; init; }
+
+            public LastCompleteInfo(CurrentStateInfo state)
+            {
+                Start = state.Start;
+                End = DateTime.Now;
+                State = state.State;
+            }
+        }
+
+        /// <summary>
+        /// Текущее состояние
+        /// </summary>
+        public CurrentStateInfo? CurrentState { get; private set; }
+
+        /// <summary>
+        /// Последняя ошибка
+        /// </summary>
+        public LastErrorInfo? LastError { get; private set; }
+
+        /// <summary>
+        /// Последняя завершенная
+        /// </summary>
+        public LastCompleteInfo? LastComplete { get; private set; }
+
+        /// <summary>
+        /// Кол-во запусков
+        /// </summary>
+        public int CountStart { get; private set; }
+
+        /// <summary>
+        /// Кол-во завершений
+        /// </summary>
+        public int CountComplete { get; private set; }
+
+        /// <summary>
+        /// Кол-во ошибок
+        /// </summary>
+        public int CountErrors { get; private set; }
+
+        /// <summary>
+        /// Максимально допустимое время выполнения работы
+        /// </summary>
+        public TimeSpan Timeout { get; set; } = TimeSpan.FromMinutes(1);
+        
+        private string WorkNameForTrace => $"Backgroud work:\"{Id}\"";
+
+        protected void SetStatusStart()
+        {
+            CurrentState = new();
+            CountStart++;
+            Trace.TraceInformation($"{WorkNameForTrace} state: starting");
+        }
+
+        protected void UpdateStatus(string newState, double? progress)
+        {
+            if (CurrentState is null)
+                return;
+
+            CurrentState.State = newState;
+            if (progress.HasValue)
+                CurrentState.Progress = progress.Value;
+
+            Trace.TraceInformation($"{WorkNameForTrace} state: {newState}");
+        }
+
+        protected void SetStatusComplete(System.Threading.Tasks.TaskStatus status)
+        {
+            if (CurrentState is null)
+                return;
+
+            LastComplete = new (CurrentState);
+            CurrentState = null;
+            CountComplete++;
+            Trace.TraceInformation($"{WorkNameForTrace} state: completed");
+        }
+
+        protected void SetLastError(string errorMessage)
+        {
+            if (CurrentState is null)
+                return;
+
+            LastError = new LastErrorInfo(CurrentState, errorMessage);
+            CurrentState = null;
+            CountErrors++;
+            Trace.TraceError($"{WorkNameForTrace} throw exception[{CountErrors}]: {errorMessage}");
+        }
+    }
+}

AsbCloudInfrastructure/Background/BackgroundWorker.cs

@@ -15,7 +15,7 @@ namespace AsbCloudInfrastructure.Background
         private static readonly TimeSpan executePeriod = TimeSpan.FromSeconds(10);
         private static readonly TimeSpan minDelay = TimeSpan.FromSeconds(2);
         private readonly IServiceProvider serviceProvider;
-        private readonly WorkQueue workQueue = new WorkQueue();
+        private readonly WorkStore workQueue = new WorkStore();
         public string? CurrentWorkId;
 
         public BackgroundWorker(IServiceProvider serviceProvider)

AsbCloudInfrastructure/Background/OrderedList.cs

@@ -0,0 +1,41 @@
+using System.Linq;
+
+namespace System.Collections.Generic
+{
+    public class OrderedList<T>: IEnumerable<T>, ICollection<T>
+        where T : notnull
+    {
+        private readonly List<T> list = new List<T>();
+
+        private readonly Func<T, object> keySelector;
+        private readonly bool isDescending = false;
+
+        private IOrderedEnumerable<T> OrdredList => isDescending
+            ? list.OrderByDescending(keySelector)
+            : list.OrderBy(keySelector);
+
+        public int Count => list.Count;
+
+        public bool IsReadOnly => false;
+
+        public OrderedList(Func<T, object> keySelector, bool isDescending = false)
+        {
+            this.keySelector = keySelector;
+            this.isDescending = isDescending;
+        }
+
+        public void Add(T item) => list.Add(item);
+
+        public void Clear()=> list.Clear();
+
+        public bool Contains(T item)=>  list.Contains(item);
+
+        public void CopyTo(T[] array, int arrayIndex)=> list.CopyTo(array, arrayIndex);
+
+        public bool Remove(T item)=> list.Remove(item);
+
+        public IEnumerator<T> GetEnumerator() => OrdredList.GetEnumerator();
+
+        IEnumerator IEnumerable.GetEnumerator() => GetEnumerator();
+    }
+}

AsbCloudInfrastructure/Background/WorkBase.cs

@@ -1,6 +1,4 @@
 using System;
-using System.Diagnostics;
-using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
 
@@ -10,14 +8,23 @@ namespace AsbCloudInfrastructure.Background
     /// Класс разовой работы.
     /// Разовая работа приоритетнее периодической.
     /// </summary>
-    public class WorkBase
+    public class WorkBase : BackgroudWorkDto
     {
-        /// <summary>
+        internal Func<string, IServiceProvider, Action<string, double?>, CancellationToken, Task> ActionAsync { get; }
-        /// Идентификатор работы. Должен быть уникальным. Используется в логах и передается в колбэки.
-        /// </summary>
-        public string Id { get; }
 
         /// <summary>
+        /// Делегат обработки ошибки.
+        /// Не должен выполняться долго.
+        /// </summary>
+        public Func<string, Exception, CancellationToken, Task>? OnErrorAsync { get; set; }
+
+        public TimeSpan OnErrorHandlerTimeout { get; set; } = TimeSpan.FromSeconds(5);
+
+        /// <summary>
+        /// Базовая работа
+        /// </summary>
+        /// <param name="id"></param>
+        /// <param name="actionAsync">
         /// Делегат работы.
         /// <para>
         /// Параметры:
@@ -31,138 +38,50 @@ namespace AsbCloudInfrastructure.Background
         ///         <description>Поставщик сервисов</description>
         ///     </item>
         ///     <item>
+        ///         <term>Action&lt;string, double?&gt;</term>
+        ///         <description>on progress callback. String - new state text. double? - optional progress 0-100%.</description>
+        ///     </item>
+        ///     <item>
         ///         <term>CancellationToken</term>
         ///         <description>Токен отмены задачи</description>
         ///     </item>
         ///   </list>
         /// </para>
-        /// </summary>
+        /// </param>
-        internal Func<string, IServiceProvider, CancellationToken, Task> ActionAsync { get; }
+        public WorkBase(string id, Func<string, IServiceProvider, Action<string, double?>, CancellationToken, Task> actionAsync)
-
-        /// <summary>
-        /// Делегат обработки ошибки.
-        /// Не должен выполняться долго.
-        /// </summary>
-        public Func<string, Exception, CancellationToken, Task>? OnErrorAsync { get; set; }
-
-        /// <summary>
-        /// максимально допустимое время выполнения работы
-        /// </summary>
-        public TimeSpan Timeout { get; set; } = TimeSpan.FromMinutes(1);
-
-        /// <summary>
-        /// Продолжительность последнего выполнения
-        /// </summary>
-        public TimeSpan? LastExecutionTime { get; private set; }
-
-        /// <summary>
-        /// Текущее время выполнения
-        /// </summary>
-        public TimeSpan? CurrentExecutionTime => CurrentStart.HasValue
-            ? DateTime.Now - CurrentStart.Value
-            : null;
-
-        /// <summary>
-        /// Время последнего запуска
-        /// </summary>
-        public DateTime? CurrentStart { get; private set; }
-
-        /// <summary>
-        /// Текстовое описание того, что происходит в задаче.
-        /// </summary>
-        public string? CurrentStatus { get; private set; }
-
-        /// <summary>
-        /// Время последнего запуска
-        /// </summary>
-        public DateTime? CurrentStatusUpdate { get; private set; }
-
-        /// <summary>
-        /// Последняя ошибка
-        /// </summary>
-        public string? LastErrorMessage { get; private set; }
-        
-        /// <summary>
-        /// Дата последнего запуска
-        /// </summary>
-        public DateTime? LastStart { get; private set; }
-
-        /// <summary>
-        /// Дата последней ошибки
-        /// </summary>
-        public DateTime? LastError { get; private set; }
-
-        /// <summary>
-        /// Дата завершения последнего выполнения
-        /// </summary>
-        public DateTime? LastComplete { get; private set; }
-
-        /// <summary>
-        /// Кол-во завершений
-        /// </summary>
-        public int CountComplete { get; private set; }
-
-        /// <summary>
-        /// Кол-во ошибок
-        /// </summary>
-        public int CountErrors { get; private set; }
-
-        private string WorkNameForTrace => $"Backgroud work:\"{Id}\"";
-
-        public WorkBase(string id, Func<string, IServiceProvider, CancellationToken, Task> actionAsync)
         {
             Id = id;
             ActionAsync = actionAsync;
         }
         
-        public async Task Start(IServiceProvider services, CancellationToken token)
+        /// <summary>
+        /// Запустить работу
+        /// </summary>
+        /// <param name="services"></param>
+        /// <param name="token"></param>
+        /// <returns>True - susess, False = Fail</returns>
+        public async Task<bool> Start(IServiceProvider services, CancellationToken token)
         {
-            CurrentStart = DateTime.Now;
+            SetStatusStart();
-            LastStart = DateTime.Now;
             try
             {
-                SetStatus(" start");
+                var task = ActionAsync(Id, services, UpdateStatus, token);
-                var task = ActionAsync(Id, services, token);
                 await task.WaitAsync(Timeout, token);
-                LastComplete = DateTime.Now;
+                SetStatusComplete(task.Status);
-                CountComplete++;
+                return true;
-                SetStatus($" {task.Status}. ExecutionTime: {CurrentExecutionTime:hh\\:mm\\:ss\\.fff}");
             }
             catch (Exception exception)
             {
-                SetError(exception.Message);
+                SetLastError(exception.Message);
                 if (OnErrorAsync is not null)
                 {
                     var task = Task.Run(
                         async () => await OnErrorAsync(Id, exception, token),
                         token);
-                    await task.WaitAsync(Timeout, token);
+                    await task.WaitAsync(OnErrorHandlerTimeout, token);
                 }
             }
-
+            return false;
-            LastExecutionTime = CurrentExecutionTime;
-            CurrentStart = null;
-            SetStatus(null);
-        }
-
-        protected void SetStatus(string? newStatus)
-        {
-            CurrentStatus = newStatus;
-            if (newStatus is not null)
-            {
-                CurrentStatusUpdate = DateTime.Now;
-                Trace.TraceInformation($"{WorkNameForTrace} state: {newStatus}");
-            }
-            else
-                CurrentStatusUpdate = null;
-        }
-
-        private void SetError(string? errorMessage)
-        {
-            CountErrors++;
-            LastErrorMessage = errorMessage;
-            LastError = DateTime.Now;
-            Trace.TraceError($"{WorkNameForTrace} throw exception[{CountErrors}]: {errorMessage}");
         }
     }
 }

AsbCloudInfrastructure/Background/WorkPeriodic.cs

@@ -18,7 +18,16 @@ namespace AsbCloudInfrastructure.Background
         /// <summary>
         /// Время следующего запуска
         /// </summary>
-        public DateTime NextStart => LastStart??DateTime.MinValue + Period;
+        public DateTime NextStart
+        {
+            get
+            {
+                var lastStart = LastComplete?.Start ?? DateTime.MinValue;
+                if (LastError?.Start > lastStart)
+                    lastStart = LastError.Start;
+                return lastStart + Period;
+            }
+        }
 
         /// <summary>
         /// Класс периодической работы
@@ -26,7 +35,7 @@ namespace AsbCloudInfrastructure.Background
         /// <param name="id">Идентификатор работы. Должен быть уникальным. Используется в логах и передается в колбэки</param>
         /// <param name="actionAsync">Делегат работы</param>
         /// <param name="period">Период выполнения задачи</param>
-        public WorkPeriodic(string id, Func<string, IServiceProvider, CancellationToken, Task> actionAsync, TimeSpan period)
+        public WorkPeriodic(string id, Func<string, IServiceProvider, Action<string, double?>, CancellationToken, Task> actionAsync, TimeSpan period)
         : base(id, actionAsync)
         {
             Period = period;

AsbCloudInfrastructure/Background/WorkStore.cs

@@ -10,11 +10,25 @@ namespace AsbCloudInfrastructure.Background
     /// </para>
     /// Не периодические задачи будут возвращаться первыми, как самые приоритетные.
     /// </summary>
-    class WorkQueue
+    class WorkStore
     {
-        private Queue<WorkBase> Primary = new(8);
         private readonly List<WorkPeriodic> Periodic = new(8);
 
+        /// <summary>
+        /// Работы выполняемые один раз
+        /// </summary>
+        public Queue<WorkBase> RunOnceQueue { get; } = new(8);
+
+        /// <summary>
+        /// Работы выполняемые периодически
+        /// </summary>
+        public IOrderedEnumerable<WorkPeriodic> Periodics => Periodic.OrderBy(work => work.NextStart);
+        
+        /// <summary>
+        /// Завершывшиеся с ошибкой
+        /// </summary>
+        public CyclycArray<WorkBase> Falled { get; } = new(16);
+
         /// <summary>
         /// Добавление работы.
         /// </summary>
@@ -25,8 +39,8 @@ namespace AsbCloudInfrastructure.Background
             if (Periodic.Any(w => w.Id == work.Id))
                 throw new ArgumentException("work.Id is not unique", nameof(work));
 
-            if (Primary.Any(w => w.Id == work.Id))
+            //if (Primary.Any(w => w.Id == work.Id))
-                throw new ArgumentException("work.Id is not unique", nameof(work));
+            //    throw new ArgumentException("work.Id is not unique", nameof(work));
 
             if (work is WorkPeriodic workPeriodic)
             {
@@ -34,7 +48,7 @@ namespace AsbCloudInfrastructure.Background
                 return;
             }
 
-            Primary.Enqueue(work);
+            //Primary.Enqueue(work);
         }
 
         /// <summary>
@@ -51,19 +65,19 @@ namespace AsbCloudInfrastructure.Background
                 return true;
             }
 
-            var work = Primary.FirstOrDefault(w => w.Id == id);
+            //var work = Primary.FirstOrDefault(w => w.Id == id);
-            if (work is not null)
+            //if (work is not null)
-            {
+            //{
-                Primary = new Queue<WorkBase>(Primary.Where(w => w.Id != id));
+            //    Primary = new Queue<WorkBase>(Primary.Where(w => w.Id != id));
-                return true;
+            //    return true;
-            }
+            //}
 
             return false;
         }
 
         public bool Contains(string id)
         {
-            var result = Periodic.Any(w => w.Id == id) || Primary.Any(w => w.Id == id);
+            var result = false;//Periodic.Any(w => w.Id == id) || Primary.Any(w => w.Id == id);
             return result;
         }
 
@@ -82,8 +96,8 @@ namespace AsbCloudInfrastructure.Background
         /// <returns></returns>
         public WorkBase? Pop()
         {
-            if (Primary.Any())
+            //if (Primary.Any())
-                return Primary.Dequeue();
+            //    return Primary.Dequeue();
 
             var work = GetNextPeriodic();
             if (work is null || work.NextStart > DateTime.Now)

AsbCloudInfrastructure/Services/DetectOperations/OperationDetectionWorkFactory.cs

@@ -46,7 +46,7 @@ namespace AsbCloudInfrastructure.Services.DetectOperations
         }
 
         // TODO: Разделить этот акшн на более мелкие части И использовать telemetryServiceData<..> вместо прямого обращения к БД.
-        private static async Task WorkAction(string _, IServiceProvider serviceProvider, CancellationToken token)
+        private static async Task WorkAction(string _, IServiceProvider serviceProvider, Action<string, double?> onProgress, CancellationToken token)
         {
             using var db = serviceProvider.GetRequiredService<IAsbCloudDbContext>();
 
@@ -75,10 +75,14 @@ namespace AsbCloudInfrastructure.Services.DetectOperations
                     });
 
             var affected = 0;
+            var count = joinedlastDetectedDates.Count();
+            var i = 0;
             foreach (var item in joinedlastDetectedDates)
             {
                 var stopwatch = Stopwatch.StartNew();
-                var newOperations = await DetectOperationsAsync(item.IdTelemetry, item.LastDate ?? DateTimeOffset.MinValue, db, token);
+                var startDate = item.LastDate ?? DateTimeOffset.MinValue;
+                onProgress($"start detecting telemetry: {item.IdTelemetry} from {startDate}", i++ / count);
+                var newOperations = await DetectOperationsAsync(item.IdTelemetry, startDate, db, token);
                 stopwatch.Stop();
                 if (newOperations.Any())
                 {

AsbCloudInfrastructure/Services/DrillingProgram/DrillingProgramService.cs

@@ -519,11 +519,12 @@ namespace AsbCloudInfrastructure.Services.DrillingProgram
                     var resultFileName = $"Программа бурения {well.Cluster} {well.Caption}.pdf";                     
                     var convertedFilesDir = Path.Combine(Path.GetTempPath(), "drillingProgram", $"{well.Cluster}_{well.Caption}");                                        
                     var tempResultFilePath = Path.Combine(convertedFilesDir, resultFileName);
-                    var workAction = async (string workId, IServiceProvider serviceProvider, CancellationToken token) =>
+                    var workAction = async (string workId, IServiceProvider serviceProvider, Action<string, double?> onProgress, CancellationToken token) =>
                     {
                         var context = serviceProvider.GetRequiredService<IAsbCloudDbContext>();
                         var fileService = serviceProvider.GetRequiredService<FileService>();
                         var files = state.Parts.Select(p => fileService.GetUrl(p.File!));
+                        onProgress($"Start converting {files.Count()} files to PDF.", null);
                         await ConvertToPdf.GetConverteAndMergedFileAsync(files, tempResultFilePath, convertedFilesDir, token);
                         await fileService.MoveAsync(idWell, null, idFileCategoryDrillingProgram, resultFileName, tempResultFilePath, token);
                     };

AsbCloudInfrastructure/Services/Email/EmailNotificationTransportService.cs

@@ -10,6 +10,7 @@ using AsbCloudApp.Exceptions;
 using AsbCloudApp.Repositories;
 using AsbCloudApp.Services.Notifications;
 using AsbCloudInfrastructure.Background;
+using DocumentFormat.OpenXml.Presentation;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
 
@@ -70,9 +71,9 @@ namespace AsbCloudInfrastructure.Services.Email
             return Task.WhenAll(tasks);
         }
         
-        private Func<string, IServiceProvider, CancellationToken, Task> MakeEmailSendWorkAction(NotificationDto notification)
+        private Func<string, IServiceProvider, Action<string, double?>, CancellationToken, Task> MakeEmailSendWorkAction(NotificationDto notification)
         {
-            return async (_, serviceProvider, token) =>
+            return async (_, serviceProvider, onProgress, token) =>
             {
                 var notificationRepository = serviceProvider.GetRequiredService<INotificationRepository>();
                 var userRepository = serviceProvider.GetRequiredService<IUserRepository>();

AsbCloudInfrastructure/Services/LimitingParameterBackgroundService.cs

@@ -28,7 +28,7 @@ namespace AsbCloudInfrastructure.Services
         }
 
         // TODO: Разделить этот акшн на более мелкие части И использовать telemetryServiceData<..> вместо прямого обращения к БД.
-        private static async Task WorkAction(string _, IServiceProvider serviceProvider, CancellationToken token)
+        private static async Task WorkAction(string _, IServiceProvider serviceProvider, Action<string, double?> onProgress, CancellationToken token)
         {
             using var db = serviceProvider.GetRequiredService<IAsbCloudDbContext>();
             var lastDetectedDates = await db.LimitingParameter
@@ -55,8 +55,11 @@ namespace AsbCloudInfrastructure.Services
                         inner.SingleOrDefault()?.LastDate,
                     });
 
+            var count = telemetryLastDetectedDates.Count();
+            var i = 0;
             foreach (var item in telemetryLastDetectedDates)
             {
+                onProgress($"Start hanling telemetry: {item.IdTelemetry} from {item.LastDate}", i++/count);
                 var newLimitingParameters = await GetLimitingParameterAsync(item.IdTelemetry, item.LastDate ?? DateTimeOffset.MinValue, db, token);
                 if (newLimitingParameters?.Any() == true)
                 {

AsbCloudInfrastructure/Services/ReportService.cs

@@ -51,7 +51,7 @@ namespace AsbCloudInfrastructure.Services
 
             var workId = $"create report by wellid:{idWell} for userid:{idUser} requested at {DateTime.Now}";
 
-            var workAction = async (string id, IServiceProvider serviceProvider, CancellationToken token) =>
+            var workAction = async (string id, IServiceProvider serviceProvider, Action<string, double?> onProgress, CancellationToken token) =>
             {
                 using var context = serviceProvider.GetRequiredService<IAsbCloudDbContext>();
                 var fileService = serviceProvider.GetRequiredService<FileService>();
@@ -64,7 +64,9 @@ namespace AsbCloudInfrastructure.Services
 
                 generator.OnProgress += (s, e) =>
                 {
-                    progressHandler.Invoke(e.Adapt<ReportProgressDto>(), id);
+                    var arg = e.Adapt<ReportProgressDto>();
+                    onProgress(arg.Operation?? string.Empty, arg.Progress);
+                    progressHandler.Invoke(arg, id);
                 };
                 generator.Make(reportFileName);
                 

AsbCloudInfrastructure/Services/SAUB/TelemetryDataCache.cs

@@ -48,9 +48,9 @@ namespace AsbCloudInfrastructure.Services.SAUB
                 instance = new TelemetryDataCache<TDto>();
                 var worker = provider.GetRequiredService<BackgroundWorker>();
                 var workId = $"Telemetry cache loading from DB {typeof(TEntity).Name}";
-                var work = new WorkBase(workId, async (workId, provider, token) => {
+                var work = new WorkBase(workId, async (workId, provider, onProgress, token) => {
                     var db = provider.GetRequiredService<IAsbCloudDbContext>();
-                    await instance.InitializeCacheFromDBAsync<TEntity>(db, token);
+                    await instance.InitializeCacheFromDBAsync<TEntity>(db, onProgress, token);
                 });
 
                 worker.Push(work);
@@ -150,7 +150,7 @@ namespace AsbCloudInfrastructure.Services.SAUB
             return new DatesRangeDto { From = from.Value, To = to };
         }
 
-        private async Task InitializeCacheFromDBAsync<TEntity>(IAsbCloudDbContext db, CancellationToken token)
+        private async Task InitializeCacheFromDBAsync<TEntity>(IAsbCloudDbContext db, Action<string, double?> onProgress, CancellationToken token)
             where TEntity : class, AsbCloudDb.Model.ITelemetryData
         {
             if (isLoading)
@@ -168,6 +168,8 @@ namespace AsbCloudInfrastructure.Services.SAUB
                 .Where(well => well.IdTelemetry != null)
                 .ToArrayAsync(token);
 
+            var count = wells.Count();
+            var i = 0;
             foreach (Well well in wells)
             {
                 var capacity = well.IdState == 1
@@ -176,7 +178,7 @@ namespace AsbCloudInfrastructure.Services.SAUB
 
                 var idTelemetry = well.IdTelemetry!.Value;
                 var hoursOffset = well.Timezone.Hours;
-
+                // TODO: remove traces
                 System.Diagnostics.Trace.TraceInformation($"cache<{typeof(TDto).Name}>: Loading for well: {well.Cluster?.Caption}/{well.Caption} (capacity:{capacity}) idTelemetry:{idTelemetry}");
                 var cacheItem = await GetOrDefaultCacheDataFromDbAsync<TEntity>(db, idTelemetry, capacity, hoursOffset, token);
                 if(cacheItem is not null)

AsbCloudInfrastructure/Services/Subsystems/SubsystemOperationTimeCalcWorkFactory.cs

@@ -36,7 +36,7 @@ namespace AsbCloudInfrastructure.Services.Subsystems
         }
 
         // TODO: Разделить этот акшн на более мелкие части И использовать telemetryServiceData<..> вместо прямого обращения к БД.
-        private static async Task WorkAction(string _, IServiceProvider serviceProvider, CancellationToken token)
+        private static async Task WorkAction(string _, IServiceProvider serviceProvider, Action<string, double?> onProgress, CancellationToken token)
         {
             using var db = serviceProvider.GetRequiredService<IAsbCloudDbContext>();
 
@@ -64,8 +64,11 @@ namespace AsbCloudInfrastructure.Services.Subsystems
                         inner.SingleOrDefault()?.LastDate,
                     });
 
+            var count = telemetryLastDetectedDates.Count();
+            var i = 0;
             foreach (var item in telemetryLastDetectedDates)
             {
+                onProgress($"Start hanling telemetry: {item.IdTelemetry} from {item.LastDate}", i++ / count);
                 var newOperationsSaub = await OperationTimeSaubAsync(item.IdTelemetry, item.LastDate ?? DateTimeOffset.MinValue, db, token);
                 if (newOperationsSaub?.Any() == true)
                 {

AsbCloudInfrastructure/Services/WellInfoService.cs

@@ -62,7 +62,7 @@ namespace AsbCloudInfrastructure.Services
             return workPeriodic;
         }
 
-        private static async Task WorkAction(string workName, IServiceProvider serviceProvider, CancellationToken token)
+        private static async Task WorkAction(string workName, IServiceProvider serviceProvider, Action<string, double?> onProgress, CancellationToken token)
         {
             var wellService = serviceProvider.GetRequiredService<IWellService>();
             var operationsStatService = serviceProvider.GetRequiredService<IOperationsStatService>();
@@ -90,11 +90,12 @@ namespace AsbCloudInfrastructure.Services
             
             var subsystemStat = await subsystemOperationTimeService
                 .GetStatByActiveWells(wellsIds, token);
-
+            var count = wells.Count();
+            var i = 0;
             WellMapInfo = wells.Select(well => {
                 var wellMapInfo = well.Adapt<WellMapInfoWithComanies>();
                 wellMapInfo.IdState = well.IdState;
-
+                onProgress($"Start updating info by well({well.Id}): {well.Caption}", i++ / count);
                 double? currentDepth = null;
 
                 TelemetryDataSaubDto? lastSaubTelemetry = null;

AsbCloudInfrastructure/Startup.cs

@@ -51,7 +51,7 @@ namespace AsbCloudInfrastructure
         static WorkPeriodic MakeMemoryMonitoringWork()
         {
             var workId = "Memory monitoring";
-            var workAction = (string _, IServiceProvider _, CancellationToken _) => {
+            var workAction = (string _, IServiceProvider _, Action<string, double?> _, CancellationToken _) => {
                 var bytes = GC.GetTotalMemory(false);
                 var bytesString = FromatBytes(bytes);
                 System.Diagnostics.Trace.TraceInformation($"Total memory allocated is {bytesString} bytes. DbContext count is:{AsbCloudDbContext.ReferenceCount}");

AsbCloudWebApi/SignalR/Services/SignalRNotificationTransportService.cs

@@ -39,9 +39,9 @@ public class SignalRNotificationTransportService : INotificationTransportService
 		return Task.CompletedTask;
 	}
 
-	private Func<string, IServiceProvider, CancellationToken, Task> MakeSignalRSendWorkAction(IEnumerable<NotificationDto> notifications)
+	private Func<string, IServiceProvider, Action<string, double?>, CancellationToken, Task> MakeSignalRSendWorkAction(IEnumerable<NotificationDto> notifications)
 	{
-		return async (_, serviceProvider, cancellationToken) =>
+		return async (_, serviceProvider, onProgress, cancellationToken) =>
 		{
 			var notificationPublisher = serviceProvider.GetRequiredService<NotificationPublisher>();