diff --git a/AsbCloudApp/Repositories/IDetectedOperationRepository.cs b/AsbCloudApp/Repositories/IDetectedOperationRepository.cs index 5a52b455..d7fe5865 100644 --- a/AsbCloudApp/Repositories/IDetectedOperationRepository.cs +++ b/AsbCloudApp/Repositories/IDetectedOperationRepository.cs @@ -1,15 +1,17 @@ -using AsbCloudApp.Data.DetectedOperation; +using System; +using AsbCloudApp.Data.DetectedOperation; using AsbCloudApp.Requests; using System.Collections.Generic; using System.Threading.Tasks; using System.Threading; +using AsbCloudApp.Services; namespace AsbCloudApp.Repositories; /// /// Таблица автоматически определенных операций /// -public interface IDetectedOperationRepository +public interface IDetectedOperationRepository : ICrudRepository { /// /// Добавление записей @@ -63,4 +65,11 @@ public interface IDetectedOperationRepository /// /// Task DeleteRange(int idUser, IEnumerable ids, CancellationToken token); + + /// + /// Получение дат последних определённых операций + /// + /// + /// + Task> GetLastDetectedDatesAsync(CancellationToken token); } diff --git a/AsbCloudApp/Services/IDetectedOperationService.cs b/AsbCloudApp/Services/IDetectedOperationService.cs index 5109fa56..998032ba 100644 --- a/AsbCloudApp/Services/IDetectedOperationService.cs +++ b/AsbCloudApp/Services/IDetectedOperationService.cs @@ -1,4 +1,5 @@ -using AsbCloudApp.Data; +using System; +using AsbCloudApp.Data; using AsbCloudApp.Data.DetectedOperation; using AsbCloudApp.Requests; using System.Collections.Generic; @@ -52,5 +53,14 @@ namespace AsbCloudApp.Services /// /// Task> GetOperationsStatAsync(DetectedOperationByWellRequest request, CancellationToken token); + + /// + /// Определение операций + /// + /// + /// + /// + /// + Task> DetectOperationsAsync(int idTelemetry, DateTimeOffset? beginDate, CancellationToken token); } } diff --git a/AsbCloudInfrastructure/Repository/DetectedOperationRepository.cs b/AsbCloudInfrastructure/Repository/DetectedOperationRepository.cs index 50ec6c9e..3cbb9c07 100644 --- a/AsbCloudInfrastructure/Repository/DetectedOperationRepository.cs +++ b/AsbCloudInfrastructure/Repository/DetectedOperationRepository.cs @@ -15,44 +15,52 @@ using System.Threading.Tasks; namespace AsbCloudInfrastructure.Repository; -public class DetectedOperationRepository : IDetectedOperationRepository +public class DetectedOperationRepository : CrudRepositoryBase, + IDetectedOperationRepository { - private readonly IAsbCloudDbContext db; private readonly ITelemetryService telemetryService; - public DetectedOperationRepository( - IAsbCloudDbContext db, + public DetectedOperationRepository(IAsbCloudDbContext context, ITelemetryService telemetryService) + : base(context) { - this.db = db; this.telemetryService = telemetryService; } public async Task Delete(int idUser, DetectedOperationByTelemetryRequest request, CancellationToken token) { var query = BuildQuery(request); - db.Set().RemoveRange(query); - return await db.SaveChangesAsync(token); + dbContext.Set().RemoveRange(query); + return await dbContext.SaveChangesAsync(token); } public async Task DeleteRange(int idUser, IEnumerable ids, CancellationToken token) { - var query = db.Set() + var query = dbContext.Set() .Where(e => ids.Contains( e.Id)); - db.Set() + dbContext.Set() .RemoveRange(query); - return await db.SaveChangesAsync(token); + return await dbContext.SaveChangesAsync(token); } + public async Task> GetLastDetectedDatesAsync(CancellationToken token) => + await dbContext.Set() + .GroupBy(o => o.IdTelemetry) + .Select(g => new + { + IdTelemetry = g.Key, + LastDate = g.Max(o => o.DateEnd) + }) + .ToDictionaryAsync(x => x.IdTelemetry, x => x.LastDate, token); + public async Task> Get(DetectedOperationByTelemetryRequest request, CancellationToken token) { var query = BuildQuery(request) .Include(o => o.OperationCategory); var entities = await query.ToArrayAsync(token); - var offset = telemetryService.GetTimezone(request.IdTelemetry).Offset; - var dtos = entities.Select(o => Convert(o, offset)); + var dtos = entities.Select(Convert); return dtos; } @@ -63,14 +71,14 @@ public class DetectedOperationRepository : IDetectedOperationRepository return 0; var entities = dtos.Select(Convert); - var dbset = db.Set(); + var dbset = dbContext.Set(); foreach(var entity in entities) { entity.Id = default; dbset.Add(entity); } - return await db.SaveChangesWithExceptionHandling(token); + return await dbContext.SaveChangesWithExceptionHandling(token); } public async Task Update(int idUser, IEnumerable dtos, CancellationToken token) @@ -89,7 +97,7 @@ public class DetectedOperationRepository : IDetectedOperationRepository if (ids.Length != dtos.Count()) throw new ArgumentInvalidException(nameof(dtos), "Все записи должны иметь уникальные Id"); - var dbSet = db.Set(); + var dbSet = dbContext.Set(); var existingEntitiesCount = await dbSet .Where(o => ids.Contains(o.Id)) @@ -106,7 +114,7 @@ public class DetectedOperationRepository : IDetectedOperationRepository for(var i = 0; i < entities.Length; i++) entries[i] = dbSet.Update(entities[i]); - var result = await db.SaveChangesWithExceptionHandling(token); + var result = await dbContext.SaveChangesWithExceptionHandling(token); for (var i = 0; i < entries.Length; i++) entries[i].State = EntityState.Detached; @@ -131,7 +139,7 @@ public class DetectedOperationRepository : IDetectedOperationRepository private IQueryable BuildQuery(DetectedOperationByTelemetryRequest request) { - var query = db.Set() + var query = dbContext.Set() .Where(o => o.IdTelemetry == request.IdTelemetry); if (request.IdsCategories.Any()) @@ -173,19 +181,21 @@ public class DetectedOperationRepository : IDetectedOperationRepository return query; } - private static DetectedOperationDto Convert(DetectedOperation entity, TimeSpan offset) + protected override DetectedOperationDto Convert(DetectedOperation src) { - var dto = entity.Adapt(); - dto.DateStart = entity.DateStart.ToOffset(offset); - dto.DateEnd = entity.DateEnd.ToOffset(offset); + var timezone = telemetryService.GetTimezone(src.IdTelemetry); + + var dto = src.Adapt(); + dto.DateStart = src.DateStart.ToOffset(timezone.Offset); + dto.DateEnd = src.DateEnd.ToOffset(timezone.Offset); return dto; } - private static DetectedOperation Convert(DetectedOperationDto dto) + protected override DetectedOperation Convert(DetectedOperationDto src) { - var entity = dto.Adapt(); - entity.DateStart = dto.DateStart.ToUniversalTime(); - entity.DateEnd = dto.DateEnd.ToUniversalTime(); + var entity = src.Adapt(); + entity.DateStart = src.DateStart.ToUniversalTime(); + entity.DateEnd = src.DateEnd.ToUniversalTime(); return entity; } } diff --git a/AsbCloudInfrastructure/Services/DetectOperations/DetectedOperationExportService.cs b/AsbCloudInfrastructure/Services/DetectOperations/DetectedOperationExportService.cs index 987f274f..3ba4350a 100644 --- a/AsbCloudInfrastructure/Services/DetectOperations/DetectedOperationExportService.cs +++ b/AsbCloudInfrastructure/Services/DetectOperations/DetectedOperationExportService.cs @@ -19,9 +19,9 @@ namespace AsbCloudInfrastructure.Services.DetectOperations; public class DetectedOperationExportService { - private readonly IAsbCloudDbContext db; - private readonly IWellService wellService; + private readonly IWellService wellService; private readonly IWellOperationCategoryRepository wellOperationCategoryRepository; + private readonly IDetectedOperationService detectedOperationService; private const int headerRowsCount = 1; private const string cellDepositName = "B1"; @@ -40,15 +40,14 @@ public class DetectedOperationExportService private const int columnIdReasonOfEnd = 9; private const int columnComment = 10; - public DetectedOperationExportService( - IAsbCloudDbContext db, - IWellService wellService, - IWellOperationCategoryRepository wellOperationCategoryRepository) + public DetectedOperationExportService(IWellService wellService, + IWellOperationCategoryRepository wellOperationCategoryRepository, + IDetectedOperationService detectedOperationService) { - this.db = db; - this.wellService = wellService; + this.wellService = wellService; this.wellOperationCategoryRepository = wellOperationCategoryRepository; - } + this.detectedOperationService = detectedOperationService; + } /// /// Экспорт excel файла с операциями по скважине @@ -68,12 +67,12 @@ public class DetectedOperationExportService if (!well.IdTelemetry.HasValue) throw new ArgumentInvalidException(nameof(idWell), $"Well {idWell} has no telemetry"); - var operations = await WorkOperationDetection.DetectOperationsAsync(well.IdTelemetry.Value, DateTime.UnixEpoch, db, token); + var operations = await detectedOperationService.DetectOperationsAsync(well.IdTelemetry.Value, DateTime.UnixEpoch, token); return await GenerateExcelFileStreamAsync(well, host, operations, token); } - private async Task GenerateExcelFileStreamAsync(WellDto well, string host, IEnumerable operationDetectorResults, + private async Task GenerateExcelFileStreamAsync(WellDto well, string host, IEnumerable operationDetectorResults, CancellationToken cancellationToken) { using var excelTemplateStream = await GetExcelTemplateStreamAsync(cancellationToken); @@ -88,7 +87,7 @@ public class DetectedOperationExportService return memoryStream; } - private void AddToWorkbook(XLWorkbook workbook, WellDto well, string host, IEnumerable operations) + private void AddToWorkbook(XLWorkbook workbook, WellDto well, string host, IEnumerable operations) { const string sheetName = "Операции"; @@ -104,14 +103,17 @@ public class DetectedOperationExportService AddToSheet(sheet, well, host, orderedOperations); } - private void AddToSheet(IXLWorksheet sheet, WellDto well, string host, IList operations) + private void AddToSheet(IXLWorksheet sheet, WellDto well, string host, IList operations) { var wellOperationCategories = wellOperationCategoryRepository.Get(true); sheet.Cell(cellDepositName).SetCellValue(well.Deposit); sheet.Cell(cellClusterName).SetCellValue(well.Cluster); sheet.Cell(cellWellName).SetCellValue(well.Caption); - sheet.Cell(cellDeltaDate).SetCellValue((TimeSpan)(Enumerable.Max(operations, (Func)(o => o.DateEnd)) - Enumerable.Min(operations, (Func)(o => o.DateStart)))); + + var deltaDate = operations.Max(o => o.DateEnd - o.DateStart); + + sheet.Cell(cellDeltaDate).SetCellValue(deltaDate); for (int i = 0; i < operations.Count; i++) { @@ -157,7 +159,7 @@ public class DetectedOperationExportService } } - private static string GetCategoryName(IEnumerable wellOperationCategories, DetectedOperation current) + private static string GetCategoryName(IEnumerable wellOperationCategories, DetectedOperationDto current) { var idCategory = current.IdCategory; if (idCategory == WellOperationCategory.IdSlide && @@ -198,7 +200,7 @@ public class DetectedOperationExportService return memoryStream; } - private static string CreateComment(DetectedOperation operation) + private static string CreateComment(DetectedOperationDto operation) { switch (operation.IdCategory) { diff --git a/AsbCloudInfrastructure/Services/DetectOperations/DetectedOperationService.cs b/AsbCloudInfrastructure/Services/DetectOperations/DetectedOperationService.cs index bd8fba77..fca85b82 100644 --- a/AsbCloudInfrastructure/Services/DetectOperations/DetectedOperationService.cs +++ b/AsbCloudInfrastructure/Services/DetectOperations/DetectedOperationService.cs @@ -1,16 +1,16 @@ -using AsbCloudApp.Data; +using System; +using AsbCloudApp.Data; using AsbCloudApp.Data.DetectedOperation; using AsbCloudApp.Repositories; using AsbCloudApp.Requests; using AsbCloudApp.Services; -using AsbCloudDb; using AsbCloudDb.Model; using Mapster; -using Microsoft.EntityFrameworkCore; using System.Collections.Generic; using System.Linq; using System.Threading; using System.Threading.Tasks; +using AsbCloudInfrastructure.Services.DetectOperations.Detectors; namespace AsbCloudInfrastructure.Services.DetectOperations; @@ -19,21 +19,29 @@ public class DetectedOperationService : IDetectedOperationService private readonly IDetectedOperationRepository operationRepository; private readonly IWellOperationCategoryRepository wellOperationCategoryRepository; private readonly IWellService wellService; - private readonly IRepositoryWellRelated operationValueService; - private readonly IScheduleRepository scheduleService; + private readonly IRepositoryWellRelated operationValueRepository; + private readonly IScheduleRepository scheduleRepository; + private readonly ITelemetryDataSaubService telemetryDataSaubService; + private static readonly DetectorAbstract[] detectors = { + new DetectorDrilling(), + new DetectorSlipsTime() + }; + public DetectedOperationService( IDetectedOperationRepository operationRepository, IWellOperationCategoryRepository wellOperationCategoryRepository, IWellService wellService, IRepositoryWellRelated operationValueRepository, - IScheduleRepository scheduleRepository) + IScheduleRepository scheduleRepository, + ITelemetryDataSaubService telemetryDataSaubService) { this.operationRepository = operationRepository; this.wellOperationCategoryRepository = wellOperationCategoryRepository; this.wellService = wellService; - this.operationValueService = operationValueRepository; - this.scheduleService = scheduleRepository; + this.operationValueRepository = operationValueRepository; + this.scheduleRepository = scheduleRepository; + this.telemetryDataSaubService = telemetryDataSaubService; } public async Task GetAsync(DetectedOperationByWellRequest request, CancellationToken token) @@ -60,8 +68,8 @@ public class DetectedOperationService : IDetectedOperationService var requestByTelemetry = new DetectedOperationByTelemetryRequest(well.IdTelemetry.Value, request); var data = await operationRepository.Get(requestByTelemetry, token); - var operationValues = await operationValueService.GetByIdWellAsync(request.IdWell, token); - var schedules = await scheduleService.GetByIdWellAsync(request.IdWell, token); + var operationValues = await operationValueRepository.GetByIdWellAsync(request.IdWell, token); + var schedules = await scheduleRepository.GetByIdWellAsync(request.IdWell, token); var dtos = data.Select(o => Convert(o, operationValues, schedules)); return dtos; } @@ -103,9 +111,7 @@ public class DetectedOperationService : IDetectedOperationService if (!operations.Any()) return Enumerable.Empty(); - - var operationValues = await operationValueService.GetByIdWellAsync(request.IdWell, token); - + var dtos = operations .GroupBy(o => (o.IdCategory, o.OperationCategory.Name)) .OrderBy(g => g.Key) @@ -126,6 +132,68 @@ public class DetectedOperationService : IDetectedOperationService return dtos; } + public async Task> DetectOperationsAsync(int idTelemetry, DateTimeOffset? beginDate, CancellationToken token) + { + const int take = 4 * 86_400; + + var detectedOperations = new List(); + DetectedOperationDto? lastDetectedOperation = null; + const int minOperationLength = 5; + const int maxDetectorsInterpolationFrameLength = 30; + const int gap = maxDetectorsInterpolationFrameLength + minOperationLength; + + while (true) + { + var request = new TelemetryDataRequest + { + GeDate = beginDate, + Take = take, + Order = 0 + }; + + var detectableTelemetries = (await telemetryDataSaubService.GetByTelemetryAsync(idTelemetry, request, token)) + .Where(t => t.BlockPosition >= 0) + .Select(t => new DetectableTelemetry + { + DateTime = t.DateTime, + Mode = t.Mode, + WellDepth = t.WellDepth, + Pressure = t.Pressure, + HookWeight = t.HookWeight, + BlockPosition = t.BlockPosition, + BitDepth = t.BitDepth, + RotorSpeed = t.RotorSpeed, + }).ToArray(); + + if (detectableTelemetries.Length < gap) + break; + + var isDetected = false; + var positionBegin = 0; + var positionEnd = detectableTelemetries.Length - gap; + while (positionEnd > positionBegin) + { + foreach (var detector in detectors) + { + if (!detector.TryDetect(idTelemetry, detectableTelemetries, positionBegin, positionEnd, lastDetectedOperation, out var result)) + continue; + + detectedOperations.Add(result!.Operation); + lastDetectedOperation = result.Operation; + isDetected = true; + positionBegin = result.TelemetryEnd; + break; + } + + positionBegin += 1; + } + + beginDate = isDetected ? lastDetectedOperation!.DateEnd : detectableTelemetries[positionEnd].DateTime; + } + + return detectedOperations; + } + public async Task DeleteAsync(DetectedOperationByWellRequest request, CancellationToken token) { var well = await wellService.GetOrDefaultAsync(request.IdWell, token); diff --git a/AsbCloudInfrastructure/Services/DetectOperations/WorkOperationDetection.cs b/AsbCloudInfrastructure/Services/DetectOperations/WorkOperationDetection.cs index aece226e..9b011970 100644 --- a/AsbCloudInfrastructure/Services/DetectOperations/WorkOperationDetection.cs +++ b/AsbCloudInfrastructure/Services/DetectOperations/WorkOperationDetection.cs @@ -1,12 +1,12 @@ -using AsbCloudDb.Model; -using Microsoft.EntityFrameworkCore; -using System; +using System; using System.Collections.Generic; using System.Diagnostics; using System.Linq; using System.Threading; using System.Threading.Tasks; -using AsbCloudInfrastructure.Services.DetectOperations.Detectors; +using AsbCloudApp.Data; +using AsbCloudApp.Repositories; +using AsbCloudApp.Services; using AsbCloudInfrastructure.Background; using Microsoft.Extensions.DependencyInjection; @@ -14,19 +14,6 @@ namespace AsbCloudInfrastructure.Services.DetectOperations; public class WorkOperationDetection: Work { - private static readonly DetectorAbstract[] detectors = new DetectorAbstract[] - { - new DetectorDrilling(), - new DetectorSlipsTime() - // new DetectorRotor(), - // new DetectorSlide(), - //new DetectorDevelopment(), - //new DetectorTemplating(), - //new DetectorStaticSurveying(), - //new DetectorFlashingBeforeConnection(), - //new DetectorFlashing(), - //new DetectorTemplatingWhileDrilling(), - }; public WorkOperationDetection() :base("Operation detection") @@ -42,115 +29,39 @@ public class WorkOperationDetection: Work protected override async Task Action(string id, IServiceProvider services, Action onProgressCallback, CancellationToken token) { - using var db = services.GetRequiredService(); - db.Database.SetCommandTimeout(TimeSpan.FromMinutes(5)); - var lastDetectedDates = await db.DetectedOperations - .GroupBy(o => o.IdTelemetry) - .Select(g => new - { - IdTelemetry = g.Key, - LastDate = g.Max(o => o.DateEnd) - }) - .ToListAsync(token); + var telemetryRepository = services.GetRequiredService>(); + var detectedOperationRepository = services.GetRequiredService(); + var detectedOperationService = services.GetRequiredService(); - var telemetryIds = await db.Telemetries - .Where(t => t.Info != null && t.TimeZone != null) - .Select(t => t.Id) - .ToListAsync(token); + var telemetryIds = (await telemetryRepository.GetAllAsync(token)) + .Select(t => t.Id); - var joinedlastDetectedDates = telemetryIds - .GroupJoin(lastDetectedDates, - t => t, - o => o.IdTelemetry, - (outer, inner) => new - { - IdTelemetry = outer, - inner.SingleOrDefault()?.LastDate, - }); + var lastDetectedDates = await detectedOperationRepository.GetLastDetectedDatesAsync(token); - var affected = 0; - var count = joinedlastDetectedDates.Count(); - var i = 0d; - foreach (var item in joinedlastDetectedDates) + var beginDatesDetectOperations = new List<(int TelemetryId, DateTimeOffset? BeginDate)>(); + + foreach (var telemetryId in telemetryIds) { - var stopwatch = Stopwatch.StartNew(); - var startDate = item.LastDate ?? DateTimeOffset.MinValue; - onProgressCallback($"start detecting telemetry: {item.IdTelemetry} from {startDate}", i++ / count); - var newOperations = await DetectOperationsAsync(item.IdTelemetry, startDate, db, token); - stopwatch.Stop(); - if (newOperations.Any()) + if (lastDetectedDates.TryGetValue(telemetryId, out var beginDate)) { - db.DetectedOperations.AddRange(newOperations); - affected += await db.SaveChangesAsync(token); + beginDatesDetectOperations.Add((telemetryId, beginDate)); + continue; } - } - } - - //todo: move this logic to DetectedOperationsService - internal static async Task> DetectOperationsAsync(int idTelemetry, DateTimeOffset begin, IAsbCloudDbContext db, CancellationToken token) - { - var query = db.TelemetryDataSaub - .AsNoTracking() - .Where(d => d.IdTelemetry == idTelemetry) - .Where(d => d.BlockPosition >= 0) - .Select(d => new DetectableTelemetry - { - DateTime = d.DateTime, - IdUser = d.IdUser, - Mode = d.Mode, - WellDepth = d.WellDepth, - Pressure = d.Pressure, - HookWeight = d.HookWeight, - BlockPosition = d.BlockPosition, - BitDepth = d.BitDepth, - RotorSpeed = d.RotorSpeed, - }) - .OrderBy(d => d.DateTime); - - var take = 4 * 86_400; // 4 дня - var startDate = begin; - var detectedOperations = new List(8); - DetectedOperation? lastDetectedOperation = null; - const int minOperationLength = 5; - const int maxDetectorsInterpolationFrameLength = 30; - const int gap = maxDetectorsInterpolationFrameLength + minOperationLength; - - while (true) - { - var data = await query - .Where(d => d.DateTime > startDate) - .Take(take) - .ToArrayAsync(token); - - if (data.Length < gap) - break; - - var isDetected = false; - var positionBegin = 0; - var positionEnd = data.Length - gap; - while (positionEnd > positionBegin) - { - foreach (var detector in detectors) - { - if (!detector.TryDetect(idTelemetry, data, positionBegin, positionEnd, lastDetectedOperation, out var result)) - continue; - - detectedOperations.Add(result!.Operation); - lastDetectedOperation = result.Operation; - isDetected = true; - positionBegin = result.TelemetryEnd; - break; - } - - positionBegin += 1; - } - - if (isDetected) - startDate = lastDetectedOperation!.DateEnd; - else - startDate = data[positionEnd].DateTime; + + beginDatesDetectOperations.Add((telemetryId, null)); } - return detectedOperations; + var count = beginDatesDetectOperations.Count; + + for (var i = 0; i < count; i++) + { + var (idTelemetry, beginDate) = beginDatesDetectOperations[i]; + + onProgressCallback($"Start detecting telemetry: {idTelemetry} from {beginDate}", i++ / count); + var detectedOperations = await detectedOperationService.DetectOperationsAsync(idTelemetry, beginDate, token); + + if (detectedOperations.Any()) + await detectedOperationRepository.InsertRangeAsync(detectedOperations, token); + } } }