on.nemtina/DD.WellWorkover.Cloud
1
0
Fork 0
forked from ddrilling/AsbCloudServer
Code Pull Requests Activity

Рефакторинг авто определения операций

Browse Source
This commit is contained in:
Степанов Дмитрий 2024-02-20 11:22:58 +03:00
parent 7919926c21
commit a77c3b4566
6 changed files with 186 additions and 176 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
AsbCloudApp/Repositories/IDetectedOperationRepository.cs
View File

@ -1,15 +1,17 @@
using AsbCloudApp.Data.DetectedOperation;
using System;
using AsbCloudApp.Data.DetectedOperation;
using AsbCloudApp.Requests;
using System.Collections.Generic;
using System.Threading.Tasks;
using System.Threading;
using AsbCloudApp.Services;
namespace AsbCloudApp.Repositories;
/// <summary>
/// Таблица автоматически определенных операций
/// </summary>
public interface IDetectedOperationRepository
public interface IDetectedOperationRepository : ICrudRepository<DetectedOperationDto>
{
/// <summary>
/// Добавление записей
@ -63,4 +65,11 @@ public interface IDetectedOperationRepository
/// <param name="token"></param>
/// <returns></returns>
Task<int> DeleteRange(int idUser, IEnumerable<int> ids, CancellationToken token);
/// <summary>
/// Получение дат последних определённых операций
/// </summary>
/// <param name="token"></param>
/// <returns></returns>
Task<IDictionary<int, DateTimeOffset>> GetLastDetectedDatesAsync(CancellationToken token);
}

12
AsbCloudApp/Services/IDetectedOperationService.cs
View File

@ -1,4 +1,5 @@
using AsbCloudApp.Data;
using System;
using AsbCloudApp.Data;
using AsbCloudApp.Data.DetectedOperation;
using AsbCloudApp.Requests;
using System.Collections.Generic;
@ -52,5 +53,14 @@ namespace AsbCloudApp.Services
/// <param name="token"></param>
/// <returns></returns>
Task<IEnumerable<DetectedOperationStatDto>> GetOperationsStatAsync(DetectedOperationByWellRequest request, CancellationToken token);
/// <summary>
/// Определение операций
/// </summary>
/// <param name="idTelemetry"></param>
/// <param name="beginDate"></param>
/// <param name="token"></param>
/// <returns></returns>
Task<IEnumerable<DetectedOperationDto>> DetectOperationsAsync(int idTelemetry, DateTimeOffset? beginDate, CancellationToken token);
}
}

60
AsbCloudInfrastructure/Repository/DetectedOperationRepository.cs
View File

@ -15,44 +15,52 @@ using System.Threading.Tasks;
namespace AsbCloudInfrastructure.Repository;
public class DetectedOperationRepository : IDetectedOperationRepository
public class DetectedOperationRepository : CrudRepositoryBase<DetectedOperationDto, DetectedOperation>,
IDetectedOperationRepository
{
private readonly IAsbCloudDbContext db;
private readonly ITelemetryService telemetryService;
public DetectedOperationRepository(
IAsbCloudDbContext db,
public DetectedOperationRepository(IAsbCloudDbContext context,
ITelemetryService telemetryService)
: base(context)
{
this.db = db;
this.telemetryService = telemetryService;
}
public async Task<int> Delete(int idUser, DetectedOperationByTelemetryRequest request, CancellationToken token)
{
var query = BuildQuery(request);
db.Set<DetectedOperation>().RemoveRange(query);
return await db.SaveChangesAsync(token);
dbContext.Set<DetectedOperation>().RemoveRange(query);
return await dbContext.SaveChangesAsync(token);
}
public async Task<int> DeleteRange(int idUser, IEnumerable<int> ids, CancellationToken token)
{
var query = db.Set<DetectedOperation>()
var query = dbContext.Set<DetectedOperation>()
.Where(e => ids.Contains( e.Id));
db.Set<DetectedOperation>()
dbContext.Set<DetectedOperation>()
.RemoveRange(query);
return await db.SaveChangesAsync(token);
return await dbContext.SaveChangesAsync(token);
}
public async Task<IDictionary<int, DateTimeOffset>> GetLastDetectedDatesAsync(CancellationToken token) =>
await dbContext.Set<DetectedOperation>()
.GroupBy(o => o.IdTelemetry)
.Select(g => new
{
IdTelemetry = g.Key,
LastDate = g.Max(o => o.DateEnd)
})
.ToDictionaryAsync(x => x.IdTelemetry, x => x.LastDate, token);
public async Task<IEnumerable<DetectedOperationDto>> Get(DetectedOperationByTelemetryRequest request, CancellationToken token)
{
var query = BuildQuery(request)
.Include(o => o.OperationCategory);
var entities = await query.ToArrayAsync(token);
var offset = telemetryService.GetTimezone(request.IdTelemetry).Offset;
var dtos = entities.Select(o => Convert(o, offset));
var dtos = entities.Select(Convert);
return dtos;
}
@ -63,14 +71,14 @@ public class DetectedOperationRepository : IDetectedOperationRepository
return 0;
var entities = dtos.Select(Convert);
var dbset = db.Set<DetectedOperation>();
var dbset = dbContext.Set<DetectedOperation>();
foreach(var entity in entities)
{
entity.Id = default;
dbset.Add(entity);
}
return await db.SaveChangesWithExceptionHandling(token);
return await dbContext.SaveChangesWithExceptionHandling(token);
}
public async Task<int> Update(int idUser, IEnumerable<DetectedOperationDto> dtos, CancellationToken token)
@ -89,7 +97,7 @@ public class DetectedOperationRepository : IDetectedOperationRepository
if (ids.Length != dtos.Count())
throw new ArgumentInvalidException(nameof(dtos), "Все записи должны иметь уникальные Id");
var dbSet = db.Set<DetectedOperation>();
var dbSet = dbContext.Set<DetectedOperation>();
var existingEntitiesCount = await dbSet
.Where(o => ids.Contains(o.Id))
@ -106,7 +114,7 @@ public class DetectedOperationRepository : IDetectedOperationRepository
for(var i = 0; i < entities.Length; i++)
entries[i] = dbSet.Update(entities[i]);
var result = await db.SaveChangesWithExceptionHandling(token);
var result = await dbContext.SaveChangesWithExceptionHandling(token);
for (var i = 0; i < entries.Length; i++)
entries[i].State = EntityState.Detached;
@ -131,7 +139,7 @@ public class DetectedOperationRepository : IDetectedOperationRepository
private IQueryable<DetectedOperation> BuildQuery(DetectedOperationByTelemetryRequest request)
{
var query = db.Set<DetectedOperation>()
var query = dbContext.Set<DetectedOperation>()
.Where(o => o.IdTelemetry == request.IdTelemetry);
if (request.IdsCategories.Any())
@ -173,19 +181,21 @@ public class DetectedOperationRepository : IDetectedOperationRepository
return query;
}
private static DetectedOperationDto Convert(DetectedOperation entity, TimeSpan offset)
protected override DetectedOperationDto Convert(DetectedOperation src)
{
var dto = entity.Adapt<DetectedOperationDto>();
dto.DateStart = entity.DateStart.ToOffset(offset);
dto.DateEnd = entity.DateEnd.ToOffset(offset);
var timezone = telemetryService.GetTimezone(src.IdTelemetry);
var dto = src.Adapt<DetectedOperationDto>();
dto.DateStart = src.DateStart.ToOffset(timezone.Offset);
dto.DateEnd = src.DateEnd.ToOffset(timezone.Offset);
return dto;
}
private static DetectedOperation Convert(DetectedOperationDto dto)
protected override DetectedOperation Convert(DetectedOperationDto src)
{
var entity = dto.Adapt<DetectedOperation>();
entity.DateStart = dto.DateStart.ToUniversalTime();
entity.DateEnd = dto.DateEnd.ToUniversalTime();
var entity = src.Adapt<DetectedOperation>();
entity.DateStart = src.DateStart.ToUniversalTime();
entity.DateEnd = src.DateEnd.ToUniversalTime();
return entity;
}
}

34
AsbCloudInfrastructure/Services/DetectOperations/DetectedOperationExportService.cs
View File

@ -19,9 +19,9 @@ namespace AsbCloudInfrastructure.Services.DetectOperations;
public class DetectedOperationExportService
{
private readonly IAsbCloudDbContext db;
private readonly IWellService wellService;
private readonly IWellService wellService;
private readonly IWellOperationCategoryRepository wellOperationCategoryRepository;
private readonly IDetectedOperationService detectedOperationService;
private const int headerRowsCount = 1;
private const string cellDepositName = "B1";
@ -40,15 +40,14 @@ public class DetectedOperationExportService
private const int columnIdReasonOfEnd = 9;
private const int columnComment = 10;
public DetectedOperationExportService(
IAsbCloudDbContext db,
IWellService wellService,
IWellOperationCategoryRepository wellOperationCategoryRepository)
public DetectedOperationExportService(IWellService wellService,
IWellOperationCategoryRepository wellOperationCategoryRepository,
IDetectedOperationService detectedOperationService)
{
this.db = db;
this.wellService = wellService;
this.wellService = wellService;
this.wellOperationCategoryRepository = wellOperationCategoryRepository;
}
this.detectedOperationService = detectedOperationService;
}
/// <summary>
/// Экспорт excel файла с операциями по скважине
@ -68,12 +67,12 @@ public class DetectedOperationExportService
if (!well.IdTelemetry.HasValue)
throw new ArgumentInvalidException(nameof(idWell), $"Well {idWell} has no telemetry");
var operations = await WorkOperationDetection.DetectOperationsAsync(well.IdTelemetry.Value, DateTime.UnixEpoch, db, token);
var operations = await detectedOperationService.DetectOperationsAsync(well.IdTelemetry.Value, DateTime.UnixEpoch, token);
return await GenerateExcelFileStreamAsync(well, host, operations, token);
}
private async Task<Stream> GenerateExcelFileStreamAsync(WellDto well, string host, IEnumerable<DetectedOperation> operationDetectorResults,
private async Task<Stream> GenerateExcelFileStreamAsync(WellDto well, string host, IEnumerable<DetectedOperationDto> operationDetectorResults,
CancellationToken cancellationToken)
{
using var excelTemplateStream = await GetExcelTemplateStreamAsync(cancellationToken);
@ -88,7 +87,7 @@ public class DetectedOperationExportService
return memoryStream;
}
private void AddToWorkbook(XLWorkbook workbook, WellDto well, string host, IEnumerable<DetectedOperation> operations)
private void AddToWorkbook(XLWorkbook workbook, WellDto well, string host, IEnumerable<DetectedOperationDto> operations)
{
const string sheetName = "Операции";
@ -104,14 +103,17 @@ public class DetectedOperationExportService
AddToSheet(sheet, well, host, orderedOperations);
}
private void AddToSheet(IXLWorksheet sheet, WellDto well, string host, IList<DetectedOperation> operations)
private void AddToSheet(IXLWorksheet sheet, WellDto well, string host, IList<DetectedOperationDto> operations)
{
var wellOperationCategories = wellOperationCategoryRepository.Get(true);
sheet.Cell(cellDepositName).SetCellValue(well.Deposit);
sheet.Cell(cellClusterName).SetCellValue(well.Cluster);
sheet.Cell(cellWellName).SetCellValue(well.Caption);
sheet.Cell(cellDeltaDate).SetCellValue((TimeSpan)(Enumerable.Max<DetectedOperation, DateTimeOffset>(operations, (Func<DetectedOperation, DateTimeOffset>)(o => o.DateEnd)) - Enumerable.Min<DetectedOperation, DateTimeOffset>(operations, (Func<DetectedOperation, DateTimeOffset>)(o => o.DateStart))));
var deltaDate = operations.Max(o => o.DateEnd - o.DateStart);
sheet.Cell(cellDeltaDate).SetCellValue(deltaDate);
for (int i = 0; i < operations.Count; i++)
{
@ -157,7 +159,7 @@ public class DetectedOperationExportService
}
}
private static string GetCategoryName(IEnumerable<WellOperationCategoryDto> wellOperationCategories, DetectedOperation current)
private static string GetCategoryName(IEnumerable<WellOperationCategoryDto> wellOperationCategories, DetectedOperationDto current)
{
var idCategory = current.IdCategory;
if (idCategory == WellOperationCategory.IdSlide &&
@ -198,7 +200,7 @@ public class DetectedOperationExportService
return memoryStream;
}
private static string CreateComment(DetectedOperation operation)
private static string CreateComment(DetectedOperationDto operation)
{
switch (operation.IdCategory)
{

94
AsbCloudInfrastructure/Services/DetectOperations/DetectedOperationService.cs
View File

@ -1,16 +1,16 @@
using AsbCloudApp.Data;
using System;
using AsbCloudApp.Data;
using AsbCloudApp.Data.DetectedOperation;
using AsbCloudApp.Repositories;
using AsbCloudApp.Requests;
using AsbCloudApp.Services;
using AsbCloudDb;
using AsbCloudDb.Model;
using Mapster;
using Microsoft.EntityFrameworkCore;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using AsbCloudInfrastructure.Services.DetectOperations.Detectors;
namespace AsbCloudInfrastructure.Services.DetectOperations;
@ -19,21 +19,29 @@ public class DetectedOperationService : IDetectedOperationService
private readonly IDetectedOperationRepository operationRepository;
private readonly IWellOperationCategoryRepository wellOperationCategoryRepository;
private readonly IWellService wellService;
private readonly IRepositoryWellRelated<OperationValueDto> operationValueService;
private readonly IScheduleRepository scheduleService;
private readonly IRepositoryWellRelated<OperationValueDto> operationValueRepository;
private readonly IScheduleRepository scheduleRepository;
private readonly ITelemetryDataSaubService telemetryDataSaubService;
private static readonly DetectorAbstract[] detectors = {
new DetectorDrilling(),
new DetectorSlipsTime()
};
public DetectedOperationService(
IDetectedOperationRepository operationRepository,
IWellOperationCategoryRepository wellOperationCategoryRepository,
IWellService wellService,
IRepositoryWellRelated<OperationValueDto> operationValueRepository,
IScheduleRepository scheduleRepository)
IScheduleRepository scheduleRepository,
ITelemetryDataSaubService telemetryDataSaubService)
{
this.operationRepository = operationRepository;
this.wellOperationCategoryRepository = wellOperationCategoryRepository;
this.wellService = wellService;
this.operationValueService = operationValueRepository;
this.scheduleService = scheduleRepository;
this.operationValueRepository = operationValueRepository;
this.scheduleRepository = scheduleRepository;
this.telemetryDataSaubService = telemetryDataSaubService;
}
public async Task<DetectedOperationListDto> GetAsync(DetectedOperationByWellRequest request, CancellationToken token)
@ -60,8 +68,8 @@ public class DetectedOperationService : IDetectedOperationService
var requestByTelemetry = new DetectedOperationByTelemetryRequest(well.IdTelemetry.Value, request);
var data = await operationRepository.Get(requestByTelemetry, token);
var operationValues = await operationValueService.GetByIdWellAsync(request.IdWell, token);
var schedules = await scheduleService.GetByIdWellAsync(request.IdWell, token);
var operationValues = await operationValueRepository.GetByIdWellAsync(request.IdWell, token);
var schedules = await scheduleRepository.GetByIdWellAsync(request.IdWell, token);
var dtos = data.Select(o => Convert(o, operationValues, schedules));
return dtos;
}
@ -103,9 +111,7 @@ public class DetectedOperationService : IDetectedOperationService
if (!operations.Any())
return Enumerable.Empty<DetectedOperationStatDto>();
var operationValues = await operationValueService.GetByIdWellAsync(request.IdWell, token);
var dtos = operations
.GroupBy(o => (o.IdCategory, o.OperationCategory.Name))
.OrderBy(g => g.Key)
@ -126,6 +132,68 @@ public class DetectedOperationService : IDetectedOperationService
return dtos;
}
public async Task<IEnumerable<DetectedOperationDto>> DetectOperationsAsync(int idTelemetry, DateTimeOffset? beginDate, CancellationToken token)
{
const int take = 4 * 86_400;
var detectedOperations = new List<DetectedOperationDto>();
DetectedOperationDto? lastDetectedOperation = null;
const int minOperationLength = 5;
const int maxDetectorsInterpolationFrameLength = 30;
const int gap = maxDetectorsInterpolationFrameLength + minOperationLength;
while (true)
{
var request = new TelemetryDataRequest
{
GeDate = beginDate,
Take = take,
Order = 0
};
var detectableTelemetries = (await telemetryDataSaubService.GetByTelemetryAsync(idTelemetry, request, token))
.Where(t => t.BlockPosition >= 0)
.Select(t => new DetectableTelemetry
{
DateTime = t.DateTime,
Mode = t.Mode,
WellDepth = t.WellDepth,
Pressure = t.Pressure,
HookWeight = t.HookWeight,
BlockPosition = t.BlockPosition,
BitDepth = t.BitDepth,
RotorSpeed = t.RotorSpeed,
}).ToArray();
if (detectableTelemetries.Length < gap)
break;
var isDetected = false;
var positionBegin = 0;
var positionEnd = detectableTelemetries.Length - gap;
while (positionEnd > positionBegin)
{
foreach (var detector in detectors)
{
if (!detector.TryDetect(idTelemetry, detectableTelemetries, positionBegin, positionEnd, lastDetectedOperation, out var result))
continue;
detectedOperations.Add(result!.Operation);
lastDetectedOperation = result.Operation;
isDetected = true;
positionBegin = result.TelemetryEnd;
break;
}
positionBegin += 1;
}
beginDate = isDetected ? lastDetectedOperation!.DateEnd : detectableTelemetries[positionEnd].DateTime;
}
return detectedOperations;
}
public async Task<int> DeleteAsync(DetectedOperationByWellRequest request, CancellationToken token)
{
var well = await wellService.GetOrDefaultAsync(request.IdWell, token);

149
AsbCloudInfrastructure/Services/DetectOperations/WorkOperationDetection.cs
View File

@ -1,12 +1,12 @@
using AsbCloudDb.Model;
using Microsoft.EntityFrameworkCore;
using System;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using AsbCloudInfrastructure.Services.DetectOperations.Detectors;
using AsbCloudApp.Data;
using AsbCloudApp.Repositories;
using AsbCloudApp.Services;
using AsbCloudInfrastructure.Background;
using Microsoft.Extensions.DependencyInjection;
@ -14,19 +14,6 @@ namespace AsbCloudInfrastructure.Services.DetectOperations;
public class WorkOperationDetection: Work
{
private static readonly DetectorAbstract[] detectors = new DetectorAbstract[]
{
new DetectorDrilling(),
new DetectorSlipsTime()
// new DetectorRotor(),
// new DetectorSlide(),
//new DetectorDevelopment(),
//new DetectorTemplating(),
//new DetectorStaticSurveying(),
//new DetectorFlashingBeforeConnection(),
//new DetectorFlashing(),
//new DetectorTemplatingWhileDrilling(),
};
public WorkOperationDetection()
:base("Operation detection")
@ -42,115 +29,39 @@ public class WorkOperationDetection: Work
protected override async Task Action(string id, IServiceProvider services, Action<string, double?> onProgressCallback, CancellationToken token)
{
using var db = services.GetRequiredService<IAsbCloudDbContext>();
db.Database.SetCommandTimeout(TimeSpan.FromMinutes(5));
var lastDetectedDates = await db.DetectedOperations
.GroupBy(o => o.IdTelemetry)
.Select(g => new
{
IdTelemetry = g.Key,
LastDate = g.Max(o => o.DateEnd)
})
.ToListAsync(token);
var telemetryRepository = services.GetRequiredService<ICrudRepository<TelemetryDto>>();
var detectedOperationRepository = services.GetRequiredService<IDetectedOperationRepository>();
var detectedOperationService = services.GetRequiredService<IDetectedOperationService>();
var telemetryIds = await db.Telemetries
.Where(t => t.Info != null && t.TimeZone != null)
.Select(t => t.Id)
.ToListAsync(token);
var telemetryIds = (await telemetryRepository.GetAllAsync(token))
.Select(t => t.Id);
var joinedlastDetectedDates = telemetryIds
.GroupJoin(lastDetectedDates,
t => t,
o => o.IdTelemetry,
(outer, inner) => new
{
IdTelemetry = outer,
inner.SingleOrDefault()?.LastDate,
});
var lastDetectedDates = await detectedOperationRepository.GetLastDetectedDatesAsync(token);
var affected = 0;
var count = joinedlastDetectedDates.Count();
var i = 0d;
foreach (var item in joinedlastDetectedDates)
var beginDatesDetectOperations = new List<(int TelemetryId, DateTimeOffset? BeginDate)>();
foreach (var telemetryId in telemetryIds)
{
var stopwatch = Stopwatch.StartNew();
var startDate = item.LastDate ?? DateTimeOffset.MinValue;
onProgressCallback($"start detecting telemetry: {item.IdTelemetry} from {startDate}", i++ / count);
var newOperations = await DetectOperationsAsync(item.IdTelemetry, startDate, db, token);
stopwatch.Stop();
if (newOperations.Any())
if (lastDetectedDates.TryGetValue(telemetryId, out var beginDate))
{
db.DetectedOperations.AddRange(newOperations);
affected += await db.SaveChangesAsync(token);
beginDatesDetectOperations.Add((telemetryId, beginDate));
continue;
}
}
}
//todo: move this logic to DetectedOperationsService
internal static async Task<IEnumerable<DetectedOperation>> DetectOperationsAsync(int idTelemetry, DateTimeOffset begin, IAsbCloudDbContext db, CancellationToken token)
{
var query = db.TelemetryDataSaub
.AsNoTracking()
.Where(d => d.IdTelemetry == idTelemetry)
.Where(d => d.BlockPosition >= 0)
.Select(d => new DetectableTelemetry
{
DateTime = d.DateTime,
IdUser = d.IdUser,
Mode = d.Mode,
WellDepth = d.WellDepth,
Pressure = d.Pressure,
HookWeight = d.HookWeight,
BlockPosition = d.BlockPosition,
BitDepth = d.BitDepth,
RotorSpeed = d.RotorSpeed,
})
.OrderBy(d => d.DateTime);
var take = 4 * 86_400; // 4 дня
var startDate = begin;
var detectedOperations = new List<DetectedOperation>(8);
DetectedOperation? lastDetectedOperation = null;
const int minOperationLength = 5;
const int maxDetectorsInterpolationFrameLength = 30;
const int gap = maxDetectorsInterpolationFrameLength + minOperationLength;
while (true)
{
var data = await query
.Where(d => d.DateTime > startDate)
.Take(take)
.ToArrayAsync(token);
if (data.Length < gap)
break;
var isDetected = false;
var positionBegin = 0;
var positionEnd = data.Length - gap;
while (positionEnd > positionBegin)
{
foreach (var detector in detectors)
{
if (!detector.TryDetect(idTelemetry, data, positionBegin, positionEnd, lastDetectedOperation, out var result))
continue;
detectedOperations.Add(result!.Operation);
lastDetectedOperation = result.Operation;
isDetected = true;
positionBegin = result.TelemetryEnd;
break;
}
positionBegin += 1;
}
if (isDetected)
startDate = lastDetectedOperation!.DateEnd;
else
startDate = data[positionEnd].DateTime;
beginDatesDetectOperations.Add((telemetryId, null));
}
return detectedOperations;
var count = beginDatesDetectOperations.Count;
for (var i = 0; i < count; i++)
{
var (idTelemetry, beginDate) = beginDatesDetectOperations[i];
onProgressCallback($"Start detecting telemetry: {idTelemetry} from {beginDate}", i++ / count);
var detectedOperations = await detectedOperationService.DetectOperationsAsync(idTelemetry, beginDate, token);
if (detectedOperations.Any())
await detectedOperationRepository.InsertRangeAsync(detectedOperations, token);
}
}
}