diff --git a/AsbCloudApp/Data/SAUB/DrillTestBaseDto.cs b/AsbCloudApp/Data/SAUB/DrillTestBaseDto.cs new file mode 100644 index 00000000..6e9f56bf --- /dev/null +++ b/AsbCloudApp/Data/SAUB/DrillTestBaseDto.cs @@ -0,0 +1,37 @@ +using System; +using System.Collections.Generic; +using System.ComponentModel.DataAnnotations; +using System.Linq; + +namespace AsbCloudApp.Data.SAUB +{ + /// + /// DTO для получения записи drill_test из панели + /// + public class DrillTestBaseDto + { + /// + /// Идентификатор drill test + /// + [Required] + public int Id { get; set; } + + /// + /// Время начала drill test + /// + [Required] + public DateTimeOffset TimeStampStart { get; set; } + + /// + /// Глубина начала drill test + /// + [Required] + public float DepthStart { get; set; } + + /// + /// Параметры теста + /// + [Required] + public IEnumerable Params { get; set; } = Enumerable.Empty(); + } +} diff --git a/AsbCloudApp/Data/SAUB/DrillTestDto.cs b/AsbCloudApp/Data/SAUB/DrillTestDto.cs index 05e0aa09..0af9cb9c 100644 --- a/AsbCloudApp/Data/SAUB/DrillTestDto.cs +++ b/AsbCloudApp/Data/SAUB/DrillTestDto.cs @@ -1,42 +1,15 @@ -using System; -using System.Collections.Generic; -using System.ComponentModel.DataAnnotations; -using System.Linq; +using System.ComponentModel.DataAnnotations; namespace AsbCloudApp.Data.SAUB { /// - /// DTO для описания записи drill_test + /// DTO для отображения записи drill_test /// - public class DrillTestDto + public class DrillTestDto : DrillTestBaseDto { - /// - /// Идентификатор drill test - /// - [Required] - public int Id { get; set; } - - /// - /// Время начала drill test - /// - [Required] - public DateTimeOffset TimeStampStart { get; set; } - - /// - /// Глубина начала drill test - /// - [Required] - public float DepthStart { get; set; } - /// /// Связанная с drill_test телеметрия /// public TelemetryDto? Telemetry { get; set; } - - /// - /// Параметры теста - /// - [Required] - public IEnumerable Params { get; set; } = Enumerable.Empty(); } } diff --git a/AsbCloudApp/Repositories/IDrillTestRepository.cs b/AsbCloudApp/Repositories/IDrillTestRepository.cs index 9c7a2f50..22f11a06 100644 --- a/AsbCloudApp/Repositories/IDrillTestRepository.cs +++ b/AsbCloudApp/Repositories/IDrillTestRepository.cs @@ -36,6 +36,6 @@ namespace AsbCloudApp.Repositories /// запись drill test /// /// - Task SaveDataAsync(int idTelemetry, DrillTestDto dto, CancellationToken token); + Task SaveDataAsync(int idTelemetry, DrillTestBaseDto dto, CancellationToken token); } } diff --git a/AsbCloudApp/Repositories/IWellOperationRepository.cs b/AsbCloudApp/Repositories/IWellOperationRepository.cs index ff3bfb8e..2ac586aa 100644 --- a/AsbCloudApp/Repositories/IWellOperationRepository.cs +++ b/AsbCloudApp/Repositories/IWellOperationRepository.cs @@ -119,18 +119,21 @@ namespace AsbCloudApp.Repositories Task GetDatesRangeAsync(int idWell, int idType, CancellationToken cancellationToken); /// - /// Валидация данных + /// Удаление полных дубликатов операций по всем скважинам /// - /// + /// + /// /// - IEnumerable Validate(IEnumerable wellOperations); + Task RemoveDuplicates(Action onProgressCallback, CancellationToken token); /// - /// Валидация данных (проверка с базой) + /// Усечение пересекающейся последующей операции по дате и глубине забоя /// - /// - /// + /// Фильтр по дате. Если хоть одна операция попадет в в фильтр, то будет обработана вся скважина, а не только эта операция + /// Фильтр по дате. Если хоть одна операция попадет в в фильтр, то будет обработана вся скважина, а не только эта операция + /// + /// /// - Task> ValidateWithDbAsync(IEnumerable wellOperations, CancellationToken cancellationToken); + Task TrimOverlapping(DateTimeOffset? geDate, DateTimeOffset leDate, Action onProgressCallback, CancellationToken token); } } \ No newline at end of file diff --git a/AsbCloudDb/Model/WellOperation.cs b/AsbCloudDb/Model/WellOperation.cs index 9926f0b1..dda4e3e4 100644 --- a/AsbCloudDb/Model/WellOperation.cs +++ b/AsbCloudDb/Model/WellOperation.cs @@ -67,6 +67,23 @@ namespace AsbCloudDb.Model [JsonIgnore] [ForeignKey(nameof(IdPlan))] public virtual WellOperation? OperationPlan { get; set; } = null!; + + public bool IsSame(WellOperation other) + { + var isSame = IdWell == other.IdWell && + IdWellSectionType == other.IdWellSectionType && + IdCategory == other.IdCategory && + IdType == other.IdType && + IdPlan == other.IdPlan && + DepthStart == other.DepthStart && + DepthEnd == other.DepthEnd && + DateStart == other.DateStart && + DurationHours == other.DurationHours && + CategoryInfo == other.CategoryInfo && + Comment == other.Comment; + + return isSame; + } } } diff --git a/AsbCloudInfrastructure/Repository/DrillTestRepository.cs b/AsbCloudInfrastructure/Repository/DrillTestRepository.cs index f5fb4ece..d8700929 100644 --- a/AsbCloudInfrastructure/Repository/DrillTestRepository.cs +++ b/AsbCloudInfrastructure/Repository/DrillTestRepository.cs @@ -63,7 +63,7 @@ namespace AsbCloudInfrastructure.Repository return dto; } - public async Task SaveDataAsync(int idTelemetry, DrillTestDto dto, CancellationToken token) + public async Task SaveDataAsync(int idTelemetry, DrillTestBaseDto dto, CancellationToken token) { var entity = dto.Adapt(); entity.IdTelemetry = idTelemetry; diff --git a/AsbCloudInfrastructure/Repository/WellOperationRepository.cs b/AsbCloudInfrastructure/Repository/WellOperationRepository.cs index 1330a282..00349a6c 100644 --- a/AsbCloudInfrastructure/Repository/WellOperationRepository.cs +++ b/AsbCloudInfrastructure/Repository/WellOperationRepository.cs @@ -10,13 +10,13 @@ using Microsoft.Extensions.Caching.Memory; using System; using System.Collections.Generic; using System.ComponentModel.DataAnnotations; +using System.Diagnostics; using System.Linq; using System.Threading; using System.Threading.Tasks; namespace AsbCloudInfrastructure.Repository; - /// /// репозиторий операций по скважине /// @@ -321,59 +321,6 @@ public class WellOperationRepository : IWellOperationRepository return dtos; } - public async Task> ValidateWithDbAsync(IEnumerable wellOperationDtos, CancellationToken token) - { - var firstOperation = wellOperationDtos - .FirstOrDefault(); - - if (firstOperation is null) - return Enumerable.Empty(); - - var request = new WellOperationRequest() - { - IdWell = firstOperation.IdWell, - OperationType = firstOperation.IdType, - }; - - var entities = await BuildQuery(request) - .AsNoTracking() - .ToArrayAsync(token); - - var wellOperationsUnion = entities.Union(wellOperationDtos).OrderBy(o => o.DateStart); - - var results = Validate(wellOperationsUnion); - return results; - } - - public IEnumerable Validate(IEnumerable wellOperationDtos) - { - var enumerator = wellOperationDtos.OrderBy(o => o.DateStart) - .GetEnumerator(); - - if (!enumerator.MoveNext()) - yield break; - - var previous = enumerator.Current; - - while(enumerator.MoveNext()) - { - var current = enumerator.Current; - var previousDateStart = previous.DateStart.ToUniversalTime(); - var currentDateStart = current.DateStart.ToUniversalTime(); - - var previousDateEnd = previous.DateStart.AddHours(previous.DurationHours).ToUniversalTime(); - - if (previousDateStart.AddDays(Gap) < currentDateStart) - { - yield return new ValidationResult( - "Разница дат между операциями не должна превышать 90 дней", - new[] { nameof(wellOperationDtos) }); - } - - previous = current; - } - } - /// public async Task InsertRangeAsync( IEnumerable wellOperationDtos, @@ -530,4 +477,161 @@ public class WellOperationRepository : IWellOperationRepository return dtos; } + + public async Task RemoveDuplicates(Action onProgressCallback, CancellationToken token) + { + IQueryable dbset = db.Set(); + var query = dbset + .GroupBy(o => new { o.IdWell, o.IdType }) + .Select(g => new { g.Key.IdWell, g.Key.IdType }); + + var groups = await query + .ToArrayAsync(token); + + var count = groups.Count(); + var i = 0; + var totalRemoved = 0; + var total = 0; + foreach (var group in groups) + { + var result = await RemoveDuplicatesInGroup(group.IdWell, group.IdType, token); + totalRemoved += result.removed; + total += result.total; + var percent = i++ / count; + var message = $"RemoveDuplicates [{i} of {count}] wellId: {group.IdWell}, opType: {group.IdType}, affected: {result.removed} of {result.total}"; + onProgressCallback?.Invoke(message, percent); + Trace.TraceInformation(message); + } + var messageDone = $"RemoveDuplicates done [{i} of {count}] totalAffected: {totalRemoved} of {total}"; + Trace.TraceInformation(messageDone); + onProgressCallback?.Invoke(messageDone, 1); + return totalRemoved; + } + + private async Task<(int removed, int total)> RemoveDuplicatesInGroup(int idWell, int idType, CancellationToken token) + { + var dbset = db.Set(); + var entities = await dbset + .Where(o => o.IdWell == idWell && o.IdType == idType) + .OrderBy(o => o.DateStart) + .ToListAsync(token); + + using var entitiesEnumerator = entities.GetEnumerator(); + + if (!entitiesEnumerator.MoveNext()) + return (0, 0); + + var preEntity = entitiesEnumerator.Current; + while (entitiesEnumerator.MoveNext()) + { + var entity = entitiesEnumerator.Current; + if (preEntity.IsSame(entity)) + dbset.Remove(entity); + else + preEntity = entity; + } + var removed = await db.SaveChangesAsync(token); + return (removed, entities.Count); + } + + public async Task TrimOverlapping(DateTimeOffset? geDate, DateTimeOffset leDate, Action onProgressCallback, CancellationToken token) + { + var leDateUtc = leDate.ToUniversalTime(); + IQueryable query = db.Set(); + if (geDate.HasValue) + { + var geDateUtc = geDate.Value.ToUniversalTime(); + query = query.Where(e => e.DateStart >= geDateUtc); + } + + var groups = await query + .GroupBy(o => new { o.IdWell, o.IdType }) + .Select(g => new{ + MaxDate = g.Max(o => o.DateStart), + g.Key.IdWell, + g.Key.IdType, + }) + .Where(g => g.MaxDate <= leDateUtc) + .ToArrayAsync(token); + + var count = groups.Count(); + var i = 0; + (int takeover, int trimmed,int total) totalResult = (0, 0, 0); + foreach (var group in groups) + { + var result = await TrimOverlapping(group.IdWell, group.IdType, token); + totalResult.takeover += result.takeover; + totalResult.trimmed += result.trimmed; + totalResult.total += result.total; + var percent = i++ / count; + var message = $"TrimOverlapping [{i} of {count}] wellId: {group.IdWell}, opType: {group.IdType}, takeover:{result.takeover}, trimmed:{result.trimmed}, of {result.total}"; + onProgressCallback?.Invoke(message, percent); + Trace.TraceInformation(message); + } + var messageDone = $"TrimOverlapping done [{i} of {count}] total takeover:{totalResult.takeover}, total trimmed:{totalResult.trimmed} of {totalResult.total}"; + Trace.TraceInformation(messageDone); + onProgressCallback?.Invoke(messageDone, 1); + return totalResult.takeover + totalResult.trimmed; + } + + private async Task<(int takeover, int trimmed, int total)> TrimOverlapping(int idWell, int idType, CancellationToken token) + { + var dbset = db.Set(); + var query = dbset + .Where(o => o.IdWell == idWell) + .Where(o => o.IdType == idType) + .OrderBy(o => o.DateStart) + .ThenBy(o => o.DepthStart); + + var entities = await query + .ToListAsync(token); + + using var entitiesEnumerator = entities.GetEnumerator(); + + if (!entitiesEnumerator.MoveNext()) + return (0, 0, 0); + + int takeover = 0; + int trimmed = 0; + var preEntity = entitiesEnumerator.Current; + while (entitiesEnumerator.MoveNext()) + { + var entity = entitiesEnumerator.Current; + var preDepth = preEntity.DepthEnd; + + if (preEntity.DepthEnd >= entity.DepthEnd) + { + dbset.Remove(entity); + takeover++; + continue; + } + + if (preEntity.DepthEnd > entity.DepthStart) + { + entity.DepthStart = preEntity.DepthEnd; + trimmed++; + } + + var preDate = preEntity.DateStart.AddHours(preEntity.DurationHours); + + if (preDate >= entity.DateStart.AddHours(entity.DurationHours)) + { + dbset.Remove(entity); + takeover++; + continue; + } + + if (preDate > entity.DateStart) + { + var entityDateEnd = entity.DateStart.AddHours(entity.DurationHours); + entity.DateStart = preDate; + entity.DurationHours = (entityDateEnd - entity.DateStart).TotalHours; + trimmed++; + } + + preEntity = entity; + } + var affected = await db.SaveChangesAsync(token); + return (takeover, trimmed, entities.Count); + } } diff --git a/AsbCloudInfrastructure/Services/ProcessMaps/Report/ProcessMapReportDataSaubStatService.cs b/AsbCloudInfrastructure/Services/ProcessMaps/Report/ProcessMapReportDataSaubStatService.cs index 9ad31d53..5c484a63 100644 --- a/AsbCloudInfrastructure/Services/ProcessMaps/Report/ProcessMapReportDataSaubStatService.cs +++ b/AsbCloudInfrastructure/Services/ProcessMaps/Report/ProcessMapReportDataSaubStatService.cs @@ -308,10 +308,10 @@ public class ProcessMapReportDataSaubStatService : IProcessMapReportDataSaubStat RotorSpeed: sumRotorSpeed / diffDepthTotal, RotorSpeedMax: maxRotorSpeed, MaxFlow: maxFlow, - SetpointUsagePressure: sumDiffDepthByPressure / diffDepthTotal, - SetpointUsageAxialLoad: sumDiffDepthByAxialLoad / diffDepthTotal, - SetpointUsageRotorTorque: sumDiffDepthByRotorTorque / diffDepthTotal, - SetpointUsageRopPlan: sumDiffDepthByRopPlan / diffDepthTotal, + SetpointUsagePressure: sumDiffDepthByPressure * 100 / diffDepthTotal, + SetpointUsageAxialLoad: sumDiffDepthByAxialLoad * 100 / diffDepthTotal, + SetpointUsageRotorTorque: sumDiffDepthByRotorTorque * 100 / diffDepthTotal, + SetpointUsageRopPlan: sumDiffDepthByRopPlan * 100/ diffDepthTotal, DrilledTime: drilledTime ); } diff --git a/AsbCloudWebApi.IntegrationTests/Controllers/WellOperationControllerTest.cs b/AsbCloudWebApi.IntegrationTests/Controllers/WellOperationControllerTest.cs index a5e0064a..46ce1d4d 100644 --- a/AsbCloudWebApi.IntegrationTests/Controllers/WellOperationControllerTest.cs +++ b/AsbCloudWebApi.IntegrationTests/Controllers/WellOperationControllerTest.cs @@ -36,56 +36,6 @@ public class WellOperationControllerTest : BaseIntegrationTest } }; - private readonly WellOperationDto[] dtosWithError = new WellOperationDto[] - { - new() - { - Id = 3, - IdWell = idWell, - IdType = 1, - DateStart = DateTimeOffset.Now, - CategoryInfo = "1", - CategoryName = "1", - Comment = "1", - Day = 1, - DepthEnd = 20, - DepthStart = 10, - DurationHours = 1, - IdCategory = 5000, - IdParentCategory = null, - IdPlan = null, - IdUser = 1, - IdWellSectionType = 1, - LastUpdateDate = DateTimeOffset.Now, - NptHours = 1, - WellSectionTypeName = null, - UserName = null - }, - new() - { - Id = 4, - IdWell = idWell, - IdType = 1, - DateStart = DateTimeOffset.Now.AddDays(1000), - CategoryInfo = "1", - CategoryName = "1", - Comment = "1", - Day = 1, - DepthEnd = 20, - DepthStart = 10, - DurationHours = 1, - IdCategory = 5000, - IdParentCategory = null, - IdPlan = null, - IdUser = 1, - IdWellSectionType = 1, - LastUpdateDate = DateTimeOffset.Now, - NptHours = 1, - WellSectionTypeName = null, - UserName = null - } - }; - private IWellOperationClient client; public WellOperationControllerTest(WebAppFactoryFixture factory) @@ -109,21 +59,6 @@ public class WellOperationControllerTest : BaseIntegrationTest Assert.Equal(HttpStatusCode.OK, response.StatusCode); } - - /// - /// Неуспешное добавление операций (без предварительной очистки данных) - /// - /// - [Fact] - public async Task InsertRange_returns_error() - { - //act - var response = await client.InsertRangeAsync(idWell, 1, false, dtosWithError); - - //assert - Assert.Equal(HttpStatusCode.BadRequest, response.StatusCode); - } - /// /// Успешное добавление операций (с предварительной очисткой данных) /// @@ -138,20 +73,6 @@ public class WellOperationControllerTest : BaseIntegrationTest Assert.Equal(HttpStatusCode.OK, response.StatusCode); } - /// - /// Неуспешное добавление операций (с предварительной очисткой данных) - /// - /// - [Fact] - public async Task InsertRangeWithDeleteBefore_returns_error() - { - //act - var response = await client.InsertRangeAsync(idWell, 1, true, dtosWithError); - - //assert - Assert.Equal(HttpStatusCode.BadRequest, response.StatusCode); - } - /// /// Успешное обновление операции /// @@ -166,21 +87,6 @@ public class WellOperationControllerTest : BaseIntegrationTest //assert Assert.Equal(HttpStatusCode.OK, response.StatusCode); } - - /// - /// Неуспешное обновление операции - /// - /// - [Fact] - public async Task UpdateAsync_returns_error() - { - //act - var dto = dtosWithError.LastOrDefault()!; - var response = await client.UpdateAsync(idWell, 1, dto, CancellationToken.None); - - //assert - Assert.Equal(HttpStatusCode.BadRequest, response.StatusCode); - } /// /// Получение плановых операций diff --git a/AsbCloudWebApi/Controllers/DrillTestController.cs b/AsbCloudWebApi/Controllers/DrillTestController.cs index 2d9585dc..8562e54c 100644 --- a/AsbCloudWebApi/Controllers/DrillTestController.cs +++ b/AsbCloudWebApi/Controllers/DrillTestController.cs @@ -58,7 +58,7 @@ public class DrillTestController : ControllerBase [HttpPost("api/telemetry/{uid}/[controller]")] public async Task PostDataAsync( string uid, - [FromBody] DrillTestDto dto, + [FromBody] DrillTestBaseDto dto, CancellationToken token) { var telemetry = telemetryService.GetOrCreateTelemetryByUid(uid); @@ -85,7 +85,6 @@ public class DrillTestController : ControllerBase /// /// [HttpGet("api/well/{idWell}/[controller]")] - [Permission] [ProducesResponseType(typeof(PhysicalFileResult), (int)HttpStatusCode.OK, "application/octet-stream")] [ProducesResponseType(StatusCodes.Status204NoContent)] public async Task GenerateReportAsync([FromRoute] int idWell, @@ -108,7 +107,6 @@ public class DrillTestController : ControllerBase /// /// [HttpGet("api/well/{idWell}/[controller]/all")] - [Permission] [ProducesResponseType(typeof(PaginationContainer), (int)HttpStatusCode.OK)] public async Task GetListAsync([FromRoute][Required] int idWell, [FromQuery] FileReportRequest request, diff --git a/AsbCloudWebApi/Controllers/WellOperationController.cs b/AsbCloudWebApi/Controllers/WellOperationController.cs index 27827a62..83f2987a 100644 --- a/AsbCloudWebApi/Controllers/WellOperationController.cs +++ b/AsbCloudWebApi/Controllers/WellOperationController.cs @@ -236,10 +236,6 @@ namespace AsbCloudWebApi.Controllers wellOperation.IdUser = User.GetUserId(); wellOperation.IdType = idType; - var validationResult = await operationRepository.ValidateWithDbAsync(new[] { wellOperation }, cancellationToken); - if (validationResult.Any()) - return this.ValidationBadRequest(validationResult); - var result = await operationRepository.InsertRangeAsync(new[] { wellOperation }, cancellationToken); return Ok(result); @@ -290,32 +286,11 @@ namespace AsbCloudWebApi.Controllers wellOperation.IdType = idType; } - - var validationResult = await Validate(wellOperations, deleteBeforeInsert, cancellationToken); - if (validationResult.Any()) - return this.ValidationBadRequest(validationResult); - var result = await operationRepository.InsertRangeAsync(wellOperations, cancellationToken); return Ok(result); } - - /// - /// Валидация данных перед вставкой / обновлением / импортом - /// - /// - /// - /// - /// - private async Task> Validate(IEnumerable wellOperations, bool deleteBeforeInsert, CancellationToken cancellationToken) - { - if (deleteBeforeInsert) - return operationRepository.Validate(wellOperations); - else - return await operationRepository.ValidateWithDbAsync(wellOperations, cancellationToken); - } - /// /// Обновляет выбранную операцию на скважине /// @@ -341,10 +316,6 @@ namespace AsbCloudWebApi.Controllers value.LastUpdateDate = DateTimeOffset.UtcNow; value.IdUser = User.GetUserId(); - var validationResult = await operationRepository.ValidateWithDbAsync(new[] { value }, token); - if (validationResult.Any()) - return this.ValidationBadRequest(validationResult); - var result = await operationRepository.UpdateAsync(value, token) .ConfigureAwait(false); return Ok(result); @@ -505,6 +476,38 @@ namespace AsbCloudWebApi.Controllers return File(stream, "application/octet-stream", fileName); } + /// + /// Удаляет полые дубликаты операций + /// + /// + /// + [HttpPost("/api/well/wellOperations/RemoveDuplicates")] + [Permission] + [Obsolete] + [ProducesResponseType(typeof(int), (int)System.Net.HttpStatusCode.OK)] + public async Task RemoveDuplicates(CancellationToken token) + { + var result = await operationRepository.RemoveDuplicates((_, _) => { }, token); + return Ok(result); + } + + /// + /// Удаляет полностью пересекающиеся операции или "подрезает" более поздние их по глубине и дате. + /// + /// + /// + /// + /// + [HttpPost("/api/well/wellOperations/TrimOverlapping")] + [Permission] + [Obsolete] + [ProducesResponseType(typeof(int), (int)System.Net.HttpStatusCode.OK)] + public async Task TrimOverlapping(DateTimeOffset? geDate, DateTimeOffset leDate, CancellationToken token) + { + var result = await operationRepository.TrimOverlapping(geDate, leDate, (_, _) => { }, token); + return Ok(result); + } + /// /// Возвращает шаблон файла импорта /// diff --git a/AsbCloudWebApi/SignalR/Clients/ITelemetryHubClient.cs b/AsbCloudWebApi/SignalR/Clients/ITelemetryHubClient.cs index 251b1a61..2e831d1e 100644 --- a/AsbCloudWebApi/SignalR/Clients/ITelemetryHubClient.cs +++ b/AsbCloudWebApi/SignalR/Clients/ITelemetryHubClient.cs @@ -20,7 +20,7 @@ namespace AsbCloudWebApi.SignalR.Clients /// /// /// - Task ReceiveDrilltestData(DrillTestDto dto, CancellationToken token); + Task ReceiveDrilltestData(DrillTestBaseDto dto, CancellationToken token); /// /// Обновление записей РТК