using AsbCloudApp.Data; using AsbCloudApp.Services; using AsbCloudDb.Model; using AsbCloudInfrastructure.Services.Cache; using Microsoft.EntityFrameworkCore; using System; using System.Collections.Generic; using System.Linq; using System.Threading; using System.Threading.Tasks; namespace AsbCloudInfrastructure.Services.Analysis { public class TelemetryAnalyticsService : ITelemetryAnalyticsService { private readonly IAsbCloudDbContext db; private readonly ITelemetryService telemetryService; private readonly CacheTable cacheOperations; private readonly TelemetryOperationDetectorService operationDetectorService; private readonly IEnumerable operations; private const int countOfRecordsForInterpolation = 12 * 60 * 60; public TelemetryAnalyticsService(IAsbCloudDbContext db, ITelemetryService telemetryService, CacheDb cacheDb) { this.db = db; this.telemetryService = telemetryService; cacheOperations = cacheDb.GetCachedTable((AsbCloudDbContext)db); operations = cacheOperations.Where(); operationDetectorService = new TelemetryOperationDetectorService(operations); } public async Task> GetWellDepthToDayAsync(int idWell, CancellationToken token = default) { var idTelemetry = telemetryService.GetIdTelemetryByIdWell(idWell); if (idTelemetry is null) return null; var timezone = telemetryService.GetTimezone((int)idTelemetry); var depthToTimeData = (from d in db.TelemetryDataSaub where d.IdTelemetry == idTelemetry select new { d.WellDepth, d.BitDepth, d.Date }); var m = (int)Math.Round(1d * depthToTimeData.Count() / 2048); if (m > 1) depthToTimeData = depthToTimeData.Where((d, i) => (((d.Date.DayOfYear * 24 + d.Date.Hour) * 60 + d.Date.Minute) * 60 + d.Date.Second) % m == 0); return await depthToTimeData.Select(d => new WellDepthToDayDto { WellDepth = d.WellDepth ?? 0.0, BitDepth = d.BitDepth ?? 0.0, Date = d.Date.ToRemoteDateTime(timezone.Hours), }).AsNoTracking().ToListAsync(token).ConfigureAwait(false); } public async Task> GetWellDepthToIntervalAsync(int idWell, int intervalSeconds, int shiftStartSec, CancellationToken token = default) { intervalSeconds = intervalSeconds == 0 ? 86400 : intervalSeconds; var telemetryId = telemetryService.GetIdTelemetryByIdWell(idWell); if (telemetryId is null) return null; var timezone = telemetryService.GetTimezone((int)telemetryId); var drillingPeriodsInfo = await db.TelemetryDataSaub .Where(t => t.IdTelemetry == telemetryId) .GroupBy(t => Math.Floor((((t.Date.DayOfYear * 24 + t.Date.Hour) * 60 + t.Date.Minute) * 60 + t.Date.Second + timezone.Hours - shiftStartSec) / intervalSeconds)) .Select(g => new { WellDepthMin = g.Min(t => t.WellDepth), WellDepthMax = g.Max(t => t.WellDepth), DateMin = g.Min(t => t.Date), DateMax = g.Max(t => t.Date), }) .OrderBy(g=>g.DateMin) .ToListAsync(token); var wellDepthToIntervalData = drillingPeriodsInfo.Select(d => new WellDepthToIntervalDto { IntervalStartDate = d.DateMin.ToRemoteDateTime(timezone.Hours), IntervalDepthProgress = (d.WellDepthMax - d.WellDepthMin) ?? 0.0 // / (d.DateMax - d.DateMin).TotalHours, }).OrderBy(d => d.IntervalStartDate).ToList(); return wellDepthToIntervalData; } public async Task> GetOperationsByWellAsync(int idWell, IEnumerable categoryIds = default, DateTime begin = default, DateTime end = default, int skip = 0, int take = 32, CancellationToken token = default) { var telemetryId = telemetryService.GetIdTelemetryByIdWell(idWell); if (telemetryId is null) return null; var operations = from a in db.TelemetryAnalysis.Include(t => t.Operation) where a.IdTelemetry == telemetryId select a; if ((categoryIds != default) && (categoryIds.Any())) operations = operations.Where(o => categoryIds.Contains(o.IdOperation)); var result = new PaginationContainer { Skip = skip, Take = take }; operations = operations.OrderBy(o => o.UnixDate); if (begin != default) { var unixBegin = (begin - new DateTime(1970, 1, 1)).TotalSeconds; operations = operations.Where(o => o.UnixDate >= unixBegin); } if (end != default) { var unixEnd = (end - new DateTime(1970, 1, 1)).TotalSeconds; operations = operations.Where(m => (m.UnixDate + m.DurationSec) <= unixEnd); } result.Count = await operations.CountAsync(token).ConfigureAwait(false); if (skip > 0) operations = operations.Skip(skip); var operationsList = await operations.Take(take) .AsNoTracking() .ToListAsync(token) .ConfigureAwait(false); if (operationsList.Count == 0) return result; foreach (var operation in operations) { var operationDto = new TelemetryOperationDto { Id = operation.Id, Name = operation.Operation.Name, BeginDate = DateTimeOffset.FromUnixTimeSeconds(operation.UnixDate).DateTime, EndDate = DateTimeOffset.FromUnixTimeSeconds(operation.UnixDate + operation.DurationSec).DateTime, StartWellDepth = operation.OperationStartDepth ?? 0.0, EndWellDepth = operation.OperationEndDepth ?? 0.0 }; result.Items.Add(operationDto); } return result; } public async Task> GetOperationsSummaryAsync(int idWell, DateTime begin = default, DateTime end = default, CancellationToken token = default) { var telemetryId = telemetryService.GetIdTelemetryByIdWell(idWell); if (telemetryId is null) return null; var unixBegin = begin == default ? 0 : (begin - new DateTime(1970, 1, 1)).TotalSeconds; var unixEnd = end == default ? (DateTime.Now - new DateTime(1970, 1, 1)).TotalSeconds : (end - new DateTime(1970, 1, 1)).TotalSeconds; return await (from a in db.TelemetryAnalysis where a.IdTelemetry == telemetryId && a.UnixDate > unixBegin && a.UnixDate < unixEnd join o in db.WellOperationCategories on a.IdOperation equals o.Id group a by new { a.IdOperation, o.Name } into g select new TelemetryOperationDurationDto { OperationName = g.Key.Name, Duration = g.Where(g => g.DurationSec > 0) .Sum(a => a.DurationSec) }).AsNoTracking().ToListAsync(token) .ConfigureAwait(false); } // This method is not finished (only half done). It returns not correct Dtos. public async Task> GetOperationsToIntervalAsync(int idWell, int intervalSeconds, int workBeginSeconds, CancellationToken token = default) { intervalSeconds = intervalSeconds == 0 ? 86400 : intervalSeconds; var telemetryId = telemetryService.GetIdTelemetryByIdWell(idWell); if (telemetryId is null) return null; var timezone = telemetryService.GetTimezone((int)telemetryId); // Get'n'Group all operations only by start date and by name (if there were several operations in interval). // Without dividing these operations duration by given interval var ops = await (from a in db.TelemetryAnalysis where a.IdTelemetry == telemetryId join o in db.WellOperationCategories on a.IdOperation equals o.Id group a by new { Interval = Math.Floor((a.UnixDate - workBeginSeconds + timezone.Hours) / intervalSeconds), o.Name } into g select new { IntervalStart = g.Min(d => d.UnixDate), OperationName = g.Key.Name, OperationDuration = g.Sum(an => an.DurationSec) }).AsNoTracking() .OrderBy(op => op.IntervalStart) .ToListAsync(token) .ConfigureAwait(false); var groupedOperationsList = new List(); if (operations is not null && operations.Any()) { var operations = ops.Select(o => (o.IntervalStart, o.OperationName, o.OperationDuration)); var splittedOperationsByInterval = DivideOperationsByIntervalLength(operations, intervalSeconds); // divides good groupedOperationsList = UniteOperationsInDto(splittedOperationsByInterval, intervalSeconds, timezone.Hours).ToList(); // unites not good } return groupedOperationsList; } public async Task AnalyzeAndSaveTelemetriesAsync(CancellationToken token = default) { var allTelemetryIds = await db.Telemetries.Select(t => t.Id).ToListAsync(token).ConfigureAwait(false); foreach (var idTelemetry in allTelemetryIds) { var analyzeStartDate = await GetLastAnalysisDateAsync(idTelemetry, token).ConfigureAwait(false); await AnalyseAndSaveTelemetryAsync(idTelemetry, analyzeStartDate, token).ConfigureAwait(false); GC.Collect(); } } private async Task AnalyseAndSaveTelemetryAsync(int idTelemetry, DateTimeOffset analyzeStartDate, CancellationToken token = default) { const int step = 10; const int take = step * 2; TelemetryAnalysis currentAnalysis = null; while (true) { var dataSaubPart = await GetDataSaubPartOrDefaultAsync(idTelemetry, analyzeStartDate, token).ConfigureAwait(false); if (dataSaubPart is null) break; var count = dataSaubPart.Count; var skip = 0; if (step > count) break; analyzeStartDate = dataSaubPart.Last().Date; for (; (skip + step) < count; skip += step) { var dataSaubPartOfPart = dataSaubPart.Skip(skip).Take(take); var telemetryAnalysis = GetDrillingAnalysis(dataSaubPartOfPart); if (currentAnalysis is not null) { if (currentAnalysis.IdOperation == telemetryAnalysis.IdOperation) currentAnalysis.DurationSec += telemetryAnalysis.DurationSec; else { currentAnalysis.OperationEndDepth = dataSaubPartOfPart.LastOrDefault()?.WellDepth; db.TelemetryAnalysis.Add(currentAnalysis); currentAnalysis = null; } } if (currentAnalysis is null) { currentAnalysis = telemetryAnalysis; currentAnalysis.OperationStartDepth = dataSaubPartOfPart.FirstOrDefault()?.WellDepth; } } await db.SaveChangesAsync(token).ConfigureAwait(false); GC.Collect(); } } public async Task GetOperationsDateRangeAsync(int idWell, CancellationToken token = default) { var idTelemetry = telemetryService.GetIdTelemetryByIdWell(idWell); if (idTelemetry is null) return null; var timezone = telemetryService.GetTimezone((int)idTelemetry); var datesRange = await (from d in db.TelemetryAnalysis where d.IdTelemetry == idTelemetry select d.UnixDate).DefaultIfEmpty() .GroupBy(g => true) .AsNoTracking() .Select(g => new { From = g.Min(), To = g.Max() }).OrderBy(gr => gr.From) .FirstOrDefaultAsync(token) .ConfigureAwait(false); var result = new DatesRangeDto { From = DateTimeOffset.FromUnixTimeSeconds(datesRange.From).ToRemoteDateTime(timezone.Hours), To = (datesRange.To == default ? DateTime.MaxValue : DateTimeOffset.FromUnixTimeSeconds(datesRange.To)).ToRemoteDateTime(timezone.Hours), }; return result; } private async Task GetLastAnalysisDateAsync(int idTelemetry, CancellationToken token = default) { var lastAnalysisInDb = await (from analysis in db.TelemetryAnalysis where analysis.IdTelemetry == idTelemetry orderby analysis.UnixDate select analysis) .LastOrDefaultAsync(token) .ConfigureAwait(false); DateTime lastAnalysisDate = new DateTime(0, DateTimeKind.Utc); if(lastAnalysisInDb is not null) lastAnalysisDate = DateTime.UnixEpoch.AddSeconds(lastAnalysisInDb.DurationSec + lastAnalysisInDb.UnixDate); return lastAnalysisDate; } private Task> GetDataSaubPartOrDefaultAsync(int idTelemetry, DateTimeOffset analyzeStartDate, CancellationToken token) => db.TelemetryDataSaub .Where(d => d.IdTelemetry == idTelemetry && d.Date > analyzeStartDate && d.BitDepth != null && d.BlockPosition != null && d.HookWeight != null && d.Pressure != null && d.RotorSpeed != null && d.WellDepth != null ) .OrderBy(d => d.Date) .Take(countOfRecordsForInterpolation) .Select(d => new DataSaubAnalyse { IdTelemetry = d.IdTelemetry, Date = d.Date, BitDepth = d.BitDepth ?? 0, BlockPosition = d.BlockPosition ?? 0, HookWeight = d.HookWeight ?? 0, Pressure = d.Pressure ?? 0, RotorSpeed = d.RotorSpeed ?? 0, WellDepth = d.WellDepth ?? 0, }) .ToListAsync(token); private static IEnumerable<(long IntervalStart, string OperationName, int OperationDuration)> DivideOperationsByIntervalLength( IEnumerable<(long IntervalStart, string OperationName, int OperationDuration)> operations, int intervalSeconds) { var splittedOperationsByInterval = new List<(long IntervalStart, string OperationName, int OperationDuration)>(); var operationDurationTimeCounter = 0; foreach (var (IntervalStart, OperationName, OperationDuration) in operations) { if (OperationDuration < (intervalSeconds - operationDurationTimeCounter)) { splittedOperationsByInterval.Add((IntervalStart, OperationName, OperationDuration)); operationDurationTimeCounter += OperationDuration; } else { // if operation duration overflows current interval it should be divided into 2 or more parts for this and next intervals var remainingIntervalTime = intervalSeconds - operationDurationTimeCounter; splittedOperationsByInterval.Add((IntervalStart, OperationName, remainingIntervalTime)); // first part of long operation var operationDurationAfterDividing = OperationDuration - remainingIntervalTime; // second part of long operation. Can be less or more than interval // If operation duration even after dividing is still more than interval, // it should be divided several times to several intervals. if (operationDurationAfterDividing > intervalSeconds) { var counter = 0; var updatedIntervalStartTime = IntervalStart + remainingIntervalTime; while (operationDurationAfterDividing > intervalSeconds) { splittedOperationsByInterval.Add((updatedIntervalStartTime + intervalSeconds * counter, OperationName, intervalSeconds)); operationDurationAfterDividing -= intervalSeconds; counter++; } splittedOperationsByInterval.Add((updatedIntervalStartTime + operationDurationAfterDividing, OperationName, operationDurationAfterDividing)); operationDurationTimeCounter = operationDurationAfterDividing; } else { splittedOperationsByInterval.Add((IntervalStart, OperationName, operationDurationAfterDividing)); operationDurationTimeCounter = operationDurationAfterDividing; } } } return splittedOperationsByInterval; } private static IEnumerable UniteOperationsInDto( IEnumerable<(long IntervalStart, string OperationName, int OperationDuration)> operations, int intervalSeconds, double timezoneOffset) { var groupedOperationsList = new List(); var groupedOperationsObj = new TelemetryOperationInfoDto { IntervalBegin = DateTimeOffset.FromUnixTimeSeconds(operations.First().IntervalStart) .ToRemoteDateTime(timezoneOffset), Operations = new List() }; var intervalEndDate = operations.First().IntervalStart + intervalSeconds; foreach (var (IntervalStart, OperationName, OperationDuration) in operations) { if (IntervalStart < intervalEndDate) { groupedOperationsObj.Operations.Add(new TelemetryOperationDetailsDto { OperationName = OperationName, DurationSec = OperationDuration }); } else { groupedOperationsList.Add(groupedOperationsObj); intervalEndDate = IntervalStart + intervalSeconds; groupedOperationsObj = new TelemetryOperationInfoDto { IntervalBegin = DateTimeOffset.FromUnixTimeSeconds(IntervalStart) .ToRemoteDateTime(timezoneOffset), Operations = new List() }; groupedOperationsObj.Operations.Add(new TelemetryOperationDetailsDto { OperationName = OperationName, DurationSec = OperationDuration }); } } groupedOperationsList.Add(groupedOperationsObj); return groupedOperationsList; } private TelemetryAnalysis GetDrillingAnalysis(IEnumerable dataSaubPartOfPart) { var dataSaubFirst = dataSaubPartOfPart.First(); var dataSaubLast = dataSaubPartOfPart.Last(); var saubWellDepths = dataSaubPartOfPart.Select(s => (Y: (double)s.WellDepth, X: (s.Date - dataSaubFirst.Date).TotalSeconds)); var saubBitDepths = dataSaubPartOfPart.Select(s => (Y: (double)s.BitDepth, X: (s.Date - dataSaubFirst.Date).TotalSeconds)); var saubBlockPositions = dataSaubPartOfPart.Select(s => (Y: (double)s.BlockPosition, X: (s.Date - dataSaubFirst.Date).TotalSeconds)); var saubRotorSpeeds = dataSaubPartOfPart.Select(s => (Y: (double)s.RotorSpeed, X: (s.Date - dataSaubFirst.Date).TotalSeconds)); var saubPressures = dataSaubPartOfPart.Select(s => (Y: (double)s.Pressure, X: (s.Date - dataSaubFirst.Date).TotalSeconds)); var saubHookWeights = dataSaubPartOfPart.Select(s => (Y: (double)s.HookWeight, X: (s.Date - dataSaubFirst.Date).TotalSeconds)); var wellDepthLine = new InterpolationLine(saubWellDepths); var bitPositionLine = new InterpolationLine(saubBitDepths); var blockPositionLine = new InterpolationLine(saubBlockPositions); var rotorSpeedLine = new InterpolationLine(saubRotorSpeeds); var pressureLine = new InterpolationLine(saubPressures); var hookWeightLine = new InterpolationLine(saubHookWeights); var drillingAnalysis = new TelemetryAnalysis { IdTelemetry = dataSaubFirst.IdTelemetry, UnixDate = (long)(dataSaubFirst.Date - DateTime.UnixEpoch).TotalSeconds, DurationSec = (int)(dataSaubLast.Date - dataSaubFirst.Date).TotalSeconds, OperationStartDepth = null, OperationEndDepth = null, IsWellDepthDecreasing = wellDepthLine.IsYDecreases(-0.0001), IsWellDepthIncreasing = wellDepthLine.IsYIncreases( 0.0001), IsBitPositionDecreasing = bitPositionLine.IsYDecreases(-0.0001), IsBitPositionIncreasing = bitPositionLine.IsYIncreases(0.0001), IsBitPositionLt20 = bitPositionLine.IsAverageYLessThanBound(20), IsBlockPositionDecreasing = blockPositionLine.IsYDecreases(-0.0001), IsBlockPositionIncreasing = blockPositionLine.IsYIncreases(0.0001), IsRotorSpeedLt5 = rotorSpeedLine.IsAverageYLessThanBound(5), IsRotorSpeedGt5 = rotorSpeedLine.IsAverageYMoreThanBound(5), IsPressureLt20 = pressureLine.IsAverageYLessThanBound(20), IsPressureGt20 = pressureLine.IsAverageYMoreThanBound(20), IsHookWeightNotChanges = hookWeightLine.IsYNotChanges(0.0001, -0.0001), IsHookWeightLt3 = hookWeightLine.IsAverageYLessThanBound(3), IdOperation = default, }; drillingAnalysis.IdOperation = operationDetectorService.DetectOperation(drillingAnalysis).Id; return drillingAnalysis; } } }