DD.WellWorkover.Cloud/AsbCloudInfrastructure/Services/Analysis/TelemetryAnalyticsService.cs

using AsbCloudApp.Data;
using AsbCloudApp.Services;
using AsbCloudDb.Model;
using AsbCloudInfrastructure.Services.Cache;
using Microsoft.EntityFrameworkCore;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;

namespace AsbCloudInfrastructure.Services.Analysis
{
    public class TelemetryAnalyticsService : ITelemetryAnalyticsService
    {
        private readonly IAsbCloudDbContext db;
        private readonly ITelemetryService telemetryService;
        private readonly CacheTable<WellOperationCategory> cacheOperations;
        private readonly TelemetryOperationDetectorService operationDetectorService;
        private readonly IEnumerable<WellOperationCategory> operations;

        private const int countOfRecordsForInterpolation = 12 * 60 * 60;

        public TelemetryAnalyticsService(IAsbCloudDbContext db, ITelemetryService telemetryService,
            CacheDb cacheDb)
        {
            this.db = db;
            this.telemetryService = telemetryService;
            cacheOperations = cacheDb.GetCachedTable<WellOperationCategory>((AsbCloudDbContext)db);
            operations = cacheOperations.Where();
            operationDetectorService = new TelemetryOperationDetectorService(operations);
        }

        public async Task<IEnumerable<WellDepthToDayDto>> GetWellDepthToDayAsync(int idWell, CancellationToken token = default)
        {
            var telemetryId = telemetryService.GetIdTelemetryByIdWell(idWell);

            if (telemetryId is null)
                return null;

            var depthToTimeData = (from d in db.TelemetryDataSaub
                                   where d.IdTelemetry == telemetryId
                                   select new
                                   {
                                       d.WellDepth,
                                       d.BitDepth,
                                       d.Date
                                   });

            var m = (int)Math.Round(1d * depthToTimeData.Count() / 2048);

            if (m > 1)
                depthToTimeData = depthToTimeData.Where((d, i) => (((d.Date.DayOfYear * 24 + d.Date.Hour) * 60 + d.Date.Minute) * 60 + d.Date.Second) % m == 0);

            return await depthToTimeData.Select(d => new WellDepthToDayDto
            {
                WellDepth = d.WellDepth ?? 0.0,
                BitDepth = d.BitDepth ?? 0.0,
                Date = d.Date
            }).AsNoTracking().ToListAsync(token).ConfigureAwait(false);
        }

        public async Task<IEnumerable<WellDepthToIntervalDto>> GetWellDepthToIntervalAsync(int idWell,
            int intervalSeconds, int workBeginSeconds, CancellationToken token = default)
        {
            intervalSeconds = intervalSeconds == 0 ? 86400 : intervalSeconds;

            var telemetryId = telemetryService.GetIdTelemetryByIdWell(idWell);

            if (telemetryId is null)
                return null;

            var timezoneOffset = telemetryService.GetTimezoneOffsetByTelemetryId((int)telemetryId);

            var drillingPeriodsInfo = await db.GetDepthToIntervalAsync((int)telemetryId, intervalSeconds,
                workBeginSeconds, timezoneOffset, token).ConfigureAwait(false);

            var wellDepthToIntervalData = drillingPeriodsInfo.Select(d => new WellDepthToIntervalDto
            {
                IntervalStartDate = d.BeginPeriodDate,
                IntervalDepthProgress = (d.MaxDepth - d.MinDepth) ?? 0.0 / intervalSeconds
            }).OrderBy(d => d.IntervalStartDate).ToList();

            return wellDepthToIntervalData;
        }

        public async Task<PaginationContainer<TelemetryOperationDto>> GetOperationsByWellAsync(int idWell,
            IEnumerable<int> categoryIds = default, DateTime begin = default,
            DateTime end = default, int skip = 0, int take = 32, CancellationToken token = default)
        {
            var telemetryId = telemetryService.GetIdTelemetryByIdWell(idWell);

            if (telemetryId is null)
                return null;

            var operations = from a in db.TelemetryAnalysis.Include(t => t.Operation)
                             where a.IdTelemetry == telemetryId
                             select a;

            if ((categoryIds != default) && (categoryIds.Any()))
                operations = operations.Where(o => categoryIds.Contains(o.IdOperation));

            var result = new PaginationContainer<TelemetryOperationDto>
            {
                Skip = skip,
                Take = take
            };

            operations = operations.OrderBy(o => o.UnixDate);

            if (begin != default)
            {
                var unixBegin = (begin - new DateTime(1970, 1, 1)).TotalSeconds;
                operations = operations.Where(o => o.UnixDate >= unixBegin);
            }

            if (end != default)
            {
                var unixEnd = (end - new DateTime(1970, 1, 1)).TotalSeconds;
                operations = operations.Where(m => (m.UnixDate + m.DurationSec) <= unixEnd);
            }

            result.Count = await operations.CountAsync(token).ConfigureAwait(false);

            if (skip > 0)
                operations = operations.Skip(skip);

            var operationsList = await operations.Take(take)
                .AsNoTracking()
                .ToListAsync(token)
                .ConfigureAwait(false);

            if (operationsList.Count == 0)
                return result;

            foreach (var operation in operations)
            {
                var operationDto = new TelemetryOperationDto
                {
                    Id = operation.Id,
                    Name = operation.Operation.Name,
                    BeginDate = DateTimeOffset.FromUnixTimeSeconds(operation.UnixDate).DateTime,
                    EndDate = DateTimeOffset.FromUnixTimeSeconds(operation.UnixDate + operation.DurationSec).DateTime,
                    StartWellDepth = operation.OperationStartDepth ?? 0.0,
                    EndWellDepth = operation.OperationEndDepth ?? 0.0
                };

                result.Items.Add(operationDto);
            }

            return result;
        }

        public async Task<IEnumerable<TelemetryOperationDurationDto>> GetOperationsSummaryAsync(int idWell,
            DateTime begin = default, DateTime end = default, CancellationToken token = default)
        {
            var telemetryId = telemetryService.GetIdTelemetryByIdWell(idWell);

            if (telemetryId is null)
                return null;

            var unixBegin = begin == default 
                            ? 0 
                            : (begin - new DateTime(1970, 1, 1)).TotalSeconds;
            var unixEnd = end == default 
                            ? (DateTime.Now - new DateTime(1970, 1, 1)).TotalSeconds 
                            : (end - new DateTime(1970, 1, 1)).TotalSeconds;

            return await (from a in db.TelemetryAnalysis
                          where a.IdTelemetry == telemetryId &&
                          a.UnixDate > unixBegin && a.UnixDate < unixEnd
                          join o in db.WellOperationCategories on a.IdOperation equals o.Id
                          group a by new { a.IdOperation, o.Name } into g
                          select new TelemetryOperationDurationDto
                          {
                              OperationName = g.Key.Name,
                              Duration = g.Where(g => g.DurationSec > 0)
                                          .Sum(a => a.DurationSec)
                          }).AsNoTracking().ToListAsync(token)
                            .ConfigureAwait(false);
        }

        // This method is not finished (only half done). It returns not correct Dtos. 
        public async Task<IEnumerable<TelemetryOperationInfoDto>> GetOperationsToIntervalAsync(int idWell,
            int intervalSeconds, int workBeginSeconds, CancellationToken token = default)
        {
            intervalSeconds = intervalSeconds == 0 ? 86400 : intervalSeconds;

            var telemetryId = telemetryService.GetIdTelemetryByIdWell(idWell);

            if (telemetryId is null)
                return null;

            var timezoneOffset = telemetryService.GetTimezoneOffsetByTelemetryId((int)telemetryId);

            // Get'n'Group all operations only by start date and by name (if there were several operations in interval).
            // Without dividing these operations duration by given interval
            var ops = await (from a in db.TelemetryAnalysis
                             where a.IdTelemetry == telemetryId
                             join o in db.WellOperationCategories on a.IdOperation equals o.Id
                             group a by new
                             {
                                 Interval = Math.Floor((a.UnixDate - workBeginSeconds + timezoneOffset) / intervalSeconds),
                                 o.Name
                             } into g
                             select new
                             {
                                 IntervalStart = g.Min(d => d.UnixDate),
                                 OperationName = g.Key.Name,
                                 OperationDuration = g.Sum(an => an.DurationSec)
                             }).AsNoTracking()
                            .OrderBy(op => op.IntervalStart)
                            .ToListAsync(token)
                            .ConfigureAwait(false);


            var groupedOperationsList = new List<TelemetryOperationInfoDto>();


            if (operations is not null && operations.Any())
            {
                var operations = ops.Select(o => (o.IntervalStart, o.OperationName, o.OperationDuration));

                var splittedOperationsByInterval = DivideOperationsByIntervalLength(operations, intervalSeconds); // divides good

                groupedOperationsList = UniteOperationsInDto(splittedOperationsByInterval, intervalSeconds).ToList(); // unites not good
            }

            return groupedOperationsList;
        }

        public async Task AnalyzeAndSaveTelemetriesAsync(CancellationToken token = default)
        {
            var allTelemetryIds = await db.Telemetries.Select(t => t.Id).ToListAsync(token).ConfigureAwait(false);

            foreach (var idTelemetry in allTelemetryIds)
            {
                var analyzeStartDate = await GetLastAnalysisDateAsync(idTelemetry, token).ConfigureAwait(false);
                await AnalyseAndSaveTelemetryAsync(idTelemetry, analyzeStartDate, token).ConfigureAwait(false);
                GC.Collect();
            }
        }

        private async Task AnalyseAndSaveTelemetryAsync(int idTelemetry, DateTime analyzeStartDate, CancellationToken token = default)
        {
            const int step = 10;
            const int take = step * 2;

            TelemetryAnalysis currentAnalysis = null;
            
            while (true)
            {
                var dataSaubPart = await GetDataSaubPartOrDefaultAsync(idTelemetry, analyzeStartDate, token).ConfigureAwait(false);
                if (dataSaubPart is null)
                    break;

                var count = dataSaubPart.Count();
                var skip = 0;

                if (step > count)
                    break;

                analyzeStartDate = dataSaubPart.Last().Date;
                for (; (skip + step) < count; skip += step)
                {
                    var dataSaubPartOfPart = dataSaubPart.Skip(skip).Take(take);
                    var telemetryAnalysis = GetDrillingAnalysis(dataSaubPartOfPart);

                    if (currentAnalysis is not null)
                    {
                        if (currentAnalysis.IdOperation == telemetryAnalysis.IdOperation)
                            currentAnalysis.DurationSec += telemetryAnalysis.DurationSec;
                        else
                        {
                            currentAnalysis.OperationEndDepth = dataSaubPartOfPart.LastOrDefault()?.WellDepth;
                            db.TelemetryAnalysis.Add(currentAnalysis);
                            currentAnalysis = null;
                        }
                    }

                    if (currentAnalysis is null)
                    {
                        currentAnalysis = telemetryAnalysis;
                        currentAnalysis.OperationStartDepth = dataSaubPartOfPart.FirstOrDefault()?.WellDepth;
                    }
                }

                await db.SaveChangesAsync(token).ConfigureAwait(false);
                GC.Collect();
            }
        }

        public async Task<DatesRangeDto> GetOperationsDateRangeAsync(int idWell, bool isUtc,
            CancellationToken token = default)
        {
            var telemetryId = telemetryService.GetIdTelemetryByIdWell(idWell);

            if (telemetryId is null)
                return null;

            var datesRange = await (from d in db.TelemetryAnalysis
                                    where d.IdTelemetry == telemetryId
                                    select d.UnixDate).DefaultIfEmpty()
                                    .GroupBy(g => true)
                                    .AsNoTracking()
                                    .Select(g => new
                                    {
                                        From = g.Min(),
                                        To = g.Max()
                                    }).OrderBy(gr => gr.From)
                                    .FirstOrDefaultAsync(token)
                                    .ConfigureAwait(false);

            var result = new DatesRangeDto
            {
                From = DateTimeOffset.FromUnixTimeSeconds(datesRange.From).DateTime,
                To = datesRange.To == default 
                    ? DateTime.MaxValue
                    : DateTimeOffset.FromUnixTimeSeconds(datesRange.To).DateTime
            };
            
            if (isUtc) 
                return result;

            result = await telemetryService.FixDatesRangeByTimeZoneAsync((int)telemetryId, result, token)
                .ConfigureAwait(false);

            return result;
        }

        private async Task<DateTime> GetLastAnalysisDateAsync(int idTelemetry, CancellationToken token = default)
        {
            var lastAnalysisInDb = await (from analysis in db.TelemetryAnalysis
                                    where analysis.IdTelemetry == idTelemetry
                                    orderby analysis.UnixDate
                                    select analysis)
                                    .LastOrDefaultAsync(token)
                                    .ConfigureAwait(false);

            DateTime lastAnalysisDate = default;

            if(lastAnalysisInDb is not null)
                lastAnalysisDate = DateTime.UnixEpoch.AddSeconds(lastAnalysisInDb.DurationSec + lastAnalysisInDb.UnixDate);
            
            return lastAnalysisDate;
        }

        private Task<List<DataSaubAnalyse>> GetDataSaubPartOrDefaultAsync(int idTelemetry, DateTime analyzeStartDate, CancellationToken token) =>
            db.TelemetryDataSaub
            .Where(d =>
                d.IdTelemetry == idTelemetry &&
                d.Date > analyzeStartDate &&
                d.BitDepth != null &&
                d.BlockPosition != null &&
                d.HookWeight != null &&
                d.Pressure != null &&
                d.RotorSpeed != null &&
                d.WellDepth != null
            )
            .OrderBy(d => d.Date)
            .Take(countOfRecordsForInterpolation)
            .Select(d => new DataSaubAnalyse {
                IdTelemetry = d.IdTelemetry,
                Date = d.Date,
                BitDepth = d.BitDepth ?? 0,
                BlockPosition = d.BlockPosition ?? 0,                
                HookWeight = d.HookWeight ?? 0,
                Pressure = d.Pressure ?? 0,
                RotorSpeed = d.RotorSpeed ?? 0,
                WellDepth = d.WellDepth ?? 0,
            })
            .ToListAsync(token);

        private static IEnumerable<(long IntervalStart, string OperationName, int OperationDuration)> DivideOperationsByIntervalLength(
            IEnumerable<(long IntervalStart, string OperationName, int OperationDuration)> operations, int intervalSeconds)
        {
            var splittedOperationsByInterval = new List<(long IntervalStart, string OperationName, int OperationDuration)>();

            var operationDurationTimeCounter = 0;

            foreach (var (IntervalStart, OperationName, OperationDuration) in operations)
            {
                if (OperationDuration < (intervalSeconds - operationDurationTimeCounter))
                {
                    splittedOperationsByInterval.Add((IntervalStart, OperationName, OperationDuration));
                    operationDurationTimeCounter += OperationDuration;
                }
                else
                {   // if operation duration overflows current interval it shoud be divided into 2 or more parts for this and next intervals
                    var remainingIntervalTime = intervalSeconds - operationDurationTimeCounter;
                    splittedOperationsByInterval.Add((IntervalStart, OperationName, remainingIntervalTime)); // first part of long operation

                    var operationDurationAfterDividing = OperationDuration - remainingIntervalTime; // second part of long operation. Can be less or more than interval

                    // If operation duration even after dividing is still more than interval,
                    // it should be divided several times to several intervals.
                    if (operationDurationAfterDividing > intervalSeconds)
                    {
                        var counter = 0;
                        var updatedIntervalStartTime = IntervalStart + remainingIntervalTime;

                        while (operationDurationAfterDividing > intervalSeconds)
                        {
                            splittedOperationsByInterval.Add((updatedIntervalStartTime + intervalSeconds * counter, OperationName, intervalSeconds));
                            operationDurationAfterDividing -= intervalSeconds;
                            counter++;
                        }

                        splittedOperationsByInterval.Add((updatedIntervalStartTime + operationDurationAfterDividing, OperationName, operationDurationAfterDividing));

                        operationDurationTimeCounter = operationDurationAfterDividing;
                    }
                    else
                    {
                        splittedOperationsByInterval.Add((IntervalStart, OperationName, operationDurationAfterDividing));
                        operationDurationTimeCounter = operationDurationAfterDividing;
                    }
                }
            }

            return splittedOperationsByInterval;
        }

        private static IEnumerable<TelemetryOperationInfoDto> UniteOperationsInDto(
            IEnumerable<(long IntervalStart, string OperationName, int OperationDuration)> operations, int intervalSeconds)
        {
            var groupedOperationsList = new List<TelemetryOperationInfoDto>();

            var groupedOperationsObj = new TelemetryOperationInfoDto
            {
                IntervalBegin = DateTimeOffset.FromUnixTimeSeconds(operations.First().IntervalStart),
                Operations = new List<TelemetryOperationDetailsDto>()
            };

            var intervalEndDate = operations.First().IntervalStart + intervalSeconds;

            foreach (var op in operations)
            {
                if (op.IntervalStart < intervalEndDate)
                {
                    groupedOperationsObj.Operations.Add(new TelemetryOperationDetailsDto
                    {
                        OperationName = op.OperationName,
                        DurationSec = op.OperationDuration
                    });
                }
                else
                {
                    groupedOperationsList.Add(groupedOperationsObj);

                    intervalEndDate = op.IntervalStart + intervalSeconds;
                    groupedOperationsObj = new TelemetryOperationInfoDto
                    {
                        IntervalBegin = DateTimeOffset.FromUnixTimeSeconds(op.IntervalStart),
                        Operations = new List<TelemetryOperationDetailsDto>()
                    };

                    groupedOperationsObj.Operations.Add(new TelemetryOperationDetailsDto
                    {
                        OperationName = op.OperationName,
                        DurationSec = op.OperationDuration
                    });
                }
            }

            groupedOperationsList.Add(groupedOperationsObj);

            return groupedOperationsList;
        }

        private TelemetryAnalysis GetDrillingAnalysis(IEnumerable<DataSaubAnalyse> dataSaubPartOfPart)
        {
            var dataSaubFirst = dataSaubPartOfPart.First();
            var dataSaubLast = dataSaubPartOfPart.Last();

            var saubWellDepths = dataSaubPartOfPart.Select(s => (Y: (double)s.WellDepth, 
                X: (s.Date - dataSaubFirst.Date).TotalSeconds));
            var saubBitDepths = dataSaubPartOfPart.Select(s => (Y: (double)s.BitDepth,
                X: (s.Date - dataSaubFirst.Date).TotalSeconds));
            var saubBlockPositions = dataSaubPartOfPart.Select(s => (Y: (double)s.BlockPosition,
                X: (s.Date - dataSaubFirst.Date).TotalSeconds));
            var saubRotorSpeeds = dataSaubPartOfPart.Select(s => (Y: (double)s.RotorSpeed,
                X: (s.Date - dataSaubFirst.Date).TotalSeconds));
            var saubPressures = dataSaubPartOfPart.Select(s => (Y: (double)s.Pressure,
                X: (s.Date - dataSaubFirst.Date).TotalSeconds));
            var saubHookWeights = dataSaubPartOfPart.Select(s => (Y: (double)s.HookWeight,
                X: (s.Date - dataSaubFirst.Date).TotalSeconds));

            var wellDepthLine = new InterpolationLine(saubWellDepths);
            var bitPositionLine = new InterpolationLine(saubBitDepths);
            var blockPositionLine = new InterpolationLine(saubBlockPositions);
            var rotorSpeedLine = new InterpolationLine(saubRotorSpeeds);
            var pressureLine = new InterpolationLine(saubPressures);
            var hookWeightLine = new InterpolationLine(saubHookWeights);

            var drillingAnalysis = new TelemetryAnalysis
            {
                IdTelemetry = dataSaubFirst.IdTelemetry,
                UnixDate = (long)(dataSaubFirst.Date - DateTime.UnixEpoch).TotalSeconds,
                DurationSec = (int)(dataSaubLast.Date - dataSaubFirst.Date).TotalSeconds,
                OperationStartDepth = null,
                OperationEndDepth = null,
                IsWellDepthDecreasing = wellDepthLine.IsYDecreases(-0.0001),
                IsWellDepthIncreasing = wellDepthLine.IsYIncreases( 0.0001),
                IsBitPositionDecreasing = bitPositionLine.IsYDecreases(-0.0001),
                IsBitPositionIncreasing = bitPositionLine.IsYIncreases(0.0001),
                IsBitPositionLt20 = bitPositionLine.IsAverageYLessThanBound(20),
                IsBlockPositionDecreasing = blockPositionLine.IsYDecreases(-0.0001),
                IsBlockPositionIncreasing = blockPositionLine.IsYIncreases(0.0001),
                IsRotorSpeedLt5 = rotorSpeedLine.IsAverageYLessThanBound(5),
                IsRotorSpeedGt5 = rotorSpeedLine.IsAverageYMoreThanBound(5),
                IsPressureLt20 = pressureLine.IsAverageYLessThanBound(20),
                IsPressureGt20 = pressureLine.IsAverageYMoreThanBound(20),
                IsHookWeightNotChanges = hookWeightLine.IsYNotChanges(0.0001, -0.0001),
                IsHookWeightLt3 = hookWeightLine.IsAverageYLessThanBound(3),
                IdOperation = default,
            };

            drillingAnalysis.IdOperation = 
                operationDetectorService.DetectOperation(drillingAnalysis).Id;

            return drillingAnalysis;
        }
    }
}