Add DRAFT telemetry merge

AsbCloudApp/Services/ITelemetryService.cs

@@ -1,4 +1,6 @@
 using AsbCloudApp.Data;
+using System.Collections.Generic;
+using System.Linq;
 
 namespace AsbCloudApp.Services
 {
@@ -9,5 +11,7 @@ namespace AsbCloudApp.Services
         double GetTimezoneOffsetByTelemetryId(int idTelemetry);
         void UpdateInfo(string uid, TelemetryInfoDto info);
         int? GetIdTelemetryByIdWell(int idWell);
+        int Merge(IEnumerable<int> telemetryIds);
+        IEnumerable<(string Key, int[] Ids)> GetRedundentRemoteUids();
     }
 }

AsbCloudInfrastructure/Services/TelemetryService.cs

@@ -3,7 +3,11 @@ using AsbCloudApp.Services;
 using AsbCloudDb.Model;
 using AsbCloudInfrastructure.Services.Cache;
 using Mapster;
+using System.Collections.Generic;
 using System.Linq;
+using System;
+using Microsoft.EntityFrameworkCore;
+using System.Threading.Tasks;
 
 namespace AsbCloudInfrastructure.Services
 {
@@ -74,5 +78,167 @@ namespace AsbCloudInfrastructure.Services
             cacheTelemetry.Refresh();
             return newTelemetryEntry.Entity;
         }
+
+        public IEnumerable<(string Key, int[] Ids)> GetRedundentRemoteUids()
+        {
+            return db.Telemetries
+                .ToList()
+                .GroupBy(t => t.RemoteUid)
+                .Where(g => g.Count() > 1)
+                .Select(g => (g.Key, g.Select(t=>t.Id).ToArray()));
+        }
+
+        public int Merge(IEnumerable<int> telemetryIds)
+        {
+            if (telemetryIds.Count() < 2)
+                throw new ArgumentException($"telemetryIds {telemetryIds} < 2. nothing to merge.", nameof(telemetryIds));
+
+            // найти телеметрию с наиболее полными справочниками и принять её за основную
+            // отделить основную от остальных
+            // связонные сущности:
+            // Telemetry
+            // TelemetryInfo
+            // TelemetryEvent
+            // TelemetryUser
+
+            // TelemetryMessage
+            // TelemetryDataSpin
+            // TelemetryDataSaub
+            // TelemetryAnalysis
+            // Well
+
+            // Оценка трудоебкости
+            var telemetriesGrade = db.Telemetries
+                .Include(t => t.Messages)
+                .Include(t => t.DataSaub)
+                .Include(t => t.DataSpin)
+                .Include(t => t.Well)
+                .Where(t => telemetryIds.Contains(t.Id))
+                .Select(t => new {
+                    t.Id,
+                    t.RemoteUid,
+                    t.Info,
+                    IdWell = t.Well != null ? t.Well.Id : int.MinValue,
+                    Records = t.Messages.Count + t.DataSaub.Count + t.DataSpin.Count,
+                    EventsAny = t.Events.Any(),
+                    UsersAny = t.Users.Any(),
+                })
+                .OrderByDescending(t=>t.Records)
+                .ToList();
+
+            var telemetryDestId = telemetriesGrade.FirstOrDefault().Id;
+            if (telemetryDestId == default)
+                return 0;
+
+            var telemetriesSrcIds = telemetryIds.Where(t => t != telemetryDestId).ToList();
+            if(!telemetriesSrcIds.Any())
+                return 0;
+
+            var telemetriesSrcIdsSql = $"({string.Join(',', telemetriesSrcIds)})";
+
+            var telemetryInfoAndUid = telemetriesGrade
+                .Where(t => t.Info != null)
+                .OrderByDescending(t => t.Id)
+                .Select(t => (t.RemoteUid, t.Info))
+                .FirstOrDefault();
+
+            var wellId = telemetriesGrade
+                .Where(t => t.IdWell > 0)
+                .OrderByDescending(t => t.Id)
+                .Select(t => t.IdWell)
+                .FirstOrDefault();
+
+            // начало изменений
+            Console.WriteLine($"Start merge telemetries ids: [{string.Join(',', telemetriesSrcIds)}] to {telemetryDestId}");
+            var sw = new System.Diagnostics.Stopwatch();
+            sw.Start();
+            var transaction = db.Database.BeginTransaction();
+            int rows = 0;
+            try
+            {
+                var telemetryDst = db.Telemetries.FirstOrDefault(t => t.Id == telemetryDestId);
+                telemetryDst.RemoteUid = telemetryInfoAndUid.RemoteUid;
+                telemetryDst.Info = telemetryInfoAndUid.Info;
+
+                if (wellId != default)
+                {
+                    var well = db.Wells.FirstOrDefault(w => w.Id == wellId);
+                    well.IdTelemetry = telemetryDestId;
+                }
+
+                // events merge
+                var telemetryDstEventsIds = db.TelemetryEvents.Where(t => t.IdTelemetry == telemetryDestId).Select(t => t.IdEvent).ToList();
+                var telemetrySrcEvents = db.TelemetryEvents
+                    .Where(t => telemetriesSrcIds.Contains(t.IdTelemetry) && !telemetryDstEventsIds.Contains(t.IdEvent))
+                    .Select(t => new TelemetryEvent
+                    {
+                        IdTelemetry = telemetryDestId,
+                        IdEvent = t.IdEvent,
+                        IdCategory = t.IdCategory,
+                        MessageTemplate = t.MessageTemplate,
+                    })
+                    .ToList();
+                var telemetryEventNewUniq = new Dictionary<int, TelemetryEvent>();
+                foreach (var telemetryEvent in telemetrySrcEvents)
+                    telemetryEventNewUniq[telemetryEvent.IdEvent] = telemetryEvent;
+                if (telemetrySrcEvents.Any())
+                    db.TelemetryEvents.AddRange(telemetryEventNewUniq.Values);
+
+                // users merge
+                var telemetryDstUsersIds = db.TelemetryUsers.Where(t => t.IdTelemetry == telemetryDestId).Select(t => t.IdUser).ToList();
+                var telemetrySrcUsers = db.TelemetryUsers
+                    .Where(t => telemetriesSrcIds.Contains(t.IdTelemetry) && !telemetryDstUsersIds.Contains(t.IdUser))
+                    .Select(t => new TelemetryUser
+                    {
+                        IdTelemetry = telemetryDestId,
+                        IdUser = t.IdUser,
+                        Level = t.Level,
+                        Name = t.Name,
+                        Patronymic = t.Patronymic,
+                        Surname = t.Surname,
+                    }).ToList();
+                var telemetryUserNewUniq = new Dictionary<int, TelemetryUser>();
+                foreach (var telemetryUser in telemetrySrcUsers)
+                    telemetryUserNewUniq[telemetryUser.IdUser] = telemetryUser;
+                if (telemetrySrcUsers.Any())
+                    db.TelemetryUsers.AddRange(telemetryUserNewUniq.Values);
+
+                db.SaveChanges();
+
+                db.Database.SetCommandTimeout(3_000); // 5 мин
+
+                db.Database.ExecuteSqlRaw($"ALTER TABLE t_telemetry_data_saub DISABLE TRIGGER ALL;");
+                rows += db.Database.ExecuteSqlRaw($"UPDATE t_telemetry_data_saub SET id_telemetry = {telemetryDestId} WHERE id_telemetry IN {telemetriesSrcIdsSql};");
+                db.Database.ExecuteSqlRaw($"ALTER TABLE t_telemetry_data_saub ENABLE TRIGGER ALL;");
+
+                db.Database.ExecuteSqlRaw($"ALTER TABLE t_telemetry_data_spin DISABLE TRIGGER ALL;");
+                rows += db.Database.ExecuteSqlRaw($"UPDATE t_telemetry_data_spin SET id_telemetry = {telemetryDestId} WHERE id_telemetry IN {telemetriesSrcIdsSql};");
+                db.Database.ExecuteSqlRaw($"ALTER TABLE t_telemetry_data_spin ENABLE TRIGGER ALL;");
+
+                db.Database.ExecuteSqlRaw($"ALTER TABLE t_telemetry_analysis DISABLE TRIGGER ALL;");
+                rows += db.Database.ExecuteSqlRaw($"UPDATE t_telemetry_analysis SET id_telemetry = {telemetryDestId} WHERE id_telemetry IN {telemetriesSrcIdsSql};");
+                db.Database.ExecuteSqlRaw($"ALTER TABLE t_telemetry_analysis ENABLE TRIGGER ALL;");
+
+                db.Database.ExecuteSqlRaw($"ALTER TABLE t_telemetry_message DISABLE TRIGGER ALL;");
+                rows += db.Database.ExecuteSqlRaw($"UPDATE t_telemetry_message SET id_telemetry = {telemetryDestId} WHERE id_telemetry IN {telemetriesSrcIdsSql};");
+                db.Database.ExecuteSqlRaw($"ALTER TABLE t_telemetry_message ENABLE TRIGGER ALL;");
+
+                rows += db.Database.ExecuteSqlRaw($"DELETE FROM t_telemetry_event WHERE id_telemetry IN {telemetriesSrcIdsSql};");
+                rows += db.Database.ExecuteSqlRaw($"DELETE FROM t_telemetry_user WHERE id_telemetry IN {telemetriesSrcIdsSql};");
+                rows += db.Database.ExecuteSqlRaw($"DELETE FROM t_telemetry WHERE id IN {telemetriesSrcIdsSql};");
+
+                transaction.Commit();
+                sw.Stop();
+                Console.WriteLine($"Successfully commited in {1d*sw.ElapsedMilliseconds/1000d: #0.00} sec.");
+            }
+            catch(Exception ex)
+            {
+                Console.WriteLine("Fail. Rollback.");
+                transaction.Rollback();
+                return 0;
+            }
+            
+            return rows;
+        }
     }
 }

AsbCloudWebApi/Controllers/AdminTelemetryController.cs

@@ -2,6 +2,8 @@
 using AsbCloudApp.Services;
 using Microsoft.AspNetCore.Authorization;
 using Microsoft.AspNetCore.Mvc;
+using System.Collections.Generic;
+using System.Linq;
 
 namespace AsbCloudWebApi.Controllers
 {
@@ -10,10 +12,36 @@ namespace AsbCloudWebApi.Controllers
     [Authorize]
     public class AdminTelemetryController : CrudController<TelemetryDto, ICrudService<TelemetryDto>>
     {
-        public AdminTelemetryController(ICrudService<TelemetryDto> service)
+        private readonly ITelemetryService telemetryService;
+
+        public AdminTelemetryController(ICrudService<TelemetryDto> service,
+            ITelemetryService telemetryService)
             : base(service)
         {
             service.Incledes.Add("Well");
+            this.telemetryService = telemetryService;
+        }
+
+
+        [HttpGet]
+        [Route("/reduntentUids")]
+        public IActionResult GetRedundentRemoteUids()
+        {
+            var result = telemetryService.GetRedundentRemoteUids().Select(i => new { i.Key, ids = i.Item2 });
+            return Ok(result);
+        }
+
+        /// <summary>
+        /// merge telemetries
+        /// </summary>
+        /// <param name="telemetriesIds">array of ids</param>
+        /// <returns></returns>
+        [HttpPost]
+        [Route("/merge")]
+        public IActionResult MergeTelemetries([FromBody] List<int> telemetriesIds)
+        {
+            var count = telemetryService.Merge(telemetriesIds);
+            return Ok(count);
         }
     }
 }

AsbCloudWebApi/Docs/Saub3.desktop

@@ -1,10 +0,0 @@
-[Desktop Entry]
-Version=1.0
-Type=Application
-Name=Saub3
-Comment=
-Exec=/home/asb/start.sh
-Icon=/home/asb/Saub3/Gui/Images/logo.png
-Path=/home/asb/Saub3/
-Terminal=false
-StartupNotify=false

ConsoleApp1/ConsoleApp1.csproj

@@ -6,10 +6,8 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="BenchmarkDotNet" Version="0.12.1" />
+    <PackageReference Include="BenchmarkDotNet" Version="0.13.1" />
-    <PackageReference Include="ClosedXML" Version="0.95.4" />
     <PackageReference Include="Mapster" Version="7.2.0" />
-    <PackageReference Include="Microsoft.AspNetCore.SignalR.Client" Version="5.0.5" />
     <PackageReference Include="System.ComponentModel.Annotations" Version="5.0.0" />
     <PackageReference Include="System.Net.Http" Version="4.3.4" />
   </ItemGroup>

ConsoleApp1/Program.cs

@@ -1,11 +1,14 @@
-//using AsbSaubReport;
+using System;
-//using AutoMapper;
-using System;
 using System.IO;
 using System.Collections.Generic;
 using System.Linq;
 using ClosedXML.Excel;
 using ClosedXML.Excel.Drawings;
+using AsbCloudApp.Data;
+using AsbCloudDb.Model;
+using Microsoft.EntityFrameworkCore;
+using AsbCloudInfrastructure.Services.Cache;
+using AsbCloudInfrastructure.Services;
 
 namespace ConsoleApp1
 {
@@ -14,128 +17,26 @@ namespace ConsoleApp1
     //    .Options;
     //var context = new AsbCloudDbContext(options);
 
-    class ImageInfo
+    
-    {
-        public int Id { get; set; }
-        public byte[] Data { get; set; }
-        public int Height { get; set; }
-        public int Width { get; set; }
-        public IXLAddress TopLeftCellAddress { get; set; }
-        public int Left { get; set; }
-        public int Top { get; set; }
-    }
-
     class Program
-    {
+    {        
-        private static IXLWorksheet CopyImagesToAnotherSheet(IEnumerable<ImageInfo> imagesInfos, 
-            IXLWorksheet resultSheet)
-        {
-            foreach (var image in imagesInfos)
-            {
-                var stream = new MemoryStream();
-                stream.Write(image.Data, 0, image.Data.Length);
-
-                resultSheet.AddPicture(stream)
-                           .WithPlacement(XLPicturePlacement.Move)
-                           .WithSize(image.Width, image.Height)
-                           .MoveTo(resultSheet.Cell(image.TopLeftCellAddress),
-                                   image.Left, image.Top);
-            }
-
-            return resultSheet;
-        }
-
-        private static void RemovePicturesFromSheet(IXLWorksheet sheet)
-        {
-            var filteredPics = sheet.Pictures.Select(p => p.Name).Distinct().ToList();
-
-            foreach (var n in filteredPics)
-                sheet.Pictures.Delete(n);
-        }
-
-        private static IXLRange GetCellsRange(IXLWorksheet sheet)
-        {
-            var firstTableCell = sheet.FirstCellUsed();
-            var lastTableCell = sheet.LastCellUsed();
-            var rngData = sheet.Range(firstTableCell.Address, lastTableCell.Address);
-
-            return rngData;
-        }
-
         static void Main(/*string[] args*/)
         {
-            var sourceExcelPaths = new List<string>
+            var options = new DbContextOptionsBuilder<AsbCloudDbContext>()
-            {
+                .UseNpgsql("Host=localhost;Database=postgres;Username=postgres;Password=q;Persist Security Info=True")
-                //@"D:\excels\excel1.xlsx",
+                .Options;
-                //@"D:\excels\excel2.xlsx"
+            var db = new AsbCloudDbContext(options);
-                //@"D:\excels\e1.xlsx",
-                @"D:\excels\e2.xlsx",
-                @"D:\excels\e3.xlsx",
-                @"D:\excels\e4.XLSX",
-                @"D:\excels\e5.XLSX",
-                @"D:\excels\e6.XLSX"
-            };
 
-            const string resultExcelPath = @"D:\excels\result.xlsx";
+            var ts = new TelemetryService(db, new CacheDb());
-
+            var groups = ts.GetRedundentRemoteUids();
-            using var resultExcelFile = new XLWorkbook(XLEventTracking.Disabled);
+            foreach(var g in groups)
-
+                ts.Merge(g.Ids);
-            const int maxAllowedColumns = 256;
+            //var sql = "UPDATE t_telemetry SET info = '{{\"a\":6}}' WHERE id = 2;\n" +
-
+            //    "UPDATE t_telemetry SET info = '{{\"a\":1}}' WHERE id = 1;\n";
-            foreach(var sourceExcelPath in sourceExcelPaths)
+            //db.Database.ExecuteSqlRaw(sql, 1);
-            {
-                using var sourceExcelFile = new XLWorkbook(sourceExcelPath, XLEventTracking.Disabled);
-
-                Console.WriteLine($"Добавляется файл {Path.GetFileName(sourceExcelPath)}");
-
-                foreach (var sheet in sourceExcelFile.Worksheets)
-                {
-                    var imagesInfos = sheet.Pictures.Select(p => new ImageInfo
-                    {
-                        Id = p.Id,
-                        Data = p.ImageStream.GetBuffer(),
-                        Height = p.Height,
-                        Width = p.Width,
-                        TopLeftCellAddress = p.TopLeftCell.Address,
-                        Left = p.Left,
-                        Top = p.Top
-                    }).ToList();
-
-                    if (sheet.Columns().Count() > maxAllowedColumns)
-                    {
-                        var resultSheet = resultExcelFile.Worksheets.Add(sheet.Name);
-
-                        var rngData = GetCellsRange(sheet);
-
-                        rngData.CopyTo(resultSheet.Cell(1, 1));
-
-                        var lastRowWithData = rngData.LastRowUsed().RangeAddress
-                            .LastAddress.RowNumber;
-
-                        for (int i = 1; i < lastRowWithData; i++)
-                        {
-                            resultSheet.Row(i).Height = sheet.Row(i).Height;
-                            resultSheet.Column(i).Width = sheet.Column(i).Width;
-                        }
-
-                        CopyImagesToAnotherSheet(imagesInfos, resultSheet);
-                    }
-                    else
-                    {
-                        RemovePicturesFromSheet(sheet);
-
-                        var resultSheet = sheet.CopyTo(resultExcelFile, sheet.Name);
-
-                        CopyImagesToAnotherSheet(imagesInfos, resultSheet);
-                    }
-                }
-            }
-
-            resultExcelFile.SaveAs(resultExcelPath);
 
             Console.WriteLine("Done. Press any key to quit.");
             Console.ReadKey();
         }
-
     }
 }