From fb38348618fb65990503ea923e2378e341e7e6f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A4=D1=80=D0=BE=D0=BB=D0=BE=D0=B2?= Date: Wed, 29 Sep 2021 10:12:54 +0500 Subject: [PATCH] Add DRAFT telemetry merge --- AsbCloudApp/Services/ITelemetryService.cs | 4 + .../Services/TelemetryService.cs | 166 ++++++++++++++++++ .../Controllers/AdminTelemetryController.cs | 30 +++- AsbCloudWebApi/Docs/Saub3.desktop | 10 -- ConsoleApp1/ConsoleApp1.csproj | 4 +- ConsoleApp1/Program.cs | 137 ++------------- 6 files changed, 219 insertions(+), 132 deletions(-) delete mode 100644 AsbCloudWebApi/Docs/Saub3.desktop diff --git a/AsbCloudApp/Services/ITelemetryService.cs b/AsbCloudApp/Services/ITelemetryService.cs index 30a26cee..57b7674d 100644 --- a/AsbCloudApp/Services/ITelemetryService.cs +++ b/AsbCloudApp/Services/ITelemetryService.cs @@ -1,4 +1,6 @@ using AsbCloudApp.Data; +using System.Collections.Generic; +using System.Linq; namespace AsbCloudApp.Services { @@ -9,5 +11,7 @@ namespace AsbCloudApp.Services double GetTimezoneOffsetByTelemetryId(int idTelemetry); void UpdateInfo(string uid, TelemetryInfoDto info); int? GetIdTelemetryByIdWell(int idWell); + int Merge(IEnumerable telemetryIds); + IEnumerable<(string Key, int[] Ids)> GetRedundentRemoteUids(); } } diff --git a/AsbCloudInfrastructure/Services/TelemetryService.cs b/AsbCloudInfrastructure/Services/TelemetryService.cs index bdb720cd..e46044c0 100644 --- a/AsbCloudInfrastructure/Services/TelemetryService.cs +++ b/AsbCloudInfrastructure/Services/TelemetryService.cs @@ -3,7 +3,11 @@ using AsbCloudApp.Services; using AsbCloudDb.Model; using AsbCloudInfrastructure.Services.Cache; using Mapster; +using System.Collections.Generic; using System.Linq; +using System; +using Microsoft.EntityFrameworkCore; +using System.Threading.Tasks; namespace AsbCloudInfrastructure.Services { @@ -74,5 +78,167 @@ namespace AsbCloudInfrastructure.Services cacheTelemetry.Refresh(); return newTelemetryEntry.Entity; } + + public IEnumerable<(string Key, int[] Ids)> GetRedundentRemoteUids() + { + return db.Telemetries + .ToList() + .GroupBy(t => t.RemoteUid) + .Where(g => g.Count() > 1) + .Select(g => (g.Key, g.Select(t=>t.Id).ToArray())); + } + + public int Merge(IEnumerable telemetryIds) + { + if (telemetryIds.Count() < 2) + throw new ArgumentException($"telemetryIds {telemetryIds} < 2. nothing to merge.", nameof(telemetryIds)); + + // найти телеметрию с наиболее полными справочниками и принять её за основную + // отделить основную от остальных + // связонные сущности: + // Telemetry + // TelemetryInfo + // TelemetryEvent + // TelemetryUser + + // TelemetryMessage + // TelemetryDataSpin + // TelemetryDataSaub + // TelemetryAnalysis + // Well + + // Оценка трудоебкости + var telemetriesGrade = db.Telemetries + .Include(t => t.Messages) + .Include(t => t.DataSaub) + .Include(t => t.DataSpin) + .Include(t => t.Well) + .Where(t => telemetryIds.Contains(t.Id)) + .Select(t => new { + t.Id, + t.RemoteUid, + t.Info, + IdWell = t.Well != null ? t.Well.Id : int.MinValue, + Records = t.Messages.Count + t.DataSaub.Count + t.DataSpin.Count, + EventsAny = t.Events.Any(), + UsersAny = t.Users.Any(), + }) + .OrderByDescending(t=>t.Records) + .ToList(); + + var telemetryDestId = telemetriesGrade.FirstOrDefault().Id; + if (telemetryDestId == default) + return 0; + + var telemetriesSrcIds = telemetryIds.Where(t => t != telemetryDestId).ToList(); + if(!telemetriesSrcIds.Any()) + return 0; + + var telemetriesSrcIdsSql = $"({string.Join(',', telemetriesSrcIds)})"; + + var telemetryInfoAndUid = telemetriesGrade + .Where(t => t.Info != null) + .OrderByDescending(t => t.Id) + .Select(t => (t.RemoteUid, t.Info)) + .FirstOrDefault(); + + var wellId = telemetriesGrade + .Where(t => t.IdWell > 0) + .OrderByDescending(t => t.Id) + .Select(t => t.IdWell) + .FirstOrDefault(); + + // начало изменений + Console.WriteLine($"Start merge telemetries ids: [{string.Join(',', telemetriesSrcIds)}] to {telemetryDestId}"); + var sw = new System.Diagnostics.Stopwatch(); + sw.Start(); + var transaction = db.Database.BeginTransaction(); + int rows = 0; + try + { + var telemetryDst = db.Telemetries.FirstOrDefault(t => t.Id == telemetryDestId); + telemetryDst.RemoteUid = telemetryInfoAndUid.RemoteUid; + telemetryDst.Info = telemetryInfoAndUid.Info; + + if (wellId != default) + { + var well = db.Wells.FirstOrDefault(w => w.Id == wellId); + well.IdTelemetry = telemetryDestId; + } + + // events merge + var telemetryDstEventsIds = db.TelemetryEvents.Where(t => t.IdTelemetry == telemetryDestId).Select(t => t.IdEvent).ToList(); + var telemetrySrcEvents = db.TelemetryEvents + .Where(t => telemetriesSrcIds.Contains(t.IdTelemetry) && !telemetryDstEventsIds.Contains(t.IdEvent)) + .Select(t => new TelemetryEvent + { + IdTelemetry = telemetryDestId, + IdEvent = t.IdEvent, + IdCategory = t.IdCategory, + MessageTemplate = t.MessageTemplate, + }) + .ToList(); + var telemetryEventNewUniq = new Dictionary(); + foreach (var telemetryEvent in telemetrySrcEvents) + telemetryEventNewUniq[telemetryEvent.IdEvent] = telemetryEvent; + if (telemetrySrcEvents.Any()) + db.TelemetryEvents.AddRange(telemetryEventNewUniq.Values); + + // users merge + var telemetryDstUsersIds = db.TelemetryUsers.Where(t => t.IdTelemetry == telemetryDestId).Select(t => t.IdUser).ToList(); + var telemetrySrcUsers = db.TelemetryUsers + .Where(t => telemetriesSrcIds.Contains(t.IdTelemetry) && !telemetryDstUsersIds.Contains(t.IdUser)) + .Select(t => new TelemetryUser + { + IdTelemetry = telemetryDestId, + IdUser = t.IdUser, + Level = t.Level, + Name = t.Name, + Patronymic = t.Patronymic, + Surname = t.Surname, + }).ToList(); + var telemetryUserNewUniq = new Dictionary(); + foreach (var telemetryUser in telemetrySrcUsers) + telemetryUserNewUniq[telemetryUser.IdUser] = telemetryUser; + if (telemetrySrcUsers.Any()) + db.TelemetryUsers.AddRange(telemetryUserNewUniq.Values); + + db.SaveChanges(); + + db.Database.SetCommandTimeout(3_000); // 5 мин + + db.Database.ExecuteSqlRaw($"ALTER TABLE t_telemetry_data_saub DISABLE TRIGGER ALL;"); + rows += db.Database.ExecuteSqlRaw($"UPDATE t_telemetry_data_saub SET id_telemetry = {telemetryDestId} WHERE id_telemetry IN {telemetriesSrcIdsSql};"); + db.Database.ExecuteSqlRaw($"ALTER TABLE t_telemetry_data_saub ENABLE TRIGGER ALL;"); + + db.Database.ExecuteSqlRaw($"ALTER TABLE t_telemetry_data_spin DISABLE TRIGGER ALL;"); + rows += db.Database.ExecuteSqlRaw($"UPDATE t_telemetry_data_spin SET id_telemetry = {telemetryDestId} WHERE id_telemetry IN {telemetriesSrcIdsSql};"); + db.Database.ExecuteSqlRaw($"ALTER TABLE t_telemetry_data_spin ENABLE TRIGGER ALL;"); + + db.Database.ExecuteSqlRaw($"ALTER TABLE t_telemetry_analysis DISABLE TRIGGER ALL;"); + rows += db.Database.ExecuteSqlRaw($"UPDATE t_telemetry_analysis SET id_telemetry = {telemetryDestId} WHERE id_telemetry IN {telemetriesSrcIdsSql};"); + db.Database.ExecuteSqlRaw($"ALTER TABLE t_telemetry_analysis ENABLE TRIGGER ALL;"); + + db.Database.ExecuteSqlRaw($"ALTER TABLE t_telemetry_message DISABLE TRIGGER ALL;"); + rows += db.Database.ExecuteSqlRaw($"UPDATE t_telemetry_message SET id_telemetry = {telemetryDestId} WHERE id_telemetry IN {telemetriesSrcIdsSql};"); + db.Database.ExecuteSqlRaw($"ALTER TABLE t_telemetry_message ENABLE TRIGGER ALL;"); + + rows += db.Database.ExecuteSqlRaw($"DELETE FROM t_telemetry_event WHERE id_telemetry IN {telemetriesSrcIdsSql};"); + rows += db.Database.ExecuteSqlRaw($"DELETE FROM t_telemetry_user WHERE id_telemetry IN {telemetriesSrcIdsSql};"); + rows += db.Database.ExecuteSqlRaw($"DELETE FROM t_telemetry WHERE id IN {telemetriesSrcIdsSql};"); + + transaction.Commit(); + sw.Stop(); + Console.WriteLine($"Successfully commited in {1d*sw.ElapsedMilliseconds/1000d: #0.00} sec."); + } + catch(Exception ex) + { + Console.WriteLine("Fail. Rollback."); + transaction.Rollback(); + return 0; + } + + return rows; + } } } diff --git a/AsbCloudWebApi/Controllers/AdminTelemetryController.cs b/AsbCloudWebApi/Controllers/AdminTelemetryController.cs index f6ade1af..17daa825 100644 --- a/AsbCloudWebApi/Controllers/AdminTelemetryController.cs +++ b/AsbCloudWebApi/Controllers/AdminTelemetryController.cs @@ -2,6 +2,8 @@ using AsbCloudApp.Services; using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; +using System.Collections.Generic; +using System.Linq; namespace AsbCloudWebApi.Controllers { @@ -10,10 +12,36 @@ namespace AsbCloudWebApi.Controllers [Authorize] public class AdminTelemetryController : CrudController> { - public AdminTelemetryController(ICrudService service) + private readonly ITelemetryService telemetryService; + + public AdminTelemetryController(ICrudService service, + ITelemetryService telemetryService) : base(service) { service.Incledes.Add("Well"); + this.telemetryService = telemetryService; + } + + + [HttpGet] + [Route("/reduntentUids")] + public IActionResult GetRedundentRemoteUids() + { + var result = telemetryService.GetRedundentRemoteUids().Select(i => new { i.Key, ids = i.Item2 }); + return Ok(result); + } + + /// + /// merge telemetries + /// + /// array of ids + /// + [HttpPost] + [Route("/merge")] + public IActionResult MergeTelemetries([FromBody] List telemetriesIds) + { + var count = telemetryService.Merge(telemetriesIds); + return Ok(count); } } } diff --git a/AsbCloudWebApi/Docs/Saub3.desktop b/AsbCloudWebApi/Docs/Saub3.desktop deleted file mode 100644 index c1b468ca..00000000 --- a/AsbCloudWebApi/Docs/Saub3.desktop +++ /dev/null @@ -1,10 +0,0 @@ -[Desktop Entry] -Version=1.0 -Type=Application -Name=Saub3 -Comment= -Exec=/home/asb/start.sh -Icon=/home/asb/Saub3/Gui/Images/logo.png -Path=/home/asb/Saub3/ -Terminal=false -StartupNotify=false diff --git a/ConsoleApp1/ConsoleApp1.csproj b/ConsoleApp1/ConsoleApp1.csproj index 42fc49a5..ea6513f8 100644 --- a/ConsoleApp1/ConsoleApp1.csproj +++ b/ConsoleApp1/ConsoleApp1.csproj @@ -6,10 +6,8 @@ - - + - diff --git a/ConsoleApp1/Program.cs b/ConsoleApp1/Program.cs index b270c901..a1688d33 100644 --- a/ConsoleApp1/Program.cs +++ b/ConsoleApp1/Program.cs @@ -1,11 +1,14 @@ -//using AsbSaubReport; -//using AutoMapper; -using System; +using System; using System.IO; using System.Collections.Generic; using System.Linq; using ClosedXML.Excel; using ClosedXML.Excel.Drawings; +using AsbCloudApp.Data; +using AsbCloudDb.Model; +using Microsoft.EntityFrameworkCore; +using AsbCloudInfrastructure.Services.Cache; +using AsbCloudInfrastructure.Services; namespace ConsoleApp1 { @@ -14,128 +17,26 @@ namespace ConsoleApp1 // .Options; //var context = new AsbCloudDbContext(options); - class ImageInfo - { - public int Id { get; set; } - public byte[] Data { get; set; } - public int Height { get; set; } - public int Width { get; set; } - public IXLAddress TopLeftCellAddress { get; set; } - public int Left { get; set; } - public int Top { get; set; } - } - + class Program - { - private static IXLWorksheet CopyImagesToAnotherSheet(IEnumerable imagesInfos, - IXLWorksheet resultSheet) - { - foreach (var image in imagesInfos) - { - var stream = new MemoryStream(); - stream.Write(image.Data, 0, image.Data.Length); - - resultSheet.AddPicture(stream) - .WithPlacement(XLPicturePlacement.Move) - .WithSize(image.Width, image.Height) - .MoveTo(resultSheet.Cell(image.TopLeftCellAddress), - image.Left, image.Top); - } - - return resultSheet; - } - - private static void RemovePicturesFromSheet(IXLWorksheet sheet) - { - var filteredPics = sheet.Pictures.Select(p => p.Name).Distinct().ToList(); - - foreach (var n in filteredPics) - sheet.Pictures.Delete(n); - } - - private static IXLRange GetCellsRange(IXLWorksheet sheet) - { - var firstTableCell = sheet.FirstCellUsed(); - var lastTableCell = sheet.LastCellUsed(); - var rngData = sheet.Range(firstTableCell.Address, lastTableCell.Address); - - return rngData; - } - + { static void Main(/*string[] args*/) { - var sourceExcelPaths = new List - { - //@"D:\excels\excel1.xlsx", - //@"D:\excels\excel2.xlsx" - //@"D:\excels\e1.xlsx", - @"D:\excels\e2.xlsx", - @"D:\excels\e3.xlsx", - @"D:\excels\e4.XLSX", - @"D:\excels\e5.XLSX", - @"D:\excels\e6.XLSX" - }; + var options = new DbContextOptionsBuilder() + .UseNpgsql("Host=localhost;Database=postgres;Username=postgres;Password=q;Persist Security Info=True") + .Options; + var db = new AsbCloudDbContext(options); - const string resultExcelPath = @"D:\excels\result.xlsx"; - - using var resultExcelFile = new XLWorkbook(XLEventTracking.Disabled); - - const int maxAllowedColumns = 256; - - foreach(var sourceExcelPath in sourceExcelPaths) - { - using var sourceExcelFile = new XLWorkbook(sourceExcelPath, XLEventTracking.Disabled); - - Console.WriteLine($"Добавляется файл {Path.GetFileName(sourceExcelPath)}"); - - foreach (var sheet in sourceExcelFile.Worksheets) - { - var imagesInfos = sheet.Pictures.Select(p => new ImageInfo - { - Id = p.Id, - Data = p.ImageStream.GetBuffer(), - Height = p.Height, - Width = p.Width, - TopLeftCellAddress = p.TopLeftCell.Address, - Left = p.Left, - Top = p.Top - }).ToList(); - - if (sheet.Columns().Count() > maxAllowedColumns) - { - var resultSheet = resultExcelFile.Worksheets.Add(sheet.Name); - - var rngData = GetCellsRange(sheet); - - rngData.CopyTo(resultSheet.Cell(1, 1)); - - var lastRowWithData = rngData.LastRowUsed().RangeAddress - .LastAddress.RowNumber; - - for (int i = 1; i < lastRowWithData; i++) - { - resultSheet.Row(i).Height = sheet.Row(i).Height; - resultSheet.Column(i).Width = sheet.Column(i).Width; - } - - CopyImagesToAnotherSheet(imagesInfos, resultSheet); - } - else - { - RemovePicturesFromSheet(sheet); - - var resultSheet = sheet.CopyTo(resultExcelFile, sheet.Name); - - CopyImagesToAnotherSheet(imagesInfos, resultSheet); - } - } - } - - resultExcelFile.SaveAs(resultExcelPath); + var ts = new TelemetryService(db, new CacheDb()); + var groups = ts.GetRedundentRemoteUids(); + foreach(var g in groups) + ts.Merge(g.Ids); + //var sql = "UPDATE t_telemetry SET info = '{{\"a\":6}}' WHERE id = 2;\n" + + // "UPDATE t_telemetry SET info = '{{\"a\":1}}' WHERE id = 1;\n"; + //db.Database.ExecuteSqlRaw(sql, 1); Console.WriteLine("Done. Press any key to quit."); Console.ReadKey(); } - } }