forked from ddrilling/AsbCloudServer
Add DRAFT telemetry merge
This commit is contained in:
Фролов
parent
2887327d5f
commit
fb38348618
@ -1,4 +1,6 @@
|
||||
using AsbCloudApp.Data;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
|
||||
namespace AsbCloudApp.Services
|
||||
{
|
||||
@ -9,5 +11,7 @@ namespace AsbCloudApp.Services
|
||||
double GetTimezoneOffsetByTelemetryId(int idTelemetry);
|
||||
void UpdateInfo(string uid, TelemetryInfoDto info);
|
||||
int? GetIdTelemetryByIdWell(int idWell);
|
||||
int Merge(IEnumerable<int> telemetryIds);
|
||||
IEnumerable<(string Key, int[] Ids)> GetRedundentRemoteUids();
|
||||
}
|
||||
}
|
||||
|
||||
@ -3,7 +3,11 @@ using AsbCloudApp.Services;
|
||||
using AsbCloudDb.Model;
|
||||
using AsbCloudInfrastructure.Services.Cache;
|
||||
using Mapster;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace AsbCloudInfrastructure.Services
|
||||
{
|
||||
@ -74,5 +78,167 @@ namespace AsbCloudInfrastructure.Services
|
||||
cacheTelemetry.Refresh();
|
||||
return newTelemetryEntry.Entity;
|
||||
}
|
||||
|
||||
public IEnumerable<(string Key, int[] Ids)> GetRedundentRemoteUids()
|
||||
{
|
||||
return db.Telemetries
|
||||
.ToList()
|
||||
.GroupBy(t => t.RemoteUid)
|
||||
.Where(g => g.Count() > 1)
|
||||
.Select(g => (g.Key, g.Select(t=>t.Id).ToArray()));
|
||||
}
|
||||
|
||||
public int Merge(IEnumerable<int> telemetryIds)
|
||||
{
|
||||
if (telemetryIds.Count() < 2)
|
||||
throw new ArgumentException($"telemetryIds {telemetryIds} < 2. nothing to merge.", nameof(telemetryIds));
|
||||
|
||||
// найти телеметрию с наиболее полными справочниками и принять её за основную
|
||||
// отделить основную от остальных
|
||||
// связонные сущности:
|
||||
// Telemetry
|
||||
// TelemetryInfo
|
||||
// TelemetryEvent
|
||||
// TelemetryUser
|
||||
|
||||
// TelemetryMessage
|
||||
// TelemetryDataSpin
|
||||
// TelemetryDataSaub
|
||||
// TelemetryAnalysis
|
||||
// Well
|
||||
|
||||
// Оценка трудоебкости
|
||||
var telemetriesGrade = db.Telemetries
|
||||
.Include(t => t.Messages)
|
||||
.Include(t => t.DataSaub)
|
||||
.Include(t => t.DataSpin)
|
||||
.Include(t => t.Well)
|
||||
.Where(t => telemetryIds.Contains(t.Id))
|
||||
.Select(t => new {
|
||||
t.Id,
|
||||
t.RemoteUid,
|
||||
t.Info,
|
||||
IdWell = t.Well != null ? t.Well.Id : int.MinValue,
|
||||
Records = t.Messages.Count + t.DataSaub.Count + t.DataSpin.Count,
|
||||
EventsAny = t.Events.Any(),
|
||||
UsersAny = t.Users.Any(),
|
||||
})
|
||||
.OrderByDescending(t=>t.Records)
|
||||
.ToList();
|
||||
|
||||
var telemetryDestId = telemetriesGrade.FirstOrDefault().Id;
|
||||
if (telemetryDestId == default)
|
||||
return 0;
|
||||
|
||||
var telemetriesSrcIds = telemetryIds.Where(t => t != telemetryDestId).ToList();
|
||||
if(!telemetriesSrcIds.Any())
|
||||
return 0;
|
||||
|
||||
var telemetriesSrcIdsSql = $"({string.Join(',', telemetriesSrcIds)})";
|
||||
|
||||
var telemetryInfoAndUid = telemetriesGrade
|
||||
.Where(t => t.Info != null)
|
||||
.OrderByDescending(t => t.Id)
|
||||
.Select(t => (t.RemoteUid, t.Info))
|
||||
.FirstOrDefault();
|
||||
|
||||
var wellId = telemetriesGrade
|
||||
.Where(t => t.IdWell > 0)
|
||||
.OrderByDescending(t => t.Id)
|
||||
.Select(t => t.IdWell)
|
||||
.FirstOrDefault();
|
||||
|
||||
// начало изменений
|
||||
Console.WriteLine($"Start merge telemetries ids: [{string.Join(',', telemetriesSrcIds)}] to {telemetryDestId}");
|
||||
var sw = new System.Diagnostics.Stopwatch();
|
||||
sw.Start();
|
||||
var transaction = db.Database.BeginTransaction();
|
||||
int rows = 0;
|
||||
try
|
||||
{
|
||||
var telemetryDst = db.Telemetries.FirstOrDefault(t => t.Id == telemetryDestId);
|
||||
telemetryDst.RemoteUid = telemetryInfoAndUid.RemoteUid;
|
||||
telemetryDst.Info = telemetryInfoAndUid.Info;
|
||||
|
||||
if (wellId != default)
|
||||
{
|
||||
var well = db.Wells.FirstOrDefault(w => w.Id == wellId);
|
||||
well.IdTelemetry = telemetryDestId;
|
||||
}
|
||||
|
||||
// events merge
|
||||
var telemetryDstEventsIds = db.TelemetryEvents.Where(t => t.IdTelemetry == telemetryDestId).Select(t => t.IdEvent).ToList();
|
||||
var telemetrySrcEvents = db.TelemetryEvents
|
||||
.Where(t => telemetriesSrcIds.Contains(t.IdTelemetry) && !telemetryDstEventsIds.Contains(t.IdEvent))
|
||||
.Select(t => new TelemetryEvent
|
||||
{
|
||||
IdTelemetry = telemetryDestId,
|
||||
IdEvent = t.IdEvent,
|
||||
IdCategory = t.IdCategory,
|
||||
MessageTemplate = t.MessageTemplate,
|
||||
})
|
||||
.ToList();
|
||||
var telemetryEventNewUniq = new Dictionary<int, TelemetryEvent>();
|
||||
foreach (var telemetryEvent in telemetrySrcEvents)
|
||||
telemetryEventNewUniq[telemetryEvent.IdEvent] = telemetryEvent;
|
||||
if (telemetrySrcEvents.Any())
|
||||
db.TelemetryEvents.AddRange(telemetryEventNewUniq.Values);
|
||||
|
||||
// users merge
|
||||
var telemetryDstUsersIds = db.TelemetryUsers.Where(t => t.IdTelemetry == telemetryDestId).Select(t => t.IdUser).ToList();
|
||||
var telemetrySrcUsers = db.TelemetryUsers
|
||||
.Where(t => telemetriesSrcIds.Contains(t.IdTelemetry) && !telemetryDstUsersIds.Contains(t.IdUser))
|
||||
.Select(t => new TelemetryUser
|
||||
{
|
||||
IdTelemetry = telemetryDestId,
|
||||
IdUser = t.IdUser,
|
||||
Level = t.Level,
|
||||
Name = t.Name,
|
||||
Patronymic = t.Patronymic,
|
||||
Surname = t.Surname,
|
||||
}).ToList();
|
||||
var telemetryUserNewUniq = new Dictionary<int, TelemetryUser>();
|
||||
foreach (var telemetryUser in telemetrySrcUsers)
|
||||
telemetryUserNewUniq[telemetryUser.IdUser] = telemetryUser;
|
||||
if (telemetrySrcUsers.Any())
|
||||
db.TelemetryUsers.AddRange(telemetryUserNewUniq.Values);
|
||||
|
||||
db.SaveChanges();
|
||||
|
||||
db.Database.SetCommandTimeout(3_000); // 5 мин
|
||||
|
||||
db.Database.ExecuteSqlRaw($"ALTER TABLE t_telemetry_data_saub DISABLE TRIGGER ALL;");
|
||||
rows += db.Database.ExecuteSqlRaw($"UPDATE t_telemetry_data_saub SET id_telemetry = {telemetryDestId} WHERE id_telemetry IN {telemetriesSrcIdsSql};");
|
||||
db.Database.ExecuteSqlRaw($"ALTER TABLE t_telemetry_data_saub ENABLE TRIGGER ALL;");
|
||||
|
||||
db.Database.ExecuteSqlRaw($"ALTER TABLE t_telemetry_data_spin DISABLE TRIGGER ALL;");
|
||||
rows += db.Database.ExecuteSqlRaw($"UPDATE t_telemetry_data_spin SET id_telemetry = {telemetryDestId} WHERE id_telemetry IN {telemetriesSrcIdsSql};");
|
||||
db.Database.ExecuteSqlRaw($"ALTER TABLE t_telemetry_data_spin ENABLE TRIGGER ALL;");
|
||||
|
||||
db.Database.ExecuteSqlRaw($"ALTER TABLE t_telemetry_analysis DISABLE TRIGGER ALL;");
|
||||
rows += db.Database.ExecuteSqlRaw($"UPDATE t_telemetry_analysis SET id_telemetry = {telemetryDestId} WHERE id_telemetry IN {telemetriesSrcIdsSql};");
|
||||
db.Database.ExecuteSqlRaw($"ALTER TABLE t_telemetry_analysis ENABLE TRIGGER ALL;");
|
||||
|
||||
db.Database.ExecuteSqlRaw($"ALTER TABLE t_telemetry_message DISABLE TRIGGER ALL;");
|
||||
rows += db.Database.ExecuteSqlRaw($"UPDATE t_telemetry_message SET id_telemetry = {telemetryDestId} WHERE id_telemetry IN {telemetriesSrcIdsSql};");
|
||||
db.Database.ExecuteSqlRaw($"ALTER TABLE t_telemetry_message ENABLE TRIGGER ALL;");
|
||||
|
||||
rows += db.Database.ExecuteSqlRaw($"DELETE FROM t_telemetry_event WHERE id_telemetry IN {telemetriesSrcIdsSql};");
|
||||
rows += db.Database.ExecuteSqlRaw($"DELETE FROM t_telemetry_user WHERE id_telemetry IN {telemetriesSrcIdsSql};");
|
||||
rows += db.Database.ExecuteSqlRaw($"DELETE FROM t_telemetry WHERE id IN {telemetriesSrcIdsSql};");
|
||||
|
||||
transaction.Commit();
|
||||
sw.Stop();
|
||||
Console.WriteLine($"Successfully commited in {1d*sw.ElapsedMilliseconds/1000d: #0.00} sec.");
|
||||
}
|
||||
catch(Exception ex)
|
||||
{
|
||||
Console.WriteLine("Fail. Rollback.");
|
||||
transaction.Rollback();
|
||||
return 0;
|
||||
}
|
||||
|
||||
return rows;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -2,6 +2,8 @@
|
||||
using AsbCloudApp.Services;
|
||||
using Microsoft.AspNetCore.Authorization;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
|
||||
namespace AsbCloudWebApi.Controllers
|
||||
{
|
||||
@ -10,10 +12,36 @@ namespace AsbCloudWebApi.Controllers
|
||||
[Authorize]
|
||||
public class AdminTelemetryController : CrudController<TelemetryDto, ICrudService<TelemetryDto>>
|
||||
{
|
||||
public AdminTelemetryController(ICrudService<TelemetryDto> service)
|
||||
private readonly ITelemetryService telemetryService;
|
||||
|
||||
public AdminTelemetryController(ICrudService<TelemetryDto> service,
|
||||
ITelemetryService telemetryService)
|
||||
: base(service)
|
||||
{
|
||||
service.Incledes.Add("Well");
|
||||
this.telemetryService = telemetryService;
|
||||
}
|
||||
|
||||
|
||||
[HttpGet]
|
||||
[Route("/reduntentUids")]
|
||||
public IActionResult GetRedundentRemoteUids()
|
||||
{
|
||||
var result = telemetryService.GetRedundentRemoteUids().Select(i => new { i.Key, ids = i.Item2 });
|
||||
return Ok(result);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// merge telemetries
|
||||
/// </summary>
|
||||
/// <param name="telemetriesIds">array of ids</param>
|
||||
/// <returns></returns>
|
||||
[HttpPost]
|
||||
[Route("/merge")]
|
||||
public IActionResult MergeTelemetries([FromBody] List<int> telemetriesIds)
|
||||
{
|
||||
var count = telemetryService.Merge(telemetriesIds);
|
||||
return Ok(count);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,10 +0,0 @@
|
||||
[Desktop Entry]
|
||||
Version=1.0
|
||||
Type=Application
|
||||
Name=Saub3
|
||||
Comment=
|
||||
Exec=/home/asb/start.sh
|
||||
Icon=/home/asb/Saub3/Gui/Images/logo.png
|
||||
Path=/home/asb/Saub3/
|
||||
Terminal=false
|
||||
StartupNotify=false
|
||||
@ -6,10 +6,8 @@
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="BenchmarkDotNet" Version="0.12.1" />
|
||||
<PackageReference Include="ClosedXML" Version="0.95.4" />
|
||||
<PackageReference Include="BenchmarkDotNet" Version="0.13.1" />
|
||||
<PackageReference Include="Mapster" Version="7.2.0" />
|
||||
<PackageReference Include="Microsoft.AspNetCore.SignalR.Client" Version="5.0.5" />
|
||||
<PackageReference Include="System.ComponentModel.Annotations" Version="5.0.0" />
|
||||
<PackageReference Include="System.Net.Http" Version="4.3.4" />
|
||||
</ItemGroup>
|
||||
|
||||
@ -1,11 +1,14 @@
|
||||
//using AsbSaubReport;
|
||||
//using AutoMapper;
|
||||
using System;
|
||||
using System;
|
||||
using System.IO;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using ClosedXML.Excel;
|
||||
using ClosedXML.Excel.Drawings;
|
||||
using AsbCloudApp.Data;
|
||||
using AsbCloudDb.Model;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using AsbCloudInfrastructure.Services.Cache;
|
||||
using AsbCloudInfrastructure.Services;
|
||||
|
||||
namespace ConsoleApp1
|
||||
{
|
||||
@ -14,128 +17,26 @@ namespace ConsoleApp1
|
||||
// .Options;
|
||||
//var context = new AsbCloudDbContext(options);
|
||||
|
||||
class ImageInfo
|
||||
{
|
||||
public int Id { get; set; }
|
||||
public byte[] Data { get; set; }
|
||||
public int Height { get; set; }
|
||||
public int Width { get; set; }
|
||||
public IXLAddress TopLeftCellAddress { get; set; }
|
||||
public int Left { get; set; }
|
||||
public int Top { get; set; }
|
||||
}
|
||||
|
||||
|
||||
class Program
|
||||
{
|
||||
private static IXLWorksheet CopyImagesToAnotherSheet(IEnumerable<ImageInfo> imagesInfos,
|
||||
IXLWorksheet resultSheet)
|
||||
{
|
||||
foreach (var image in imagesInfos)
|
||||
{
|
||||
var stream = new MemoryStream();
|
||||
stream.Write(image.Data, 0, image.Data.Length);
|
||||
|
||||
resultSheet.AddPicture(stream)
|
||||
.WithPlacement(XLPicturePlacement.Move)
|
||||
.WithSize(image.Width, image.Height)
|
||||
.MoveTo(resultSheet.Cell(image.TopLeftCellAddress),
|
||||
image.Left, image.Top);
|
||||
}
|
||||
|
||||
return resultSheet;
|
||||
}
|
||||
|
||||
private static void RemovePicturesFromSheet(IXLWorksheet sheet)
|
||||
{
|
||||
var filteredPics = sheet.Pictures.Select(p => p.Name).Distinct().ToList();
|
||||
|
||||
foreach (var n in filteredPics)
|
||||
sheet.Pictures.Delete(n);
|
||||
}
|
||||
|
||||
private static IXLRange GetCellsRange(IXLWorksheet sheet)
|
||||
{
|
||||
var firstTableCell = sheet.FirstCellUsed();
|
||||
var lastTableCell = sheet.LastCellUsed();
|
||||
var rngData = sheet.Range(firstTableCell.Address, lastTableCell.Address);
|
||||
|
||||
return rngData;
|
||||
}
|
||||
|
||||
{
|
||||
static void Main(/*string[] args*/)
|
||||
{
|
||||
var sourceExcelPaths = new List<string>
|
||||
{
|
||||
//@"D:\excels\excel1.xlsx",
|
||||
//@"D:\excels\excel2.xlsx"
|
||||
//@"D:\excels\e1.xlsx",
|
||||
@"D:\excels\e2.xlsx",
|
||||
@"D:\excels\e3.xlsx",
|
||||
@"D:\excels\e4.XLSX",
|
||||
@"D:\excels\e5.XLSX",
|
||||
@"D:\excels\e6.XLSX"
|
||||
};
|
||||
var options = new DbContextOptionsBuilder<AsbCloudDbContext>()
|
||||
.UseNpgsql("Host=localhost;Database=postgres;Username=postgres;Password=q;Persist Security Info=True")
|
||||
.Options;
|
||||
var db = new AsbCloudDbContext(options);
|
||||
|
||||
const string resultExcelPath = @"D:\excels\result.xlsx";
|
||||
|
||||
using var resultExcelFile = new XLWorkbook(XLEventTracking.Disabled);
|
||||
|
||||
const int maxAllowedColumns = 256;
|
||||
|
||||
foreach(var sourceExcelPath in sourceExcelPaths)
|
||||
{
|
||||
using var sourceExcelFile = new XLWorkbook(sourceExcelPath, XLEventTracking.Disabled);
|
||||
|
||||
Console.WriteLine($"Добавляется файл {Path.GetFileName(sourceExcelPath)}");
|
||||
|
||||
foreach (var sheet in sourceExcelFile.Worksheets)
|
||||
{
|
||||
var imagesInfos = sheet.Pictures.Select(p => new ImageInfo
|
||||
{
|
||||
Id = p.Id,
|
||||
Data = p.ImageStream.GetBuffer(),
|
||||
Height = p.Height,
|
||||
Width = p.Width,
|
||||
TopLeftCellAddress = p.TopLeftCell.Address,
|
||||
Left = p.Left,
|
||||
Top = p.Top
|
||||
}).ToList();
|
||||
|
||||
if (sheet.Columns().Count() > maxAllowedColumns)
|
||||
{
|
||||
var resultSheet = resultExcelFile.Worksheets.Add(sheet.Name);
|
||||
|
||||
var rngData = GetCellsRange(sheet);
|
||||
|
||||
rngData.CopyTo(resultSheet.Cell(1, 1));
|
||||
|
||||
var lastRowWithData = rngData.LastRowUsed().RangeAddress
|
||||
.LastAddress.RowNumber;
|
||||
|
||||
for (int i = 1; i < lastRowWithData; i++)
|
||||
{
|
||||
resultSheet.Row(i).Height = sheet.Row(i).Height;
|
||||
resultSheet.Column(i).Width = sheet.Column(i).Width;
|
||||
}
|
||||
|
||||
CopyImagesToAnotherSheet(imagesInfos, resultSheet);
|
||||
}
|
||||
else
|
||||
{
|
||||
RemovePicturesFromSheet(sheet);
|
||||
|
||||
var resultSheet = sheet.CopyTo(resultExcelFile, sheet.Name);
|
||||
|
||||
CopyImagesToAnotherSheet(imagesInfos, resultSheet);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
resultExcelFile.SaveAs(resultExcelPath);
|
||||
var ts = new TelemetryService(db, new CacheDb());
|
||||
var groups = ts.GetRedundentRemoteUids();
|
||||
foreach(var g in groups)
|
||||
ts.Merge(g.Ids);
|
||||
//var sql = "UPDATE t_telemetry SET info = '{{\"a\":6}}' WHERE id = 2;\n" +
|
||||
// "UPDATE t_telemetry SET info = '{{\"a\":1}}' WHERE id = 1;\n";
|
||||
//db.Database.ExecuteSqlRaw(sql, 1);
|
||||
|
||||
Console.WriteLine("Done. Press any key to quit.");
|
||||
Console.ReadKey();
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user