using AsbCloudApp.Data; using AsbCloudApp.Data.DailyReport; using AsbCloudDb; using AsbCloudDb.Model; using AsbCloudInfrastructure.Services.DailyReport; using ClosedXML.Excel; using DocumentFormat.OpenXml.Wordprocessing; using Microsoft.EntityFrameworkCore; using System; using System.IO; using System.Linq; using System.Threading; using System.Threading.Tasks; namespace ConsoleApp1 { class Program { private static AsbCloudDbContext db = ServiceFactory.Context; public delegate void OnProgress(int handled, int total); public static async Task<(int oldCount, int newCount)> RediceSamplingAsync(int idTelemetry, int ratio, int maxDateGapSec, OnProgress onProgress, CancellationToken token) { const int ramLimit = 10 * 1024 * 1024; const int dataItemSize = 345; // by profiler const int itemsCountLimit = ramLimit / dataItemSize; // ~ 90_000 const double maxWellDepthGap = 0.1; ratio = ratio > 0 ? ratio : 5; maxDateGapSec = maxDateGapSec > 0 ? maxDateGapSec : 9; var dbset = db.Set(); var oldCount = await dbset.Where(t => t.IdTelemetry == idTelemetry).CountAsync(token); onProgress?.Invoke(0, oldCount); var sqlSelect = "select " + " * " + "from " + " (select " + " *, " + " rank() over win1 as row_num, " + " lag(\"date\", 1) over win1 as lag_date, " + " lag(\"mode\", 1) over win1 as lag_mode, " + " lag(mse_state, 1) over win1 as lag_mse_state, " + " lag(well_depth, 1) over win1 as lag_well_depth, " + " lag(id_feed_regulator, 1) over win1 as lag_id_feed_regulator " + " from t_telemetry_data_saub " + $" where id_telemetry = {idTelemetry} and \"date\" > {{0}}" + " window win1 as (order by \"date\") " + " ) as ttt " + "where " + $" (row_num % {ratio}) = 0 " + " or \"mode\" != lag_mode " + $" or(\"date\" - lag_date) >= interval '{maxDateGapSec} second' " + $" or well_depth - lag_well_depth > {maxWellDepthGap:#0,0#} " + " or mse_state != lag_mse_state " + " or id_feed_regulator != lag_id_feed_regulator " + "order by \"date\" "; var sqlDelete = "delete " + "from t_telemetry_data_saub " + $"where id_telemetry = {idTelemetry} and \"date\" between {{0}} and {{1}};"; var startDate = DateTimeOffset.MinValue; var deleted = 0; var saved = 0; do { var query = dbset .FromSqlRaw(sqlSelect, startDate) .AsNoTracking(); var data = await query .Take(itemsCountLimit) .ToArrayAsync(token); var currentDataCount = data.Length; if (currentDataCount == 0) break; var lastDate = data.Last().DateTime; var currentDeleted = await db.Database.ExecuteSqlRawAsync(sqlDelete, new object[]{ startDate, lastDate}.AsEnumerable(), token); if (currentDeleted == 0) break; await db.Database.ExecInsertAsync(dbset, data, token); startDate = lastDate; deleted += currentDeleted; saved += currentDataCount; onProgress?.Invoke(deleted, oldCount); } while (true); return (oldCount, saved); } // use ServiceFactory to make services static void Main(/*string[] args*/) { var i = 0; var sw = new System.Diagnostics.Stopwatch(); sw.Start(); var result = RediceSamplingAsync(94, 5, 5, (p, t) => { Console.WriteLine($"{i++:0000}\t{p:00_000_000}\t{t:00_000_000}\t{1d*p/t:00.00}"); }, CancellationToken.None).Result; sw.Stop(); Console.WriteLine($"result: saved {result.newCount} old = {result.oldCount} ratio = {1d*result.oldCount/result.newCount}"); Console.WriteLine($"total time: {sw.ElapsedMilliseconds} ms"); } } }