diff --git a/AsbCloudDb/EFExtentions.cs b/AsbCloudDb/EFExtentions.cs index db66be04..f0523294 100644 --- a/AsbCloudDb/EFExtentions.cs +++ b/AsbCloudDb/EFExtentions.cs @@ -69,6 +69,15 @@ namespace AsbCloudDb return database.ExecuteSqlRawAsync(query, token); } + public static Task ExecInsertAsync(this Microsoft.EntityFrameworkCore.Infrastructure.DatabaseFacade database, DbSet dbSet, IEnumerable items, CancellationToken token) + where T : class + { + var factory = GetQueryStringFactory(dbSet); + var query = factory.MakeInsertSql(items); + + return database.ExecuteSqlRawAsync(query, token); + } + public static string GetTableName(this DbSet dbSet) where T : class { @@ -154,6 +163,14 @@ namespace AsbCloudDb return builder.ToString(); } + public string MakeInsertSql(IEnumerable items) + { + var builder = new StringBuilder(insertHeader, 7); + BuildRows(builder, items); + builder.Append(';'); + return builder.ToString(); + } + private StringBuilder BuildRows(StringBuilder builder, IEnumerable items) { var list = items.ToList(); diff --git a/ConsoleApp1/Program.cs b/ConsoleApp1/Program.cs index c1fe0dc8..3bb591fc 100644 --- a/ConsoleApp1/Program.cs +++ b/ConsoleApp1/Program.cs @@ -1,5 +1,6 @@ using AsbCloudApp.Data; using AsbCloudApp.Data.DailyReport; +using AsbCloudDb; using AsbCloudDb.Model; using AsbCloudInfrastructure.Services.DailyReport; using ClosedXML.Excel; @@ -18,17 +19,21 @@ namespace ConsoleApp1 { private static AsbCloudDbContext db = ServiceFactory.Context; - public static async Task<(int oldCount, int newCount)> RediceSamplingAsync(int idTelemetry, int ratio, int maxDateGapSec, CancellationToken token) + public delegate void OnProgress(int handled, int total); + + public static async Task<(int oldCount, int newCount)> RediceSamplingAsync(int idTelemetry, int ratio, int maxDateGapSec, OnProgress onProgress, CancellationToken token) { - const int ramLimit = 30 * 1024 * 1024; + const int ramLimit = 10 * 1024 * 1024; const int dataItemSize = 345; // by profiler const int itemsCountLimit = ramLimit / dataItemSize; // ~ 90_000 - const int maxWellDepthGap = 1; + const double maxWellDepthGap = 0.1; ratio = ratio > 0 ? ratio : 5; maxDateGapSec = maxDateGapSec > 0 ? maxDateGapSec : 9; var dbset = db.Set(); + var oldCount = await dbset.Where(t => t.IdTelemetry == idTelemetry).CountAsync(token); + onProgress?.Invoke(0, oldCount); var sqlSelect = "select " + @@ -49,8 +54,8 @@ namespace ConsoleApp1 "where " + $" (row_num % {ratio}) = 0 " + " or \"mode\" != lag_mode " + - $" or(\"date\" - lag_date) > interval '{maxDateGapSec} second' " + - $" or well_depth - lag_well_depth > {maxWellDepthGap} " + + $" or(\"date\" - lag_date) >= interval '{maxDateGapSec} second' " + + $" or well_depth - lag_well_depth > {maxWellDepthGap:#0,0#} " + " or mse_state != lag_mse_state " + " or id_feed_regulator != lag_id_feed_regulator " + "order by \"date\" "; @@ -60,7 +65,6 @@ namespace ConsoleApp1 $"where id_telemetry = {idTelemetry} and \"date\" between {{0}} and {{1}};"; var startDate = DateTimeOffset.MinValue; - var iteration = 0; var deleted = 0; var saved = 0; do @@ -83,22 +87,27 @@ namespace ConsoleApp1 if (currentDeleted == 0) break; - dbset.AddRange(data); - await db.SaveChangesAsync(token); + await db.Database.ExecInsertAsync(dbset, data, token); startDate = lastDate; deleted += currentDeleted; saved += currentDataCount; - iteration++; + onProgress?.Invoke(deleted, oldCount); } while (true); - return (deleted, saved); + return (oldCount, saved); } // use ServiceFactory to make services static void Main(/*string[] args*/) { - RediceSamplingAsync(183, 5, 10, CancellationToken.None).Wait(); + var i = 0; + var sw = new System.Diagnostics.Stopwatch(); + sw.Start(); + var result = RediceSamplingAsync(94, 5, 5, (p, t) => { Console.WriteLine($"{i++:0000}\t{p:00_000_000}\t{t:00_000_000}\t{1d*p/t:00.00}"); }, CancellationToken.None).Result; + sw.Stop(); + Console.WriteLine($"result: saved {result.newCount} old = {result.oldCount} ratio = {1d*result.oldCount/result.newCount}"); + Console.WriteLine($"total time: {sw.ElapsedMilliseconds} ms"); } } }