poc debagged.

AsbCloudDb/EFExtentions.cs

@@ -69,6 +69,15 @@ namespace AsbCloudDb
             return database.ExecuteSqlRawAsync(query, token);
         }
 
+        public static Task<int> ExecInsertAsync<T>(this Microsoft.EntityFrameworkCore.Infrastructure.DatabaseFacade database, DbSet<T> dbSet, IEnumerable<T> items, CancellationToken token)
+            where T : class
+        {
+            var factory = GetQueryStringFactory(dbSet);
+            var query = factory.MakeInsertSql(items);
+
+            return database.ExecuteSqlRawAsync(query, token);
+        }
+
         public static string GetTableName<T>(this DbSet<T> dbSet)
         where T : class
         {
@@ -154,6 +163,14 @@ namespace AsbCloudDb
             return builder.ToString();
         }
 
+        public string MakeInsertSql(IEnumerable<T> items)
+        {
+            var builder = new StringBuilder(insertHeader, 7);
+            BuildRows(builder, items);
+            builder.Append(';');
+            return builder.ToString();
+        }
+
         private StringBuilder BuildRows(StringBuilder builder, IEnumerable<T> items)
         {
             var list = items.ToList();

ConsoleApp1/Program.cs

@@ -1,5 +1,6 @@
 using AsbCloudApp.Data;
 using AsbCloudApp.Data.DailyReport;
+using AsbCloudDb;
 using AsbCloudDb.Model;
 using AsbCloudInfrastructure.Services.DailyReport;
 using ClosedXML.Excel;
@@ -18,17 +19,21 @@ namespace ConsoleApp1
     {
         private static AsbCloudDbContext db = ServiceFactory.Context;
 
-        public static async Task<(int oldCount, int newCount)> RediceSamplingAsync(int idTelemetry, int ratio, int maxDateGapSec, CancellationToken token)
+        public delegate void OnProgress(int handled, int total);
+
+        public static async Task<(int oldCount, int newCount)> RediceSamplingAsync(int idTelemetry, int ratio, int maxDateGapSec, OnProgress onProgress, CancellationToken token)
         {
-            const int ramLimit = 30 * 1024 * 1024;
+            const int ramLimit = 10 * 1024 * 1024;
             const int dataItemSize = 345; // by profiler
             const int itemsCountLimit = ramLimit / dataItemSize; // ~ 90_000
-            const int maxWellDepthGap = 1;
+            const double maxWellDepthGap = 0.1;
 
             ratio = ratio > 0 ? ratio : 5;
             maxDateGapSec = maxDateGapSec > 0 ? maxDateGapSec : 9;
 
             var dbset = db.Set<TelemetryDataSaub>();
+            var oldCount = await dbset.Where(t => t.IdTelemetry == idTelemetry).CountAsync(token);
+            onProgress?.Invoke(0, oldCount);
 
             var sqlSelect =
                 "select " +
@@ -49,8 +54,8 @@ namespace ConsoleApp1
                 "where " +
                 $"   (row_num % {ratio}) = 0 " +
                 "   or \"mode\" != lag_mode " +
-                $"   or(\"date\" - lag_date) > interval '{maxDateGapSec} second' " +
+                $"   or(\"date\" - lag_date) >= interval '{maxDateGapSec} second' " +
-                $"   or well_depth - lag_well_depth > {maxWellDepthGap} " +
+                $"   or well_depth - lag_well_depth > {maxWellDepthGap:#0,0#} " +
                 "   or mse_state != lag_mse_state " +
                 "   or id_feed_regulator != lag_id_feed_regulator " +
                 "order by \"date\" ";
@@ -60,7 +65,6 @@ namespace ConsoleApp1
                 $"where id_telemetry = {idTelemetry} and \"date\" between {{0}} and {{1}};";
 
             var startDate = DateTimeOffset.MinValue;
-            var iteration = 0;
             var deleted = 0;
             var saved = 0;
             do
@@ -83,22 +87,27 @@ namespace ConsoleApp1
                 if (currentDeleted == 0)
                     break;
 
-                dbset.AddRange(data);
+                await db.Database.ExecInsertAsync(dbset, data, token);
-                await db.SaveChangesAsync(token);
 
                 startDate = lastDate;
                 deleted += currentDeleted;
                 saved += currentDataCount;
-                iteration++;
+                onProgress?.Invoke(deleted, oldCount);
             } while (true);
             
-            return (deleted, saved);
+            return (oldCount, saved);
         }
 
         // use ServiceFactory to make services
         static void Main(/*string[] args*/)
         {
-            RediceSamplingAsync(183, 5, 10, CancellationToken.None).Wait();
+            var i = 0;
+            var sw = new System.Diagnostics.Stopwatch();
+            sw.Start();
+            var result = RediceSamplingAsync(94, 5, 5, (p, t) => { Console.WriteLine($"{i++:0000}\t{p:00_000_000}\t{t:00_000_000}\t{1d*p/t:00.00}"); }, CancellationToken.None).Result;
+            sw.Stop();
+            Console.WriteLine($"result: saved {result.newCount} old = {result.oldCount} ratio = {1d*result.oldCount/result.newCount}");
+            Console.WriteLine($"total time: {sw.ElapsedMilliseconds} ms");
         }
     }
 }