Persistence.API/Controllers/TimestampedSetController.cs

@@ -0,0 +1,63 @@
+using Microsoft.AspNetCore.Authorization;
+using Microsoft.AspNetCore.Mvc;
+using Persistence.Models;
+using Persistence.Repositories;
+using Persistence.Repository.Repositories;
+using System.Net;
+
+namespace Persistence.API.Controllers;
+
+[ApiController]
+[Authorize]
+[Route("api/[controller]/{idDiscriminator}")]
+public class TimestampedSetController : ControllerBase 
+{
+    private readonly ITimestampedSetRepository repository;
+
+    public TimestampedSetController(ITimestampedSetRepository repository)
+    {
+        this.repository = repository;
+    }
+
+    [HttpPost]
+    [ProducesResponseType(typeof(int), (int)HttpStatusCode.OK)]
+    public async Task<IActionResult> InsertRange([FromRoute]Guid idDiscriminator, [FromBody]IEnumerable<TimestampedSetDto> sets, CancellationToken token)
+    {
+        var result = await repository.InsertRange(idDiscriminator, sets, token);
+        return Ok(result);
+    }
+
+    [HttpGet]
+    [ProducesResponseType(typeof(IEnumerable<TimestampedSetDto>), (int)HttpStatusCode.OK)]
+    public async Task<IActionResult> Get(Guid idDiscriminator, DateTimeOffset? geTimestamp, [FromQuery]IEnumerable<string>? props, int skip, int take, CancellationToken token)
+    {
+        var result = await repository.Get(idDiscriminator, geTimestamp, props, skip, take, token);
+        return Ok(result);
+    }
+
+    [HttpGet("last")]
+    [ProducesResponseType(typeof(IEnumerable<TimestampedSetDto>), (int)HttpStatusCode.OK)]
+    public async Task<IActionResult> GetLast(Guid idDiscriminator, [FromQuery]IEnumerable<string>? props, int take, CancellationToken token)
+    {
+        var result = await repository.GetLast(idDiscriminator, props, take, token);
+        return Ok(result);
+    }
+
+    [HttpGet("datesRange")]
+    [ProducesResponseType(typeof(DatesRangeDto), (int)HttpStatusCode.OK)]
+    [ProducesResponseType((int)HttpStatusCode.NoContent)]
+    public async Task<IActionResult> GetDatesRange(Guid idDiscriminator, CancellationToken token)
+    {
+        var result = await repository.GetDatesRange(idDiscriminator, token);
+        return Ok(result);
+    }
+
+    [HttpGet("count")]
+    [ProducesResponseType(typeof(DatesRangeDto), (int)HttpStatusCode.OK)]
+    [ProducesResponseType((int)HttpStatusCode.NoContent)]
+    public async Task<IActionResult> Count(Guid idDiscriminator, CancellationToken token)
+    {
+        var result = await repository.Count(idDiscriminator, token);
+        return Ok(result);
+    }
+}

Persistence.Database.Postgres/PersistenceDbContext.cs

@@ -1,4 +1,6 @@
 using Microsoft.EntityFrameworkCore;
+using Npgsql;
+using Persistence.Database.Entity;
 using System.Data.Common;
 
 namespace Persistence.Database.Model;
@@ -6,6 +8,8 @@ public partial class PersistenceDbContext : DbContext, IPersistenceDbContext
 {
     public DbSet<DataSaub> DataSaub => Set<DataSaub>();
 
+    public DbSet<TimestampedSet> TimestampedSets => Set<TimestampedSet>();
+
     public PersistenceDbContext()
         : base()
     {
@@ -30,7 +34,9 @@ public partial class PersistenceDbContext : DbContext, IPersistenceDbContext
     {
         modelBuilder.HasPostgresExtension("adminpack")
             .HasAnnotation("Relational:Collation", "Russian_Russia.1251");
-    }
-
 
+        modelBuilder.Entity<TimestampedSet>()
+            .Property(e => e.Set)
+            .HasJsonConversion();
+    }
 }

Persistence.Database/EFExtensions.cs

@@ -0,0 +1,48 @@
+using Microsoft.EntityFrameworkCore.ChangeTracking;
+using Microsoft.EntityFrameworkCore.Metadata;
+using Microsoft.EntityFrameworkCore;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Text.Json.Serialization;
+using System.Text.Json;
+using System.Threading.Tasks;
+
+namespace Persistence.Database;
+
+public static class EFExtensions
+{
+    private static readonly JsonSerializerOptions jsonSerializerOptions = new()
+    {
+        AllowTrailingCommas = true,
+        WriteIndented = true,
+        NumberHandling = JsonNumberHandling.AllowReadingFromString | JsonNumberHandling.AllowNamedFloatingPointLiterals,
+    };
+
+    public static Microsoft.EntityFrameworkCore.Metadata.Builders.PropertyBuilder<TProperty> HasJsonConversion<TProperty>(
+        this Microsoft.EntityFrameworkCore.Metadata.Builders.PropertyBuilder<TProperty> builder)
+        => HasJsonConversion(builder, jsonSerializerOptions);
+
+    public static Microsoft.EntityFrameworkCore.Metadata.Builders.PropertyBuilder<TProperty> HasJsonConversion<TProperty>(
+        this Microsoft.EntityFrameworkCore.Metadata.Builders.PropertyBuilder<TProperty> builder,
+        JsonSerializerOptions jsonSerializerOptions)
+    {
+        builder.HasConversion(
+                    s => JsonSerializer.Serialize(s, jsonSerializerOptions),
+                    s => JsonSerializer.Deserialize<TProperty>(s, jsonSerializerOptions)!);
+
+        ValueComparer<TProperty> valueComparer = new(
+            (a, b) =>
+                (a != null) && (b != null)
+                    ? a.GetHashCode() == b.GetHashCode()
+                    : (a == null) && (b == null),
+            i => (i == null) ? -1 : i.GetHashCode(),
+            i => i);
+
+        builder.Metadata.SetValueComparer(valueComparer);
+        return builder;
+    }
+
+}
+

Persistence.Database/Entity/TimestampedSet.cs

@@ -0,0 +1,11 @@
+using Microsoft.EntityFrameworkCore;
+using System.ComponentModel.DataAnnotations.Schema;
+
+namespace Persistence.Database.Entity;
+
+[Comment("Общая таблица данных временных рядов")]
+[PrimaryKey(nameof(IdDiscriminator), nameof(Timestamp))]
+public record TimestampedSet(
+    [property: Comment("Дискриминатор ссылка на тип сохраняемых данных")] Guid IdDiscriminator, 
+    [property: Comment("Отметка времени, строго в UTC")] DateTimeOffset Timestamp, 
+    [property: Column(TypeName = "jsonb"), Comment("Набор сохраняемых данных")] IDictionary<string, object> Set);

Persistence.Database/IPersistenceDbContext.cs

@@ -1,8 +1,13 @@
 using Microsoft.EntityFrameworkCore;
+using Persistence.Database.Entity;
 using Persistence.Database.Model;
+using System.Diagnostics.CodeAnalysis;
 
 namespace Persistence.Database;
 public interface IPersistenceDbContext : IDisposable
 {
     DbSet<DataSaub> DataSaub { get; }
+    DbSet<TimestampedSet> TimestampedSets { get; }
+    DbSet<TEntity> Set<[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors | DynamicallyAccessedMemberTypes.NonPublicConstructors | DynamicallyAccessedMemberTypes.PublicFields | DynamicallyAccessedMemberTypes.NonPublicFields | DynamicallyAccessedMemberTypes.PublicProperties | DynamicallyAccessedMemberTypes.NonPublicProperties | DynamicallyAccessedMemberTypes.Interfaces)] TEntity>() where TEntity : class;
+    Task<int> SaveChangesAsync(CancellationToken cancellationToken);
 }

Persistence.IntegrationTests/Clients/ITimestampedSetClient.cs

@@ -0,0 +1,23 @@
+using Persistence.Models;
+using Refit;
+
+namespace Persistence.IntegrationTests.Clients;
+public interface ITimestampedSetClient
+{
+    private const string baseUrl = "/api/TimestampedSet/{idDiscriminator}";
+
+    [Post(baseUrl)]
+    Task<IApiResponse<int>> InsertRange(Guid idDiscriminator, IEnumerable<TimestampedSetDto> sets);
+
+    [Get(baseUrl)]
+    Task<IApiResponse<IEnumerable<TimestampedSetDto>>> Get(Guid idDiscriminator, [Query]DateTimeOffset? geTimestamp, [Query]IEnumerable<string>? props, int skip, int take);
+
+    [Get($"{baseUrl}/last")]
+    Task<IApiResponse<IEnumerable<TimestampedSetDto>>> GetLast(Guid idDiscriminator, [Query] IEnumerable<string>? props, int take);
+
+    [Get($"{baseUrl}/count")]
+    Task<IApiResponse<int>> Count(Guid idDiscriminator);
+
+    [Get($"{baseUrl}/datesRange")]
+    Task<IApiResponse<DatesRangeDto?>> GetDatesRange(Guid idDiscriminator);
+}

Persistence.IntegrationTests/Controllers/TimestampedSetControllerTest.cs

@@ -0,0 +1,219 @@
+using Microsoft.AspNetCore.Mvc;
+using Persistence.IntegrationTests.Clients;
+using Persistence.Models;
+using Xunit;
+
+namespace Persistence.IntegrationTests.Controllers;
+public class TimestampedSetControllerTest : BaseIntegrationTest
+{
+    private readonly ITimestampedSetClient client;
+
+    public TimestampedSetControllerTest(WebAppFactoryFixture factory) : base(factory)
+    {
+
+        client = factory.GetAuthorizedHttpClient<ITimestampedSetClient>(string.Empty).Result;
+    }
+
+    [Fact]
+    public async Task InsertRange()
+    {
+        // arrange
+        Guid idDiscriminator = Guid.NewGuid();
+        IEnumerable<TimestampedSetDto> testSets = Generate(10, DateTimeOffset.Now.ToOffset(TimeSpan.FromHours(7)));
+
+        // act
+        var response = await client.InsertRange(idDiscriminator, testSets);
+
+        // assert
+        Assert.Equal(System.Net.HttpStatusCode.OK, response.StatusCode);
+        Assert.Equal(testSets.Count(), response.Content);
+    }
+
+    [Fact]
+    public async Task Get_without_filter()
+    {
+        // arrange
+        Guid idDiscriminator = Guid.NewGuid();
+        int count = 10;
+        IEnumerable<TimestampedSetDto> testSets = Generate(count, DateTimeOffset.Now.ToOffset(TimeSpan.FromHours(7)));
+        var insertResponse = await client.InsertRange(idDiscriminator, testSets);
+
+        // act
+        var response = await client.Get(idDiscriminator, null, null, 0, int.MaxValue);
+
+        // assert
+        Assert.Equal(System.Net.HttpStatusCode.OK, response.StatusCode);
+        Assert.NotNull(response.Content);
+        var items = response.Content!;
+        Assert.Equal(count, items.Count());
+    }
+
+    [Fact]
+    public async Task Get_with_filter_props()
+    {
+        // arrange
+        Guid idDiscriminator = Guid.NewGuid();
+        int count = 10;
+        IEnumerable<TimestampedSetDto> testSets = Generate(count, DateTimeOffset.Now.ToOffset(TimeSpan.FromHours(7)));
+        var insertResponse = await client.InsertRange(idDiscriminator, testSets);
+        string[] props = ["A"];
+
+        // act
+        var response = await client.Get(idDiscriminator, null, props, 0, int.MaxValue);
+
+        // assert
+        Assert.Equal(System.Net.HttpStatusCode.OK, response.StatusCode);
+        Assert.NotNull(response.Content);
+        var items = response.Content!;
+        Assert.Equal(count, items.Count());
+        foreach ( var item in items )
+        {
+            Assert.Single(item.Set);
+            var kv = item.Set.First();
+            Assert.Equal("A", kv.Key);
+        }            
+    }
+
+    [Fact]
+    public async Task Get_geDate()
+    {
+        // arrange
+        Guid idDiscriminator = Guid.NewGuid();
+        int count = 10;
+        var dateMin = DateTimeOffset.Now;
+        var dateMax = DateTimeOffset.Now.AddSeconds(count);
+        IEnumerable<TimestampedSetDto> testSets = Generate(count, dateMin.ToOffset(TimeSpan.FromHours(7)));        
+        var insertResponse = await client.InsertRange(idDiscriminator, testSets);
+        var tail = testSets.OrderBy(t => t.Timestamp).Skip(count / 2).Take(int.MaxValue);
+        var geDate = tail.First().Timestamp;
+        var tolerance = TimeSpan.FromSeconds(1);
+        var expectedCount = tail.Count();
+
+        // act
+        var response = await client.Get(idDiscriminator, geDate, null, 0, int.MaxValue);
+
+        // assert
+        Assert.Equal(System.Net.HttpStatusCode.OK, response.StatusCode);
+        Assert.NotNull(response.Content);
+        var items = response.Content!;
+        Assert.Equal(expectedCount, items.Count());
+        var minDate = items.Min(t => t.Timestamp);
+        Assert.Equal(geDate, geDate, tolerance);
+    }
+
+    [Fact]
+    public async Task Get_with_skip_take()
+    {
+        // arrange
+        Guid idDiscriminator = Guid.NewGuid();
+        int count = 10;
+        IEnumerable<TimestampedSetDto> testSets = Generate(count, DateTimeOffset.Now.ToOffset(TimeSpan.FromHours(7)));
+        var insertResponse = await client.InsertRange(idDiscriminator, testSets);
+        var expectedCount = count / 2;
+
+        // act
+        var response = await client.Get(idDiscriminator, null, null, 2, expectedCount);
+
+        // assert
+        Assert.Equal(System.Net.HttpStatusCode.OK, response.StatusCode);
+        Assert.NotNull(response.Content);
+        var items = response.Content!;
+        Assert.Equal(expectedCount, items.Count());
+    }
+
+
+    [Fact]
+    public async Task Get_with_big_skip_take()
+    {
+        // arrange
+        Guid idDiscriminator = Guid.NewGuid();
+        var expectedCount = 1;
+        int count = 10 + expectedCount;
+        IEnumerable<TimestampedSetDto> testSets = Generate(count, DateTimeOffset.Now.ToOffset(TimeSpan.FromHours(7)));
+        var insertResponse = await client.InsertRange(idDiscriminator, testSets);
+
+        // act
+        var response = await client.Get(idDiscriminator, null, null, count - expectedCount, count);
+
+        // assert
+        Assert.Equal(System.Net.HttpStatusCode.OK, response.StatusCode);
+        Assert.NotNull(response.Content);
+        var items = response.Content!;
+        Assert.Equal(expectedCount, items.Count());
+    }
+
+    [Fact]
+    public async Task GetLast()
+    {
+        // arrange
+        Guid idDiscriminator = Guid.NewGuid();
+        int count = 10;
+        IEnumerable<TimestampedSetDto> testSets = Generate(count, DateTimeOffset.Now.ToOffset(TimeSpan.FromHours(7)));
+        var insertResponse = await client.InsertRange(idDiscriminator, testSets);
+        var expectedCount = 8;
+
+        // act
+        var response = await client.GetLast(idDiscriminator, null, expectedCount);
+
+        // assert
+        Assert.Equal(System.Net.HttpStatusCode.OK, response.StatusCode);
+        Assert.NotNull(response.Content);
+        var items = response.Content!;
+        Assert.Equal(expectedCount, items.Count());
+    }
+
+    [Fact]
+    public async Task GetDatesRange()
+    {
+        // arrange
+        Guid idDiscriminator = Guid.NewGuid();
+        int count = 10;
+        var dateMin = DateTimeOffset.Now;
+        var dateMax = DateTimeOffset.Now.AddSeconds(count-1);
+        IEnumerable<TimestampedSetDto> testSets = Generate(count, dateMin.ToOffset(TimeSpan.FromHours(7)));
+        var insertResponse = await client.InsertRange(idDiscriminator, testSets);
+        var tolerance = TimeSpan.FromSeconds(1);
+
+        // act
+        var response = await client.GetDatesRange(idDiscriminator);
+
+        // assert
+        Assert.Equal(System.Net.HttpStatusCode.OK, response.StatusCode);
+        Assert.NotNull(response.Content);
+        var range = response.Content!;
+        Assert.Equal(dateMin, range.From, tolerance);
+        Assert.Equal(dateMax, range.To, tolerance);
+    }
+
+    [Fact]
+    public async Task Count()
+    {
+        // arrange
+        Guid idDiscriminator = Guid.NewGuid();
+        int count = 144;
+        IEnumerable<TimestampedSetDto> testSets = Generate(count, DateTimeOffset.Now.ToOffset(TimeSpan.FromHours(7)));
+        var insertResponse = await client.InsertRange(idDiscriminator, testSets);
+
+        // act
+        var response = await client.Count(idDiscriminator);
+
+        // assert
+        Assert.Equal(System.Net.HttpStatusCode.OK, response.StatusCode);
+        Assert.Equal(count, response.Content);
+    }
+
+    private static IEnumerable<TimestampedSetDto> Generate(int n, DateTimeOffset from)
+    {
+        for (int i = 0; i < n; i++)
+            yield return new TimestampedSetDto
+            (
+                from.AddSeconds(i),
+                new Dictionary<string, object>{
+                    {"A", i },
+                    {"B", i * 1.1 },
+                    {"C", $"Any{i}" },
+                    {"D", DateTimeOffset.Now},
+                }
+            );
+    }
+}

Persistence.IntegrationTests/WebAppFactoryFixture.cs

@@ -4,6 +4,7 @@ using Microsoft.EntityFrameworkCore;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
 using Persistence.API;
+using Persistence.Database;
 using Persistence.Database.Model;
 using Persistence.Database.Postgres;
 using Refit;
@@ -53,6 +54,8 @@ public class WebAppFactoryFixture : WebApplicationFactory<Startup>
             services.AddDbContext<PersistenceDbContext>(options =>
                options.UseNpgsql(connectionString));
 
+            services.AddScoped<IPersistenceDbContext>(provider => provider.GetRequiredService<PersistenceDbContext>());
+
             var serviceProvider = services.BuildServiceProvider();
 
             using var scope = serviceProvider.CreateScope();

Persistence.Repository/DependencyInjection.cs

@@ -16,6 +16,7 @@ public static class DependencyInjection
         MapsterSetup();
 
         services.AddTransient<ITimeSeriesDataRepository<DataSaubDto>, TimeSeriesDataCachedRepository<DataSaub, DataSaubDto>>();
+        services.AddTransient<ITimestampedSetRepository, TimestampedSetRepository>();
 
         return services;
     }

Persistence.Repository/Repositories/TimestampedSetRepository.cs

@@ -0,0 +1,110 @@
+using Microsoft.EntityFrameworkCore;
+using Persistence.Database;
+using Persistence.Database.Entity;
+using Persistence.Models;
+using Persistence.Repositories;
+
+namespace Persistence.Repository.Repositories;
+public class TimestampedSetRepository : ITimestampedSetRepository
+{
+    private readonly IPersistenceDbContext db;
+
+    public TimestampedSetRepository(IPersistenceDbContext db)
+    {
+        this.db = db;
+    }
+
+    public Task<int> InsertRange(Guid idDiscriminator, IEnumerable<TimestampedSetDto> sets, CancellationToken token)
+    {
+        var entities = sets.Select(set => new TimestampedSet(idDiscriminator, set.Timestamp.ToUniversalTime(), set.Set));
+        var dbSet = db.Set<TimestampedSet>();
+        dbSet.AddRange(entities);
+        return db.SaveChangesAsync(token);
+    }
+
+    public async Task<IEnumerable<TimestampedSetDto>> Get(Guid idDiscriminator, DateTimeOffset? geTimestamp, IEnumerable<string>? props, int skip, int take, CancellationToken token)
+    {
+        var dbSet = db.Set<TimestampedSet>();
+        var query = dbSet.Where(entity => entity.IdDiscriminator == idDiscriminator);
+
+        if (geTimestamp.HasValue)
+            query = ApplyGeTimestamp(query, geTimestamp.Value);
+
+        var data = await Materialize(query, token);
+
+        if (props is not null && props.Any())
+            data = ApplyPropsFilter(data, props);
+
+        return data;
+    }
+
+    public async Task<IEnumerable<TimestampedSetDto>> GetLast(Guid idDiscriminator, IEnumerable<string>? props, int take, CancellationToken token)
+    {
+        var dbSet = db.Set<TimestampedSet>();
+        var query = dbSet.Where(entity => entity.IdDiscriminator == idDiscriminator);
+
+        query = query.OrderByDescending(entity => entity.Timestamp)
+            .Take(take)
+            .OrderBy(entity => entity.Timestamp);
+
+        var data = await Materialize(query, token);
+
+        if (props is not null && props.Any())
+            data = ApplyPropsFilter(data, props);
+
+        return data;
+    }
+
+    public Task<int> Count(Guid idDiscriminator, CancellationToken token)
+    {
+        var dbSet = db.Set<TimestampedSet>();
+        var query = dbSet.Where(entity => entity.IdDiscriminator == idDiscriminator);
+        return query.CountAsync(token);
+    }
+
+    public async Task<DatesRangeDto?> GetDatesRange(Guid idDiscriminator, CancellationToken token)
+    {
+        var query = db.Set<TimestampedSet>()
+            .GroupBy(entity => entity.IdDiscriminator)
+            .Select(group => new 
+            { 
+                Min = group.Min(entity => entity.Timestamp),
+                Max = group.Max(entity => entity.Timestamp),
+            });
+        
+        var item = await query.FirstOrDefaultAsync(token);
+        if (item is null)
+            return null;
+
+        return new DatesRangeDto
+        {
+            From = item.Min,
+            To = item.Max,
+        };
+    }
+
+    private static async Task<IEnumerable<TimestampedSetDto>> Materialize(IQueryable<TimestampedSet> query, CancellationToken token)
+    {
+        var dtoQuery = query.Select(entity => new TimestampedSetDto(entity.Timestamp, entity.Set));
+        var dtos = await dtoQuery.ToArrayAsync(token);
+        return dtos;
+    }
+
+    private static IQueryable<TimestampedSet> ApplyGeTimestamp(IQueryable<TimestampedSet> query, DateTimeOffset geTimestamp)
+    {
+        var geTimestampUtc = geTimestamp.ToUniversalTime();
+        return query.Where(entity => entity.Timestamp >= geTimestampUtc);
+    }
+
+    private static IEnumerable<TimestampedSetDto> ApplyPropsFilter(IEnumerable<TimestampedSetDto> query, IEnumerable<string> props)
+    {
+        var newQuery = query
+            .Select(entity => new TimestampedSetDto(
+                entity.Timestamp,
+                entity.Set
+                    .Where(prop => props.Contains(prop.Key))
+                    .ToDictionary(prop => prop.Key, prop => prop.Value)
+            ));
+        return newQuery;
+    }
+}

Persistence/Models/TimestampedSetDto.cs

@@ -0,0 +1,8 @@
+namespace Persistence.Models;
+
+/// <summary>
+/// набор данных с отметкой времени
+/// </summary>
+/// <param name="Timestamp">отметка времени</param>
+/// <param name="Set">набор данных</param>
+public record TimestampedSetDto(DateTimeOffset Timestamp, IDictionary<string, object> Set);

Persistence/Repositories/ITimestampedSetRepository.cs

@@ -0,0 +1,11 @@
+using Persistence.Models;
+
+namespace Persistence.Repositories;
+public interface ITimestampedSetRepository
+{
+    Task<int> Count(Guid idDiscriminator, CancellationToken token);
+    Task<IEnumerable<TimestampedSetDto>> Get(Guid idDiscriminator, DateTimeOffset? geTimestamp, IEnumerable<string>? props, int skip, int take, CancellationToken token);
+    Task<DatesRangeDto?> GetDatesRange(Guid idDiscriminator, CancellationToken token);
+    Task<IEnumerable<TimestampedSetDto>> GetLast(Guid idDiscriminator, IEnumerable<string>? props, int take, CancellationToken token);
+    Task<int> InsertRange(Guid idDiscriminator, IEnumerable<TimestampedSetDto> sets, CancellationToken token);
+}