diff --git a/Persistence.API/Controllers/TimestampedSetController.cs b/Persistence.API/Controllers/TimestampedSetController.cs new file mode 100644 index 0000000..3d7ce63 --- /dev/null +++ b/Persistence.API/Controllers/TimestampedSetController.cs @@ -0,0 +1,63 @@ +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Mvc; +using Persistence.Models; +using Persistence.Repositories; +using Persistence.Repository.Repositories; +using System.Net; + +namespace Persistence.API.Controllers; + +[ApiController] +[Authorize] +[Route("api/[controller]/{idDiscriminator}")] +public class TimestampedSetController : ControllerBase +{ + private readonly ITimestampedSetRepository repository; + + public TimestampedSetController(ITimestampedSetRepository repository) + { + this.repository = repository; + } + + [HttpPost] + [ProducesResponseType(typeof(int), (int)HttpStatusCode.OK)] + public async Task InsertRange([FromRoute]Guid idDiscriminator, [FromBody]IEnumerable sets, CancellationToken token) + { + var result = await repository.InsertRange(idDiscriminator, sets, token); + return Ok(result); + } + + [HttpGet] + [ProducesResponseType(typeof(IEnumerable), (int)HttpStatusCode.OK)] + public async Task Get(Guid idDiscriminator, DateTimeOffset? geTimestamp, [FromQuery]IEnumerable? props, int skip, int take, CancellationToken token) + { + var result = await repository.Get(idDiscriminator, geTimestamp, props, skip, take, token); + return Ok(result); + } + + [HttpGet("last")] + [ProducesResponseType(typeof(IEnumerable), (int)HttpStatusCode.OK)] + public async Task GetLast(Guid idDiscriminator, [FromQuery]IEnumerable? props, int take, CancellationToken token) + { + var result = await repository.GetLast(idDiscriminator, props, take, token); + return Ok(result); + } + + [HttpGet("datesRange")] + [ProducesResponseType(typeof(DatesRangeDto), (int)HttpStatusCode.OK)] + [ProducesResponseType((int)HttpStatusCode.NoContent)] + public async Task GetDatesRange(Guid idDiscriminator, CancellationToken token) + { + var result = await repository.GetDatesRange(idDiscriminator, token); + return Ok(result); + } + + [HttpGet("count")] + [ProducesResponseType(typeof(DatesRangeDto), (int)HttpStatusCode.OK)] + [ProducesResponseType((int)HttpStatusCode.NoContent)] + public async Task Count(Guid idDiscriminator, CancellationToken token) + { + var result = await repository.Count(idDiscriminator, token); + return Ok(result); + } +} diff --git a/Persistence.Database.Postgres/PersistenceDbContext.cs b/Persistence.Database.Postgres/PersistenceDbContext.cs index f68be73..22462ed 100644 --- a/Persistence.Database.Postgres/PersistenceDbContext.cs +++ b/Persistence.Database.Postgres/PersistenceDbContext.cs @@ -1,4 +1,6 @@ using Microsoft.EntityFrameworkCore; +using Npgsql; +using Persistence.Database.Entity; using System.Data.Common; namespace Persistence.Database.Model; @@ -6,10 +8,12 @@ public partial class PersistenceDbContext : DbContext, IPersistenceDbContext { public DbSet DataSaub => Set(); + public DbSet TimestampedSets => Set(); + public PersistenceDbContext() : base() { - + } public PersistenceDbContext(DbContextOptions options) @@ -30,7 +34,9 @@ public partial class PersistenceDbContext : DbContext, IPersistenceDbContext { modelBuilder.HasPostgresExtension("adminpack") .HasAnnotation("Relational:Collation", "Russian_Russia.1251"); + + modelBuilder.Entity() + .Property(e => e.Set) + .HasJsonConversion(); } - - } diff --git a/Persistence.Database/EFExtensions.cs b/Persistence.Database/EFExtensions.cs new file mode 100644 index 0000000..c424aa3 --- /dev/null +++ b/Persistence.Database/EFExtensions.cs @@ -0,0 +1,48 @@ +using Microsoft.EntityFrameworkCore.ChangeTracking; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Text.Json.Serialization; +using System.Text.Json; +using System.Threading.Tasks; + +namespace Persistence.Database; + +public static class EFExtensions +{ + private static readonly JsonSerializerOptions jsonSerializerOptions = new() + { + AllowTrailingCommas = true, + WriteIndented = true, + NumberHandling = JsonNumberHandling.AllowReadingFromString | JsonNumberHandling.AllowNamedFloatingPointLiterals, + }; + + public static Microsoft.EntityFrameworkCore.Metadata.Builders.PropertyBuilder HasJsonConversion( + this Microsoft.EntityFrameworkCore.Metadata.Builders.PropertyBuilder builder) + => HasJsonConversion(builder, jsonSerializerOptions); + + public static Microsoft.EntityFrameworkCore.Metadata.Builders.PropertyBuilder HasJsonConversion( + this Microsoft.EntityFrameworkCore.Metadata.Builders.PropertyBuilder builder, + JsonSerializerOptions jsonSerializerOptions) + { + builder.HasConversion( + s => JsonSerializer.Serialize(s, jsonSerializerOptions), + s => JsonSerializer.Deserialize(s, jsonSerializerOptions)!); + + ValueComparer valueComparer = new( + (a, b) => + (a != null) && (b != null) + ? a.GetHashCode() == b.GetHashCode() + : (a == null) && (b == null), + i => (i == null) ? -1 : i.GetHashCode(), + i => i); + + builder.Metadata.SetValueComparer(valueComparer); + return builder; + } + +} + diff --git a/Persistence.Database/Entity/TimestampedSet.cs b/Persistence.Database/Entity/TimestampedSet.cs new file mode 100644 index 0000000..c9a0dda --- /dev/null +++ b/Persistence.Database/Entity/TimestampedSet.cs @@ -0,0 +1,11 @@ +using Microsoft.EntityFrameworkCore; +using System.ComponentModel.DataAnnotations.Schema; + +namespace Persistence.Database.Entity; + +[Comment("Общая таблица данных временных рядов")] +[PrimaryKey(nameof(IdDiscriminator), nameof(Timestamp))] +public record TimestampedSet( + [property: Comment("Дискриминатор ссылка на тип сохраняемых данных")] Guid IdDiscriminator, + [property: Comment("Отметка времени, строго в UTC")] DateTimeOffset Timestamp, + [property: Column(TypeName = "jsonb"), Comment("Набор сохраняемых данных")] IDictionary Set); diff --git a/Persistence.Database/IPersistenceDbContext.cs b/Persistence.Database/IPersistenceDbContext.cs index 66f34ff..d23656e 100644 --- a/Persistence.Database/IPersistenceDbContext.cs +++ b/Persistence.Database/IPersistenceDbContext.cs @@ -1,8 +1,13 @@ using Microsoft.EntityFrameworkCore; +using Persistence.Database.Entity; using Persistence.Database.Model; +using System.Diagnostics.CodeAnalysis; namespace Persistence.Database; public interface IPersistenceDbContext : IDisposable { DbSet DataSaub { get; } + DbSet TimestampedSets { get; } + DbSet Set<[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors | DynamicallyAccessedMemberTypes.NonPublicConstructors | DynamicallyAccessedMemberTypes.PublicFields | DynamicallyAccessedMemberTypes.NonPublicFields | DynamicallyAccessedMemberTypes.PublicProperties | DynamicallyAccessedMemberTypes.NonPublicProperties | DynamicallyAccessedMemberTypes.Interfaces)] TEntity>() where TEntity : class; + Task SaveChangesAsync(CancellationToken cancellationToken); } diff --git a/Persistence.IntegrationTests/Clients/ITimestampedSetClient.cs b/Persistence.IntegrationTests/Clients/ITimestampedSetClient.cs new file mode 100644 index 0000000..96e2500 --- /dev/null +++ b/Persistence.IntegrationTests/Clients/ITimestampedSetClient.cs @@ -0,0 +1,23 @@ +using Persistence.Models; +using Refit; + +namespace Persistence.IntegrationTests.Clients; +public interface ITimestampedSetClient +{ + private const string baseUrl = "/api/TimestampedSet/{idDiscriminator}"; + + [Post(baseUrl)] + Task> InsertRange(Guid idDiscriminator, IEnumerable sets); + + [Get(baseUrl)] + Task>> Get(Guid idDiscriminator, [Query]DateTimeOffset? geTimestamp, [Query]IEnumerable? props, int skip, int take); + + [Get($"{baseUrl}/last")] + Task>> GetLast(Guid idDiscriminator, [Query] IEnumerable? props, int take); + + [Get($"{baseUrl}/count")] + Task> Count(Guid idDiscriminator); + + [Get($"{baseUrl}/datesRange")] + Task> GetDatesRange(Guid idDiscriminator); +} diff --git a/Persistence.IntegrationTests/Controllers/TimestampedSetControllerTest.cs b/Persistence.IntegrationTests/Controllers/TimestampedSetControllerTest.cs new file mode 100644 index 0000000..12ebb94 --- /dev/null +++ b/Persistence.IntegrationTests/Controllers/TimestampedSetControllerTest.cs @@ -0,0 +1,219 @@ +using Microsoft.AspNetCore.Mvc; +using Persistence.IntegrationTests.Clients; +using Persistence.Models; +using Xunit; + +namespace Persistence.IntegrationTests.Controllers; +public class TimestampedSetControllerTest : BaseIntegrationTest +{ + private readonly ITimestampedSetClient client; + + public TimestampedSetControllerTest(WebAppFactoryFixture factory) : base(factory) + { + + client = factory.GetAuthorizedHttpClient(string.Empty).Result; + } + + [Fact] + public async Task InsertRange() + { + // arrange + Guid idDiscriminator = Guid.NewGuid(); + IEnumerable testSets = Generate(10, DateTimeOffset.Now.ToOffset(TimeSpan.FromHours(7))); + + // act + var response = await client.InsertRange(idDiscriminator, testSets); + + // assert + Assert.Equal(System.Net.HttpStatusCode.OK, response.StatusCode); + Assert.Equal(testSets.Count(), response.Content); + } + + [Fact] + public async Task Get_without_filter() + { + // arrange + Guid idDiscriminator = Guid.NewGuid(); + int count = 10; + IEnumerable testSets = Generate(count, DateTimeOffset.Now.ToOffset(TimeSpan.FromHours(7))); + var insertResponse = await client.InsertRange(idDiscriminator, testSets); + + // act + var response = await client.Get(idDiscriminator, null, null, 0, int.MaxValue); + + // assert + Assert.Equal(System.Net.HttpStatusCode.OK, response.StatusCode); + Assert.NotNull(response.Content); + var items = response.Content!; + Assert.Equal(count, items.Count()); + } + + [Fact] + public async Task Get_with_filter_props() + { + // arrange + Guid idDiscriminator = Guid.NewGuid(); + int count = 10; + IEnumerable testSets = Generate(count, DateTimeOffset.Now.ToOffset(TimeSpan.FromHours(7))); + var insertResponse = await client.InsertRange(idDiscriminator, testSets); + string[] props = ["A"]; + + // act + var response = await client.Get(idDiscriminator, null, props, 0, int.MaxValue); + + // assert + Assert.Equal(System.Net.HttpStatusCode.OK, response.StatusCode); + Assert.NotNull(response.Content); + var items = response.Content!; + Assert.Equal(count, items.Count()); + foreach ( var item in items ) + { + Assert.Single(item.Set); + var kv = item.Set.First(); + Assert.Equal("A", kv.Key); + } + } + + [Fact] + public async Task Get_geDate() + { + // arrange + Guid idDiscriminator = Guid.NewGuid(); + int count = 10; + var dateMin = DateTimeOffset.Now; + var dateMax = DateTimeOffset.Now.AddSeconds(count); + IEnumerable testSets = Generate(count, dateMin.ToOffset(TimeSpan.FromHours(7))); + var insertResponse = await client.InsertRange(idDiscriminator, testSets); + var tail = testSets.OrderBy(t => t.Timestamp).Skip(count / 2).Take(int.MaxValue); + var geDate = tail.First().Timestamp; + var tolerance = TimeSpan.FromSeconds(1); + var expectedCount = tail.Count(); + + // act + var response = await client.Get(idDiscriminator, geDate, null, 0, int.MaxValue); + + // assert + Assert.Equal(System.Net.HttpStatusCode.OK, response.StatusCode); + Assert.NotNull(response.Content); + var items = response.Content!; + Assert.Equal(expectedCount, items.Count()); + var minDate = items.Min(t => t.Timestamp); + Assert.Equal(geDate, geDate, tolerance); + } + + [Fact] + public async Task Get_with_skip_take() + { + // arrange + Guid idDiscriminator = Guid.NewGuid(); + int count = 10; + IEnumerable testSets = Generate(count, DateTimeOffset.Now.ToOffset(TimeSpan.FromHours(7))); + var insertResponse = await client.InsertRange(idDiscriminator, testSets); + var expectedCount = count / 2; + + // act + var response = await client.Get(idDiscriminator, null, null, 2, expectedCount); + + // assert + Assert.Equal(System.Net.HttpStatusCode.OK, response.StatusCode); + Assert.NotNull(response.Content); + var items = response.Content!; + Assert.Equal(expectedCount, items.Count()); + } + + + [Fact] + public async Task Get_with_big_skip_take() + { + // arrange + Guid idDiscriminator = Guid.NewGuid(); + var expectedCount = 1; + int count = 10 + expectedCount; + IEnumerable testSets = Generate(count, DateTimeOffset.Now.ToOffset(TimeSpan.FromHours(7))); + var insertResponse = await client.InsertRange(idDiscriminator, testSets); + + // act + var response = await client.Get(idDiscriminator, null, null, count - expectedCount, count); + + // assert + Assert.Equal(System.Net.HttpStatusCode.OK, response.StatusCode); + Assert.NotNull(response.Content); + var items = response.Content!; + Assert.Equal(expectedCount, items.Count()); + } + + [Fact] + public async Task GetLast() + { + // arrange + Guid idDiscriminator = Guid.NewGuid(); + int count = 10; + IEnumerable testSets = Generate(count, DateTimeOffset.Now.ToOffset(TimeSpan.FromHours(7))); + var insertResponse = await client.InsertRange(idDiscriminator, testSets); + var expectedCount = 8; + + // act + var response = await client.GetLast(idDiscriminator, null, expectedCount); + + // assert + Assert.Equal(System.Net.HttpStatusCode.OK, response.StatusCode); + Assert.NotNull(response.Content); + var items = response.Content!; + Assert.Equal(expectedCount, items.Count()); + } + + [Fact] + public async Task GetDatesRange() + { + // arrange + Guid idDiscriminator = Guid.NewGuid(); + int count = 10; + var dateMin = DateTimeOffset.Now; + var dateMax = DateTimeOffset.Now.AddSeconds(count-1); + IEnumerable testSets = Generate(count, dateMin.ToOffset(TimeSpan.FromHours(7))); + var insertResponse = await client.InsertRange(idDiscriminator, testSets); + var tolerance = TimeSpan.FromSeconds(1); + + // act + var response = await client.GetDatesRange(idDiscriminator); + + // assert + Assert.Equal(System.Net.HttpStatusCode.OK, response.StatusCode); + Assert.NotNull(response.Content); + var range = response.Content!; + Assert.Equal(dateMin, range.From, tolerance); + Assert.Equal(dateMax, range.To, tolerance); + } + + [Fact] + public async Task Count() + { + // arrange + Guid idDiscriminator = Guid.NewGuid(); + int count = 144; + IEnumerable testSets = Generate(count, DateTimeOffset.Now.ToOffset(TimeSpan.FromHours(7))); + var insertResponse = await client.InsertRange(idDiscriminator, testSets); + + // act + var response = await client.Count(idDiscriminator); + + // assert + Assert.Equal(System.Net.HttpStatusCode.OK, response.StatusCode); + Assert.Equal(count, response.Content); + } + + private static IEnumerable Generate(int n, DateTimeOffset from) + { + for (int i = 0; i < n; i++) + yield return new TimestampedSetDto + ( + from.AddSeconds(i), + new Dictionary{ + {"A", i }, + {"B", i * 1.1 }, + {"C", $"Any{i}" }, + {"D", DateTimeOffset.Now}, + } + ); + } +} diff --git a/Persistence.IntegrationTests/WebAppFactoryFixture.cs b/Persistence.IntegrationTests/WebAppFactoryFixture.cs index 8ac7bf2..740d3c5 100644 --- a/Persistence.IntegrationTests/WebAppFactoryFixture.cs +++ b/Persistence.IntegrationTests/WebAppFactoryFixture.cs @@ -4,6 +4,7 @@ using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Persistence.API; +using Persistence.Database; using Persistence.Database.Model; using Persistence.Database.Postgres; using Refit; @@ -53,6 +54,8 @@ public class WebAppFactoryFixture : WebApplicationFactory services.AddDbContext(options => options.UseNpgsql(connectionString)); + services.AddScoped(provider => provider.GetRequiredService()); + var serviceProvider = services.BuildServiceProvider(); using var scope = serviceProvider.CreateScope(); diff --git a/Persistence.Repository/DependencyInjection.cs b/Persistence.Repository/DependencyInjection.cs index 8e2f759..3a27ed1 100644 --- a/Persistence.Repository/DependencyInjection.cs +++ b/Persistence.Repository/DependencyInjection.cs @@ -16,6 +16,7 @@ public static class DependencyInjection MapsterSetup(); services.AddTransient, TimeSeriesDataCachedRepository>(); + services.AddTransient(); return services; } diff --git a/Persistence.Repository/Repositories/TimestampedSetRepository.cs b/Persistence.Repository/Repositories/TimestampedSetRepository.cs new file mode 100644 index 0000000..a81f7f1 --- /dev/null +++ b/Persistence.Repository/Repositories/TimestampedSetRepository.cs @@ -0,0 +1,110 @@ +using Microsoft.EntityFrameworkCore; +using Persistence.Database; +using Persistence.Database.Entity; +using Persistence.Models; +using Persistence.Repositories; + +namespace Persistence.Repository.Repositories; +public class TimestampedSetRepository : ITimestampedSetRepository +{ + private readonly IPersistenceDbContext db; + + public TimestampedSetRepository(IPersistenceDbContext db) + { + this.db = db; + } + + public Task InsertRange(Guid idDiscriminator, IEnumerable sets, CancellationToken token) + { + var entities = sets.Select(set => new TimestampedSet(idDiscriminator, set.Timestamp.ToUniversalTime(), set.Set)); + var dbSet = db.Set(); + dbSet.AddRange(entities); + return db.SaveChangesAsync(token); + } + + public async Task> Get(Guid idDiscriminator, DateTimeOffset? geTimestamp, IEnumerable? props, int skip, int take, CancellationToken token) + { + var dbSet = db.Set(); + var query = dbSet.Where(entity => entity.IdDiscriminator == idDiscriminator); + + if (geTimestamp.HasValue) + query = ApplyGeTimestamp(query, geTimestamp.Value); + + var data = await Materialize(query, token); + + if (props is not null && props.Any()) + data = ApplyPropsFilter(data, props); + + return data; + } + + public async Task> GetLast(Guid idDiscriminator, IEnumerable? props, int take, CancellationToken token) + { + var dbSet = db.Set(); + var query = dbSet.Where(entity => entity.IdDiscriminator == idDiscriminator); + + query = query.OrderByDescending(entity => entity.Timestamp) + .Take(take) + .OrderBy(entity => entity.Timestamp); + + var data = await Materialize(query, token); + + if (props is not null && props.Any()) + data = ApplyPropsFilter(data, props); + + return data; + } + + public Task Count(Guid idDiscriminator, CancellationToken token) + { + var dbSet = db.Set(); + var query = dbSet.Where(entity => entity.IdDiscriminator == idDiscriminator); + return query.CountAsync(token); + } + + public async Task GetDatesRange(Guid idDiscriminator, CancellationToken token) + { + var query = db.Set() + .GroupBy(entity => entity.IdDiscriminator) + .Select(group => new + { + Min = group.Min(entity => entity.Timestamp), + Max = group.Max(entity => entity.Timestamp), + }); + + var item = await query.FirstOrDefaultAsync(token); + if (item is null) + return null; + + return new DatesRangeDto + { + From = item.Min, + To = item.Max, + }; + } + + private static async Task> Materialize(IQueryable query, CancellationToken token) + { + var dtoQuery = query.Select(entity => new TimestampedSetDto(entity.Timestamp, entity.Set)); + var dtos = await dtoQuery.ToArrayAsync(token); + return dtos; + } + + private static IQueryable ApplyGeTimestamp(IQueryable query, DateTimeOffset geTimestamp) + { + var geTimestampUtc = geTimestamp.ToUniversalTime(); + return query.Where(entity => entity.Timestamp >= geTimestampUtc); + } + + private static IEnumerable ApplyPropsFilter(IEnumerable query, IEnumerable props) + { + var newQuery = query + .Select(entity => new TimestampedSetDto( + entity.Timestamp, + entity.Set + .Where(prop => props.Contains(prop.Key)) + .ToDictionary(prop => prop.Key, prop => prop.Value) + )); + return newQuery; + } +} diff --git a/Persistence/Models/TimestampedSetDto.cs b/Persistence/Models/TimestampedSetDto.cs new file mode 100644 index 0000000..3235a4e --- /dev/null +++ b/Persistence/Models/TimestampedSetDto.cs @@ -0,0 +1,8 @@ +namespace Persistence.Models; + +/// +/// набор данных с отметкой времени +/// +/// отметка времени +/// набор данных +public record TimestampedSetDto(DateTimeOffset Timestamp, IDictionary Set); diff --git a/Persistence/Repositories/ITimestampedSetRepository.cs b/Persistence/Repositories/ITimestampedSetRepository.cs new file mode 100644 index 0000000..3e853f1 --- /dev/null +++ b/Persistence/Repositories/ITimestampedSetRepository.cs @@ -0,0 +1,11 @@ +using Persistence.Models; + +namespace Persistence.Repositories; +public interface ITimestampedSetRepository +{ + Task Count(Guid idDiscriminator, CancellationToken token); + Task> Get(Guid idDiscriminator, DateTimeOffset? geTimestamp, IEnumerable? props, int skip, int take, CancellationToken token); + Task GetDatesRange(Guid idDiscriminator, CancellationToken token); + Task> GetLast(Guid idDiscriminator, IEnumerable? props, int take, CancellationToken token); + Task InsertRange(Guid idDiscriminator, IEnumerable sets, CancellationToken token); +} \ No newline at end of file