diff --git a/Persistence.API/Controllers/TimestampedSetController.cs b/Persistence.API/Controllers/TimestampedSetController.cs new file mode 100644 index 0000000..3d7ce63 --- /dev/null +++ b/Persistence.API/Controllers/TimestampedSetController.cs @@ -0,0 +1,63 @@ +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Mvc; +using Persistence.Models; +using Persistence.Repositories; +using Persistence.Repository.Repositories; +using System.Net; + +namespace Persistence.API.Controllers; + +[ApiController] +[Authorize] +[Route("api/[controller]/{idDiscriminator}")] +public class TimestampedSetController : ControllerBase +{ + private readonly ITimestampedSetRepository repository; + + public TimestampedSetController(ITimestampedSetRepository repository) + { + this.repository = repository; + } + + [HttpPost] + [ProducesResponseType(typeof(int), (int)HttpStatusCode.OK)] + public async Task InsertRange([FromRoute]Guid idDiscriminator, [FromBody]IEnumerable sets, CancellationToken token) + { + var result = await repository.InsertRange(idDiscriminator, sets, token); + return Ok(result); + } + + [HttpGet] + [ProducesResponseType(typeof(IEnumerable), (int)HttpStatusCode.OK)] + public async Task Get(Guid idDiscriminator, DateTimeOffset? geTimestamp, [FromQuery]IEnumerable? props, int skip, int take, CancellationToken token) + { + var result = await repository.Get(idDiscriminator, geTimestamp, props, skip, take, token); + return Ok(result); + } + + [HttpGet("last")] + [ProducesResponseType(typeof(IEnumerable), (int)HttpStatusCode.OK)] + public async Task GetLast(Guid idDiscriminator, [FromQuery]IEnumerable? props, int take, CancellationToken token) + { + var result = await repository.GetLast(idDiscriminator, props, take, token); + return Ok(result); + } + + [HttpGet("datesRange")] + [ProducesResponseType(typeof(DatesRangeDto), (int)HttpStatusCode.OK)] + [ProducesResponseType((int)HttpStatusCode.NoContent)] + public async Task GetDatesRange(Guid idDiscriminator, CancellationToken token) + { + var result = await repository.GetDatesRange(idDiscriminator, token); + return Ok(result); + } + + [HttpGet("count")] + [ProducesResponseType(typeof(DatesRangeDto), (int)HttpStatusCode.OK)] + [ProducesResponseType((int)HttpStatusCode.NoContent)] + public async Task Count(Guid idDiscriminator, CancellationToken token) + { + var result = await repository.Count(idDiscriminator, token); + return Ok(result); + } +} diff --git a/Persistence.Client/Clients/ITimestampedSetClient.cs b/Persistence.Client/Clients/ITimestampedSetClient.cs new file mode 100644 index 0000000..96e2500 --- /dev/null +++ b/Persistence.Client/Clients/ITimestampedSetClient.cs @@ -0,0 +1,23 @@ +using Persistence.Models; +using Refit; + +namespace Persistence.IntegrationTests.Clients; +public interface ITimestampedSetClient +{ + private const string baseUrl = "/api/TimestampedSet/{idDiscriminator}"; + + [Post(baseUrl)] + Task> InsertRange(Guid idDiscriminator, IEnumerable sets); + + [Get(baseUrl)] + Task>> Get(Guid idDiscriminator, [Query]DateTimeOffset? geTimestamp, [Query]IEnumerable? props, int skip, int take); + + [Get($"{baseUrl}/last")] + Task>> GetLast(Guid idDiscriminator, [Query] IEnumerable? props, int take); + + [Get($"{baseUrl}/count")] + Task> Count(Guid idDiscriminator); + + [Get($"{baseUrl}/datesRange")] + Task> GetDatesRange(Guid idDiscriminator); +} diff --git a/Persistence.Database.Postgres/PersistenceDbContext.cs b/Persistence.Database.Postgres/PersistenceDbContext.cs index 58d201c..053df12 100644 --- a/Persistence.Database.Postgres/PersistenceDbContext.cs +++ b/Persistence.Database.Postgres/PersistenceDbContext.cs @@ -1,4 +1,6 @@ using Microsoft.EntityFrameworkCore; +using Npgsql; +using Persistence.Database.Entity; using System.Data.Common; namespace Persistence.Database.Model; @@ -8,10 +10,12 @@ public partial class PersistenceDbContext : DbContext, IPersistenceDbContext public DbSet Setpoint => Set(); + public DbSet TimestampedSets => Set(); + public PersistenceDbContext() : base() { - + } public PersistenceDbContext(DbContextOptions options) @@ -32,7 +36,9 @@ public partial class PersistenceDbContext : DbContext, IPersistenceDbContext { modelBuilder.HasPostgresExtension("adminpack") .HasAnnotation("Relational:Collation", "Russian_Russia.1251"); + + modelBuilder.Entity() + .Property(e => e.Set) + .HasJsonConversion(); } - - } diff --git a/Persistence.Database/EFExtensions.cs b/Persistence.Database/EFExtensions.cs new file mode 100644 index 0000000..d60d768 --- /dev/null +++ b/Persistence.Database/EFExtensions.cs @@ -0,0 +1,41 @@ +using Microsoft.EntityFrameworkCore.ChangeTracking; +using System.Text.Json.Serialization; +using System.Text.Json; + +namespace Persistence.Database; + +public static class EFExtensions +{ + private static readonly JsonSerializerOptions jsonSerializerOptions = new() + { + AllowTrailingCommas = true, + WriteIndented = true, + NumberHandling = JsonNumberHandling.AllowReadingFromString | JsonNumberHandling.AllowNamedFloatingPointLiterals, + }; + + public static Microsoft.EntityFrameworkCore.Metadata.Builders.PropertyBuilder HasJsonConversion( + this Microsoft.EntityFrameworkCore.Metadata.Builders.PropertyBuilder builder) + => HasJsonConversion(builder, jsonSerializerOptions); + + public static Microsoft.EntityFrameworkCore.Metadata.Builders.PropertyBuilder HasJsonConversion( + this Microsoft.EntityFrameworkCore.Metadata.Builders.PropertyBuilder builder, + JsonSerializerOptions jsonSerializerOptions) + { + builder.HasConversion( + s => JsonSerializer.Serialize(s, jsonSerializerOptions), + s => JsonSerializer.Deserialize(s, jsonSerializerOptions)!); + + ValueComparer valueComparer = new( + (a, b) => + (a != null) && (b != null) + ? a.GetHashCode() == b.GetHashCode() + : (a == null) && (b == null), + i => (i == null) ? -1 : i.GetHashCode(), + i => i); + + builder.Metadata.SetValueComparer(valueComparer); + return builder; + } + +} + diff --git a/Persistence.Database/Entity/TimestampedSet.cs b/Persistence.Database/Entity/TimestampedSet.cs new file mode 100644 index 0000000..c9a0dda --- /dev/null +++ b/Persistence.Database/Entity/TimestampedSet.cs @@ -0,0 +1,11 @@ +using Microsoft.EntityFrameworkCore; +using System.ComponentModel.DataAnnotations.Schema; + +namespace Persistence.Database.Entity; + +[Comment("Общая таблица данных временных рядов")] +[PrimaryKey(nameof(IdDiscriminator), nameof(Timestamp))] +public record TimestampedSet( + [property: Comment("Дискриминатор ссылка на тип сохраняемых данных")] Guid IdDiscriminator, + [property: Comment("Отметка времени, строго в UTC")] DateTimeOffset Timestamp, + [property: Column(TypeName = "jsonb"), Comment("Набор сохраняемых данных")] IDictionary Set); diff --git a/Persistence.Database/IPersistenceDbContext.cs b/Persistence.Database/IPersistenceDbContext.cs index 66f34ff..d23656e 100644 --- a/Persistence.Database/IPersistenceDbContext.cs +++ b/Persistence.Database/IPersistenceDbContext.cs @@ -1,8 +1,13 @@ using Microsoft.EntityFrameworkCore; +using Persistence.Database.Entity; using Persistence.Database.Model; +using System.Diagnostics.CodeAnalysis; namespace Persistence.Database; public interface IPersistenceDbContext : IDisposable { DbSet DataSaub { get; } + DbSet TimestampedSets { get; } + DbSet Set<[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors | DynamicallyAccessedMemberTypes.NonPublicConstructors | DynamicallyAccessedMemberTypes.PublicFields | DynamicallyAccessedMemberTypes.NonPublicFields | DynamicallyAccessedMemberTypes.PublicProperties | DynamicallyAccessedMemberTypes.NonPublicProperties | DynamicallyAccessedMemberTypes.Interfaces)] TEntity>() where TEntity : class; + Task SaveChangesAsync(CancellationToken cancellationToken); } diff --git a/Persistence.IntegrationTests/Controllers/TimestampedSetControllerTest.cs b/Persistence.IntegrationTests/Controllers/TimestampedSetControllerTest.cs new file mode 100644 index 0000000..12ebb94 --- /dev/null +++ b/Persistence.IntegrationTests/Controllers/TimestampedSetControllerTest.cs @@ -0,0 +1,219 @@ +using Microsoft.AspNetCore.Mvc; +using Persistence.IntegrationTests.Clients; +using Persistence.Models; +using Xunit; + +namespace Persistence.IntegrationTests.Controllers; +public class TimestampedSetControllerTest : BaseIntegrationTest +{ + private readonly ITimestampedSetClient client; + + public TimestampedSetControllerTest(WebAppFactoryFixture factory) : base(factory) + { + + client = factory.GetAuthorizedHttpClient(string.Empty).Result; + } + + [Fact] + public async Task InsertRange() + { + // arrange + Guid idDiscriminator = Guid.NewGuid(); + IEnumerable testSets = Generate(10, DateTimeOffset.Now.ToOffset(TimeSpan.FromHours(7))); + + // act + var response = await client.InsertRange(idDiscriminator, testSets); + + // assert + Assert.Equal(System.Net.HttpStatusCode.OK, response.StatusCode); + Assert.Equal(testSets.Count(), response.Content); + } + + [Fact] + public async Task Get_without_filter() + { + // arrange + Guid idDiscriminator = Guid.NewGuid(); + int count = 10; + IEnumerable testSets = Generate(count, DateTimeOffset.Now.ToOffset(TimeSpan.FromHours(7))); + var insertResponse = await client.InsertRange(idDiscriminator, testSets); + + // act + var response = await client.Get(idDiscriminator, null, null, 0, int.MaxValue); + + // assert + Assert.Equal(System.Net.HttpStatusCode.OK, response.StatusCode); + Assert.NotNull(response.Content); + var items = response.Content!; + Assert.Equal(count, items.Count()); + } + + [Fact] + public async Task Get_with_filter_props() + { + // arrange + Guid idDiscriminator = Guid.NewGuid(); + int count = 10; + IEnumerable testSets = Generate(count, DateTimeOffset.Now.ToOffset(TimeSpan.FromHours(7))); + var insertResponse = await client.InsertRange(idDiscriminator, testSets); + string[] props = ["A"]; + + // act + var response = await client.Get(idDiscriminator, null, props, 0, int.MaxValue); + + // assert + Assert.Equal(System.Net.HttpStatusCode.OK, response.StatusCode); + Assert.NotNull(response.Content); + var items = response.Content!; + Assert.Equal(count, items.Count()); + foreach ( var item in items ) + { + Assert.Single(item.Set); + var kv = item.Set.First(); + Assert.Equal("A", kv.Key); + } + } + + [Fact] + public async Task Get_geDate() + { + // arrange + Guid idDiscriminator = Guid.NewGuid(); + int count = 10; + var dateMin = DateTimeOffset.Now; + var dateMax = DateTimeOffset.Now.AddSeconds(count); + IEnumerable testSets = Generate(count, dateMin.ToOffset(TimeSpan.FromHours(7))); + var insertResponse = await client.InsertRange(idDiscriminator, testSets); + var tail = testSets.OrderBy(t => t.Timestamp).Skip(count / 2).Take(int.MaxValue); + var geDate = tail.First().Timestamp; + var tolerance = TimeSpan.FromSeconds(1); + var expectedCount = tail.Count(); + + // act + var response = await client.Get(idDiscriminator, geDate, null, 0, int.MaxValue); + + // assert + Assert.Equal(System.Net.HttpStatusCode.OK, response.StatusCode); + Assert.NotNull(response.Content); + var items = response.Content!; + Assert.Equal(expectedCount, items.Count()); + var minDate = items.Min(t => t.Timestamp); + Assert.Equal(geDate, geDate, tolerance); + } + + [Fact] + public async Task Get_with_skip_take() + { + // arrange + Guid idDiscriminator = Guid.NewGuid(); + int count = 10; + IEnumerable testSets = Generate(count, DateTimeOffset.Now.ToOffset(TimeSpan.FromHours(7))); + var insertResponse = await client.InsertRange(idDiscriminator, testSets); + var expectedCount = count / 2; + + // act + var response = await client.Get(idDiscriminator, null, null, 2, expectedCount); + + // assert + Assert.Equal(System.Net.HttpStatusCode.OK, response.StatusCode); + Assert.NotNull(response.Content); + var items = response.Content!; + Assert.Equal(expectedCount, items.Count()); + } + + + [Fact] + public async Task Get_with_big_skip_take() + { + // arrange + Guid idDiscriminator = Guid.NewGuid(); + var expectedCount = 1; + int count = 10 + expectedCount; + IEnumerable testSets = Generate(count, DateTimeOffset.Now.ToOffset(TimeSpan.FromHours(7))); + var insertResponse = await client.InsertRange(idDiscriminator, testSets); + + // act + var response = await client.Get(idDiscriminator, null, null, count - expectedCount, count); + + // assert + Assert.Equal(System.Net.HttpStatusCode.OK, response.StatusCode); + Assert.NotNull(response.Content); + var items = response.Content!; + Assert.Equal(expectedCount, items.Count()); + } + + [Fact] + public async Task GetLast() + { + // arrange + Guid idDiscriminator = Guid.NewGuid(); + int count = 10; + IEnumerable testSets = Generate(count, DateTimeOffset.Now.ToOffset(TimeSpan.FromHours(7))); + var insertResponse = await client.InsertRange(idDiscriminator, testSets); + var expectedCount = 8; + + // act + var response = await client.GetLast(idDiscriminator, null, expectedCount); + + // assert + Assert.Equal(System.Net.HttpStatusCode.OK, response.StatusCode); + Assert.NotNull(response.Content); + var items = response.Content!; + Assert.Equal(expectedCount, items.Count()); + } + + [Fact] + public async Task GetDatesRange() + { + // arrange + Guid idDiscriminator = Guid.NewGuid(); + int count = 10; + var dateMin = DateTimeOffset.Now; + var dateMax = DateTimeOffset.Now.AddSeconds(count-1); + IEnumerable testSets = Generate(count, dateMin.ToOffset(TimeSpan.FromHours(7))); + var insertResponse = await client.InsertRange(idDiscriminator, testSets); + var tolerance = TimeSpan.FromSeconds(1); + + // act + var response = await client.GetDatesRange(idDiscriminator); + + // assert + Assert.Equal(System.Net.HttpStatusCode.OK, response.StatusCode); + Assert.NotNull(response.Content); + var range = response.Content!; + Assert.Equal(dateMin, range.From, tolerance); + Assert.Equal(dateMax, range.To, tolerance); + } + + [Fact] + public async Task Count() + { + // arrange + Guid idDiscriminator = Guid.NewGuid(); + int count = 144; + IEnumerable testSets = Generate(count, DateTimeOffset.Now.ToOffset(TimeSpan.FromHours(7))); + var insertResponse = await client.InsertRange(idDiscriminator, testSets); + + // act + var response = await client.Count(idDiscriminator); + + // assert + Assert.Equal(System.Net.HttpStatusCode.OK, response.StatusCode); + Assert.Equal(count, response.Content); + } + + private static IEnumerable Generate(int n, DateTimeOffset from) + { + for (int i = 0; i < n; i++) + yield return new TimestampedSetDto + ( + from.AddSeconds(i), + new Dictionary{ + {"A", i }, + {"B", i * 1.1 }, + {"C", $"Any{i}" }, + {"D", DateTimeOffset.Now}, + } + ); + } +} diff --git a/Persistence.IntegrationTests/WebAppFactoryFixture.cs b/Persistence.IntegrationTests/WebAppFactoryFixture.cs index b5f6f97..1d99a2a 100644 --- a/Persistence.IntegrationTests/WebAppFactoryFixture.cs +++ b/Persistence.IntegrationTests/WebAppFactoryFixture.cs @@ -5,6 +5,7 @@ using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; using Persistence.API; +using Persistence.Database; using Persistence.Client; using Persistence.Database.Model; using Persistence.Database.Postgres; @@ -42,6 +43,9 @@ public class WebAppFactoryFixture : WebApplicationFactory services.AddSingleton(); var serviceProvider = services.BuildServiceProvider(); + services.AddScoped(provider => provider.GetRequiredService()); + + var serviceProvider = services.BuildServiceProvider(); using var scope = serviceProvider.CreateScope(); var scopedServices = scope.ServiceProvider; diff --git a/Persistence.Repository/DependencyInjection.cs b/Persistence.Repository/DependencyInjection.cs index 27063cb..e79c555 100644 --- a/Persistence.Repository/DependencyInjection.cs +++ b/Persistence.Repository/DependencyInjection.cs @@ -17,6 +17,8 @@ public static class DependencyInjection services.AddTransient, TimeSeriesDataRepository>(); services.AddTransient(); + services.AddTransient, TimeSeriesDataCachedRepository>(); + services.AddTransient(); return services; } diff --git a/Persistence.Repository/Repositories/TimestampedSetRepository.cs b/Persistence.Repository/Repositories/TimestampedSetRepository.cs new file mode 100644 index 0000000..a190d71 --- /dev/null +++ b/Persistence.Repository/Repositories/TimestampedSetRepository.cs @@ -0,0 +1,117 @@ +using Microsoft.EntityFrameworkCore; +using Persistence.Database; +using Persistence.Database.Entity; +using Persistence.Models; +using Persistence.Repositories; + +namespace Persistence.Repository.Repositories; + +/// +/// Репозиторий для хранения разных наборов данных временных рядов. +/// idDiscriminator - идентифицирует конкретный набор данных, прим.: циклы измерения АСИБР, или отчет о DrillTest. +/// idDiscriminator формируют клиенты и только им известно что они обозначают. +/// Так как данные приходят редко, то их прореживания для построения графиков не предусмотрено. +/// +public class TimestampedSetRepository : ITimestampedSetRepository +{ + private readonly IPersistenceDbContext db; + + public TimestampedSetRepository(IPersistenceDbContext db) + { + this.db = db; + } + + public Task InsertRange(Guid idDiscriminator, IEnumerable sets, CancellationToken token) + { + var entities = sets.Select(set => new TimestampedSet(idDiscriminator, set.Timestamp.ToUniversalTime(), set.Set)); + var dbSet = db.Set(); + dbSet.AddRange(entities); + return db.SaveChangesAsync(token); + } + + public async Task> Get(Guid idDiscriminator, DateTimeOffset? geTimestamp, IEnumerable? props, int skip, int take, CancellationToken token) + { + var dbSet = db.Set(); + var query = dbSet.Where(entity => entity.IdDiscriminator == idDiscriminator); + + if (geTimestamp.HasValue) + query = ApplyGeTimestamp(query, geTimestamp.Value); + + var data = await Materialize(query, token); + + if (props is not null && props.Any()) + data = ApplyPropsFilter(data, props); + + return data; + } + + public async Task> GetLast(Guid idDiscriminator, IEnumerable? props, int take, CancellationToken token) + { + var dbSet = db.Set(); + var query = dbSet.Where(entity => entity.IdDiscriminator == idDiscriminator); + + query = query.OrderByDescending(entity => entity.Timestamp) + .Take(take) + .OrderBy(entity => entity.Timestamp); + + var data = await Materialize(query, token); + + if (props is not null && props.Any()) + data = ApplyPropsFilter(data, props); + + return data; + } + + public Task Count(Guid idDiscriminator, CancellationToken token) + { + var dbSet = db.Set(); + var query = dbSet.Where(entity => entity.IdDiscriminator == idDiscriminator); + return query.CountAsync(token); + } + + public async Task GetDatesRange(Guid idDiscriminator, CancellationToken token) + { + var query = db.Set() + .GroupBy(entity => entity.IdDiscriminator) + .Select(group => new + { + Min = group.Min(entity => entity.Timestamp), + Max = group.Max(entity => entity.Timestamp), + }); + + var item = await query.FirstOrDefaultAsync(token); + if (item is null) + return null; + + return new DatesRangeDto + { + From = item.Min, + To = item.Max, + }; + } + + private static async Task> Materialize(IQueryable query, CancellationToken token) + { + var dtoQuery = query.Select(entity => new TimestampedSetDto(entity.Timestamp, entity.Set)); + var dtos = await dtoQuery.ToArrayAsync(token); + return dtos; + } + + private static IQueryable ApplyGeTimestamp(IQueryable query, DateTimeOffset geTimestamp) + { + var geTimestampUtc = geTimestamp.ToUniversalTime(); + return query.Where(entity => entity.Timestamp >= geTimestampUtc); + } + + private static IEnumerable ApplyPropsFilter(IEnumerable query, IEnumerable props) + { + var newQuery = query + .Select(entity => new TimestampedSetDto( + entity.Timestamp, + entity.Set + .Where(prop => props.Contains(prop.Key)) + .ToDictionary(prop => prop.Key, prop => prop.Value) + )); + return newQuery; + } +} diff --git a/Persistence/Models/TimestampedSetDto.cs b/Persistence/Models/TimestampedSetDto.cs new file mode 100644 index 0000000..3235a4e --- /dev/null +++ b/Persistence/Models/TimestampedSetDto.cs @@ -0,0 +1,8 @@ +namespace Persistence.Models; + +/// +/// набор данных с отметкой времени +/// +/// отметка времени +/// набор данных +public record TimestampedSetDto(DateTimeOffset Timestamp, IDictionary Set); diff --git a/Persistence/Repositories/ITimestampedSetRepository.cs b/Persistence/Repositories/ITimestampedSetRepository.cs new file mode 100644 index 0000000..2966a82 --- /dev/null +++ b/Persistence/Repositories/ITimestampedSetRepository.cs @@ -0,0 +1,59 @@ +using Persistence.Models; + +namespace Persistence.Repositories; + +/// +/// Репозиторий для хранения разных наборов данных временных рядов. +/// idDiscriminator - идентифицирует конкретный набор данных, прим.: циклы измерения АСИБР, или отчет о DrillTest. +/// idDiscriminator формируют клиенты и только им известно что они обозначают. +/// Так как данные приходят редко, то их прореживания для построения графиков не предусмотрено. +/// +public interface ITimestampedSetRepository +{ + /// + /// Количество записей по указанному набору в БД. Для пагинации. + /// + /// + /// + /// + Task Count(Guid idDiscriminator, CancellationToken token); + + /// + /// Получение данных с фильтрацией. Значение фильтра null - отключен + /// + /// Идентификатор набора + /// Фильтр позднее даты + /// Фильтр свойств набора. Можно запросить только некоторые свойства из набора + /// + /// + /// + /// + Task> Get(Guid idDiscriminator, DateTimeOffset? geTimestamp, IEnumerable? props, int skip, int take, CancellationToken token); + + /// + /// Диапазон дат за которые есть данные + /// + /// + /// + /// + Task GetDatesRange(Guid idDiscriminator, CancellationToken token); + + /// + /// Получить последние данные + /// + /// + /// + /// + /// + /// + Task> GetLast(Guid idDiscriminator, IEnumerable? props, int take, CancellationToken token); + + /// + /// Добавление новых данных + /// + /// + /// + /// + /// + Task InsertRange(Guid idDiscriminator, IEnumerable sets, CancellationToken token); +} \ No newline at end of file