on.nemtina/persistence
1
0
Fork 0
Code Issues Pull Requests 1 Packages Projects Releases Wiki Activity

Compare commits

base: on.nemtina:TimestampedSetRepository
Branches Tags
on.nemtina:master
on.nemtina:featute/ChangeLog
on.nemtina:TimestampedSetRepository
on.nemtina:PersistenceClient
on.nemtina:ChangeLog
..
compare: on.nemtina:master
Branches Tags
on.nemtina:featute/ChangeLog
on.nemtina:TimestampedSetRepository
on.nemtina:master
on.nemtina:PersistenceClient
on.nemtina:ChangeLog

No commits in common. "TimestampedSetRepository" and "master" have entirely different histories.
Timestampe ... master

12 changed files with 3 additions and 561 deletions
Show Stats Expand all files Collapse all files

63
Persistence.API/Controllers/TimestampedSetController.cs
View File

@ -1,63 +0,0 @@
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc;
using Persistence.Models;
using Persistence.Repositories;
using Persistence.Repository.Repositories;
using System.Net;
namespace Persistence.API.Controllers;
[ApiController]
[Authorize]
[Route("api/[controller]/{idDiscriminator}")]
public class TimestampedSetController : ControllerBase
{
private readonly ITimestampedSetRepository repository;
public TimestampedSetController(ITimestampedSetRepository repository)
{
this.repository = repository;
}
[HttpPost]
[ProducesResponseType(typeof(int), (int)HttpStatusCode.OK)]
public async Task<IActionResult> InsertRange([FromRoute]Guid idDiscriminator, [FromBody]IEnumerable<TimestampedSetDto> sets, CancellationToken token)
{
var result = await repository.InsertRange(idDiscriminator, sets, token);
return Ok(result);
}
[HttpGet]
[ProducesResponseType(typeof(IEnumerable<TimestampedSetDto>), (int)HttpStatusCode.OK)]
public async Task<IActionResult> Get(Guid idDiscriminator, DateTimeOffset? geTimestamp, [FromQuery]IEnumerable<string>? props, int skip, int take, CancellationToken token)
{
var result = await repository.Get(idDiscriminator, geTimestamp, props, skip, take, token);
return Ok(result);
}
[HttpGet("last")]
[ProducesResponseType(typeof(IEnumerable<TimestampedSetDto>), (int)HttpStatusCode.OK)]
public async Task<IActionResult> GetLast(Guid idDiscriminator, [FromQuery]IEnumerable<string>? props, int take, CancellationToken token)
{
var result = await repository.GetLast(idDiscriminator, props, take, token);
return Ok(result);
}
[HttpGet("datesRange")]
[ProducesResponseType(typeof(DatesRangeDto), (int)HttpStatusCode.OK)]
[ProducesResponseType((int)HttpStatusCode.NoContent)]
public async Task<IActionResult> GetDatesRange(Guid idDiscriminator, CancellationToken token)
{
var result = await repository.GetDatesRange(idDiscriminator, token);
return Ok(result);
}
[HttpGet("count")]
[ProducesResponseType(typeof(DatesRangeDto), (int)HttpStatusCode.OK)]
[ProducesResponseType((int)HttpStatusCode.NoContent)]
public async Task<IActionResult> Count(Guid idDiscriminator, CancellationToken token)
{
var result = await repository.Count(idDiscriminator, token);
return Ok(result);
}
}

23
Persistence.Client/Clients/ITimestampedSetClient.cs
View File

@ -1,23 +0,0 @@
using Persistence.Models;
using Refit;
namespace Persistence.IntegrationTests.Clients;
public interface ITimestampedSetClient
{
private const string baseUrl = "/api/TimestampedSet/{idDiscriminator}";
[Post(baseUrl)]
Task<IApiResponse<int>> InsertRange(Guid idDiscriminator, IEnumerable<TimestampedSetDto> sets);
[Get(baseUrl)]
Task<IApiResponse<IEnumerable<TimestampedSetDto>>> Get(Guid idDiscriminator, [Query]DateTimeOffset? geTimestamp, [Query]IEnumerable<string>? props, int skip, int take);
[Get($"{baseUrl}/last")]
Task<IApiResponse<IEnumerable<TimestampedSetDto>>> GetLast(Guid idDiscriminator, [Query] IEnumerable<string>? props, int take);
[Get($"{baseUrl}/count")]
Task<IApiResponse<int>> Count(Guid idDiscriminator);
[Get($"{baseUrl}/datesRange")]
Task<IApiResponse<DatesRangeDto?>> GetDatesRange(Guid idDiscriminator);
}

12
Persistence.Database.Postgres/PersistenceDbContext.cs
View File

@ -1,6 +1,4 @@
using Microsoft.EntityFrameworkCore;
using Npgsql;
using Persistence.Database.Entity;
using System.Data.Common;
namespace Persistence.Database.Model;
@ -10,12 +8,10 @@ public partial class PersistenceDbContext : DbContext, IPersistenceDbContext
public DbSet<Setpoint> Setpoint => Set<Setpoint>();
public DbSet<TimestampedSet> TimestampedSets => Set<TimestampedSet>();
public PersistenceDbContext()
: base()
{
}
public PersistenceDbContext(DbContextOptions<PersistenceDbContext> options)
@ -36,9 +32,7 @@ public partial class PersistenceDbContext : DbContext, IPersistenceDbContext
{
modelBuilder.HasPostgresExtension("adminpack")
.HasAnnotation("Relational:Collation", "Russian_Russia.1251");
modelBuilder.Entity<TimestampedSet>()
.Property(e => e.Set)
.HasJsonConversion();
}
}

41
Persistence.Database/EFExtensions.cs
View File

@ -1,41 +0,0 @@
using Microsoft.EntityFrameworkCore.ChangeTracking;
using System.Text.Json.Serialization;
using System.Text.Json;
namespace Persistence.Database;
public static class EFExtensions
{
private static readonly JsonSerializerOptions jsonSerializerOptions = new()
{
AllowTrailingCommas = true,
WriteIndented = true,
NumberHandling = JsonNumberHandling.AllowReadingFromString | JsonNumberHandling.AllowNamedFloatingPointLiterals,
};
public static Microsoft.EntityFrameworkCore.Metadata.Builders.PropertyBuilder<TProperty> HasJsonConversion<TProperty>(
this Microsoft.EntityFrameworkCore.Metadata.Builders.PropertyBuilder<TProperty> builder)
=> HasJsonConversion(builder, jsonSerializerOptions);
public static Microsoft.EntityFrameworkCore.Metadata.Builders.PropertyBuilder<TProperty> HasJsonConversion<TProperty>(
this Microsoft.EntityFrameworkCore.Metadata.Builders.PropertyBuilder<TProperty> builder,
JsonSerializerOptions jsonSerializerOptions)
{
builder.HasConversion(
s => JsonSerializer.Serialize(s, jsonSerializerOptions),
s => JsonSerializer.Deserialize<TProperty>(s, jsonSerializerOptions)!);
ValueComparer<TProperty> valueComparer = new(
(a, b) =>
(a != null) && (b != null)
? a.GetHashCode() == b.GetHashCode()
: (a == null) && (b == null),
i => (i == null) ? -1 : i.GetHashCode(),
i => i);
builder.Metadata.SetValueComparer(valueComparer);
return builder;
}
}

11
Persistence.Database/Entity/TimestampedSet.cs
View File

@ -1,11 +0,0 @@
using Microsoft.EntityFrameworkCore;
using System.ComponentModel.DataAnnotations.Schema;
namespace Persistence.Database.Entity;
[Comment("Общая таблица данных временных рядов")]
[PrimaryKey(nameof(IdDiscriminator), nameof(Timestamp))]
public record TimestampedSet(
[property: Comment("Дискриминатор ссылка на тип сохраняемых данных")] Guid IdDiscriminator,
[property: Comment("Отметка времени, строго в UTC")] DateTimeOffset Timestamp,
[property: Column(TypeName = "jsonb"), Comment("Набор сохраняемых данных")] IDictionary<string, object> Set);

5
Persistence.Database/IPersistenceDbContext.cs
View File

@ -1,13 +1,8 @@
using Microsoft.EntityFrameworkCore;
using Persistence.Database.Entity;
using Persistence.Database.Model;
using System.Diagnostics.CodeAnalysis;
namespace Persistence.Database;
public interface IPersistenceDbContext : IDisposable
{
DbSet<DataSaub> DataSaub { get; }
DbSet<TimestampedSet> TimestampedSets { get; }
DbSet<TEntity> Set<[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors | DynamicallyAccessedMemberTypes.NonPublicConstructors | DynamicallyAccessedMemberTypes.PublicFields | DynamicallyAccessedMemberTypes.NonPublicFields | DynamicallyAccessedMemberTypes.PublicProperties | DynamicallyAccessedMemberTypes.NonPublicProperties | DynamicallyAccessedMemberTypes.Interfaces)] TEntity>() where TEntity : class;
Task<int> SaveChangesAsync(CancellationToken cancellationToken);
}

219
Persistence.IntegrationTests/Controllers/TimestampedSetControllerTest.cs
View File

@ -1,219 +0,0 @@
using Microsoft.AspNetCore.Mvc;
using Persistence.IntegrationTests.Clients;
using Persistence.Models;
using Xunit;
namespace Persistence.IntegrationTests.Controllers;
public class TimestampedSetControllerTest : BaseIntegrationTest
{
private readonly ITimestampedSetClient client;
public TimestampedSetControllerTest(WebAppFactoryFixture factory) : base(factory)
{
client = factory.GetAuthorizedHttpClient<ITimestampedSetClient>(string.Empty).Result;
}
[Fact]
public async Task InsertRange()
{
// arrange
Guid idDiscriminator = Guid.NewGuid();
IEnumerable<TimestampedSetDto> testSets = Generate(10, DateTimeOffset.Now.ToOffset(TimeSpan.FromHours(7)));
// act
var response = await client.InsertRange(idDiscriminator, testSets);
// assert
Assert.Equal(System.Net.HttpStatusCode.OK, response.StatusCode);
Assert.Equal(testSets.Count(), response.Content);
}
[Fact]
public async Task Get_without_filter()
{
// arrange
Guid idDiscriminator = Guid.NewGuid();
int count = 10;
IEnumerable<TimestampedSetDto> testSets = Generate(count, DateTimeOffset.Now.ToOffset(TimeSpan.FromHours(7)));
var insertResponse = await client.InsertRange(idDiscriminator, testSets);
// act
var response = await client.Get(idDiscriminator, null, null, 0, int.MaxValue);
// assert
Assert.Equal(System.Net.HttpStatusCode.OK, response.StatusCode);
Assert.NotNull(response.Content);
var items = response.Content!;
Assert.Equal(count, items.Count());
}
[Fact]
public async Task Get_with_filter_props()
{
// arrange
Guid idDiscriminator = Guid.NewGuid();
int count = 10;
IEnumerable<TimestampedSetDto> testSets = Generate(count, DateTimeOffset.Now.ToOffset(TimeSpan.FromHours(7)));
var insertResponse = await client.InsertRange(idDiscriminator, testSets);
string[] props = ["A"];
// act
var response = await client.Get(idDiscriminator, null, props, 0, int.MaxValue);
// assert
Assert.Equal(System.Net.HttpStatusCode.OK, response.StatusCode);
Assert.NotNull(response.Content);
var items = response.Content!;
Assert.Equal(count, items.Count());
foreach ( var item in items )
{
Assert.Single(item.Set);
var kv = item.Set.First();
Assert.Equal("A", kv.Key);
}
}
[Fact]
public async Task Get_geDate()
{
// arrange
Guid idDiscriminator = Guid.NewGuid();
int count = 10;
var dateMin = DateTimeOffset.Now;
var dateMax = DateTimeOffset.Now.AddSeconds(count);
IEnumerable<TimestampedSetDto> testSets = Generate(count, dateMin.ToOffset(TimeSpan.FromHours(7)));
var insertResponse = await client.InsertRange(idDiscriminator, testSets);
var tail = testSets.OrderBy(t => t.Timestamp).Skip(count / 2).Take(int.MaxValue);
var geDate = tail.First().Timestamp;
var tolerance = TimeSpan.FromSeconds(1);
var expectedCount = tail.Count();
// act
var response = await client.Get(idDiscriminator, geDate, null, 0, int.MaxValue);
// assert
Assert.Equal(System.Net.HttpStatusCode.OK, response.StatusCode);
Assert.NotNull(response.Content);
var items = response.Content!;
Assert.Equal(expectedCount, items.Count());
var minDate = items.Min(t => t.Timestamp);
Assert.Equal(geDate, geDate, tolerance);
}
[Fact]
public async Task Get_with_skip_take()
{
// arrange
Guid idDiscriminator = Guid.NewGuid();
int count = 10;
IEnumerable<TimestampedSetDto> testSets = Generate(count, DateTimeOffset.Now.ToOffset(TimeSpan.FromHours(7)));
var insertResponse = await client.InsertRange(idDiscriminator, testSets);
var expectedCount = count / 2;
// act
var response = await client.Get(idDiscriminator, null, null, 2, expectedCount);
// assert
Assert.Equal(System.Net.HttpStatusCode.OK, response.StatusCode);
Assert.NotNull(response.Content);
var items = response.Content!;
Assert.Equal(expectedCount, items.Count());
}
[Fact]
public async Task Get_with_big_skip_take()
{
// arrange
Guid idDiscriminator = Guid.NewGuid();
var expectedCount = 1;
int count = 10 + expectedCount;
IEnumerable<TimestampedSetDto> testSets = Generate(count, DateTimeOffset.Now.ToOffset(TimeSpan.FromHours(7)));
var insertResponse = await client.InsertRange(idDiscriminator, testSets);
// act
var response = await client.Get(idDiscriminator, null, null, count - expectedCount, count);
// assert
Assert.Equal(System.Net.HttpStatusCode.OK, response.StatusCode);
Assert.NotNull(response.Content);
var items = response.Content!;
Assert.Equal(expectedCount, items.Count());
}
[Fact]
public async Task GetLast()
{
// arrange
Guid idDiscriminator = Guid.NewGuid();
int count = 10;
IEnumerable<TimestampedSetDto> testSets = Generate(count, DateTimeOffset.Now.ToOffset(TimeSpan.FromHours(7)));
var insertResponse = await client.InsertRange(idDiscriminator, testSets);
var expectedCount = 8;
// act
var response = await client.GetLast(idDiscriminator, null, expectedCount);
// assert
Assert.Equal(System.Net.HttpStatusCode.OK, response.StatusCode);
Assert.NotNull(response.Content);
var items = response.Content!;
Assert.Equal(expectedCount, items.Count());
}
[Fact]
public async Task GetDatesRange()
{
// arrange
Guid idDiscriminator = Guid.NewGuid();
int count = 10;
var dateMin = DateTimeOffset.Now;
var dateMax = DateTimeOffset.Now.AddSeconds(count-1);
IEnumerable<TimestampedSetDto> testSets = Generate(count, dateMin.ToOffset(TimeSpan.FromHours(7)));
var insertResponse = await client.InsertRange(idDiscriminator, testSets);
var tolerance = TimeSpan.FromSeconds(1);
// act
var response = await client.GetDatesRange(idDiscriminator);
// assert
Assert.Equal(System.Net.HttpStatusCode.OK, response.StatusCode);
Assert.NotNull(response.Content);
var range = response.Content!;
Assert.Equal(dateMin, range.From, tolerance);
Assert.Equal(dateMax, range.To, tolerance);
}
[Fact]
public async Task Count()
{
// arrange
Guid idDiscriminator = Guid.NewGuid();
int count = 144;
IEnumerable<TimestampedSetDto> testSets = Generate(count, DateTimeOffset.Now.ToOffset(TimeSpan.FromHours(7)));
var insertResponse = await client.InsertRange(idDiscriminator, testSets);
// act
var response = await client.Count(idDiscriminator);
// assert
Assert.Equal(System.Net.HttpStatusCode.OK, response.StatusCode);
Assert.Equal(count, response.Content);
}
private static IEnumerable<TimestampedSetDto> Generate(int n, DateTimeOffset from)
{
for (int i = 0; i < n; i++)
yield return new TimestampedSetDto
(
from.AddSeconds(i),
new Dictionary<string, object>{
{"A", i },
{"B", i * 1.1 },
{"C", $"Any{i}" },
{"D", DateTimeOffset.Now},
}
);
}
}

4
Persistence.IntegrationTests/WebAppFactoryFixture.cs
View File

@ -5,7 +5,6 @@ using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.DependencyInjection.Extensions;
using Persistence.API;
using Persistence.Database;
using Persistence.Client;
using Persistence.Database.Model;
using Persistence.Database.Postgres;
@ -43,9 +42,6 @@ public class WebAppFactoryFixture : WebApplicationFactory<Startup>
services.AddSingleton<PersistenceClientFactory>();
var serviceProvider = services.BuildServiceProvider();
services.AddScoped<IPersistenceDbContext>(provider => provider.GetRequiredService<PersistenceDbContext>());
var serviceProvider = services.BuildServiceProvider();
using var scope = serviceProvider.CreateScope();
var scopedServices = scope.ServiceProvider;

2
Persistence.Repository/DependencyInjection.cs
View File

@ -17,8 +17,6 @@ public static class DependencyInjection
services.AddTransient<ITimeSeriesDataRepository<DataSaubDto>, TimeSeriesDataRepository<DataSaub, DataSaubDto>>();
services.AddTransient<ISetpointRepository, SetpointRepository>();
services.AddTransient<ITimeSeriesDataRepository<DataSaubDto>, TimeSeriesDataCachedRepository<DataSaub, DataSaubDto>>();
services.AddTransient<ITimestampedSetRepository, TimestampedSetRepository>();
return services;
}

117
Persistence.Repository/Repositories/TimestampedSetRepository.cs
View File

@ -1,117 +0,0 @@
using Microsoft.EntityFrameworkCore;
using Persistence.Database;
using Persistence.Database.Entity;
using Persistence.Models;
using Persistence.Repositories;
namespace Persistence.Repository.Repositories;
/// <summary>
/// Репозиторий для хранения разных наборов данных временных рядов.
/// idDiscriminator - идентифицирует конкретный набор данных, прим.: циклы измерения АСИБР, или отчет о DrillTest.
/// idDiscriminator формируют клиенты и только им известно что они обозначают.
/// Так как данные приходят редко, то их прореживания для построения графиков не предусмотрено.
/// </summary>
public class TimestampedSetRepository : ITimestampedSetRepository
{
private readonly IPersistenceDbContext db;
public TimestampedSetRepository(IPersistenceDbContext db)
{
this.db = db;
}
public Task<int> InsertRange(Guid idDiscriminator, IEnumerable<TimestampedSetDto> sets, CancellationToken token)
{
var entities = sets.Select(set => new TimestampedSet(idDiscriminator, set.Timestamp.ToUniversalTime(), set.Set));
var dbSet = db.Set<TimestampedSet>();
dbSet.AddRange(entities);
return db.SaveChangesAsync(token);
}
public async Task<IEnumerable<TimestampedSetDto>> Get(Guid idDiscriminator, DateTimeOffset? geTimestamp, IEnumerable<string>? props, int skip, int take, CancellationToken token)
{
var dbSet = db.Set<TimestampedSet>();
var query = dbSet.Where(entity => entity.IdDiscriminator == idDiscriminator);
if (geTimestamp.HasValue)
query = ApplyGeTimestamp(query, geTimestamp.Value);
var data = await Materialize(query, token);
if (props is not null && props.Any())
data = ApplyPropsFilter(data, props);
return data;
}
public async Task<IEnumerable<TimestampedSetDto>> GetLast(Guid idDiscriminator, IEnumerable<string>? props, int take, CancellationToken token)
{
var dbSet = db.Set<TimestampedSet>();
var query = dbSet.Where(entity => entity.IdDiscriminator == idDiscriminator);
query = query.OrderByDescending(entity => entity.Timestamp)
.Take(take)
.OrderBy(entity => entity.Timestamp);
var data = await Materialize(query, token);
if (props is not null && props.Any())
data = ApplyPropsFilter(data, props);
return data;
}
public Task<int> Count(Guid idDiscriminator, CancellationToken token)
{
var dbSet = db.Set<TimestampedSet>();
var query = dbSet.Where(entity => entity.IdDiscriminator == idDiscriminator);
return query.CountAsync(token);
}
public async Task<DatesRangeDto?> GetDatesRange(Guid idDiscriminator, CancellationToken token)
{
var query = db.Set<TimestampedSet>()
.GroupBy(entity => entity.IdDiscriminator)
.Select(group => new
{
Min = group.Min(entity => entity.Timestamp),
Max = group.Max(entity => entity.Timestamp),
});
var item = await query.FirstOrDefaultAsync(token);
if (item is null)
return null;
return new DatesRangeDto
{
From = item.Min,
To = item.Max,
};
}
private static async Task<IEnumerable<TimestampedSetDto>> Materialize(IQueryable<TimestampedSet> query, CancellationToken token)
{
var dtoQuery = query.Select(entity => new TimestampedSetDto(entity.Timestamp, entity.Set));
var dtos = await dtoQuery.ToArrayAsync(token);
return dtos;
}
private static IQueryable<TimestampedSet> ApplyGeTimestamp(IQueryable<TimestampedSet> query, DateTimeOffset geTimestamp)
{
var geTimestampUtc = geTimestamp.ToUniversalTime();
return query.Where(entity => entity.Timestamp >= geTimestampUtc);
}
private static IEnumerable<TimestampedSetDto> ApplyPropsFilter(IEnumerable<TimestampedSetDto> query, IEnumerable<string> props)
{
var newQuery = query
.Select(entity => new TimestampedSetDto(
entity.Timestamp,
entity.Set
.Where(prop => props.Contains(prop.Key))
.ToDictionary(prop => prop.Key, prop => prop.Value)
));
return newQuery;
}
}

8
Persistence/Models/TimestampedSetDto.cs
View File

@ -1,8 +0,0 @@
namespace Persistence.Models;
/// <summary>
/// набор данных с отметкой времени
/// </summary>
/// <param name="Timestamp">отметка времени</param>
/// <param name="Set">набор данных</param>
public record TimestampedSetDto(DateTimeOffset Timestamp, IDictionary<string, object> Set);

59
Persistence/Repositories/ITimestampedSetRepository.cs
View File

@ -1,59 +0,0 @@
using Persistence.Models;
namespace Persistence.Repositories;
/// <summary>
/// Репозиторий для хранения разных наборов данных временных рядов.
/// idDiscriminator - идентифицирует конкретный набор данных, прим.: циклы измерения АСИБР, или отчет о DrillTest.
/// idDiscriminator формируют клиенты и только им известно что они обозначают.
/// Так как данные приходят редко, то их прореживания для построения графиков не предусмотрено.
/// </summary>
public interface ITimestampedSetRepository
{
/// <summary>
/// Количество записей по указанному набору в БД. Для пагинации.
/// </summary>
/// <param name="idDiscriminator"></param>
/// <param name="token"></param>
/// <returns></returns>
Task<int> Count(Guid idDiscriminator, CancellationToken token);
/// <summary>
/// Получение данных с фильтрацией. Значение фильтра null - отключен
/// </summary>
/// <param name="idDiscriminator">Идентификатор набора</param>
/// <param name="geTimestamp">Фильтр позднее даты</param>
/// <param name="props">Фильтр свойств набора. Можно запросить только некоторые свойства из набора</param>
/// <param name="skip"></param>
/// <param name="take"></param>
/// <param name="token"></param>
/// <returns></returns>
Task<IEnumerable<TimestampedSetDto>> Get(Guid idDiscriminator, DateTimeOffset? geTimestamp, IEnumerable<string>? props, int skip, int take, CancellationToken token);
/// <summary>
/// Диапазон дат за которые есть данные
/// </summary>
/// <param name="idDiscriminator"></param>
/// <param name="token"></param>
/// <returns></returns>
Task<DatesRangeDto?> GetDatesRange(Guid idDiscriminator, CancellationToken token);
/// <summary>
/// Получить последние данные
/// </summary>
/// <param name="idDiscriminator"></param>
/// <param name="props"></param>
/// <param name="take"></param>
/// <param name="token"></param>
/// <returns></returns>
Task<IEnumerable<TimestampedSetDto>> GetLast(Guid idDiscriminator, IEnumerable<string>? props, int take, CancellationToken token);
/// <summary>
/// Добавление новых данных
/// </summary>
/// <param name="idDiscriminator"></param>
/// <param name="sets"></param>
/// <param name="token"></param>
/// <returns></returns>
Task<int> InsertRange(Guid idDiscriminator, IEnumerable<TimestampedSetDto> sets, CancellationToken token);
}