diff --git a/DD.Persistence.API/Controllers/DataSourceSystemController.cs b/DD.Persistence.API/Controllers/DataSourceSystemController.cs index 5e551a9..896af1f 100644 --- a/DD.Persistence.API/Controllers/DataSourceSystemController.cs +++ b/DD.Persistence.API/Controllers/DataSourceSystemController.cs @@ -14,9 +14,9 @@ namespace DD.Persistence.API.Controllers; [Route("api/[controller]")] public class DataSourceSystemController : ControllerBase { - private readonly IRelatedDataRepository dataSourceSystemRepository; + private readonly IDataSourceSystemRepository dataSourceSystemRepository; - public DataSourceSystemController(IRelatedDataRepository dataSourceSystemRepository) + public DataSourceSystemController(IDataSourceSystemRepository dataSourceSystemRepository) { this.dataSourceSystemRepository = dataSourceSystemRepository; } diff --git a/DD.Persistence.API/Controllers/TimestampedValuesController.cs b/DD.Persistence.API/Controllers/TimestampedValuesController.cs index 66206ad..759d88a 100644 --- a/DD.Persistence.API/Controllers/TimestampedValuesController.cs +++ b/DD.Persistence.API/Controllers/TimestampedValuesController.cs @@ -1,6 +1,7 @@ using DD.Persistence.Models; using DD.Persistence.Models.Common; -using DD.Persistence.Repositories; +using DD.Persistence.Services.Interfaces; +using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; using System.Net; @@ -10,13 +11,13 @@ namespace DD.Persistence.API.Controllers; /// Хранение наборов данных с отметкой времени. /// [ApiController] -//[Authorize] +[Authorize] [Route("api/[controller]/{discriminatorId}")] public class TimestampedValuesController : ControllerBase { - private readonly ITimestampedValuesRepository timestampedValuesRepository; + private readonly ITimestampedValuesService timestampedValuesRepository; - public TimestampedValuesController(ITimestampedValuesRepository repository) + public TimestampedValuesController(ITimestampedValuesService repository) { this.timestampedValuesRepository = repository; } @@ -47,6 +48,8 @@ public class TimestampedValuesController : ControllerBase /// /// [HttpGet] + [ProducesResponseType(typeof(IEnumerable), (int)HttpStatusCode.OK)] + [ProducesResponseType((int)HttpStatusCode.NoContent)] public async Task>> Get([FromRoute] Guid discriminatorId, DateTimeOffset? timestampBegin, [FromQuery] string[]? columnNames, int skip, int take, CancellationToken token) { var result = await timestampedValuesRepository.Get(discriminatorId, timestampBegin, columnNames, skip, take, token); @@ -61,6 +64,8 @@ public class TimestampedValuesController : ControllerBase /// Фильтр позднее даты /// [HttpGet("gtdate")] + [ProducesResponseType(typeof(IEnumerable), (int)HttpStatusCode.OK)] + [ProducesResponseType((int)HttpStatusCode.NoContent)] public async Task>> GetGtDate([FromRoute] Guid discriminatorId, DateTimeOffset timestampBegin, CancellationToken token) { var result = await timestampedValuesRepository.GetGtDate(discriminatorId, timestampBegin, token); @@ -75,6 +80,8 @@ public class TimestampedValuesController : ControllerBase /// /// [HttpGet("first")] + [ProducesResponseType(typeof(IEnumerable), (int)HttpStatusCode.OK)] + [ProducesResponseType((int)HttpStatusCode.NoContent)] public async Task>> GetFirst([FromRoute] Guid discriminatorId, int take, CancellationToken token) { var result = await timestampedValuesRepository.GetFirst(discriminatorId, take, token); @@ -89,6 +96,8 @@ public class TimestampedValuesController : ControllerBase /// /// [HttpGet("last")] + [ProducesResponseType(typeof(IEnumerable), (int)HttpStatusCode.OK)] + [ProducesResponseType((int)HttpStatusCode.NoContent)] public async Task>> GetLast([FromRoute] Guid discriminatorId, int take, CancellationToken token) { var result = await timestampedValuesRepository.GetLast(discriminatorId, take, token); @@ -105,6 +114,8 @@ public class TimestampedValuesController : ControllerBase /// /// [HttpGet("resampled")] + [ProducesResponseType(typeof(IEnumerable), (int)HttpStatusCode.OK)] + [ProducesResponseType((int)HttpStatusCode.NoContent)] public async Task>> GetResampledData([FromRoute] Guid discriminatorId, DateTimeOffset timestampBegin, double intervalSec = 600d, int approxPointsCount = 1024, CancellationToken token = default) { var result = await timestampedValuesRepository.GetResampledData(discriminatorId, timestampBegin, intervalSec, approxPointsCount, token); diff --git a/DD.Persistence.API/DependencyInjection.cs b/DD.Persistence.API/DependencyInjection.cs index 6b0754c..b543841 100644 --- a/DD.Persistence.API/DependencyInjection.cs +++ b/DD.Persistence.API/DependencyInjection.cs @@ -53,6 +53,7 @@ public static class DependencyInjection public static void AddServices(this IServiceCollection services) { services.AddTransient(); + services.AddTransient(); } #region Authentication diff --git a/DD.Persistence.Database/Entity/ValuesIdentity.cs b/DD.Persistence.Database/Entity/DataScheme.cs similarity index 60% rename from DD.Persistence.Database/Entity/ValuesIdentity.cs rename to DD.Persistence.Database/Entity/DataScheme.cs index dc7f607..7936d0c 100644 --- a/DD.Persistence.Database/Entity/ValuesIdentity.cs +++ b/DD.Persistence.Database/Entity/DataScheme.cs @@ -4,11 +4,11 @@ using System.ComponentModel.DataAnnotations.Schema; namespace DD.Persistence.Database.Entity; -public class ValuesIdentity +public class DataScheme { [Key, Comment("Дискриминатор системы"),] public Guid DiscriminatorId { get; set; } - [Comment("Идентификаторы"), Column(TypeName = "jsonb")] - public string[] Identity { get; set; } = []; + [Comment("Наименования полей в порядке индексации"), Column(TypeName = "jsonb")] + public string[] PropNames { get; set; } = []; } diff --git a/DD.Persistence.Database/Entity/TimestampedValues.cs b/DD.Persistence.Database/Entity/TimestampedValues.cs index 05c7a57..28c8946 100644 --- a/DD.Persistence.Database/Entity/TimestampedValues.cs +++ b/DD.Persistence.Database/Entity/TimestampedValues.cs @@ -18,5 +18,5 @@ public class TimestampedValues : ITimestampedItem public required object[] Values { get; set; } [Required, ForeignKey(nameof(DiscriminatorId)), Comment("Идентификаторы")] - public virtual ValuesIdentity? ValuesIdentity { get; set; } + public virtual DataScheme? DataScheme { get; set; } } diff --git a/DD.Persistence.Database/PersistenceDbContext.cs b/DD.Persistence.Database/PersistenceDbContext.cs index 18a01cc..5a9a13c 100644 --- a/DD.Persistence.Database/PersistenceDbContext.cs +++ b/DD.Persistence.Database/PersistenceDbContext.cs @@ -11,7 +11,7 @@ public class PersistenceDbContext : DbContext { public DbSet Setpoint => Set(); - public DbSet ValuesIdentities => Set(); + public DbSet DataSchemes => Set(); public DbSet TimestampedValues => Set(); @@ -31,8 +31,8 @@ public class PersistenceDbContext : DbContext protected override void OnModelCreating(ModelBuilder modelBuilder) { - modelBuilder.Entity() - .Property(e => e.Identity) + modelBuilder.Entity() + .Property(e => e.PropNames) .HasJsonConversion(); modelBuilder.Entity() diff --git a/DD.Persistence.IntegrationTests/Controllers/TimestampedValuesControllerTest.cs b/DD.Persistence.IntegrationTests/Controllers/TimestampedValuesControllerTest.cs index b166631..71a49d5 100644 --- a/DD.Persistence.IntegrationTests/Controllers/TimestampedValuesControllerTest.cs +++ b/DD.Persistence.IntegrationTests/Controllers/TimestampedValuesControllerTest.cs @@ -13,9 +13,9 @@ using Xunit; namespace DD.Persistence.IntegrationTests.Controllers; public class TimestampedValuesControllerTest : BaseIntegrationTest { - private static readonly string SystemCacheKey = $"{typeof(ValuesIdentity).FullName}CacheKey"; private readonly ITimestampedValuesClient timestampedValuesClient; private readonly IMemoryCache memoryCache; + private IEnumerable discriminatorIds = []; public TimestampedValuesControllerTest(WebAppFactoryFixture factory) : base(factory) { @@ -32,6 +32,7 @@ public class TimestampedValuesControllerTest : BaseIntegrationTest public async Task AddRange_returns_success() { var discriminatorId = Guid.NewGuid(); + discriminatorIds.Append(discriminatorId); await AddRange(discriminatorId); } @@ -43,6 +44,7 @@ public class TimestampedValuesControllerTest : BaseIntegrationTest Cleanup(); var discriminatorId = Guid.NewGuid(); + discriminatorIds.Append(discriminatorId); //act var response = await timestampedValuesClient.Get(discriminatorId, null, null, 0, 1, CancellationToken.None); @@ -59,6 +61,8 @@ public class TimestampedValuesControllerTest : BaseIntegrationTest Cleanup(); var discriminatorId = Guid.NewGuid(); + discriminatorIds.Append(discriminatorId); + var timestampBegin = DateTimeOffset.UtcNow.AddDays(-1); var columnNames = new List() { "A", "C" }; var skip = 5; @@ -94,6 +98,8 @@ public class TimestampedValuesControllerTest : BaseIntegrationTest //arrange Cleanup(); var discriminatorId = Guid.NewGuid(); + discriminatorIds.Append(discriminatorId); + var timestampBegin = DateTimeOffset.UtcNow.AddDays(-1); //act @@ -109,6 +115,8 @@ public class TimestampedValuesControllerTest : BaseIntegrationTest //arrange Cleanup(); var discriminatorId = Guid.NewGuid(); + discriminatorIds.Append(discriminatorId); + var dtos = await AddRange(discriminatorId); var timestampBegin = DateTimeOffset.UtcNow.AddSeconds(-5); @@ -130,6 +138,8 @@ public class TimestampedValuesControllerTest : BaseIntegrationTest //arrange Cleanup(); var discriminatorId = Guid.NewGuid(); + discriminatorIds.Append(discriminatorId); + var take = 1; //act @@ -145,6 +155,8 @@ public class TimestampedValuesControllerTest : BaseIntegrationTest //arrange Cleanup(); var discriminatorId = Guid.NewGuid(); + discriminatorIds.Append(discriminatorId); + var dtos = await AddRange(discriminatorId); var take = 1; @@ -172,7 +184,10 @@ public class TimestampedValuesControllerTest : BaseIntegrationTest { //arrange Cleanup(); + var discriminatorId = Guid.NewGuid(); + discriminatorIds.Append(discriminatorId); + var take = 1; //act @@ -187,7 +202,10 @@ public class TimestampedValuesControllerTest : BaseIntegrationTest { //arrange Cleanup(); + var discriminatorId = Guid.NewGuid(); + discriminatorIds.Append(discriminatorId); + var dtos = await AddRange(discriminatorId); var take = 1; @@ -215,7 +233,10 @@ public class TimestampedValuesControllerTest : BaseIntegrationTest { //arrange Cleanup(); + var discriminatorId = Guid.NewGuid(); + discriminatorIds.Append(discriminatorId); + var timestampBegin = DateTimeOffset.UtcNow; //act @@ -230,7 +251,10 @@ public class TimestampedValuesControllerTest : BaseIntegrationTest { //arrange Cleanup(); + var discriminatorId = Guid.NewGuid(); + discriminatorIds.Append(discriminatorId); + var count = 2048; var timestampBegin = DateTimeOffset.UtcNow; var dtos = await AddRange(discriminatorId, count); @@ -253,7 +277,9 @@ public class TimestampedValuesControllerTest : BaseIntegrationTest { //arrange Cleanup(); + var discriminatorId = Guid.NewGuid(); + discriminatorIds.Append(discriminatorId); //act var response = await timestampedValuesClient.Count(discriminatorId, CancellationToken.None); @@ -267,7 +293,10 @@ public class TimestampedValuesControllerTest : BaseIntegrationTest { //arrange Cleanup(); + var discriminatorId = Guid.NewGuid(); + discriminatorIds.Append(discriminatorId); + var dtos = await AddRange(discriminatorId); //act @@ -283,7 +312,9 @@ public class TimestampedValuesControllerTest : BaseIntegrationTest { //arrange Cleanup(); + var discriminatorId = Guid.NewGuid(); + discriminatorIds.Append(discriminatorId); //act var response = await timestampedValuesClient.GetDatesRange(discriminatorId, CancellationToken.None); @@ -297,7 +328,10 @@ public class TimestampedValuesControllerTest : BaseIntegrationTest { //arrange Cleanup(); + var discriminatorId = Guid.NewGuid(); + discriminatorIds.Append(discriminatorId); + var dtos = await AddRange(discriminatorId); //act @@ -364,8 +398,8 @@ public class TimestampedValuesControllerTest : BaseIntegrationTest private void Cleanup() { - memoryCache.Remove(SystemCacheKey); + discriminatorIds = []; dbContext.CleanupDbSet(); - dbContext.CleanupDbSet(); + dbContext.CleanupDbSet(); } } diff --git a/DD.Persistence.Models/DataSchemeDto.cs b/DD.Persistence.Models/DataSchemeDto.cs new file mode 100644 index 0000000..d1efdd5 --- /dev/null +++ b/DD.Persistence.Models/DataSchemeDto.cs @@ -0,0 +1,17 @@ +namespace DD.Persistence.Models; + +/// +/// Схема для набора данных +/// +public class DataSchemeDto +{ + /// + /// Дискриминатор + /// + public Guid DiscriminatorId { get; set; } + + /// + /// Наименования полей + /// + public string[] PropNames { get; set; } = []; +} diff --git a/DD.Persistence.Models/ValuesIdentityDto.cs b/DD.Persistence.Models/ValuesIdentityDto.cs deleted file mode 100644 index 36d91fc..0000000 --- a/DD.Persistence.Models/ValuesIdentityDto.cs +++ /dev/null @@ -1,17 +0,0 @@ -namespace DD.Persistence.Models; - -/// -/// Набор идентификаторов для набора данных -/// -public class ValuesIdentityDto -{ - /// - /// Дискриминатор системы - /// - public Guid DiscriminatorId { get; set; } - - /// - /// Идентификаторы - /// - public string[] Identity { get; set; } = []; -} diff --git a/DD.Persistence.Repository/DependencyInjection.cs b/DD.Persistence.Repository/DependencyInjection.cs index 498849a..d0a5d37 100644 --- a/DD.Persistence.Repository/DependencyInjection.cs +++ b/DD.Persistence.Repository/DependencyInjection.cs @@ -41,10 +41,8 @@ public static class DependencyInjection services.AddTransient(); services.AddTransient(); services.AddTransient(); - services.AddTransient, - RelatedDataCachedRepository>(); - services.AddTransient, - RelatedDataCachedRepository>(); + services.AddTransient(); + services.AddTransient(); return services; } diff --git a/DD.Persistence.Repository/QueryBuilders.cs b/DD.Persistence.Repository/QueryBuilders.cs index 5fb3eb5..53e72a4 100644 --- a/DD.Persistence.Repository/QueryBuilders.cs +++ b/DD.Persistence.Repository/QueryBuilders.cs @@ -3,6 +3,7 @@ using DD.Persistence.Models.Requests; using DD.Persistence.Models.Common; using DD.Persistence.ModelsAbstractions; using DD.Persistence.Database.EntityAbstractions; +using DD.Persistence.Extensions; namespace DD.Persistence.Repository; diff --git a/DD.Persistence.Repository/Repositories/DataSchemeRepository.cs b/DD.Persistence.Repository/Repositories/DataSchemeRepository.cs new file mode 100644 index 0000000..3fe99d2 --- /dev/null +++ b/DD.Persistence.Repository/Repositories/DataSchemeRepository.cs @@ -0,0 +1,34 @@ +using DD.Persistence.Database.Entity; +using DD.Persistence.Models; +using DD.Persistence.Repositories; +using Mapster; +using Microsoft.EntityFrameworkCore; + +namespace DD.Persistence.Repository.Repositories; +public class DataSchemeRepository : IDataSchemeRepository +{ + protected DbContext db; + public DataSchemeRepository(DbContext db) + { + this.db = db; + } + protected virtual IQueryable GetQueryReadOnly() => db.Set(); + + public virtual async Task Add(DataSchemeDto dataSourceSystemDto, CancellationToken token) + { + var entity = dataSourceSystemDto.Adapt(); + + await db.Set().AddAsync(entity, token); + await db.SaveChangesAsync(token); + } + + public virtual async Task GetByDiscriminator(Guid discriminatorId, CancellationToken token) + { + var query = GetQueryReadOnly() + .Where(e => e.DiscriminatorId == discriminatorId); + var entity = await query.ToArrayAsync(); + var dto = entity.Select(e => e.Adapt()).FirstOrDefault(); + + return dto; + } +} diff --git a/DD.Persistence.Repository/Repositories/DataSourceSystemRepository.cs b/DD.Persistence.Repository/Repositories/DataSourceSystemRepository.cs new file mode 100644 index 0000000..d8b6c0a --- /dev/null +++ b/DD.Persistence.Repository/Repositories/DataSourceSystemRepository.cs @@ -0,0 +1,33 @@ +using DD.Persistence.Database.Entity; +using DD.Persistence.Models; +using DD.Persistence.Repositories; +using Mapster; +using Microsoft.EntityFrameworkCore; + +namespace DD.Persistence.Repository.Repositories; +public class DataSourceSystemRepository : IDataSourceSystemRepository +{ + protected DbContext db; + public DataSourceSystemRepository(DbContext db) + { + this.db = db; + } + protected virtual IQueryable GetQueryReadOnly() => db.Set(); + + public virtual async Task Add(DataSourceSystemDto dataSourceSystemDto, CancellationToken token) + { + var entity = dataSourceSystemDto.Adapt(); + + await db.Set().AddAsync(entity, token); + await db.SaveChangesAsync(token); + } + + public virtual async Task> Get(CancellationToken token) + { + var query = GetQueryReadOnly(); + var entities = await query.ToArrayAsync(token); + var dtos = entities.Select(e => e.Adapt()); + + return dtos; + } +} diff --git a/DD.Persistence.Repository/Repositories/RelatedDataRepository.cs b/DD.Persistence.Repository/Repositories/RelatedDataRepository.cs deleted file mode 100644 index 7efc329..0000000 --- a/DD.Persistence.Repository/Repositories/RelatedDataRepository.cs +++ /dev/null @@ -1,33 +0,0 @@ -using DD.Persistence.Repositories; -using Mapster; -using Microsoft.EntityFrameworkCore; - -namespace DD.Persistence.Repository.Repositories; -public class RelatedDataRepository : IRelatedDataRepository - where TDto : class, new() - where TEntity : class, new() -{ - protected DbContext db; - public RelatedDataRepository(DbContext db) - { - this.db = db; - } - protected virtual IQueryable GetQueryReadOnly() => db.Set(); - - public virtual async Task Add(TDto dataSourceSystemDto, CancellationToken token) - { - var entity = dataSourceSystemDto.Adapt(); - - await db.Set().AddAsync(entity, token); - await db.SaveChangesAsync(token); - } - - public virtual async Task> Get(CancellationToken token) - { - var query = GetQueryReadOnly(); - var entities = await query.ToArrayAsync(token); - var dtos = entities.Select(e => e.Adapt()); - - return dtos; - } -} diff --git a/DD.Persistence.Repository/Repositories/TechMessagesRepository.cs b/DD.Persistence.Repository/Repositories/TechMessagesRepository.cs index 6d4f3b1..1a84f25 100644 --- a/DD.Persistence.Repository/Repositories/TechMessagesRepository.cs +++ b/DD.Persistence.Repository/Repositories/TechMessagesRepository.cs @@ -1,4 +1,5 @@ using DD.Persistence.Database.Entity; +using DD.Persistence.Extensions; using DD.Persistence.Models; using DD.Persistence.Models.Common; using DD.Persistence.Models.Requests; @@ -10,10 +11,10 @@ namespace DD.Persistence.Repository.Repositories { public class TechMessagesRepository : ITechMessagesRepository { - private readonly IRelatedDataRepository sourceSystemRepository; + private readonly IDataSourceSystemRepository sourceSystemRepository; private DbContext db; - public TechMessagesRepository(DbContext db, IRelatedDataRepository sourceSystemRepository) + public TechMessagesRepository(DbContext db, IDataSourceSystemRepository sourceSystemRepository) { this.db = db; this.sourceSystemRepository = sourceSystemRepository; diff --git a/DD.Persistence.Repository/Repositories/TimestampedValuesRepository.cs b/DD.Persistence.Repository/Repositories/TimestampedValuesRepository.cs index 9e1597c..b2f8203 100644 --- a/DD.Persistence.Repository/Repositories/TimestampedValuesRepository.cs +++ b/DD.Persistence.Repository/Repositories/TimestampedValuesRepository.cs @@ -2,32 +2,25 @@ using DD.Persistence.Models; using DD.Persistence.Models.Common; using DD.Persistence.Repositories; -using DD.Persistence.Repository.Extensions; using Microsoft.EntityFrameworkCore; namespace DD.Persistence.Repository.Repositories; public class TimestampedValuesRepository : ITimestampedValuesRepository { private readonly DbContext db; - private readonly IRelatedDataRepository relatedDataRepository; - public TimestampedValuesRepository(DbContext db, IRelatedDataRepository relatedDataRepository) + public TimestampedValuesRepository(DbContext db) { this.db = db; - this.relatedDataRepository = relatedDataRepository; } - protected virtual IQueryable GetQueryReadOnly() => this.db.Set() - .Include(e => e.ValuesIdentity); + protected virtual IQueryable GetQueryReadOnly() => this.db.Set(); public async virtual Task AddRange(Guid discriminatorId, IEnumerable dtos, CancellationToken token) { var timestampedValuesEntities = new List(); foreach (var dto in dtos) { - var keys = dto.Values.Keys.ToArray(); - await CreateValuesIdentityIfNotExist(discriminatorId, keys, token); - var timestampedValuesEntity = new TimestampedValues() { DiscriminatorId = discriminatorId, @@ -44,7 +37,7 @@ public class TimestampedValuesRepository : ITimestampedValuesRepository return result; } - public async virtual Task> Get(Guid discriminatorId, DateTimeOffset? timestampBegin, IEnumerable? columnNames, int skip, int take, CancellationToken token) + public async virtual Task>> Get(Guid discriminatorId, DateTimeOffset? timestampBegin, IEnumerable? columnNames, int skip, int take, CancellationToken token) { var query = GetQueryReadOnly() .Where(entity => entity.DiscriminatorId == discriminatorId); @@ -59,68 +52,79 @@ public class TimestampedValuesRepository : ITimestampedValuesRepository .OrderBy(item => item.Timestamp) .Skip(skip) .Take(take); - var data = await Materialize(discriminatorId, query, token); + var entities = await query.ToArrayAsync(token); - // Фильтрация по запрашиваемым полям - if (!columnNames.IsNullOrEmpty()) - { - data = ReduceSetColumnsByNames(data, columnNames!); - } + var result = entities.Select(e => Tuple.Create( + e.Timestamp, + e.Values + )); - return data; + return result; } - public async virtual Task> GetFirst(Guid discriminatorId, int takeCount, CancellationToken token) + public async virtual Task>> GetFirst(Guid discriminatorId, int takeCount, CancellationToken token) { var query = GetQueryReadOnly() .OrderBy(e => e.Timestamp) .Take(takeCount); + var entities = await query.ToArrayAsync(token); - var dtos = await Materialize(discriminatorId, query, token); + var result = entities.Select(e => Tuple.Create( + e.Timestamp, + e.Values + )); - return dtos; + return result; } - public async virtual Task> GetLast(Guid discriminatorId, int takeCount, CancellationToken token) + public async virtual Task>> GetLast(Guid discriminatorId, int takeCount, CancellationToken token) { var query = GetQueryReadOnly() .OrderByDescending(e => e.Timestamp) .Take(takeCount); + var entities = await query.ToArrayAsync(token); - var dtos = await Materialize(discriminatorId, query, token); + var result = entities.Select(e => Tuple.Create( + e.Timestamp, + e.Values + )); - return dtos; + return result; } - public async virtual Task> GetResampledData( + public async virtual Task>> GetResampledData( Guid discriminatorId, DateTimeOffset dateBegin, double intervalSec = 600d, int approxPointsCount = 1024, CancellationToken token = default) { - var dtos = await GetGtDate(discriminatorId, dateBegin, token); + var result = await GetGtDate(discriminatorId, dateBegin, token); var dateEnd = dateBegin.AddSeconds(intervalSec); - dtos = dtos - .Where(i => i.Timestamp <= dateEnd); + result = result + .Where(i => i.Item1 <= dateEnd); - var ratio = dtos.Count() / approxPointsCount; + var ratio = result.Count() / approxPointsCount; if (ratio > 1) - dtos = dtos + result = result .Where((_, index) => index % ratio == 0); - return dtos; + return result; } - public async virtual Task> GetGtDate(Guid discriminatorId, DateTimeOffset timestampBegin, CancellationToken token) + public async virtual Task>> GetGtDate(Guid discriminatorId, DateTimeOffset timestampBegin, CancellationToken token) { var query = GetQueryReadOnly() .Where(e => e.Timestamp > timestampBegin); + var entities = await query.ToArrayAsync(token); - var dtos = await Materialize(discriminatorId, query, token); + var result = entities.Select(e => Tuple.Create( + e.Timestamp, + e.Values + )); - return dtos; + return result; } public async virtual Task GetDatesRange(Guid discriminatorId, CancellationToken token) @@ -156,37 +160,12 @@ public class TimestampedValuesRepository : ITimestampedValuesRepository return query.CountAsync(token); } - private async Task> Materialize(Guid discriminatorId, IQueryable query, CancellationToken token) - { - var valuesIdentities = await relatedDataRepository.Get(token); - var valuesIdentity = valuesIdentities? - .FirstOrDefault(e => e.DiscriminatorId == discriminatorId); - if (valuesIdentity == null) - return []; - - var entities = await query.ToArrayAsync(token); - - var dtos = entities.Select(entity => - { - var dto = new TimestampedValuesDto() - { - Timestamp = entity.Timestamp.ToUniversalTime() - }; - - for (var i = 0; i < valuesIdentity.Identity.Count(); i++) - { - var key = valuesIdentity.Identity[i]; - var value = entity.Values[i]; - - dto.Values.Add(key, value); - } - - return dto; - }); - - return dtos; - } - + /// + /// Применить фильтр по дате + /// + /// + /// + /// private IQueryable ApplyGeTimestamp(IQueryable query, DateTimeOffset timestampBegin) { var geTimestampUtc = timestampBegin.ToUniversalTime(); @@ -196,46 +175,4 @@ public class TimestampedValuesRepository : ITimestampedValuesRepository return result; } - - private IEnumerable ReduceSetColumnsByNames(IEnumerable dtos, IEnumerable columnNames) - { - var result = dtos.Select(dto => - { - var reducedValues = dto.Values - .Where(v => columnNames.Contains(v.Key)) - .ToDictionary(); - dto.Values = reducedValues; - - return dto; - }); - - return result; - } - - private async Task CreateValuesIdentityIfNotExist(Guid discriminatorId, string[] keys, CancellationToken token) - { - var valuesIdentities = await relatedDataRepository.Get(token); - var valuesIdentity = valuesIdentities? - .FirstOrDefault(e => e.DiscriminatorId == discriminatorId); - - if (valuesIdentity is null) - { - valuesIdentity = new ValuesIdentityDto() - { - DiscriminatorId = discriminatorId, - Identity = keys - }; - await relatedDataRepository.Add(valuesIdentity, token); - - return; - } - - if (!valuesIdentity.Identity.SequenceEqual(keys)) - { - var expectedIdentity = string.Join(", ", valuesIdentity.Identity); - var actualIdentity = string.Join(", ", keys); - throw new InvalidOperationException($"Для системы {discriminatorId.ToString()} " + - $"характерен набор данных: [{expectedIdentity}], однако был передан набор: [{actualIdentity}]"); - } - } } diff --git a/DD.Persistence.Repository/RepositoriesCached/DataSchemeCachedRepository.cs b/DD.Persistence.Repository/RepositoriesCached/DataSchemeCachedRepository.cs new file mode 100644 index 0000000..aeb144a --- /dev/null +++ b/DD.Persistence.Repository/RepositoriesCached/DataSchemeCachedRepository.cs @@ -0,0 +1,30 @@ +using DD.Persistence.Models; +using DD.Persistence.Repository.Repositories; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Caching.Memory; + +namespace DD.Persistence.Repository.RepositoriesCached; +public class DataSchemeCachedRepository : DataSchemeRepository +{ + private readonly IMemoryCache memoryCache; + + public DataSchemeCachedRepository(DbContext db, IMemoryCache memoryCache) : base(db) + { + this.memoryCache = memoryCache; + } + + public override async Task Add(DataSchemeDto dataSourceSystemDto, CancellationToken token) + { + await base.Add(dataSourceSystemDto, token); + + memoryCache.Set(dataSourceSystemDto.DiscriminatorId, dataSourceSystemDto); + } + + public override async Task GetByDiscriminator(Guid discriminatorId, CancellationToken token) + { + var result = memoryCache.Get(discriminatorId) + ?? await base.GetByDiscriminator(discriminatorId, token); + + return result; + } +} diff --git a/DD.Persistence.Repository/RepositoriesCached/RelatedDataCachedRepository.cs b/DD.Persistence.Repository/RepositoriesCached/DataSourceSystemCachedRepository.cs similarity index 56% rename from DD.Persistence.Repository/RepositoriesCached/RelatedDataCachedRepository.cs rename to DD.Persistence.Repository/RepositoriesCached/DataSourceSystemCachedRepository.cs index 6f596ba..87e487e 100644 --- a/DD.Persistence.Repository/RepositoriesCached/RelatedDataCachedRepository.cs +++ b/DD.Persistence.Repository/RepositoriesCached/DataSourceSystemCachedRepository.cs @@ -1,27 +1,27 @@ -using DD.Persistence.Repository.Repositories; +using DD.Persistence.Database.Entity; +using DD.Persistence.Models; +using DD.Persistence.Repository.Repositories; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Caching.Memory; namespace DD.Persistence.Repository.RepositoriesCached; -public class RelatedDataCachedRepository : RelatedDataRepository - where TEntity : class, new() - where TDto : class, new() +public class DataSourceSystemCachedRepository : DataSourceSystemRepository { - private static readonly string SystemCacheKey = $"{typeof(TEntity).FullName}CacheKey"; + private static readonly string SystemCacheKey = $"{typeof(DataSourceSystem).FullName}CacheKey"; private readonly IMemoryCache memoryCache; private readonly TimeSpan? AbsoluteExpirationRelativeToNow = TimeSpan.FromMinutes(60); - public RelatedDataCachedRepository(DbContext db, IMemoryCache memoryCache) : base(db) + public DataSourceSystemCachedRepository(DbContext db, IMemoryCache memoryCache) : base(db) { this.memoryCache = memoryCache; } - public override async Task Add(TDto dataSourceSystemDto, CancellationToken token) + public override async Task Add(DataSourceSystemDto dataSourceSystemDto, CancellationToken token) { await base.Add(dataSourceSystemDto, token); memoryCache.Remove(SystemCacheKey); } - public override async Task> Get(CancellationToken token) + public override async Task> Get(CancellationToken token) { var systems = await memoryCache.GetOrCreateAsync(SystemCacheKey, async (cacheEntry) => { diff --git a/DD.Persistence.Repository/RepositoriesCached/TimestampedValuesCachedRepository.cs b/DD.Persistence.Repository/RepositoriesCached/TimestampedValuesCachedRepository.cs index 7bdaa98..f81f7a0 100644 --- a/DD.Persistence.Repository/RepositoriesCached/TimestampedValuesCachedRepository.cs +++ b/DD.Persistence.Repository/RepositoriesCached/TimestampedValuesCachedRepository.cs @@ -12,7 +12,7 @@ // private const int CacheItemsCount = 3600; -// public TimestampedValuesCachedRepository(DbContext db, IRelatedDataRepository relatedDataRepository) : base(db, relatedDataRepository) +// public TimestampedValuesCachedRepository(DbContext db, IDataSourceSystemRepository relatedDataRepository) : base(db, relatedDataRepository) // { // //Task.Run(async () => // //{ diff --git a/DD.Persistence/EFExtensions.cs b/DD.Persistence/Extensions/EFExtensions.cs similarity index 98% rename from DD.Persistence/EFExtensions.cs rename to DD.Persistence/Extensions/EFExtensions.cs index af6cd15..f1d1fb8 100644 --- a/DD.Persistence/EFExtensions.cs +++ b/DD.Persistence/Extensions/EFExtensions.cs @@ -2,7 +2,7 @@ using System.Linq.Expressions; using System.Reflection; -namespace DD.Persistence; +namespace DD.Persistence.Extensions; public static class EFExtensions { struct TypeAccessor @@ -23,7 +23,7 @@ public static class EFExtensions private static ConcurrentDictionary> TypePropSelectors { get; set; } = new(); private static MethodInfo GetExtOrderMethod(string methodName) - => typeof(System.Linq.Queryable) + => typeof(Queryable) .GetMethods() .Where(m => m.Name == methodName && m.IsGenericMethodDefinition && diff --git a/DD.Persistence.Repository/Extensions/IEnumerableExtensions.cs b/DD.Persistence/Extensions/IEnumerableExtensions.cs similarity index 93% rename from DD.Persistence.Repository/Extensions/IEnumerableExtensions.cs rename to DD.Persistence/Extensions/IEnumerableExtensions.cs index e307513..702a08b 100644 --- a/DD.Persistence.Repository/Extensions/IEnumerableExtensions.cs +++ b/DD.Persistence/Extensions/IEnumerableExtensions.cs @@ -1,4 +1,4 @@ -namespace DD.Persistence.Repository.Extensions; +namespace DD.Persistence.Extensions; public static class IEnumerableExtensions { diff --git a/DD.Persistence/Repositories/IDataSchemeRepository.cs b/DD.Persistence/Repositories/IDataSchemeRepository.cs new file mode 100644 index 0000000..ff6ef4b --- /dev/null +++ b/DD.Persistence/Repositories/IDataSchemeRepository.cs @@ -0,0 +1,25 @@ +using DD.Persistence.Models; + +namespace DD.Persistence.Repositories; + +/// +/// Репозиторий для работы со схемами наборов данных +/// +public interface IDataSchemeRepository +{ + /// + /// Добавить схему + /// + /// + /// + /// + Task Add(DataSchemeDto dataSourceSystemDto, CancellationToken token); + + /// + /// Вычитать схему + /// + /// Дискриминатор системы + /// + /// + Task GetByDiscriminator(Guid discriminatorId, CancellationToken token); +} \ No newline at end of file diff --git a/DD.Persistence/Repositories/IDataSourceSystemRepository.cs b/DD.Persistence/Repositories/IDataSourceSystemRepository.cs new file mode 100644 index 0000000..21d7b1c --- /dev/null +++ b/DD.Persistence/Repositories/IDataSourceSystemRepository.cs @@ -0,0 +1,23 @@ +using DD.Persistence.Models; + +namespace DD.Persistence.Repositories; + +/// +/// Репозиторий для работы с системами - источниками данных +/// +public interface IDataSourceSystemRepository +{ + /// + /// Добавить систему - источник данных + /// + /// + /// + /// + public Task Add(DataSourceSystemDto dataSourceSystemDto, CancellationToken token); + + /// + /// Получить список систем - источников данных + /// + /// + public Task> Get(CancellationToken token); +} diff --git a/DD.Persistence/Repositories/IRelatedDataRepository.cs b/DD.Persistence/Repositories/IRelatedDataRepository.cs deleted file mode 100644 index e7eae87..0000000 --- a/DD.Persistence/Repositories/IRelatedDataRepository.cs +++ /dev/null @@ -1,23 +0,0 @@ -namespace DD.Persistence.Repositories; - -/// -/// Интерфейс по работе с простой структурой данных, подразумевающей наличие связи с более сложной -/// В контексте TechMessagesRepository это системы - источники данных -/// В контексте TimestampedValuesRepository это идентификационные наборы (ключи для значений в соответствии с индексами в хранимых массивах) -/// -public interface IRelatedDataRepository -{ - /// - /// Добавить данные - /// - /// - /// - /// - public Task Add(TDto dataSourceSystemDto, CancellationToken token); - - /// - /// Получить список данных - /// - /// - public Task> Get(CancellationToken token); -} diff --git a/DD.Persistence/Repositories/ITimestampedValuesRepository.cs b/DD.Persistence/Repositories/ITimestampedValuesRepository.cs index abb860c..6bd2866 100644 --- a/DD.Persistence/Repositories/ITimestampedValuesRepository.cs +++ b/DD.Persistence/Repositories/ITimestampedValuesRepository.cs @@ -4,14 +4,14 @@ using DD.Persistence.RepositoriesAbstractions; namespace DD.Persistence.Repositories; /// -/// Интерфейс по работе с временными данными +/// Репозиторий для работы с временными данными /// public interface ITimestampedValuesRepository : ISyncRepository, ITimeSeriesBaseRepository { /// /// Добавление записей /// - /// + /// Дискриминатор (идентификатор) набора /// /// /// @@ -35,7 +35,7 @@ public interface ITimestampedValuesRepository : ISyncRepository, ITimeSeriesBase /// /// /// - Task> Get(Guid idDiscriminator, DateTimeOffset? geTimestamp, IEnumerable? columnNames, int skip, int take, CancellationToken token); + Task>> Get(Guid idDiscriminator, DateTimeOffset? geTimestamp, IEnumerable? columnNames, int skip, int take, CancellationToken token); /// /// Получение данных с начала @@ -44,7 +44,7 @@ public interface ITimestampedValuesRepository : ISyncRepository, ITimeSeriesBase /// Количество /// /// - Task> GetFirst(Guid discriminatorId, int takeCount, CancellationToken token); + Task>> GetFirst(Guid discriminatorId, int takeCount, CancellationToken token); /// /// Получение данных с конца @@ -53,5 +53,5 @@ public interface ITimestampedValuesRepository : ISyncRepository, ITimeSeriesBase /// Количество /// /// - Task> GetLast(Guid discriminatorId, int takeCount, CancellationToken token); + Task>> GetLast(Guid discriminatorId, int takeCount, CancellationToken token); } diff --git a/DD.Persistence/RepositoriesAbstractions/ISyncRepository.cs b/DD.Persistence/RepositoriesAbstractions/ISyncRepository.cs index d60e0ce..95bfee2 100644 --- a/DD.Persistence/RepositoriesAbstractions/ISyncRepository.cs +++ b/DD.Persistence/RepositoriesAbstractions/ISyncRepository.cs @@ -6,7 +6,7 @@ namespace DD.Persistence.RepositoriesAbstractions; /// /// Интерфейс по работе с данными /// -public interface ISyncRepository +public interface ISyncRepository // ToDo: исчерпывающая абстракция { /// /// Получить данные, начиная с определенной даты @@ -15,7 +15,7 @@ public interface ISyncRepository /// дата начала /// /// - Task> GetGtDate(Guid discriminatorId, DateTimeOffset dateBegin, CancellationToken token); + Task>> GetGtDate(Guid discriminatorId, DateTimeOffset dateBegin, CancellationToken token); /// diff --git a/DD.Persistence/RepositoriesAbstractions/ITimeSeriesBaseRepository.cs b/DD.Persistence/RepositoriesAbstractions/ITimeSeriesBaseRepository.cs index b8c6791..04b8e1a 100644 --- a/DD.Persistence/RepositoriesAbstractions/ITimeSeriesBaseRepository.cs +++ b/DD.Persistence/RepositoriesAbstractions/ITimeSeriesBaseRepository.cs @@ -5,7 +5,7 @@ namespace DD.Persistence.RepositoriesAbstractions; /// /// Интерфейс по работе с прореженными данными /// -public interface ITimeSeriesBaseRepository +public interface ITimeSeriesBaseRepository // ToDo: исчерпывающая абстракция { /// /// Получить список объектов с прореживанием @@ -16,7 +16,7 @@ public interface ITimeSeriesBaseRepository /// /// /// - Task> GetResampledData( + Task>> GetResampledData( Guid discriminatorId, DateTimeOffset dateBegin, double intervalSec = 600d, diff --git a/DD.Persistence/Services/Interfaces/ITimestampedValuesService.cs b/DD.Persistence/Services/Interfaces/ITimestampedValuesService.cs new file mode 100644 index 0000000..8e6d606 --- /dev/null +++ b/DD.Persistence/Services/Interfaces/ITimestampedValuesService.cs @@ -0,0 +1,85 @@ +using DD.Persistence.Models; +using DD.Persistence.Models.Common; + +namespace DD.Persistence.Services.Interfaces; + +/// +/// Сервис для работы с временными данными +/// +public interface ITimestampedValuesService +{ + /// + /// Добавление записей + /// + /// + /// + /// + /// + Task AddRange(Guid discriminatorId, IEnumerable dtos, CancellationToken token); + + /// + /// Количество записей по указанному набору в БД. Для пагинации + /// + /// Дискриминатор (идентификатор) набора + /// + /// + Task Count(Guid discriminatorId, CancellationToken token); + + /// + /// Получение данных с фильтрацией. Значение фильтра null - отключен + /// + /// Дискриминатор (идентификатор) набора + /// Фильтр позднее даты + /// Фильтр свойств набора. Можно запросить только некоторые свойства из набора + /// + /// + /// + /// + Task> Get(Guid discriminatorId, DateTimeOffset? geTimestamp, IEnumerable? columnNames, int skip, int take, CancellationToken token); + + /// + /// Получение данных с начала + /// + /// Дискриминатор (идентификатор) набора + /// Количество + /// + /// + Task> GetFirst(Guid discriminatorId, int takeCount, CancellationToken token); + + /// + /// Получить данные, начиная с определенной даты + /// + /// + /// дата начала + /// + /// + Task> GetGtDate(Guid discriminatorId, DateTimeOffset timestampBegin, CancellationToken token); + + /// + /// Получение данных с конца + /// + /// Дискриминатор (идентификатор) набора + /// Количество + /// + /// + Task> GetLast(Guid discriminatorId, int takeCount, CancellationToken token); + + /// + /// Получить список объектов с прореживанием + /// + /// + /// дата начала + /// + /// + /// + /// + Task> GetResampledData(Guid discriminatorId, DateTimeOffset timestampBegin, double intervalSec = 600, int approxPointsCount = 1024, CancellationToken token = default); + + /// + /// Получить диапазон дат, для которых есть данные + /// + /// + /// + /// + Task GetDatesRange(Guid discriminatorId, CancellationToken token); +} \ No newline at end of file diff --git a/DD.Persistence/Services/TimestampedValuesService.cs b/DD.Persistence/Services/TimestampedValuesService.cs new file mode 100644 index 0000000..2b7fc6f --- /dev/null +++ b/DD.Persistence/Services/TimestampedValuesService.cs @@ -0,0 +1,201 @@ +using DD.Persistence.Extensions; +using DD.Persistence.Models; +using DD.Persistence.Models.Common; +using DD.Persistence.Repositories; +using DD.Persistence.Services.Interfaces; + +namespace DD.Persistence.Services; + +/// +public class TimestampedValuesService : ITimestampedValuesService +{ + private readonly ITimestampedValuesRepository timestampedValuesRepository; + private readonly IDataSchemeRepository dataSchemeRepository; + + /// + public TimestampedValuesService(ITimestampedValuesRepository timestampedValuesRepository, IDataSchemeRepository relatedDataRepository) + { + this.timestampedValuesRepository = timestampedValuesRepository; + this.dataSchemeRepository = relatedDataRepository; + } + + /// + public async Task AddRange(Guid discriminatorId, IEnumerable dtos, CancellationToken token) + { + foreach (var dto in dtos) + { + var keys = dto.Values.Keys.ToArray(); + await CreateSystemSpecificationIfNotExist(discriminatorId, keys, token); + } + + var result = await timestampedValuesRepository.AddRange(discriminatorId, dtos, token); + + return result; + } + + /// + public async Task> Get(Guid discriminatorId, DateTimeOffset? geTimestamp, IEnumerable? columnNames, int skip, int take, CancellationToken token) + { + var result = await timestampedValuesRepository.Get(discriminatorId, geTimestamp, columnNames, skip, take, token); + + var dtos = await Materialize(discriminatorId, result, token); + + if (!columnNames.IsNullOrEmpty()) + { + dtos = ReduceSetColumnsByNames(dtos, columnNames!); + } + + return dtos; + } + + /// + public async Task> GetFirst(Guid discriminatorId, int takeCount, CancellationToken token) + { + var result = await timestampedValuesRepository.GetFirst(discriminatorId, takeCount, token); + + var dtos = await Materialize(discriminatorId, result, token); + + return dtos; + } + + /// + public async Task> GetLast(Guid discriminatorId, int takeCount, CancellationToken token) + { + var result = await timestampedValuesRepository.GetLast(discriminatorId, takeCount, token); + + var dtos = await Materialize(discriminatorId, result, token); + + return dtos; + } + + /// + public async Task> GetResampledData( + Guid discriminatorId, + DateTimeOffset beginTimestamp, + double intervalSec = 600d, + int approxPointsCount = 1024, + CancellationToken token = default) + { + var result = await timestampedValuesRepository.GetResampledData(discriminatorId, beginTimestamp, intervalSec, approxPointsCount, token); + + var dtos = await Materialize(discriminatorId, result, token); + + return dtos; + } + + /// + public async Task> GetGtDate(Guid discriminatorId, DateTimeOffset beginTimestamp, CancellationToken token) + { + var result = await timestampedValuesRepository.GetGtDate(discriminatorId, beginTimestamp, token); + + var dtos = await Materialize(discriminatorId, result, token); + + return dtos; + } + + /// + public async Task Count(Guid discriminatorId, CancellationToken token) + { + var result = await timestampedValuesRepository.Count(discriminatorId, token); + + return result; + } + + /// + public async virtual Task GetDatesRange(Guid discriminatorId, CancellationToken token) + { + var result = await timestampedValuesRepository.GetDatesRange(discriminatorId, token); + + return result; + } + + /// + /// Преобразовать результат запроса в набор dto + /// + /// + /// + /// + /// + private async Task> Materialize(Guid discriminatorId, IEnumerable> queryResult, CancellationToken token) + { + var systemSpecification = await dataSchemeRepository.GetByDiscriminator(discriminatorId, token); + if (systemSpecification is null) + { + return []; + } + + var dtos = queryResult.Select(entity => + { + var dto = new TimestampedValuesDto() + { + Timestamp = entity.Item1.ToUniversalTime() + }; + + var identity = systemSpecification!.PropNames; + for (var i = 0; i < identity.Count(); i++) + { + var key = identity[i]; + var value = entity.Item2[i]; + + dto.Values.Add(key, value); + } + + return dto; + }); + + return dtos; + } + + /// + /// Создать спецификацию, при отсутствии таковой + /// + /// Дискриминатор системы + /// Набор наименований полей + /// + /// + /// Некорректный набор наименований полей + private async Task CreateSystemSpecificationIfNotExist(Guid discriminatorId, string[] fieldNames, CancellationToken token) + { + var systemSpecification = await dataSchemeRepository.GetByDiscriminator(discriminatorId, token); + if (systemSpecification is null) + { + systemSpecification = new DataSchemeDto() + { + DiscriminatorId = discriminatorId, + PropNames = fieldNames + }; + await dataSchemeRepository.Add(systemSpecification, token); + + return; + } + + if (!systemSpecification.PropNames.SequenceEqual(fieldNames)) + { + var expectedFieldNames = string.Join(", ", systemSpecification.PropNames); + var actualFieldNames = string.Join(", ", fieldNames); + throw new InvalidOperationException($"Для системы {discriminatorId.ToString()} " + + $"характерен набор данных: [{expectedFieldNames}], однако был передан набор: [{actualFieldNames}]"); + } + } + + /// + /// Отсеить лишние поля в соответствии с заданным фильтром + /// + /// + /// Поля, которые необходимо оставить + /// + private IEnumerable ReduceSetColumnsByNames(IEnumerable dtos, IEnumerable fieldNames) + { + var result = dtos.Select(dto => + { + var reducedValues = dto.Values + .Where(v => fieldNames.Contains(v.Key)) + .ToDictionary(); + dto.Values = reducedValues; + + return dto; + }); + + return result; + } +}