on.nemtina/persistence
1
0
Fork 0
Code Issues Pull Requests 3 Actions Packages Projects Releases Wiki Activity

Доработки
Some checks failed
Unit tests / test (push) Failing after 55s
Details

Browse Source
This commit is contained in:
Roman Efremov 2025-01-17 17:21:54 +05:00
parent 556fdcbca0
commit fd276f5a43
16 changed files with 771 additions and 526 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

125
DD.Persistence.API/Controllers/TimestampedValuesController.cs
View File

@ -15,26 +15,26 @@ namespace DD.Persistence.API.Controllers;
[Route("api/[controller]/{discriminatorId}")]
public class TimestampedValuesController : ControllerBase
{
private readonly ITimestampedValuesRepository repository;
private readonly ITimestampedValuesRepository timestampedValuesRepository;
public TimestampedValuesController(ITimestampedValuesRepository repository)
{
this.repository = repository;
this.timestampedValuesRepository = repository;
}
/// <summary>
/// Записать новые данные
/// Записать новые данные.
/// Предполагается что данные с одним дискриминатором имеют одинаковую структуру
/// </summary>
/// <param name="discriminatorId">Дискриминатор (идентификатор) набора</param>
/// <param name="sets"></param>
/// <param name="dtos"></param>
/// <param name="token"></param>
/// <returns>кол-во затронутых записей</returns>
[HttpPost]
[ProducesResponseType(typeof(int), (int)HttpStatusCode.OK)]
public async Task<IActionResult> AddRange([FromRoute] Guid discriminatorId, [FromBody] IEnumerable<TimestampedValuesDto> sets, CancellationToken token)
[ProducesResponseType(typeof(int), (int)HttpStatusCode.Created)]
public async Task<IActionResult> AddRange([FromRoute] Guid discriminatorId, [FromBody] IEnumerable<TimestampedValuesDto> dtos, CancellationToken token)
{
var result = await repository.AddRange(discriminatorId, sets, token);
var result = await timestampedValuesRepository.AddRange(discriminatorId, dtos, token);
return Ok(result);
}
@ -42,81 +42,100 @@ public class TimestampedValuesController : ControllerBase
/// Получение данных с фильтрацией. Значение фильтра null - отключен
/// </summary>
/// <param name="discriminatorId">Дискриминатор (идентификатор) набора</param>
/// <param name="geTimestamp">Фильтр позднее даты</param>
/// <param name="columnNames">Фильтр свойств набора. Можно запросить только некоторые свойства из набора</param>
/// <param name="timestampBegin">Фильтр позднее даты</param>
/// <param name="columnNames">Фильтр свойств набора</param>
/// <param name="skip"></param>
/// <param name="take"></param>
/// <param name="token"></param>
/// <returns>Фильтрованный набор данных с сортировкой по отметке времени</returns>
[HttpGet]
[ProducesResponseType(typeof(IEnumerable<TimestampedValuesDto>), (int)HttpStatusCode.OK)]
public async Task<IActionResult> Get([FromRoute] Guid discriminatorId, DateTimeOffset? geTimestamp, [FromQuery] IEnumerable<string>? columnNames, int skip, int take, CancellationToken token)
public async Task<ActionResult<IEnumerable<TimestampedValuesDto>>> Get([FromRoute] Guid discriminatorId, DateTimeOffset? timestampBegin, [FromQuery] string[]? columnNames, int skip, int take, CancellationToken token)
{
var result = await repository.Get(discriminatorId, geTimestamp, columnNames, skip, take, token);
return Ok(result);
var result = await timestampedValuesRepository.Get(discriminatorId, timestampBegin, columnNames, skip, take, token);
return result.Any() ? Ok(result) : NoContent();
}
/// <summary>
/// Получить последние данные
/// Получить данные, начиная с заданной отметки времени
/// </summary>
/// <param name="discriminatorId">Дискриминатор (идентификатор) набора</param>
/// <param name="timestampBegin">Фильтр позднее даты</param>
/// <param name="token"></param>
[HttpGet("gtdate")]
public async Task<ActionResult<IEnumerable<TimestampedValuesDto>>> GetGtDate([FromRoute] Guid discriminatorId, DateTimeOffset timestampBegin, CancellationToken token)
{
var result = await timestampedValuesRepository.GetGtDate(discriminatorId, timestampBegin, token);
return result.Any() ? Ok(result) : NoContent();
}
/// <summary>
/// Получить данные c начала
/// </summary>
/// <param name="discriminatorId">Дискриминатор (идентификатор) набора</param>
/// <param name="columnNames">Фильтр свойств набора. Можно запросить только некоторые свойства из набора</param>
/// <param name="take"></param>
/// <param name="token"></param>
/// <returns>Фильтрованный набор данных с сортировкой по отметке времени</returns>
[HttpGet("last")]
[ProducesResponseType(typeof(IEnumerable<TimestampedValuesDto>), (int)HttpStatusCode.OK)]
public async Task<IActionResult> GetLast([FromRoute] Guid discriminatorId, [FromQuery] IEnumerable<string>? columnNames, int take, CancellationToken token)
[HttpGet("first")]
public async Task<ActionResult<IEnumerable<TimestampedValuesDto>>> GetFirst([FromRoute] Guid discriminatorId, int take, CancellationToken token)
{
var result = await repository.GetLast(discriminatorId, columnNames, take, token);
return Ok(result);
var result = await timestampedValuesRepository.GetFirst(discriminatorId, take, token);
return result.Any() ? Ok(result) : NoContent();
}
/// <summary>
/// Диапазон дат за которые есть данные
/// </summary>
/// <param name="discriminatorId"></param>
/// <param name="token"></param>
/// <returns>Дата первой и последней записи</returns>
[HttpGet("datesRange")]
[ProducesResponseType(typeof(DatesRangeDto), (int)HttpStatusCode.OK)]
[ProducesResponseType((int)HttpStatusCode.NoContent)]
public async Task<IActionResult> GetDatesRange([FromRoute] Guid discriminatorId, CancellationToken token)
{
var result = await repository.GetDatesRange(discriminatorId, token);
return Ok(result);
}
/// <summary>
/// Количество записей по указанному набору в БД. Для пагинации.
/// Получить данные c конца
/// </summary>
/// <param name="discriminatorId">Дискриминатор (идентификатор) набора</param>
/// <param name="take"></param>
/// <param name="token"></param>
/// <returns></returns>
[HttpGet("count")]
[ProducesResponseType(typeof(int), (int)HttpStatusCode.OK)]
[ProducesResponseType((int)HttpStatusCode.NoContent)]
public async Task<IActionResult> Count([FromRoute] Guid discriminatorId, CancellationToken token)
[HttpGet("last")]
public async Task<ActionResult<IEnumerable<TimestampedValuesDto>>> GetLast([FromRoute] Guid discriminatorId, int take, CancellationToken token)
{
var result = await repository.Count(discriminatorId, token);
return Ok(result);
var result = await timestampedValuesRepository.GetLast(discriminatorId, take, token);
return result.Any() ? Ok(result) : NoContent();
}
/// <summary>
/// Получить список объектов с прореживанием, удовлетворяющий диапазону дат
/// </summary>
/// <param name="discriminatorId"></param>
/// <param name="dateBegin"></param>
/// <param name="discriminatorId">Дискриминатор (идентификатор) набора</param>
/// <param name="timestampBegin">Фильтр позднее даты</param>
/// <param name="intervalSec"></param>
/// <param name="approxPointsCount"></param>
/// <param name="token"></param>
/// <returns></returns>
[HttpGet("resampled")]
[ProducesResponseType(typeof(IEnumerable<TimestampedValuesDto>), (int)HttpStatusCode.OK)]
[ProducesResponseType((int)HttpStatusCode.NoContent)]
public async Task<IActionResult> GetResampledData([FromRoute] Guid discriminatorId, DateTimeOffset dateBegin, double intervalSec = 600d, int approxPointsCount = 1024, CancellationToken token = default)
public async Task<ActionResult<IEnumerable<TimestampedValuesDto>>> GetResampledData([FromRoute] Guid discriminatorId, DateTimeOffset timestampBegin, double intervalSec = 600d, int approxPointsCount = 1024, CancellationToken token = default)
{
var result = await repository.GetResampledData(discriminatorId, dateBegin, intervalSec, approxPointsCount, token);
var result = await timestampedValuesRepository.GetResampledData(discriminatorId, timestampBegin, intervalSec, approxPointsCount, token);
return result.Any() ? Ok(result) : NoContent();
}
/// <summary>
/// Получить количество записей по указанному набору в БД. Для пагинации
/// </summary>
/// <param name="discriminatorId">Дискриминатор (идентификатор) набора</param>
/// <param name="token"></param>
[HttpGet("count")]
public async Task<ActionResult<int>> Count([FromRoute] Guid discriminatorId, CancellationToken token)
{
var result = await timestampedValuesRepository.Count(discriminatorId, token);
return Ok(result);
}
/// <summary>
/// Получить диапазон дат, в пределах которых хранятся даные
/// </summary>
/// <param name="discriminatorId"></param>
/// <param name="token"></param>
[HttpGet("datesRange")]
public async Task<ActionResult<DatesRangeDto>> GetDatesRange([FromRoute] Guid discriminatorId, CancellationToken token)
{
var result = await timestampedValuesRepository.GetDatesRange(discriminatorId, token);
return Ok(result);
}
}

4
DD.Persistence.Client/Clients/Base/BaseClient.cs
View File

@ -12,13 +12,13 @@ public abstract class BaseClient
this.logger = logger;
}
public async Task<T?> ExecuteGetResponse<T>(Func<Task<IApiResponse<T>>> getMethod, CancellationToken token)
public async Task<T> ExecuteGetResponse<T>(Func<Task<IApiResponse<T>>> getMethod, CancellationToken token)
{
var response = await getMethod.Invoke().WaitAsync(token);
if (response.IsSuccessStatusCode)
{
return response.Content;
return response.Content!;
}
var exception = response.GetPersistenceException();

102
DD.Persistence.Client/Clients/Interfaces/ITimestampedValuesClient.cs
View File

@ -10,61 +10,71 @@ namespace DD.Persistence.Client.Clients.Interfaces;
/// </summary>
public interface ITimestampedValuesClient : IDisposable
{
/// <summary>
/// Записать новые данные
/// </summary>
/// <param name="discriminatorId"></param>
/// <param name="sets"></param>
/// <param name="token"></param>
/// <returns></returns>
Task<int> AddRange(Guid discriminatorId, IEnumerable<TimestampedValuesDto> sets, CancellationToken token);
/// <summary>
/// Количество записей по указанному набору в БД. Для пагинации
/// </summary>
/// <param name="discriminatorId"></param>
/// <param name="token"></param>
/// <returns></returns>
Task<int> Count(Guid discriminatorId, CancellationToken token);
/// <summary>
/// Получение данных с фильтрацией. Значение фильтра null - отключен
/// </summary>
/// <param name="discriminatorId"></param>
/// <param name="geTimestamp"></param>
/// <param name="columnNames"></param>
/// <param name="skip"></param>
/// <param name="take"></param>
/// <param name="token"></param>
/// <returns></returns>
Task<IEnumerable<TimestampedValuesDto>> Get(Guid discriminatorId, DateTimeOffset? geTimestamp, IEnumerable<string>? columnNames, int skip, int take, CancellationToken token);
/// <summary>
/// Диапазон дат за которые есть данные
/// </summary>
/// <param name="discriminatorId"></param>
/// <param name="token"></param>
/// <returns></returns>
Task<DatesRangeDto?> GetDatesRange(Guid discriminatorId, CancellationToken token);
/// <summary>
/// Записать новые данные
/// Предполагается что данные с одним дискриминатором имеют одинаковую структуру
/// </summary>
/// <param name="discriminatorId">Дискриминатор (идентификатор) набора</param>
/// <param name="dtos"></param>
/// <param name="token"></param>
Task<int> AddRange(Guid discriminatorId, IEnumerable<TimestampedValuesDto> dtos, CancellationToken token);
/// <summary>
/// Получить последние данные
/// Получить данные с фильтрацией. Значение фильтра null - отключен
/// </summary>
/// <param name="discriminatorId"></param>
/// <param name="columnNames"></param>
/// <param name="discriminatorId">Дискриминатор (идентификатор) набора</param>
/// <param name="timestampBegin">Фильтр позднее даты</param>
/// <param name="columnNames">Фильтр свойств набора</param>
/// <param name="skip"></param>
/// <param name="take"></param>
/// <param name="token"></param>
/// <returns></returns>
Task<IEnumerable<TimestampedValuesDto>> GetLast(Guid discriminatorId, IEnumerable<string>? columnNames, int take, CancellationToken token);
Task<IEnumerable<TimestampedValuesDto>> Get(Guid discriminatorId, DateTimeOffset? timestampBegin, IEnumerable<string>? columnNames, int skip, int take, CancellationToken token);
/// <summary>
/// Получить список объектов с прореживанием, удовлетворяющий диапазону дат
/// Получить данные, начиная с заданной отметки времени
/// </summary>
/// <param name="discriminatorId"></param>
/// <param name="dateBegin"></param>
/// <param name="discriminatorId">Дискриминатор (идентификатор) набора</param>
/// <param name="timestampBegin">Фильтр позднее даты</param>
/// <param name="token"></param>
Task<IEnumerable<TimestampedValuesDto>> GetGtDate(Guid discriminatorId, DateTimeOffset timestampBegin, CancellationToken token);
/// <summary>
/// Получить данные с начала
/// </summary>
/// <param name="discriminatorId">Дискриминатор (идентификатор) набора</param>
/// <param name="take"></param>
/// <param name="token"></param>
Task<IEnumerable<TimestampedValuesDto>> GetFirst(Guid discriminatorId, int take, CancellationToken token);
/// <summary>
/// Получить данные с конца
/// </summary>
/// <param name="discriminatorId">Дискриминатор (идентификатор) набора</param>
/// <param name="take"></param>
/// <param name="token"></param>
Task<IEnumerable<TimestampedValuesDto>> GetLast(Guid discriminatorId, int take, CancellationToken token);
/// <summary>
/// Получить данные с прореживанием, удовлетворяющем диапазону дат
/// </summary>
/// <param name="discriminatorId">Дискриминатор (идентификатор) набора</param>
/// <param name="timestampBegin"></param>
/// <param name="intervalSec"></param>
/// <param name="approxPointsCount"></param>
/// <param name="token"></param>
/// <returns></returns>
Task<IEnumerable<TimestampedValuesDto>> GetResampledData(Guid discriminatorId, DateTimeOffset dateBegin, double intervalSec = 600d, int approxPointsCount = 1024, CancellationToken token = default);
Task<IEnumerable<TimestampedValuesDto>> GetResampledData(Guid discriminatorId, DateTimeOffset timestampBegin, double intervalSec = 600d, int approxPointsCount = 1024, CancellationToken token = default);
/// <summary>
/// Количество записей по указанному набору в БД. Для пагинации
/// </summary>
/// <param name="discriminatorId">Дискриминатор (идентификатор) набора</param>
/// <param name="token"></param>
Task<int> Count(Guid discriminatorId, CancellationToken token);
/// <summary>
/// Диапазон дат, в пределах которых осуществляется хранение данных
/// </summary>
/// <param name="discriminatorId">Дискриминатор (идентификатор) набора</param>
/// <param name="token"></param>
Task<DatesRangeDto?> GetDatesRange(Guid discriminatorId, CancellationToken token);
}

4
DD.Persistence.Client/Clients/Interfaces/Refit/IRefitClient.cs
View File

@ -5,6 +5,10 @@ using System.Text;
using System.Threading.Tasks;
namespace DD.Persistence.Client.Clients.Interfaces.Refit;
/// <summary>
/// Базовый Refit интерфейс
/// </summary>
public interface IRefitClient
{
}

45
DD.Persistence.Client/Clients/Interfaces/Refit/IRefitTimestampedValuesClient.cs
View File

@ -4,25 +4,58 @@ using Refit;
namespace DD.Persistence.Client.Clients.Interfaces.Refit;
/// <summary>
/// Refit интерфейс для TimestampedValuesController
/// </summary>
public interface IRefitTimestampedValuesClient : IRefitClient, IDisposable
{
private const string baseUrl = "/api/TimestampedValues/{discriminatorId}";
/// <summary>
/// Записать новые данные
/// </summary>
[Post(baseUrl)]
Task<IApiResponse<int>> AddRange(Guid discriminatorId, IEnumerable<TimestampedValuesDto> sets, CancellationToken token);
Task<IApiResponse<int>> AddRange(Guid discriminatorId, IEnumerable<TimestampedValuesDto> dtos, CancellationToken token);
/// <summary>
/// Получение данных с фильтрацией
/// </summary>
[Get(baseUrl)]
Task<IApiResponse<IEnumerable<TimestampedValuesDto>>> Get(Guid discriminatorId, [Query] DateTimeOffset? geTimestamp, [Query] IEnumerable<string>? columnNames, int skip, int take, CancellationToken token);
Task<IApiResponse<IEnumerable<TimestampedValuesDto>>> Get(Guid discriminatorId, DateTimeOffset? timestampBegin, [Query(CollectionFormat.Multi)] IEnumerable<string>? columnNames, int skip, int take, CancellationToken token);
/// <summary>
/// Получить данные, начиная с заданной отметки времени
/// </summary>
[Get($"{baseUrl}/gtdate")]
Task<IApiResponse<IEnumerable<TimestampedValuesDto>>> GetGtDate(Guid discriminatorId, DateTimeOffset timestampBegin, CancellationToken token);
/// <summary>
/// Получить данные c начала
/// </summary>
[Get($"{baseUrl}/first")]
Task<IApiResponse<IEnumerable<TimestampedValuesDto>>> GetFirst(Guid discriminatorId, int take, CancellationToken token);
/// <summary>
/// Получить данные c конца
/// </summary>
[Get($"{baseUrl}/last")]
Task<IApiResponse<IEnumerable<TimestampedValuesDto>>> GetLast(Guid discriminatorId, [Query] IEnumerable<string>? columnNames, int take, CancellationToken token);
Task<IApiResponse<IEnumerable<TimestampedValuesDto>>> GetLast(Guid discriminatorId, int take, CancellationToken token);
/// <summary>
/// Получить список объектов с прореживанием, удовлетворяющий диапазону временных отметок
/// </summary>
[Get($"{baseUrl}/resampled")]
Task<IApiResponse<IEnumerable<TimestampedValuesDto>>> GetResampledData(Guid discriminatorId, DateTimeOffset timestampBegin, double intervalSec = 600d, int approxPointsCount = 1024, CancellationToken token = default);
/// <summary>
/// Получить количество записей по указанному набору в БД. Для пагинации
/// </summary>
[Get($"{baseUrl}/count")]
Task<IApiResponse<int>> Count(Guid discriminatorId, CancellationToken token);
/// <summary>
/// Получить диапазон дат, в пределах которых хранятся даные
/// </summary>
[Get($"{baseUrl}/datesRange")]
Task<IApiResponse<DatesRangeDto?>> GetDatesRange(Guid discriminatorId, CancellationToken token);
[Get($"{baseUrl}/resampled")]
Task<IApiResponse<IEnumerable<TimestampedValuesDto>>> GetResampledData(Guid discriminatorId, DateTimeOffset dateBegin, double intervalSec = 600d, int approxPointsCount = 1024, CancellationToken token = default);
}

73
DD.Persistence.Client/Clients/TimestampedValuesClient.cs
View File

@ -1,20 +1,24 @@
using Microsoft.Extensions.Logging;
using DD.Persistence.Client.Clients.Base;
using DD.Persistence.Client.Clients.Base;
using DD.Persistence.Client.Clients.Interfaces;
using DD.Persistence.Client.Clients.Interfaces.Refit;
using DD.Persistence.Models;
using DD.Persistence.Models.Common;
using Microsoft.Extensions.Logging;
namespace DD.Persistence.Client.Clients;
/// <inheritdoc/>
public class TimestampedValuesClient : BaseClient, ITimestampedValuesClient
{
private readonly IRefitTimestampedValuesClient refitTimestampedSetClient;
/// <inheritdoc/>
public TimestampedValuesClient(IRefitClientFactory<IRefitTimestampedValuesClient> refitTimestampedSetClientFactory, ILogger<TimestampedValuesClient> logger) : base(logger)
{
this.refitTimestampedSetClient = refitTimestampedSetClientFactory.Create();
}
/// <inheritdoc/>
public async Task<int> AddRange(Guid discriminatorId, IEnumerable<TimestampedValuesDto> sets, CancellationToken token)
{
var result = await ExecutePostResponse(
@ -23,38 +27,42 @@ public class TimestampedValuesClient : BaseClient, ITimestampedValuesClient
return result;
}
/// <inheritdoc/>
public async Task<IEnumerable<TimestampedValuesDto>> Get(Guid discriminatorId, DateTimeOffset? geTimestamp, IEnumerable<string>? columnNames, int skip, int take, CancellationToken token)
{
var result = await ExecuteGetResponse(
async () => await refitTimestampedSetClient.Get(discriminatorId, geTimestamp, columnNames, skip, take, token), token);
return result;
}
return result!;
}
/// <inheritdoc/>
public async Task<IEnumerable<TimestampedValuesDto>> GetGtDate(Guid discriminatorId, DateTimeOffset timestampBegin, CancellationToken token)
{
var result = await ExecuteGetResponse(
async () => await refitTimestampedSetClient.GetGtDate(discriminatorId, timestampBegin, token), token);
public async Task<IEnumerable<TimestampedValuesDto>> GetLast(Guid discriminatorId, IEnumerable<string>? columnNames, int take, CancellationToken token)
return result;
}
/// <inheritdoc/>
public async Task<IEnumerable<TimestampedValuesDto>> GetFirst(Guid discriminatorId, int take, CancellationToken token)
{
var result = await ExecuteGetResponse(
async () => await refitTimestampedSetClient.GetFirst(discriminatorId, take, token), token);
return result;
}
/// <inheritdoc/>
public async Task<IEnumerable<TimestampedValuesDto>> GetLast(Guid discriminatorId, int take, CancellationToken token)
{
var result = await ExecuteGetResponse(
async () => await refitTimestampedSetClient.GetLast(discriminatorId, columnNames, take, token), token);
async () => await refitTimestampedSetClient.GetLast(discriminatorId, take, token), token);
return result!;
}
public async Task<int> Count(Guid discriminatorId, CancellationToken token)
{
var result = await ExecuteGetResponse(
async () => await refitTimestampedSetClient.Count(discriminatorId, token), token);
return result;
}
public async Task<DatesRangeDto?> GetDatesRange(Guid discriminatorId, CancellationToken token)
{
var result = await ExecuteGetResponse(
async () => await refitTimestampedSetClient.GetDatesRange(discriminatorId, token), token);
return result;
return result;
}
/// <inheritdoc/>
public async Task<IEnumerable<TimestampedValuesDto>> GetResampledData(Guid discriminatorId, DateTimeOffset dateBegin, double intervalSec = 600, int approxPointsCount = 1024, CancellationToken token = default)
{
var result = await ExecuteGetResponse(
@ -63,6 +71,25 @@ public class TimestampedValuesClient : BaseClient, ITimestampedValuesClient
return result;
}
/// <inheritdoc/>
public async Task<int> Count(Guid discriminatorId, CancellationToken token)
{
var result = await ExecuteGetResponse(
async () => await refitTimestampedSetClient.Count(discriminatorId, token), token);
return result;
}
/// <inheritdoc/>
public async Task<DatesRangeDto?> GetDatesRange(Guid discriminatorId, CancellationToken token)
{
var result = await ExecuteGetResponse(
async () => await refitTimestampedSetClient.GetDatesRange(discriminatorId, token), token);
return result;
}
/// <inheritdoc/>
public void Dispose()
{
refitTimestampedSetClient.Dispose();

2
DD.Persistence.IntegrationTests/Controllers/TechMessagesControllerTest.cs
View File

@ -15,7 +15,7 @@ namespace DD.Persistence.IntegrationTests.Controllers
{
public class TechMessagesControllerTest : BaseIntegrationTest
{
private static readonly string SystemCacheKey = $"{typeof(Database.Entity.DataSourceSystem).FullName}CacheKey";
private static readonly string SystemCacheKey = $"{typeof(DataSourceSystem).FullName}CacheKey";
private readonly ITechMessagesClient techMessagesClient;
private readonly IMemoryCache memoryCache;
public TechMessagesControllerTest(WebAppFactoryFixture factory) : base(factory)

224
DD.Persistence.IntegrationTests/Controllers/TimestampedSetControllerTest.cs
View File

@ -1,224 +0,0 @@
using Microsoft.Extensions.DependencyInjection;
using DD.Persistence.Client;
using DD.Persistence.Client.Clients.Interfaces;
using DD.Persistence.Models;
using Xunit;
using DD.Persistence.Client.Clients.Interfaces.Refit;
using DD.Persistence.Client.Clients;
using Microsoft.Extensions.Logging;
using System.Text.Json;
using System.Text;
using Microsoft.Extensions.Primitives;
using System.Dynamic;
using Newtonsoft.Json.Linq;
namespace DD.Persistence.IntegrationTests.Controllers;
public class TimestampedSetControllerTest : BaseIntegrationTest
{
private readonly ITimestampedValuesClient client;
public TimestampedSetControllerTest(WebAppFactoryFixture factory) : base(factory)
{
var refitClientFactory = scope.ServiceProvider
.GetRequiredService<IRefitClientFactory<IRefitTimestampedValuesClient>>();
var logger = scope.ServiceProvider.GetRequiredService<ILogger<TimestampedValuesClient>>();
client = scope.ServiceProvider
.GetRequiredService<ITimestampedValuesClient>();
}
[Fact]
public async Task InsertRange()
{
// arrange
Guid idDiscriminator = Guid.NewGuid();
IEnumerable<TimestampedValuesDto> testSets = Generate(10, DateTimeOffset.Now.ToOffset(TimeSpan.FromHours(7)));
// act
var response = await client.AddRange(idDiscriminator, testSets, CancellationToken.None);
// assert
Assert.Equal(testSets.Count(), response);
}
[Fact]
public async Task Get_without_filter()
{
// arrange
Guid idDiscriminator = Guid.NewGuid();
int count = 10;
IEnumerable<TimestampedValuesDto> testSets = Generate(count, DateTimeOffset.Now.ToOffset(TimeSpan.FromHours(7)));
await client.AddRange(idDiscriminator, testSets, CancellationToken.None);
// act
var response = await client.Get(idDiscriminator, null, null, 0, int.MaxValue, CancellationToken.None);
// assert
Assert.NotNull(response);
Assert.Equal(count, response.Count());
}
[Fact]
public async Task Get_with_filter_props()
{
// arrange
Guid idDiscriminator = Guid.NewGuid();
int count = 10;
IEnumerable<TimestampedValuesDto> testSets = Generate(count, DateTimeOffset.Now.ToOffset(TimeSpan.FromHours(7)));
await client.AddRange(idDiscriminator, testSets, CancellationToken.None);
string[] props = ["A"];
// act
var response = await client.Get(idDiscriminator, null, props, 0, int.MaxValue, new CancellationToken());
// assert
Assert.NotNull(response);
Assert.Equal(count, response.Count());
foreach (var item in response)
{
Assert.Single(item.Values!);
var kv = item.Values!.First();
Assert.Equal("A", ((KeyValuePair<string, object>) kv).Key);
}
}
[Fact]
public async Task Get_geDate()
{
// arrange
Guid idDiscriminator = Guid.NewGuid();
int count = 10;
var dateMin = DateTimeOffset.Now;
var dateMax = DateTimeOffset.Now.AddSeconds(count);
IEnumerable<TimestampedValuesDto> testSets = Generate(count, dateMin.ToOffset(TimeSpan.FromHours(7)));
var insertResponse = await client.AddRange(idDiscriminator, testSets, CancellationToken.None);
var tail = testSets.OrderBy(t => t.Timestamp).Skip(count / 2).Take(int.MaxValue);
var geDate = tail.First().Timestamp;
var tolerance = TimeSpan.FromSeconds(1);
var expectedCount = tail.Count();
// act
var response = await client.Get(idDiscriminator, geDate, null, 0, int.MaxValue, CancellationToken.None);
// assert
Assert.NotNull(response);
Assert.Equal(expectedCount, response.Count());
var minDate = response.Min(t => t.Timestamp);
Assert.Equal(geDate, geDate, tolerance);
}
[Fact]
public async Task Get_with_skip_take()
{
// arrange
Guid idDiscriminator = Guid.NewGuid();
int count = 10;
IEnumerable<TimestampedValuesDto> testSets = Generate(count, DateTimeOffset.Now.ToOffset(TimeSpan.FromHours(7)));
await client.AddRange(idDiscriminator, testSets, CancellationToken.None);
var expectedCount = count / 2;
// act
var response = await client.Get(idDiscriminator, null, null, 2, expectedCount, new CancellationToken());
// assert
Assert.NotNull(response);
Assert.Equal(expectedCount, response.Count());
}
[Fact]
public async Task Get_with_big_skip_take()
{
// arrange
Guid idDiscriminator = Guid.NewGuid();
var expectedCount = 1;
int count = 10 + expectedCount;
IEnumerable<TimestampedValuesDto> testSets = Generate(count, DateTimeOffset.Now.ToOffset(TimeSpan.FromHours(7)));
await client.AddRange(idDiscriminator, testSets, CancellationToken.None);
// act
var response = await client.Get(idDiscriminator, null, null, count - expectedCount, count, new CancellationToken());
// assert
Assert.NotNull(response);
Assert.Equal(expectedCount, response.Count());
}
[Fact]
public async Task GetLast()
{
// arrange
Guid idDiscriminator = Guid.NewGuid();
int count = 10;
IEnumerable<TimestampedValuesDto> testSets = Generate(count, DateTimeOffset.Now.ToOffset(TimeSpan.FromHours(7)));
await client.AddRange(idDiscriminator, testSets, CancellationToken.None);
var expectedCount = 8;
// act
var response = await client.GetLast(idDiscriminator, null, expectedCount, new CancellationToken());
// assert
Assert.NotNull(response);
Assert.Equal(expectedCount, response.Count());
}
[Fact]
public async Task GetDatesRange()
{
// arrange
Guid idDiscriminator = Guid.NewGuid();
int count = 10;
var dateMin = DateTimeOffset.Now;
var dateMax = DateTimeOffset.Now.AddSeconds(count - 1);
IEnumerable<TimestampedValuesDto> testSets = Generate(count, dateMin.ToOffset(TimeSpan.FromHours(7)));
await client.AddRange(idDiscriminator, testSets, CancellationToken.None);
var tolerance = TimeSpan.FromSeconds(1);
// act
var response = await client.GetDatesRange(idDiscriminator, new CancellationToken());
// assert
Assert.NotNull(response);
Assert.Equal(dateMin, response.From, tolerance);
Assert.Equal(dateMax, response.To, tolerance);
}
[Fact]
public async Task Count()
{
// arrange
Guid idDiscriminator = Guid.NewGuid();
int count = 144;
IEnumerable<TimestampedValuesDto> testSets = Generate(count, DateTimeOffset.Now.ToOffset(TimeSpan.FromHours(7)));
await client.AddRange(idDiscriminator, testSets, CancellationToken.None);
// act
var response = await client.Count(idDiscriminator, new CancellationToken());
// assert
Assert.Equal(count, response);
}
private static IEnumerable<TimestampedValuesDto> Generate(int n, DateTimeOffset from)
{
var result = new List<TimestampedValuesDto>();
for (int i = 0; i < n; i++)
{
var t = new object[] {
new { A = i },
new { B = i * 1.1 },
new { C = $"Any{i}" },
new { D = DateTimeOffset.Now }
};
string jsonString = JsonSerializer.Serialize(t);
var values = JsonSerializer.Deserialize<object[]>(jsonString);
yield return new TimestampedValuesDto()
{
Timestamp = from.AddSeconds(i),
Values = values
};
}
}
}

371
DD.Persistence.IntegrationTests/Controllers/TimestampedValuesControllerTest.cs Normal file
View File

@ -0,0 +1,371 @@
using DD.Persistence.Client;
using DD.Persistence.Client.Clients;
using DD.Persistence.Client.Clients.Interfaces;
using DD.Persistence.Client.Clients.Interfaces.Refit;
using DD.Persistence.Database.Entity;
using DD.Persistence.Models;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using System.Text.Json;
using Xunit;
namespace DD.Persistence.IntegrationTests.Controllers;
public class TimestampedValuesControllerTest : BaseIntegrationTest
{
private static readonly string SystemCacheKey = $"{typeof(ValuesIdentity).FullName}CacheKey";
private readonly ITimestampedValuesClient timestampedValuesClient;
private readonly IMemoryCache memoryCache;
public TimestampedValuesControllerTest(WebAppFactoryFixture factory) : base(factory)
{
var refitClientFactory = scope.ServiceProvider
.GetRequiredService<IRefitClientFactory<IRefitTimestampedValuesClient>>();
var logger = scope.ServiceProvider.GetRequiredService<ILogger<TimestampedValuesClient>>();
timestampedValuesClient = scope.ServiceProvider
.GetRequiredService<ITimestampedValuesClient>();
memoryCache = scope.ServiceProvider.GetRequiredService<IMemoryCache>();
}
[Fact]
public async Task AddRange_returns_success()
{
var discriminatorId = Guid.NewGuid();
await AddRange(discriminatorId);
}
[Fact]
public async Task Get_returns_success()
{
//arrange
Cleanup();
var discriminatorId = Guid.NewGuid();
//act
var response = await timestampedValuesClient.Get(discriminatorId, null, null, 0, 1, CancellationToken.None);
//assert
Assert.Null(response);
}
[Fact]
public async Task Get_AfterSave_returns_success()
{
//arrange
Cleanup();
var discriminatorId = Guid.NewGuid();
var timestampBegin = DateTimeOffset.UtcNow.AddDays(-1);
var columnNames = new List<string>() { "A", "C" };
var skip = 5;
var take = 5;
var dtos = await AddRange(discriminatorId);
//act
var response = await timestampedValuesClient.Get(discriminatorId, timestampBegin, columnNames, skip, take, CancellationToken.None);
//assert
Assert.NotNull(response);
Assert.NotEmpty(response);
var actualCount = response.Count();
Assert.Equal(take, actualCount);
var actualColumnNames = response.SelectMany(e => e.Values.Keys).Distinct().ToList();
Assert.Equal(columnNames, actualColumnNames);
var expectedValueKind = JsonValueKind.Number;
var actualValueKind = ((JsonElement) response.First().Values["A"]).ValueKind;
Assert.Equal(expectedValueKind, actualValueKind);
expectedValueKind = JsonValueKind.String;
actualValueKind = ((JsonElement)response.First().Values["C"]).ValueKind;
Assert.Equal(expectedValueKind, actualValueKind);
}
[Fact]
public async Task GetGtDate_returns_success()
{
//arrange
Cleanup();
var discriminatorId = Guid.NewGuid();
var timestampBegin = DateTimeOffset.UtcNow.AddDays(-1);
//act
var response = await timestampedValuesClient.GetGtDate(discriminatorId, timestampBegin, CancellationToken.None);
//assert
Assert.Null(response);
}
[Fact]
public async Task GetGtDate_AfterSave_returns_success()
{
//arrange
Cleanup();
var discriminatorId = Guid.NewGuid();
var dtos = await AddRange(discriminatorId);
var timestampBegin = DateTimeOffset.UtcNow.AddSeconds(-5);
//act
var response = await timestampedValuesClient.GetGtDate(discriminatorId, timestampBegin, CancellationToken.None);
//assert
Assert.NotNull(response);
Assert.NotEmpty(response);
var expectedCount = dtos.Count(dto => dto.Timestamp.ToUniversalTime() > timestampBegin);
var actualCount = response.Count();
Assert.Equal(expectedCount, actualCount);
}
[Fact]
public async Task GetFirst_returns_success()
{
//arrange
Cleanup();
var discriminatorId = Guid.NewGuid();
var take = 1;
//act
var response = await timestampedValuesClient.GetFirst(discriminatorId, take, CancellationToken.None);
//assert
Assert.Null(response);
}
[Fact]
public async Task GetFirst_AfterSave_returns_success()
{
//arrange
Cleanup();
var discriminatorId = Guid.NewGuid();
var dtos = await AddRange(discriminatorId);
var take = 1;
//act
var response = await timestampedValuesClient.GetFirst(discriminatorId, take, CancellationToken.None);
//assert
Assert.NotNull(response);
Assert.NotEmpty(response);
var expectedTimestampString = dtos
.OrderBy(dto => dto.Timestamp)
.First().Timestamp
.ToUniversalTime()
.ToString();
var actualTimestampString = response
.First().Timestamp
.ToUniversalTime()
.ToString();
Assert.Equal(expectedTimestampString, actualTimestampString);
}
[Fact]
public async Task GetLast_returns_success()
{
//arrange
Cleanup();
var discriminatorId = Guid.NewGuid();
var take = 1;
//act
var response = await timestampedValuesClient.GetLast(discriminatorId, take, CancellationToken.None);
//assert
Assert.Null(response);
}
[Fact]
public async Task GetLast_AfterSave_returns_success()
{
//arrange
Cleanup();
var discriminatorId = Guid.NewGuid();
var dtos = await AddRange(discriminatorId);
var take = 1;
//act
var response = await timestampedValuesClient.GetLast(discriminatorId, take, CancellationToken.None);
//assert
Assert.NotNull(response);
Assert.NotEmpty(response);
var expectedTimestampString = dtos
.OrderByDescending(dto => dto.Timestamp)
.First().Timestamp
.ToUniversalTime()
.ToString();
var actualTimestampString = response
.First().Timestamp
.ToUniversalTime()
.ToString();
Assert.Equal(expectedTimestampString, actualTimestampString);
}
[Fact]
public async Task GetResampledData_returns_success()
{
//arrange
Cleanup();
var discriminatorId = Guid.NewGuid();
var timestampBegin = DateTimeOffset.UtcNow;
//act
var response = await timestampedValuesClient.GetResampledData(discriminatorId, timestampBegin);
//assert
Assert.Null(response);
}
[Fact]
public async Task GetResampledData_AfterSave_returns_success()
{
//arrange
Cleanup();
var discriminatorId = Guid.NewGuid();
var count = 2048;
var timestampBegin = DateTimeOffset.UtcNow;
var dtos = await AddRange(discriminatorId, count);
//act
var response = await timestampedValuesClient.GetResampledData(discriminatorId, timestampBegin, count);
//assert
Assert.NotNull(response);
Assert.NotEmpty(response);
var expectedCount = count / 2;
var actualCount = response.Count();
Assert.Equal(expectedCount, actualCount);
}
[Fact]
public async Task Count_returns_success()
{
//arrange
Cleanup();
var discriminatorId = Guid.NewGuid();
//act
var response = await timestampedValuesClient.Count(discriminatorId, CancellationToken.None);
//assert
Assert.Equal(0, response);
}
[Fact]
public async Task Count_AfterSave_returns_success()
{
//arrange
Cleanup();
var discriminatorId = Guid.NewGuid();
var dtos = await AddRange(discriminatorId);
//act
var response = await timestampedValuesClient.Count(discriminatorId, CancellationToken.None);
//assert
var expectedCount = dtos.Count();
Assert.Equal(expectedCount, response);
}
[Fact]
public async Task GetDatesRange_returns_success()
{
//arrange
Cleanup();
var discriminatorId = Guid.NewGuid();
//act
var response = await timestampedValuesClient.GetDatesRange(discriminatorId, CancellationToken.None);
//assert
Assert.Null(response);
}
[Fact]
public async Task GetDatesRange_AfterSave_returns_success()
{
//arrange
Cleanup();
var discriminatorId = Guid.NewGuid();
var dtos = await AddRange(discriminatorId);
//act
var response = await timestampedValuesClient.GetDatesRange(discriminatorId, CancellationToken.None);
//assert
Assert.NotNull(response);
var expectedDateFromString = dtos
.OrderBy(dto => dto.Timestamp)
.First().Timestamp
.ToUniversalTime()
.ToString();
var actualDateFromString = response.From
.ToUniversalTime()
.ToString();
Assert.Equal(expectedDateFromString, actualDateFromString);
var expectedDateToString = dtos
.OrderByDescending(dto => dto.Timestamp)
.First().Timestamp
.ToUniversalTime()
.ToString();
var actualDateToString = response.To
.ToUniversalTime()
.ToString();
Assert.Equal(expectedDateToString, actualDateToString);
}
private async Task<IEnumerable<TimestampedValuesDto>> AddRange(Guid discriminatorId, int countToCreate = 10)
{
// arrange
IEnumerable<TimestampedValuesDto> generatedDtos = Generate(countToCreate, DateTimeOffset.Now.ToOffset(TimeSpan.FromHours(7)));
// act
var response = await timestampedValuesClient.AddRange(discriminatorId, generatedDtos, CancellationToken.None);
// assert
Assert.Equal(generatedDtos.Count(), response);
return generatedDtos;
}
private static IEnumerable<TimestampedValuesDto> Generate(int countToCreate, DateTimeOffset from)
{
var result = new List<TimestampedValuesDto>();
for (int i = 0; i < countToCreate; i++)
{
var values = new Dictionary<string, object>()
{
{ "A", i },
{ "B", i * 1.1 },
{ "C", $"Any{i}" },
{ "D", DateTimeOffset.Now },
};
yield return new TimestampedValuesDto()
{
Timestamp = from.AddSeconds(i),
Values = values
};
}
}
private void Cleanup()
{
memoryCache.Remove(SystemCacheKey);
dbContext.CleanupDbSet<TimestampedValues>();
dbContext.CleanupDbSet<ValuesIdentity>();
}
}

15
DD.Persistence.Repository/Extensions/IEnumerableExtensions.cs
View File

@ -14,4 +14,19 @@ public static class IEnumerableExtensions
action(item);
}
}
public static bool IsNullOrEmpty<T>(this IEnumerable<T>? enumerable)
{
if (enumerable == null)
{
return true;
}
var collection = enumerable as ICollection<T>;
if (collection != null)
{
return collection.Count < 1;
}
return !enumerable.Any();
}
}

224
DD.Persistence.Repository/Repositories/TimestampedValuesRepository.cs
View File

@ -3,12 +3,7 @@ using DD.Persistence.Models;
using DD.Persistence.Models.Common;
using DD.Persistence.Repositories;
using DD.Persistence.Repository.Extensions;
using Mapster;
using Microsoft.EntityFrameworkCore;
using Newtonsoft.Json.Linq;
using System.Linq;
using System.Text.Json;
using System.Text.Json.Nodes;
namespace DD.Persistence.Repository.Repositories;
public class TimestampedValuesRepository : ITimestampedValuesRepository
@ -25,54 +20,19 @@ public class TimestampedValuesRepository : ITimestampedValuesRepository
protected virtual IQueryable<TimestampedValues> GetQueryReadOnly() => this.db.Set<TimestampedValues>()
.Include(e => e.ValuesIdentity);
public virtual async Task<DatesRangeDto?> GetDatesRange(Guid discriminatorId, CancellationToken token)
{
var query = GetQueryReadOnly()
.GroupBy(entity => entity.DiscriminatorId)
.Select(group => new
{
Min = group.Min(entity => entity.Timestamp),
Max = group.Max(entity => entity.Timestamp),
});
var item = await query.FirstOrDefaultAsync(token);
if (item is null)
return null;
return new DatesRangeDto
{
From = item.Min,
To = item.Max,
};
}
public virtual async Task<IEnumerable<TimestampedValuesDto>> GetGtDate(Guid discriminatorId, DateTimeOffset date, CancellationToken token)
{
var query = GetQueryReadOnly().Where(e => e.Timestamp > date);
var entities = await query.ToArrayAsync(token);
var dtos = entities.Select(e => e.Adapt<TimestampedValuesDto>());
return dtos;
}
public virtual async Task<int> AddRange(Guid discriminatorId, IEnumerable<TimestampedValuesDto> dtos, CancellationToken token)
public async virtual Task<int> AddRange(Guid discriminatorId, IEnumerable<TimestampedValuesDto> dtos, CancellationToken token)
{
var timestampedValuesEntities = new List<TimestampedValues>();
foreach (var dto in dtos)
{
var values = dto.Values
.SelectMany(v => JsonSerializer.Deserialize<Dictionary<string, object>>(v.ToString()!)!)
.ToDictionary();
var keys = values.Keys.ToArray();
var keys = dto.Values.Keys.ToArray();
await CreateValuesIdentityIfNotExist(discriminatorId, keys, token);
var timestampedValuesEntity = new TimestampedValues()
{
DiscriminatorId = discriminatorId,
Timestamp = dto.Timestamp.ToUniversalTime(),
Values = values.Values.ToArray()
Values = dto.Values.Values.ToArray()
};
timestampedValuesEntities.Add(timestampedValuesEntity);
}
@ -84,32 +44,54 @@ public class TimestampedValuesRepository : ITimestampedValuesRepository
return result;
}
protected async Task<IEnumerable<TimestampedValuesDto>> GetLastAsync(int takeCount, CancellationToken token)
public async virtual Task<IEnumerable<TimestampedValuesDto>> Get(Guid discriminatorId, DateTimeOffset? timestampBegin, IEnumerable<string>? columnNames, int skip, int take, CancellationToken token)
{
var query = GetQueryReadOnly()
.Where(entity => entity.DiscriminatorId == discriminatorId);
// Фильтрация по дате
if (timestampBegin.HasValue)
{
query = ApplyGeTimestamp(query, timestampBegin.Value);
}
query = query
.OrderBy(item => item.Timestamp)
.Skip(skip)
.Take(take);
var data = await Materialize(discriminatorId, query, token);
// Фильтрация по запрашиваемым полям
if (!columnNames.IsNullOrEmpty())
{
data = ReduceSetColumnsByNames(data, columnNames!);
}
return data;
}
public async virtual Task<IEnumerable<TimestampedValuesDto>> GetFirst(Guid discriminatorId, int takeCount, CancellationToken token)
{
var query = GetQueryReadOnly()
.OrderBy(e => e.Timestamp)
.Take(takeCount);
var dtos = await Materialize(discriminatorId, query, token);
return dtos;
}
public async virtual Task<IEnumerable<TimestampedValuesDto>> GetLast(Guid discriminatorId, int takeCount, CancellationToken token)
{
var query = GetQueryReadOnly()
.OrderByDescending(e => e.Timestamp)
.Take(takeCount);
var entities = await query.ToArrayAsync(token);
var dtos = entities.Select(e => e.Adapt<TimestampedValuesDto>());
var dtos = await Materialize(discriminatorId, query, token);
return dtos;
}
protected async Task<TimestampedValuesDto?> GetFirstAsync(CancellationToken token)
{
var query = GetQueryReadOnly()
.OrderBy(e => e.Timestamp);
var entity = await query.FirstOrDefaultAsync(token);
if (entity == null)
return null;
var dto = entity.Adapt<TimestampedValuesDto>();
return dto;
}
public async virtual Task<IEnumerable<TimestampedValuesDto>> GetResampledData(
Guid discriminatorId,
DateTimeOffset dateBegin,
@ -131,99 +113,103 @@ public class TimestampedValuesRepository : ITimestampedValuesRepository
return dtos;
}
public async Task<IEnumerable<TimestampedValuesDto>> Get(Guid discriminatorId, DateTimeOffset? geTimestamp, IEnumerable<string>? columnNames, int skip, int take, CancellationToken token)
public async virtual Task<IEnumerable<TimestampedValuesDto>> GetGtDate(Guid discriminatorId, DateTimeOffset timestampBegin, CancellationToken token)
{
var dbSet = db.Set<TimestampedValues>();
var query = dbSet.Where(entity => entity.DiscriminatorId == discriminatorId);
var query = GetQueryReadOnly()
.Where(e => e.Timestamp > timestampBegin);
if (geTimestamp.HasValue)
query = ApplyGeTimestamp(query, geTimestamp.Value);
var dtos = await Materialize(discriminatorId, query, token);
query = query
.OrderBy(item => item.Timestamp)
.Skip(skip)
.Take(take);
var data = await Materialize(discriminatorId, query, token);
if (columnNames is not null && columnNames.Any())
data = ReduceSetColumnsByNames(data, columnNames);
return data;
return dtos;
}
public async Task<IEnumerable<TimestampedValuesDto>> GetLast(Guid discriminatorId, IEnumerable<string>? columnNames, int take, CancellationToken token)
public async virtual Task<DatesRangeDto?> GetDatesRange(Guid discriminatorId, CancellationToken token)
{
var dbSet = db.Set<TimestampedValues>();
var query = dbSet.Where(entity => entity.DiscriminatorId == discriminatorId);
var query = GetQueryReadOnly()
.GroupBy(entity => entity.DiscriminatorId)
.Select(group => new
{
Min = group.Min(entity => entity.Timestamp),
Max = group.Max(entity => entity.Timestamp),
});
query = query.OrderByDescending(entity => entity.Timestamp)
.Take(take)
.OrderBy(entity => entity.Timestamp);
var item = await query.FirstOrDefaultAsync(token);
if (item is null)
{
return null;
}
var data = await Materialize(discriminatorId, query, token);
var dto = new DatesRangeDto
{
From = item.Min,
To = item.Max,
};
if (columnNames is not null && columnNames.Any())
data = ReduceSetColumnsByNames(data, columnNames);
return data;
return dto;
}
public Task<int> Count(Guid discriminatorId, CancellationToken token)
public virtual Task<int> Count(Guid discriminatorId, CancellationToken token)
{
var dbSet = db.Set<TimestampedValues>();
var query = dbSet.Where(entity => entity.DiscriminatorId == discriminatorId);
return query.CountAsync(token);
}
private async Task<IEnumerable<TimestampedValuesDto>> Materialize(Guid discriminatorId, IQueryable<TimestampedValues> query, CancellationToken token)
{
var dtoQuery = query.Select(entity => new TimestampedValuesDto()
{
Timestamp = entity.Timestamp,
Values = entity.Values
});
var valuesIdentities = await relatedDataRepository.Get(token);
var valuesIdentity = valuesIdentities?
.FirstOrDefault(e => e.DiscriminatorId == discriminatorId);
if (valuesIdentity == null)
return [];
var dtos = await dtoQuery.ToArrayAsync(token);
foreach(var dto in dtos)
var entities = await query.ToArrayAsync(token);
var dtos = entities.Select(entity =>
{
var valuesIdentities = await relatedDataRepository.Get(token);
var valuesIdentity = valuesIdentities?
.FirstOrDefault(e => e.DiscriminatorId == discriminatorId);
if (valuesIdentity == null)
return []; // ToDo: какая логика должна быть?
var dto = new TimestampedValuesDto()
{
Timestamp = entity.Timestamp.ToUniversalTime()
};
for (var i = 0; i < valuesIdentity.Identity.Count(); i++)
{
var key = valuesIdentity.Identity[i];
var value = dto.Values[i];
var value = entity.Values[i];
dto.Values[i] = new { key = value }; // ToDo: вывод?
dto.Values.Add(key, value);
}
}
return dto;
});
return dtos;
}
private IQueryable<TimestampedValues> ApplyGeTimestamp(IQueryable<TimestampedValues> query, DateTimeOffset geTimestamp)
private IQueryable<TimestampedValues> ApplyGeTimestamp(IQueryable<TimestampedValues> query, DateTimeOffset timestampBegin)
{
var geTimestampUtc = geTimestamp.ToUniversalTime();
return query.Where(entity => entity.Timestamp >= geTimestampUtc);
var geTimestampUtc = timestampBegin.ToUniversalTime();
var result = query
.Where(entity => entity.Timestamp >= geTimestampUtc);
return result;
}
private static IEnumerable<TimestampedValuesDto> ReduceSetColumnsByNames(IEnumerable<TimestampedValuesDto> query, IEnumerable<string> columnNames)
private IEnumerable<TimestampedValuesDto> ReduceSetColumnsByNames(IEnumerable<TimestampedValuesDto> dtos, IEnumerable<string> columnNames)
{
var newQuery = query;
//.Select(entity => new TimestampedValuesDto()
//{
// Timestamp = entity.Timestamp,
// Values = entity.Values?
// .Where(prop => columnNames.Contains(
// JsonSerializer.Deserialize<Dictionary<string, object>>(prop.ToString()!)?
// .FirstOrDefault().Key
// )).ToArray()
//});
return newQuery;
var result = dtos.Select(dto =>
{
var reducedValues = dto.Values
.Where(v => columnNames.Contains(v.Key))
.ToDictionary();
dto.Values = reducedValues;
return dto;
});
return result;
}
private async Task CreateValuesIdentityIfNotExist(Guid discriminatorId, string[] keys, CancellationToken token)
@ -232,7 +218,7 @@ public class TimestampedValuesRepository : ITimestampedValuesRepository
var valuesIdentity = valuesIdentities?
.FirstOrDefault(e => e.DiscriminatorId == discriminatorId);
if (valuesIdentity == null)
if (valuesIdentity is null)
{
valuesIdentity = new ValuesIdentityDto()
{

8
DD.Persistence.Repository/RepositoriesCached/RelatedDataCachedRepository.cs
View File

@ -1,16 +1,14 @@
using Microsoft.EntityFrameworkCore;
using DD.Persistence.Repository.Repositories;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Caching.Memory;
using DD.Persistence.Models;
using DD.Persistence.Repository.Repositories;
namespace DD.Persistence.Repository.RepositoriesCached;
public class RelatedDataCachedRepository<TDto, TEntity> : RelatedDataRepository<TDto, TEntity>
where TEntity : class, new()
where TDto : class, new()
{
private static readonly string SystemCacheKey = $"{typeof(Database.Entity.DataSourceSystem).FullName}CacheKey";
private static readonly string SystemCacheKey = $"{typeof(TEntity).FullName}CacheKey";
private readonly IMemoryCache memoryCache;
private const int CacheExpirationInMinutes = 60;
private readonly TimeSpan? AbsoluteExpirationRelativeToNow = TimeSpan.FromMinutes(60);
public RelatedDataCachedRepository(DbContext db, IMemoryCache memoryCache) : base(db)

55
DD.Persistence.Repository/RepositoriesCached/TimestampedValuesCachedRepository.cs
View File

@ -1,43 +1,41 @@
//using Microsoft.EntityFrameworkCore;
//using DD.Persistence.Models;
//using DD.Persistence.Models.Common;
//using DD.Persistence.ModelsAbstractions;
//using DD.Persistence.Database.EntityAbstractions;
//using DD.Persistence.Repositories;
//using Microsoft.EntityFrameworkCore;
//namespace DD.Persistence.Repository.Repositories;
//public class TimestampedValuesCachedRepository<TEntity, TDto> : TimeSeriesDataRepository<TEntity, TDto>
// where TEntity : class, ITimestampedItem, new()
// where TDto : class, ITimestampAbstractDto, new()
//public class TimestampedValuesCachedRepository : TimestampedValuesRepository
//{
// public static TDto? FirstByDate { get; private set; }
// public static CyclicArray<TDto> LastData { get; } = new CyclicArray<TDto>(CacheItemsCount);
// public static TimestampedValuesDto? FirstByDate { get; private set; }
// public static CyclicArray<TimestampedValuesDto> LastData { get; } = new CyclicArray<TimestampedValuesDto>(CacheItemsCount);
// private const int CacheItemsCount = 3600;
// public TimestampedValuesCachedRepository(DbContext db) : base(db)
// public TimestampedValuesCachedRepository(DbContext db, IRelatedDataRepository<ValuesIdentityDto> relatedDataRepository) : base(db, relatedDataRepository)
// {
// Task.Run(async () =>
// {
// var firstDateItem = await base.GetFirstAsync(CancellationToken.None);
// if (firstDateItem == null)
// {
// return;
// }
// //Task.Run(async () =>
// //{
// // var firstDateItem = await base.GetFirst(CancellationToken.None);
// // if (firstDateItem == null)
// // {
// // return;
// // }
// FirstByDate = firstDateItem;
// // FirstByDate = firstDateItem;
// var dtos = await base.GetLastAsync(CacheItemsCount, CancellationToken.None);
// dtos = dtos.OrderBy(d => d.Timestamp);
// LastData.AddRange(dtos);
// }).Wait();
// // var dtos = await base.GetLast(CacheItemsCount, CancellationToken.None);
// // dtos = dtos.OrderBy(d => d.Timestamp);
// // LastData.AddRange(dtos);
// //}).Wait();
// }
// public override async Task<IEnumerable<TDto>> GetGtDate(DateTimeOffset dateBegin, CancellationToken token)
// public override async Task<IEnumerable<TimestampedValuesDto>> GetGtDate(Guid discriminatorId, DateTimeOffset dateBegin, CancellationToken token)
// {
// if (LastData.Count == 0 || LastData[0].Timestamp > dateBegin)
// {
// var dtos = await base.GetGtDate(dateBegin, token);
// var dtos = await base.GetGtDate(discriminatorId, dateBegin, token);
// return dtos;
// }
@ -47,9 +45,9 @@
// return items;
// }
// public override async Task<int> AddRange(IEnumerable<TDto> dtos, CancellationToken token)
// public override async Task<int> AddRange(Guid discriminatorId, IEnumerable<TimestampedValuesDto> dtos, CancellationToken token)
// {
// var result = await base.AddRange(dtos, token);
// var result = await base.AddRange(discriminatorId, dtos, token);
// if (result > 0)
// {
@ -62,7 +60,7 @@
// return result;
// }
// public override async Task<DatesRangeDto?> GetDatesRange(CancellationToken token)
// public override async Task<DatesRangeDto?> GetDatesRange(Guid discriminatorId, CancellationToken token)
// {
// if (FirstByDate == null)
// return null;
@ -77,7 +75,8 @@
// });
// }
// public override async Task<IEnumerable<TDto>> GetResampledData(
// public override async Task<IEnumerable<TimestampedValuesDto>> GetResampledData(
// Guid discriminatorId,
// DateTimeOffset dateBegin,
// double intervalSec = 600d,
// int approxPointsCount = 1024,
@ -86,7 +85,7 @@
// var dtos = LastData.Where(i => i.Timestamp >= dateBegin);
// if (LastData.Count == 0 || LastData[0].Timestamp > dateBegin)
// {
// dtos = await base.GetGtDate(dateBegin, token);
// dtos = await base.GetGtDate(discriminatorId, dateBegin, token);
// }
// var dateEnd = dateBegin.AddSeconds(intervalSec);

2
DD.Persistence/Models/TimestampedValuesDto.cs
View File

@ -15,5 +15,5 @@ public class TimestampedValuesDto : ITimestampAbstractDto
/// <summary>
/// Набор данных
/// </summary>
public object[] Values { get; set; } = [];
public Dictionary<string, object> Values { get; set; } = [];
}

12
DD.Persistence/Repositories/IRelatedDataRepository.cs
View File

@ -1,14 +1,14 @@
using DD.Persistence.Models;
namespace DD.Persistence.Repositories;
namespace DD.Persistence.Repositories;
/// <summary>
/// Интерфейс по работе с системами - источниками данных
/// Интерфейс по работе с простой структурой данных, подразумевающей наличие связи с более сложной
/// В контексте TechMessagesRepository это системы - источники данных
/// В контексте TimestampedValuesRepository это идентификационные наборы (ключи для значений в соответствии с индексами в хранимых массивах)
/// </summary>
public interface IRelatedDataRepository<TDto>
{
/// <summary>
/// Добавить систему
/// Добавить данные
/// </summary>
/// <param name="dataSourceSystemDto"></param>
/// <param name="token"></param>
@ -16,7 +16,7 @@ public interface IRelatedDataRepository<TDto>
public Task Add(TDto dataSourceSystemDto, CancellationToken token);
/// <summary>
/// Получить список систем
/// Получить список данных
/// </summary>
/// <returns></returns>
public Task<IEnumerable<TDto>> Get(CancellationToken token);

31
DD.Persistence/Repositories/ITimestampedValuesRepository.cs
View File

@ -1,5 +1,4 @@
using DD.Persistence.Models;
using DD.Persistence.ModelsAbstractions;
using DD.Persistence.RepositoriesAbstractions;
namespace DD.Persistence.Repositories;
@ -19,23 +18,13 @@ public interface ITimestampedValuesRepository : ISyncRepository, ITimeSeriesBase
Task<int> AddRange(Guid idDiscriminator, IEnumerable<TimestampedValuesDto> dtos, CancellationToken token);
/// <summary>
/// Количество записей по указанному набору в БД. Для пагинации.
/// Количество записей по указанному набору в БД. Для пагинации
/// </summary>
/// <param name="idDiscriminator">Дискриминатор (идентификатор) набора</param>
/// <param name="token"></param>
/// <returns></returns>
Task<int> Count(Guid idDiscriminator, CancellationToken token);
/// <summary>
/// Получить последние данные
/// </summary>
/// <param name="idDiscriminator">Дискриминатор (идентификатор) набора</param>
/// <param name="columnNames">Фильтр свойств набора. Можно запросить только некоторые свойства из набора</param>
/// <param name="take"></param>
/// <param name="token"></param>
/// <returns></returns>
Task<IEnumerable<TimestampedValuesDto>> GetLast(Guid idDiscriminator, IEnumerable<string>? columnNames, int take, CancellationToken token);
/// <summary>
/// Получение данных с фильтрацией. Значение фильтра null - отключен
/// </summary>
@ -47,4 +36,22 @@ public interface ITimestampedValuesRepository : ISyncRepository, ITimeSeriesBase
/// <param name="token"></param>
/// <returns></returns>
Task<IEnumerable<TimestampedValuesDto>> Get(Guid idDiscriminator, DateTimeOffset? geTimestamp, IEnumerable<string>? columnNames, int skip, int take, CancellationToken token);
/// <summary>
/// Получение данных с начала
/// </summary>
/// <param name="discriminatorId">Дискриминатор (идентификатор) набора</param>
/// <param name="takeCount">Количество</param>
/// <param name="token"></param>
/// <returns></returns>
Task<IEnumerable<TimestampedValuesDto>> GetFirst(Guid discriminatorId, int takeCount, CancellationToken token);
/// <summary>
/// Получение данных с конца
/// </summary>
/// <param name="discriminatorId">Дискриминатор (идентификатор) набора</param>
/// <param name="takeCount">Количество</param>
/// <param name="token"></param>
/// <returns></returns>
Task<IEnumerable<TimestampedValuesDto>> GetLast(Guid discriminatorId, int takeCount, CancellationToken token);
}