Aктуализировать функционал по работе с временными рядами под единую сущность
Some checks failed
Unit tests / test (push) Failing after 1m5s

This commit is contained in:
Roman Efremov 2025-01-14 17:56:59 +05:00
parent 750788d550
commit ce0ad70369
24 changed files with 480 additions and 678 deletions

View File

@ -1,20 +0,0 @@
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc;
using DD.Persistence.Models;
using DD.Persistence.Repositories;
namespace DD.Persistence.API.Controllers;
/// <summary>
/// Работа с временными данными
/// </summary>
[ApiController]
[Authorize]
[Route("api/[controller]")]
public class DataSaubController : TimeSeriesController<DataSaubDto>
{
public DataSaubController(ITimestampedValuesRepository<DataSaubDto> timeSeriesDataRepository) : base(timeSeriesDataRepository)
{
}
}

View File

@ -1,76 +0,0 @@
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc;
using DD.Persistence.Repositories;
using DD.Persistence.ModelsAbstractions;
namespace DD.Persistence.API.Controllers;
[ApiController]
[Authorize]
[Route("api/[controller]")]
public class TimeSeriesController<TDto> : ControllerBase, ITimeSeriesDataApi<TDto>
where TDto : class, ITimestampAbstractDto, new()
{
private readonly ITimestampedValuesRepository<TDto> timeSeriesDataRepository;
public TimeSeriesController(ITimestampedValuesRepository<TDto> timeSeriesDataRepository)
{
this.timeSeriesDataRepository = timeSeriesDataRepository;
}
/// <summary>
/// Получить список объектов, удовлетворяющий диапазону дат
/// </summary>
/// <param name="dateBegin"></param>
/// <param name="token"></param>
/// <returns></returns>
[HttpGet]
[ProducesResponseType(StatusCodes.Status200OK)]
public async Task<IActionResult> Get(DateTimeOffset dateBegin, CancellationToken token)
{
var result = await timeSeriesDataRepository.GetGtDate(dateBegin, token);
return Ok(result);
}
/// <summary>
/// Получить диапазон дат, для которых есть данные в репозитории
/// </summary>
/// <param name="token"></param>
/// <returns></returns>
[HttpGet("datesRange")]
public async Task<IActionResult> GetDatesRange(CancellationToken token)
{
var result = await timeSeriesDataRepository.GetDatesRange(token);
return Ok(result);
}
/// <summary>
/// Получить список объектов с прореживанием, удовлетворяющий диапазону дат
/// </summary>
/// <param name="dateBegin"></param>
/// <param name="intervalSec"></param>
/// <param name="approxPointsCount"></param>
/// <param name="token"></param>
/// <returns></returns>
[HttpGet("resampled")]
public async Task<IActionResult> GetResampledData(DateTimeOffset dateBegin, double intervalSec = 600d, int approxPointsCount = 1024, CancellationToken token = default)
{
var result = await timeSeriesDataRepository.GetResampledData(dateBegin, intervalSec, approxPointsCount, token);
return Ok(result);
}
/// <summary>
/// Добавить записи
/// </summary>
/// <param name="dtos"></param>
/// <param name="token"></param>
/// <returns></returns>
[HttpPost]
public async Task<IActionResult> AddRange(IEnumerable<TDto> dtos, CancellationToken token)
{
var result = await timeSeriesDataRepository.AddRange(dtos, token);
return Ok(result);
}
}

View File

@ -1,9 +1,8 @@
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc;
using DD.Persistence.Models;
using DD.Persistence.Repositories;
using System.Net;
using DD.Persistence.Models;
using DD.Persistence.Models.Common;
using DD.Persistence.Repositories;
using Microsoft.AspNetCore.Mvc;
using System.Net;
namespace DD.Persistence.API.Controllers;
@ -12,13 +11,13 @@ namespace DD.Persistence.API.Controllers;
/// Не оптимизировано под большие данные.
/// </summary>
[ApiController]
[Authorize]
[Route("api/[controller]/{idDiscriminator}")]
//[Authorize]
[Route("api/[controller]/{discriminatorId}")]
public class TimestampedSetController : ControllerBase
{
private readonly ITimestampedSetRepository repository;
private readonly ITimestampedValuesRepository<TimestampedValuesDto> repository;
public TimestampedSetController(ITimestampedSetRepository repository)
public TimestampedSetController(ITimestampedValuesRepository<TimestampedValuesDto> repository)
{
this.repository = repository;
}
@ -27,22 +26,22 @@ public class TimestampedSetController : ControllerBase
/// Записать новые данные
/// Предполагается что данные с одним дискриминатором имеют одинаковую структуру
/// </summary>
/// <param name="idDiscriminator">Дискриминатор (идентификатор) набора</param>
/// <param name="discriminatorId">Дискриминатор (идентификатор) набора</param>
/// <param name="sets"></param>
/// <param name="token"></param>
/// <returns>кол-во затронутых записей</returns>
[HttpPost]
[ProducesResponseType(typeof(int), (int)HttpStatusCode.OK)]
public async Task<IActionResult> AddRange([FromRoute] Guid idDiscriminator, [FromBody] IEnumerable<TimestampedSetDto> sets, CancellationToken token)
public async Task<IActionResult> AddRange([FromRoute] Guid discriminatorId, [FromBody] IEnumerable<TimestampedValuesDto> sets, CancellationToken token)
{
var result = await repository.AddRange(idDiscriminator, sets, token);
var result = await repository.AddRange(discriminatorId, sets, token);
return Ok(result);
}
/// <summary>
/// Получение данных с фильтрацией. Значение фильтра null - отключен
/// </summary>
/// <param name="idDiscriminator">Дискриминатор (идентификатор) набора</param>
/// <param name="discriminatorId">Дискриминатор (идентификатор) набора</param>
/// <param name="geTimestamp">Фильтр позднее даты</param>
/// <param name="columnNames">Фильтр свойств набора. Можно запросить только некоторые свойства из набора</param>
/// <param name="skip"></param>
@ -50,56 +49,74 @@ public class TimestampedSetController : ControllerBase
/// <param name="token"></param>
/// <returns>Фильтрованный набор данных с сортировкой по отметке времени</returns>
[HttpGet]
[ProducesResponseType(typeof(IEnumerable<TimestampedSetDto>), (int)HttpStatusCode.OK)]
public async Task<IActionResult> Get(Guid idDiscriminator, DateTimeOffset? geTimestamp, [FromQuery] IEnumerable<string>? columnNames, int skip, int take, CancellationToken token)
[ProducesResponseType(typeof(IEnumerable<TimestampedValuesDto>), (int)HttpStatusCode.OK)]
public async Task<IActionResult> Get([FromRoute] Guid discriminatorId, DateTimeOffset? geTimestamp, [FromQuery] IEnumerable<string>? columnNames, int skip, int take, CancellationToken token)
{
var result = await repository.Get(idDiscriminator, geTimestamp, columnNames, skip, take, token);
var result = await repository.Get(discriminatorId, geTimestamp, columnNames, skip, take, token);
return Ok(result);
}
/// <summary>
/// Получить последние данные
/// </summary>
/// <param name="idDiscriminator">Дискриминатор (идентификатор) набора</param>
/// <param name="discriminatorId">Дискриминатор (идентификатор) набора</param>
/// <param name="columnNames">Фильтр свойств набора. Можно запросить только некоторые свойства из набора</param>
/// <param name="take"></param>
/// <param name="token"></param>
/// <returns>Фильтрованный набор данных с сортировкой по отметке времени</returns>
[HttpGet("last")]
[ProducesResponseType(typeof(IEnumerable<TimestampedSetDto>), (int)HttpStatusCode.OK)]
public async Task<IActionResult> GetLast(Guid idDiscriminator, [FromQuery] IEnumerable<string>? columnNames, int take, CancellationToken token)
[ProducesResponseType(typeof(IEnumerable<TimestampedValuesDto>), (int)HttpStatusCode.OK)]
public async Task<IActionResult> GetLast([FromRoute] Guid discriminatorId, [FromQuery] IEnumerable<string>? columnNames, int take, CancellationToken token)
{
var result = await repository.GetLast(idDiscriminator, columnNames, take, token);
var result = await repository.GetLast(discriminatorId, columnNames, take, token);
return Ok(result);
}
/// <summary>
/// Диапазон дат за которые есть данные
/// </summary>
/// <param name="idDiscriminator"></param>
/// <param name="discriminatorId"></param>
/// <param name="token"></param>
/// <returns>Дата первой и последней записи</returns>
[HttpGet("datesRange")]
[ProducesResponseType(typeof(DatesRangeDto), (int)HttpStatusCode.OK)]
[ProducesResponseType((int)HttpStatusCode.NoContent)]
public async Task<IActionResult> GetDatesRange(Guid idDiscriminator, CancellationToken token)
public async Task<IActionResult> GetDatesRange([FromRoute] Guid discriminatorId, CancellationToken token)
{
var result = await repository.GetDatesRange(idDiscriminator, token);
var result = await repository.GetDatesRange(discriminatorId, token);
return Ok(result);
}
/// <summary>
/// Количество записей по указанному набору в БД. Для пагинации.
/// </summary>
/// <param name="idDiscriminator">Дискриминатор (идентификатор) набора</param>
/// <param name="discriminatorId">Дискриминатор (идентификатор) набора</param>
/// <param name="token"></param>
/// <returns></returns>
[HttpGet("count")]
[ProducesResponseType(typeof(int), (int)HttpStatusCode.OK)]
[ProducesResponseType((int)HttpStatusCode.NoContent)]
public async Task<IActionResult> Count(Guid idDiscriminator, CancellationToken token)
public async Task<IActionResult> Count([FromRoute] Guid discriminatorId, CancellationToken token)
{
var result = await repository.Count(idDiscriminator, token);
var result = await repository.Count(discriminatorId, token);
return Ok(result);
}
/// <summary>
/// Получить список объектов с прореживанием, удовлетворяющий диапазону дат
/// </summary>
/// <param name="discriminatorId"></param>
/// <param name="dateBegin"></param>
/// <param name="intervalSec"></param>
/// <param name="approxPointsCount"></param>
/// <param name="token"></param>
/// <returns></returns>
[HttpGet("resampled")]
[ProducesResponseType(typeof(IEnumerable<TimestampedValuesDto>), (int)HttpStatusCode.OK)]
[ProducesResponseType((int)HttpStatusCode.NoContent)]
public async Task<IActionResult> GetResampledData([FromRoute] Guid discriminatorId, DateTimeOffset dateBegin, double intervalSec = 600d, int approxPointsCount = 1024, CancellationToken token = default)
{
var result = await repository.GetResampledData(discriminatorId, dateBegin, intervalSec, approxPointsCount, token);
return Ok(result);
}
}

View File

@ -1,6 +1,6 @@
{
"DbConnection": {
"Host": "postgres",
"Host": "localhost",
"Port": 5432,
"Database": "persistence",
"Username": "postgres",

View File

@ -18,7 +18,7 @@ public interface ITimestampedSetClient : IDisposable
/// <param name="sets"></param>
/// <param name="token"></param>
/// <returns></returns>
Task<int> AddRange(Guid idDiscriminator, IEnumerable<TimestampedSetDto> sets, CancellationToken token);
Task<int> AddRange(Guid idDiscriminator, IEnumerable<TimestampedValuesDto> sets, CancellationToken token);
/// <summary>
/// Количество записей по указанному набору в БД. Для пагинации
@ -38,7 +38,7 @@ public interface ITimestampedSetClient : IDisposable
/// <param name="take"></param>
/// <param name="token"></param>
/// <returns></returns>
Task<IEnumerable<TimestampedSetDto>> Get(Guid idDiscriminator, DateTimeOffset? geTimestamp, IEnumerable<string>? columnNames, int skip, int take, CancellationToken token);
Task<IEnumerable<TimestampedValuesDto>> Get(Guid idDiscriminator, DateTimeOffset? geTimestamp, IEnumerable<string>? columnNames, int skip, int take, CancellationToken token);
/// <summary>
/// Диапазон дат за которые есть данные
@ -56,5 +56,5 @@ public interface ITimestampedSetClient : IDisposable
/// <param name="take"></param>
/// <param name="token"></param>
/// <returns></returns>
Task<IEnumerable<TimestampedSetDto>> GetLast(Guid idDiscriminator, IEnumerable<string>? columnNames, int take, CancellationToken token);
Task<IEnumerable<TimestampedValuesDto>> GetLast(Guid idDiscriminator, IEnumerable<string>? columnNames, int take, CancellationToken token);
}

View File

@ -1,25 +0,0 @@
using DD.Persistence.Models;
using DD.Persistence.Models.Common;
using Refit;
namespace DD.Persistence.Client.Clients.Interfaces.Refit;
public interface IRefitTimestampedSetClient : IRefitClient, IDisposable
{
private const string baseUrl = "/api/TimestampedSet/{idDiscriminator}";
[Post(baseUrl)]
Task<IApiResponse<int>> AddRange(Guid idDiscriminator, IEnumerable<TimestampedSetDto> sets, CancellationToken token);
[Get(baseUrl)]
Task<IApiResponse<IEnumerable<TimestampedSetDto>>> Get(Guid idDiscriminator, [Query] DateTimeOffset? geTimestamp, [Query] IEnumerable<string>? columnNames, int skip, int take, CancellationToken token);
[Get($"{baseUrl}/last")]
Task<IApiResponse<IEnumerable<TimestampedSetDto>>> GetLast(Guid idDiscriminator, [Query] IEnumerable<string>? columnNames, int take, CancellationToken token);
[Get($"{baseUrl}/count")]
Task<IApiResponse<int>> Count(Guid idDiscriminator, CancellationToken token);
[Get($"{baseUrl}/datesRange")]
Task<IApiResponse<DatesRangeDto?>> GetDatesRange(Guid idDiscriminator, CancellationToken token);
}

View File

@ -0,0 +1,28 @@
using DD.Persistence.Models;
using DD.Persistence.Models.Common;
using Refit;
namespace DD.Persistence.Client.Clients.Interfaces.Refit;
public interface IRefitTimestampedValuesClient : IRefitClient, IDisposable
{
private const string baseUrl = "/api/TimestampedSet/{idDiscriminator}";
[Post(baseUrl)]
Task<IApiResponse<int>> AddRange(Guid idDiscriminator, IEnumerable<TimestampedValuesDto> sets, CancellationToken token);
[Get(baseUrl)]
Task<IApiResponse<IEnumerable<TimestampedValuesDto>>> Get(Guid idDiscriminator, [Query] DateTimeOffset? geTimestamp, [Query] IEnumerable<string>? columnNames, int skip, int take, CancellationToken token);
[Get($"{baseUrl}/last")]
Task<IApiResponse<IEnumerable<TimestampedValuesDto>>> GetLast(Guid discriminatorId, [Query] IEnumerable<string>? columnNames, int take, CancellationToken token);
[Get($"{baseUrl}/count")]
Task<IApiResponse<int>> Count(Guid discriminatorId, CancellationToken token);
[Get($"{baseUrl}/datesRange")]
Task<IApiResponse<DatesRangeDto?>> GetDatesRange(Guid discriminatorId, CancellationToken token);
[Get($"{baseUrl}/resampled")]
Task<IApiResponse<IEnumerable<TimestampedValuesDto>>> GetResampledData(DateTimeOffset dateBegin, double intervalSec = 600d, int approxPointsCount = 1024, CancellationToken token = default);
}

View File

@ -8,14 +8,14 @@ using DD.Persistence.Models.Common;
namespace DD.Persistence.Client.Clients;
public class TimestampedSetClient : BaseClient, ITimestampedSetClient
{
private readonly IRefitTimestampedSetClient refitTimestampedSetClient;
private readonly IRefitTimestampedValuesClient refitTimestampedSetClient;
public TimestampedSetClient(IRefitClientFactory<IRefitTimestampedSetClient> refitTimestampedSetClientFactory, ILogger<TimestampedSetClient> logger) : base(logger)
public TimestampedSetClient(IRefitClientFactory<IRefitTimestampedValuesClient> refitTimestampedSetClientFactory, ILogger<TimestampedSetClient> logger) : base(logger)
{
this.refitTimestampedSetClient = refitTimestampedSetClientFactory.Create();
}
public async Task<int> AddRange(Guid idDiscriminator, IEnumerable<TimestampedSetDto> sets, CancellationToken token)
public async Task<int> AddRange(Guid idDiscriminator, IEnumerable<TimestampedValuesDto> sets, CancellationToken token)
{
var result = await ExecutePostResponse(
async () => await refitTimestampedSetClient.AddRange(idDiscriminator, sets, token), token);
@ -23,7 +23,7 @@ public class TimestampedSetClient : BaseClient, ITimestampedSetClient
return result;
}
public async Task<IEnumerable<TimestampedSetDto>> Get(Guid idDiscriminator, DateTimeOffset? geTimestamp, IEnumerable<string>? columnNames, int skip, int take, CancellationToken token)
public async Task<IEnumerable<TimestampedValuesDto>> Get(Guid idDiscriminator, DateTimeOffset? geTimestamp, IEnumerable<string>? columnNames, int skip, int take, CancellationToken token)
{
var result = await ExecuteGetResponse(
async () => await refitTimestampedSetClient.Get(idDiscriminator, geTimestamp, columnNames, skip, take, token), token);
@ -31,7 +31,7 @@ public class TimestampedSetClient : BaseClient, ITimestampedSetClient
return result!;
}
public async Task<IEnumerable<TimestampedSetDto>> GetLast(Guid idDiscriminator, IEnumerable<string>? columnNames, int take, CancellationToken token)
public async Task<IEnumerable<TimestampedValuesDto>> GetLast(Guid idDiscriminator, IEnumerable<string>? columnNames, int take, CancellationToken token)
{
var result = await ExecuteGetResponse(
async () => await refitTimestampedSetClient.GetLast(idDiscriminator, columnNames, take, token), token);

View File

@ -22,7 +22,6 @@ public static class DependencyInjection
services.AddTransient<IDataSourceSystemClient, DataSourceSystemClient>();
services.AddTransient<ISetpointClient, SetpointClient>();
services.AddTransient<ITechMessagesClient, TechMessagesClient>();
services.AddTransient<ITimeSeriesClient<DataSaubDto>, TimeSeriesClient<DataSaubDto>>();
services.AddTransient<ITimestampedSetClient, TimestampedSetClient>();
services.AddTransient<IWitsDataClient, WitsDataClient>();
return services;

View File

@ -12,7 +12,7 @@ using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata;
namespace DD.Persistence.Database.Postgres.Migrations
{
[DbContext(typeof(PersistencePostgresContext))]
[Migration("20241220062251_Init")]
[Migration("20250114100429_Init")]
partial class Init
{
/// <inheritdoc />
@ -20,7 +20,7 @@ namespace DD.Persistence.Database.Postgres.Migrations
{
#pragma warning disable 612, 618
modelBuilder
.HasAnnotation("ProductVersion", "8.0.10")
.HasAnnotation("ProductVersion", "9.0.0")
.HasAnnotation("Relational:MaxIdentifierLength", 63);
NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder);
@ -105,27 +105,24 @@ namespace DD.Persistence.Database.Postgres.Migrations
b.ToTable("TechMessage");
});
modelBuilder.Entity("DD.Persistence.Database.Entity.TimestampedSet", b =>
modelBuilder.Entity("DD.Persistence.Database.Entity.TimestampedValues", b =>
{
b.Property<Guid>("IdDiscriminator")
b.Property<Guid>("DiscriminatorId")
.HasColumnType("uuid")
.HasComment("Дискриминатор ссылка на тип сохраняемых данных");
.HasComment("Дискриминатор системы");
b.Property<DateTimeOffset>("Timestamp")
.HasColumnType("timestamp with time zone")
.HasComment("Отметка времени, строго в UTC");
.HasComment("Временная отметка");
b.Property<string>("Set")
b.Property<string>("Values")
.IsRequired()
.HasColumnType("jsonb")
.HasComment("Набор сохраняемых данных");
.HasComment("Данные");
b.HasKey("IdDiscriminator", "Timestamp");
b.HasKey("DiscriminatorId", "Timestamp");
b.ToTable("TimestampedSets", t =>
{
t.HasComment("Общая таблица данных временных рядов");
});
b.ToTable("TimestampedSets");
});
modelBuilder.Entity("DD.Persistence.Database.Model.ChangeLog", b =>
@ -181,96 +178,13 @@ namespace DD.Persistence.Database.Postgres.Migrations
b.ToTable("ChangeLog");
});
modelBuilder.Entity("DD.Persistence.Database.Model.DataSaub", b =>
{
b.Property<DateTimeOffset>("Date")
.HasColumnType("timestamp with time zone")
.HasColumnName("date");
b.Property<double?>("AxialLoad")
.HasColumnType("double precision")
.HasColumnName("axialLoad");
b.Property<double?>("BitDepth")
.HasColumnType("double precision")
.HasColumnName("bitDepth");
b.Property<double?>("BlockPosition")
.HasColumnType("double precision")
.HasColumnName("blockPosition");
b.Property<double?>("BlockSpeed")
.HasColumnType("double precision")
.HasColumnName("blockSpeed");
b.Property<double?>("Flow")
.HasColumnType("double precision")
.HasColumnName("flow");
b.Property<double?>("HookWeight")
.HasColumnType("double precision")
.HasColumnName("hookWeight");
b.Property<int>("IdFeedRegulator")
.HasColumnType("integer")
.HasColumnName("idFeedRegulator");
b.Property<int?>("Mode")
.HasColumnType("integer")
.HasColumnName("mode");
b.Property<double?>("Mse")
.HasColumnType("double precision")
.HasColumnName("mse");
b.Property<short>("MseState")
.HasColumnType("smallint")
.HasColumnName("mseState");
b.Property<double?>("Pressure")
.HasColumnType("double precision")
.HasColumnName("pressure");
b.Property<double?>("Pump0Flow")
.HasColumnType("double precision")
.HasColumnName("pump0Flow");
b.Property<double?>("Pump1Flow")
.HasColumnType("double precision")
.HasColumnName("pump1Flow");
b.Property<double?>("Pump2Flow")
.HasColumnType("double precision")
.HasColumnName("pump2Flow");
b.Property<double?>("RotorSpeed")
.HasColumnType("double precision")
.HasColumnName("rotorSpeed");
b.Property<double?>("RotorTorque")
.HasColumnType("double precision")
.HasColumnName("rotorTorque");
b.Property<string>("User")
.HasColumnType("text")
.HasColumnName("user");
b.Property<double?>("WellDepth")
.HasColumnType("double precision")
.HasColumnName("wellDepth");
b.HasKey("Date");
b.ToTable("DataSaub");
});
modelBuilder.Entity("DD.Persistence.Database.Model.Setpoint", b =>
{
b.Property<Guid>("Key")
.HasColumnType("uuid")
.HasComment("Ключ");
b.Property<DateTimeOffset>("Created")
b.Property<DateTimeOffset>("Timestamp")
.HasColumnType("timestamp with time zone")
.HasComment("Дата создания уставки");
@ -283,7 +197,7 @@ namespace DD.Persistence.Database.Postgres.Migrations
.HasColumnType("jsonb")
.HasComment("Значение уставки");
b.HasKey("Key", "Created");
b.HasKey("Key", "Timestamp");
b.ToTable("Setpoint");
});

View File

@ -32,35 +32,6 @@ namespace DD.Persistence.Database.Postgres.Migrations
table.PrimaryKey("PK_ChangeLog", x => x.Id);
});
migrationBuilder.CreateTable(
name: "DataSaub",
columns: table => new
{
date = table.Column<DateTimeOffset>(type: "timestamp with time zone", nullable: false),
mode = table.Column<int>(type: "integer", nullable: true),
user = table.Column<string>(type: "text", nullable: true),
wellDepth = table.Column<double>(type: "double precision", nullable: true),
bitDepth = table.Column<double>(type: "double precision", nullable: true),
blockPosition = table.Column<double>(type: "double precision", nullable: true),
blockSpeed = table.Column<double>(type: "double precision", nullable: true),
pressure = table.Column<double>(type: "double precision", nullable: true),
axialLoad = table.Column<double>(type: "double precision", nullable: true),
hookWeight = table.Column<double>(type: "double precision", nullable: true),
rotorTorque = table.Column<double>(type: "double precision", nullable: true),
rotorSpeed = table.Column<double>(type: "double precision", nullable: true),
flow = table.Column<double>(type: "double precision", nullable: true),
mseState = table.Column<short>(type: "smallint", nullable: false),
idFeedRegulator = table.Column<int>(type: "integer", nullable: false),
mse = table.Column<double>(type: "double precision", nullable: true),
pump0Flow = table.Column<double>(type: "double precision", nullable: true),
pump1Flow = table.Column<double>(type: "double precision", nullable: true),
pump2Flow = table.Column<double>(type: "double precision", nullable: true)
},
constraints: table =>
{
table.PrimaryKey("PK_DataSaub", x => x.date);
});
migrationBuilder.CreateTable(
name: "DataSourceSystem",
columns: table => new
@ -93,28 +64,27 @@ namespace DD.Persistence.Database.Postgres.Migrations
columns: table => new
{
Key = table.Column<Guid>(type: "uuid", nullable: false, comment: "Ключ"),
Created = table.Column<DateTimeOffset>(type: "timestamp with time zone", nullable: false, comment: "Дата создания уставки"),
Timestamp = table.Column<DateTimeOffset>(type: "timestamp with time zone", nullable: false, comment: "Дата создания уставки"),
Value = table.Column<object>(type: "jsonb", nullable: false, comment: "Значение уставки"),
IdUser = table.Column<Guid>(type: "uuid", nullable: false, comment: "Id автора последнего изменения")
},
constraints: table =>
{
table.PrimaryKey("PK_Setpoint", x => new { x.Key, x.Created });
table.PrimaryKey("PK_Setpoint", x => new { x.Key, x.Timestamp });
});
migrationBuilder.CreateTable(
name: "TimestampedSets",
columns: table => new
{
IdDiscriminator = table.Column<Guid>(type: "uuid", nullable: false, comment: "Дискриминатор ссылка на тип сохраняемых данных"),
Timestamp = table.Column<DateTimeOffset>(type: "timestamp with time zone", nullable: false, comment: "Отметка времени, строго в UTC"),
Set = table.Column<string>(type: "jsonb", nullable: false, comment: "Набор сохраняемых данных")
Timestamp = table.Column<DateTimeOffset>(type: "timestamp with time zone", nullable: false, comment: "Временная отметка"),
DiscriminatorId = table.Column<Guid>(type: "uuid", nullable: false, comment: "Дискриминатор системы"),
Values = table.Column<string>(type: "jsonb", nullable: false, comment: "Данные")
},
constraints: table =>
{
table.PrimaryKey("PK_TimestampedSets", x => new { x.IdDiscriminator, x.Timestamp });
},
comment: "Общая таблица данных временных рядов");
table.PrimaryKey("PK_TimestampedSets", x => new { x.DiscriminatorId, x.Timestamp });
});
migrationBuilder.CreateTable(
name: "TechMessage",
@ -150,9 +120,6 @@ namespace DD.Persistence.Database.Postgres.Migrations
migrationBuilder.DropTable(
name: "ChangeLog");
migrationBuilder.DropTable(
name: "DataSaub");
migrationBuilder.DropTable(
name: "ParameterData");

View File

@ -17,7 +17,7 @@ namespace DD.Persistence.Database.Postgres.Migrations
{
#pragma warning disable 612, 618
modelBuilder
.HasAnnotation("ProductVersion", "8.0.10")
.HasAnnotation("ProductVersion", "9.0.0")
.HasAnnotation("Relational:MaxIdentifierLength", 63);
NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder);
@ -102,27 +102,24 @@ namespace DD.Persistence.Database.Postgres.Migrations
b.ToTable("TechMessage");
});
modelBuilder.Entity("DD.Persistence.Database.Entity.TimestampedSet", b =>
modelBuilder.Entity("DD.Persistence.Database.Entity.TimestampedValues", b =>
{
b.Property<Guid>("IdDiscriminator")
b.Property<Guid>("DiscriminatorId")
.HasColumnType("uuid")
.HasComment("Дискриминатор ссылка на тип сохраняемых данных");
.HasComment("Дискриминатор системы");
b.Property<DateTimeOffset>("Timestamp")
.HasColumnType("timestamp with time zone")
.HasComment("Отметка времени, строго в UTC");
.HasComment("Временная отметка");
b.Property<string>("Set")
b.Property<string>("Values")
.IsRequired()
.HasColumnType("jsonb")
.HasComment("Набор сохраняемых данных");
.HasComment("Данные");
b.HasKey("IdDiscriminator", "Timestamp");
b.HasKey("DiscriminatorId", "Timestamp");
b.ToTable("TimestampedSets", t =>
{
t.HasComment("Общая таблица данных временных рядов");
});
b.ToTable("TimestampedSets");
});
modelBuilder.Entity("DD.Persistence.Database.Model.ChangeLog", b =>
@ -178,96 +175,13 @@ namespace DD.Persistence.Database.Postgres.Migrations
b.ToTable("ChangeLog");
});
modelBuilder.Entity("DD.Persistence.Database.Model.DataSaub", b =>
{
b.Property<DateTimeOffset>("Date")
.HasColumnType("timestamp with time zone")
.HasColumnName("date");
b.Property<double?>("AxialLoad")
.HasColumnType("double precision")
.HasColumnName("axialLoad");
b.Property<double?>("BitDepth")
.HasColumnType("double precision")
.HasColumnName("bitDepth");
b.Property<double?>("BlockPosition")
.HasColumnType("double precision")
.HasColumnName("blockPosition");
b.Property<double?>("BlockSpeed")
.HasColumnType("double precision")
.HasColumnName("blockSpeed");
b.Property<double?>("Flow")
.HasColumnType("double precision")
.HasColumnName("flow");
b.Property<double?>("HookWeight")
.HasColumnType("double precision")
.HasColumnName("hookWeight");
b.Property<int>("IdFeedRegulator")
.HasColumnType("integer")
.HasColumnName("idFeedRegulator");
b.Property<int?>("Mode")
.HasColumnType("integer")
.HasColumnName("mode");
b.Property<double?>("Mse")
.HasColumnType("double precision")
.HasColumnName("mse");
b.Property<short>("MseState")
.HasColumnType("smallint")
.HasColumnName("mseState");
b.Property<double?>("Pressure")
.HasColumnType("double precision")
.HasColumnName("pressure");
b.Property<double?>("Pump0Flow")
.HasColumnType("double precision")
.HasColumnName("pump0Flow");
b.Property<double?>("Pump1Flow")
.HasColumnType("double precision")
.HasColumnName("pump1Flow");
b.Property<double?>("Pump2Flow")
.HasColumnType("double precision")
.HasColumnName("pump2Flow");
b.Property<double?>("RotorSpeed")
.HasColumnType("double precision")
.HasColumnName("rotorSpeed");
b.Property<double?>("RotorTorque")
.HasColumnType("double precision")
.HasColumnName("rotorTorque");
b.Property<string>("User")
.HasColumnType("text")
.HasColumnName("user");
b.Property<double?>("WellDepth")
.HasColumnType("double precision")
.HasColumnName("wellDepth");
b.HasKey("Date");
b.ToTable("DataSaub");
});
modelBuilder.Entity("DD.Persistence.Database.Model.Setpoint", b =>
{
b.Property<Guid>("Key")
.HasColumnType("uuid")
.HasComment("Ключ");
b.Property<DateTimeOffset>("Created")
b.Property<DateTimeOffset>("Timestamp")
.HasColumnType("timestamp with time zone")
.HasComment("Дата создания уставки");
@ -280,7 +194,7 @@ namespace DD.Persistence.Database.Postgres.Migrations
.HasColumnType("jsonb")
.HasComment("Значение уставки");
b.HasKey("Key", "Created");
b.HasKey("Key", "Timestamp");
b.ToTable("Setpoint");
});

View File

@ -9,11 +9,9 @@ namespace DD.Persistence.Database;
/// </summary>
public class PersistenceDbContext : DbContext
{
public DbSet<DataSaub> DataSaub => Set<DataSaub>();
public DbSet<Setpoint> Setpoint => Set<Setpoint>();
public DbSet<TimestampedSet> TimestampedSets => Set<TimestampedSet>();
public DbSet<TimestampedValues> TimestampedSets => Set<TimestampedValues>();
public DbSet<ChangeLog> ChangeLog => Set<ChangeLog>();
@ -31,8 +29,8 @@ public class PersistenceDbContext : DbContext
protected override void OnModelCreating(ModelBuilder modelBuilder)
{
modelBuilder.Entity<TimestampedSet>()
.Property(e => e.Set)
modelBuilder.Entity<TimestampedValues>()
.Property(e => e.Values)
.HasJsonConversion();
modelBuilder.Entity<ChangeLog>()

View File

@ -1,85 +1,85 @@
using DD.Persistence.Database.Model;
using DD.Persistence.Models;
using Xunit;
//using DD.Persistence.Database.Model;
//using DD.Persistence.Models;
//using Xunit;
namespace DD.Persistence.IntegrationTests.Controllers;
public class DataSaubControllerTest : TimeSeriesBaseControllerTest<DataSaub, DataSaubDto>
{
private readonly DataSaubDto dto = new()
{
AxialLoad = 1,
BitDepth = 2,
BlockPosition = 3,
BlockSpeed = 4,
Date = DateTimeOffset.UtcNow,
Flow = 5,
HookWeight = 6,
IdFeedRegulator = 8,
Mode = 9,
Mse = 10,
MseState = 11,
Pressure = 12,
Pump0Flow = 13,
Pump1Flow = 14,
Pump2Flow = 15,
RotorSpeed = 16,
RotorTorque = 17,
User = string.Empty,
WellDepth = 18,
};
//namespace DD.Persistence.IntegrationTests.Controllers;
//public class DataSaubControllerTest : TimeSeriesBaseControllerTest<DataSaub, DataSaubDto>
//{
// private readonly DataSaubDto dto = new()
// {
// AxialLoad = 1,
// BitDepth = 2,
// BlockPosition = 3,
// BlockSpeed = 4,
// Date = DateTimeOffset.UtcNow,
// Flow = 5,
// HookWeight = 6,
// IdFeedRegulator = 8,
// Mode = 9,
// Mse = 10,
// MseState = 11,
// Pressure = 12,
// Pump0Flow = 13,
// Pump1Flow = 14,
// Pump2Flow = 15,
// RotorSpeed = 16,
// RotorTorque = 17,
// User = string.Empty,
// WellDepth = 18,
// };
private readonly DataSaub entity = new()
{
AxialLoad = 1,
BitDepth = 2,
BlockPosition = 3,
BlockSpeed = 4,
Timestamp = DateTimeOffset.UtcNow,
Flow = 5,
HookWeight = 6,
IdFeedRegulator = 8,
Mode = 9,
Mse = 10,
MseState = 11,
Pressure = 12,
Pump0Flow = 13,
Pump1Flow = 14,
Pump2Flow = 15,
RotorSpeed = 16,
RotorTorque = 17,
User = string.Empty,
WellDepth = 18,
};
// private readonly DataSaub entity = new()
// {
// AxialLoad = 1,
// BitDepth = 2,
// BlockPosition = 3,
// BlockSpeed = 4,
// Timestamp = DateTimeOffset.UtcNow,
// Flow = 5,
// HookWeight = 6,
// IdFeedRegulator = 8,
// Mode = 9,
// Mse = 10,
// MseState = 11,
// Pressure = 12,
// Pump0Flow = 13,
// Pump1Flow = 14,
// Pump2Flow = 15,
// RotorSpeed = 16,
// RotorTorque = 17,
// User = string.Empty,
// WellDepth = 18,
// };
public DataSaubControllerTest(WebAppFactoryFixture factory) : base(factory)
{
}
// public DataSaubControllerTest(WebAppFactoryFixture factory) : base(factory)
// {
// }
[Fact]
public async Task InsertRange_returns_success()
{
await InsertRangeSuccess(dto);
}
// [Fact]
// public async Task InsertRange_returns_success()
// {
// await InsertRangeSuccess(dto);
// }
[Fact]
public async Task Get_returns_success()
{
var beginDate = DateTimeOffset.UtcNow.AddDays(-1);
var endDate = DateTimeOffset.UtcNow;
await GetSuccess(beginDate, endDate, entity);
}
// [Fact]
// public async Task Get_returns_success()
// {
// var beginDate = DateTimeOffset.UtcNow.AddDays(-1);
// var endDate = DateTimeOffset.UtcNow;
// await GetSuccess(beginDate, endDate, entity);
// }
[Fact]
public async Task GetDatesRange_returns_success()
{
await GetDatesRangeSuccess(entity);
}
// [Fact]
// public async Task GetDatesRange_returns_success()
// {
// await GetDatesRangeSuccess(entity);
// }
[Fact]
public async Task GetResampledData_returns_success()
{
await GetResampledDataSuccess(entity);
}
}
// [Fact]
// public async Task GetResampledData_returns_success()
// {
// await GetResampledDataSuccess(entity);
// }
//}

View File

@ -15,7 +15,7 @@ public class TimestampedSetControllerTest : BaseIntegrationTest
public TimestampedSetControllerTest(WebAppFactoryFixture factory) : base(factory)
{
var refitClientFactory = scope.ServiceProvider
.GetRequiredService<IRefitClientFactory<IRefitTimestampedSetClient>>();
.GetRequiredService<IRefitClientFactory<IRefitTimestampedValuesClient>>();
var logger = scope.ServiceProvider.GetRequiredService<ILogger<TimestampedSetClient>>();
client = scope.ServiceProvider
@ -27,7 +27,7 @@ public class TimestampedSetControllerTest : BaseIntegrationTest
{
// arrange
Guid idDiscriminator = Guid.NewGuid();
IEnumerable<TimestampedSetDto> testSets = Generate(10, DateTimeOffset.Now.ToOffset(TimeSpan.FromHours(7)));
IEnumerable<TimestampedValuesDto> testSets = Generate(10, DateTimeOffset.Now.ToOffset(TimeSpan.FromHours(7)));
// act
var response = await client.AddRange(idDiscriminator, testSets, CancellationToken.None);
@ -42,7 +42,7 @@ public class TimestampedSetControllerTest : BaseIntegrationTest
// arrange
Guid idDiscriminator = Guid.NewGuid();
int count = 10;
IEnumerable<TimestampedSetDto> testSets = Generate(count, DateTimeOffset.Now.ToOffset(TimeSpan.FromHours(7)));
IEnumerable<TimestampedValuesDto> testSets = Generate(count, DateTimeOffset.Now.ToOffset(TimeSpan.FromHours(7)));
await client.AddRange(idDiscriminator, testSets, CancellationToken.None);
// act
@ -59,7 +59,7 @@ public class TimestampedSetControllerTest : BaseIntegrationTest
// arrange
Guid idDiscriminator = Guid.NewGuid();
int count = 10;
IEnumerable<TimestampedSetDto> testSets = Generate(count, DateTimeOffset.Now.ToOffset(TimeSpan.FromHours(7)));
IEnumerable<TimestampedValuesDto> testSets = Generate(count, DateTimeOffset.Now.ToOffset(TimeSpan.FromHours(7)));
await client.AddRange(idDiscriminator, testSets, CancellationToken.None);
string[] props = ["A"];
@ -71,9 +71,9 @@ public class TimestampedSetControllerTest : BaseIntegrationTest
Assert.Equal(count, response.Count());
foreach (var item in response)
{
Assert.Single(item.Set);
var kv = item.Set.First();
Assert.Equal("A", kv.Key);
Assert.Single(item.Values);
var kv = item.Values.First();
Assert.Equal("A", ((KeyValuePair<string, object>) kv).Key);
}
}
@ -85,7 +85,7 @@ public class TimestampedSetControllerTest : BaseIntegrationTest
int count = 10;
var dateMin = DateTimeOffset.Now;
var dateMax = DateTimeOffset.Now.AddSeconds(count);
IEnumerable<TimestampedSetDto> testSets = Generate(count, dateMin.ToOffset(TimeSpan.FromHours(7)));
IEnumerable<TimestampedValuesDto> testSets = Generate(count, dateMin.ToOffset(TimeSpan.FromHours(7)));
var insertResponse = await client.AddRange(idDiscriminator, testSets, CancellationToken.None);
var tail = testSets.OrderBy(t => t.Timestamp).Skip(count / 2).Take(int.MaxValue);
var geDate = tail.First().Timestamp;
@ -108,7 +108,7 @@ public class TimestampedSetControllerTest : BaseIntegrationTest
// arrange
Guid idDiscriminator = Guid.NewGuid();
int count = 10;
IEnumerable<TimestampedSetDto> testSets = Generate(count, DateTimeOffset.Now.ToOffset(TimeSpan.FromHours(7)));
IEnumerable<TimestampedValuesDto> testSets = Generate(count, DateTimeOffset.Now.ToOffset(TimeSpan.FromHours(7)));
await client.AddRange(idDiscriminator, testSets, CancellationToken.None);
var expectedCount = count / 2;
@ -128,7 +128,7 @@ public class TimestampedSetControllerTest : BaseIntegrationTest
Guid idDiscriminator = Guid.NewGuid();
var expectedCount = 1;
int count = 10 + expectedCount;
IEnumerable<TimestampedSetDto> testSets = Generate(count, DateTimeOffset.Now.ToOffset(TimeSpan.FromHours(7)));
IEnumerable<TimestampedValuesDto> testSets = Generate(count, DateTimeOffset.Now.ToOffset(TimeSpan.FromHours(7)));
await client.AddRange(idDiscriminator, testSets, CancellationToken.None);
// act
@ -145,7 +145,7 @@ public class TimestampedSetControllerTest : BaseIntegrationTest
// arrange
Guid idDiscriminator = Guid.NewGuid();
int count = 10;
IEnumerable<TimestampedSetDto> testSets = Generate(count, DateTimeOffset.Now.ToOffset(TimeSpan.FromHours(7)));
IEnumerable<TimestampedValuesDto> testSets = Generate(count, DateTimeOffset.Now.ToOffset(TimeSpan.FromHours(7)));
await client.AddRange(idDiscriminator, testSets, CancellationToken.None);
var expectedCount = 8;
@ -165,7 +165,7 @@ public class TimestampedSetControllerTest : BaseIntegrationTest
int count = 10;
var dateMin = DateTimeOffset.Now;
var dateMax = DateTimeOffset.Now.AddSeconds(count - 1);
IEnumerable<TimestampedSetDto> testSets = Generate(count, dateMin.ToOffset(TimeSpan.FromHours(7)));
IEnumerable<TimestampedValuesDto> testSets = Generate(count, dateMin.ToOffset(TimeSpan.FromHours(7)));
await client.AddRange(idDiscriminator, testSets, CancellationToken.None);
var tolerance = TimeSpan.FromSeconds(1);
@ -184,7 +184,7 @@ public class TimestampedSetControllerTest : BaseIntegrationTest
// arrange
Guid idDiscriminator = Guid.NewGuid();
int count = 144;
IEnumerable<TimestampedSetDto> testSets = Generate(count, DateTimeOffset.Now.ToOffset(TimeSpan.FromHours(7)));
IEnumerable<TimestampedValuesDto> testSets = Generate(count, DateTimeOffset.Now.ToOffset(TimeSpan.FromHours(7)));
await client.AddRange(idDiscriminator, testSets, CancellationToken.None);
// act
@ -194,18 +194,20 @@ public class TimestampedSetControllerTest : BaseIntegrationTest
Assert.Equal(count, response);
}
private static IEnumerable<TimestampedSetDto> Generate(int n, DateTimeOffset from)
private static IEnumerable<TimestampedValuesDto> Generate(int n, DateTimeOffset from)
{
for (int i = 0; i < n; i++)
yield return new TimestampedSetDto
(
from.AddSeconds(i),
new Dictionary<string, object>{
yield return new TimestampedValuesDto()
{
Timestamp = from.AddSeconds(i),
Values = new Dictionary<string, object> {
{"A", i },
{"B", i * 1.1 },
{"C", $"Any{i}" },
{"D", DateTimeOffset.Now},
{"D", DateTimeOffset.Now}
}
);
.Select(e => (object) e)
.ToArray()
};
}
}

View File

@ -36,11 +36,9 @@ public static class DependencyInjection
MapsterSetup();
services.AddTransient<ITimestampedValuesRepository<DataSaubDto>, TagBagDataRepository>();
services.AddTransient<ISetpointRepository, SetpointRepository>();
//services.AddTransient<ITimestampedValuesRepository<DataSaubDto>, TimeSeriesDataCachedRepository<DataSaub, DataSaubDto>>();
services.AddTransient<IChangeLogRepository, ChangeLogRepository>();
services.AddTransient<ITimestampedSetRepository, TimestampedSetRepository>();
services.AddTransient<ITimestampedValuesRepository<TimestampedValuesDto>, TimestampedValuesRepository<TimestampedValuesDto>>();
services.AddTransient<ITechMessagesRepository, TechMessagesRepository>();
services.AddTransient<IParameterRepository, ParameterRepository>();
services.AddTransient<IDataSourceSystemRepository, DataSourceSystemCachedRepository>();

View File

@ -0,0 +1,17 @@
namespace DD.Persistence.Repository.Extensions;
public static class IEnumerableExtensions
{
public static void ForEach<T>(this IEnumerable<T> source, Action<T> action)
{
if (source == null)
throw new ArgumentNullException(nameof(source));
if (action == null)
throw new ArgumentNullException(nameof(action));
foreach (var item in source)
{
action(item);
}
}
}

View File

@ -1,122 +1,122 @@
using Microsoft.EntityFrameworkCore;
using DD.Persistence.Database.Entity;
using DD.Persistence.Models;
using DD.Persistence.Repositories;
using DD.Persistence.Models.Common;
//using Microsoft.EntityFrameworkCore;
//using DD.Persistence.Database.Entity;
//using DD.Persistence.Models;
//using DD.Persistence.Repositories;
//using DD.Persistence.Models.Common;
namespace DD.Persistence.Repository.Repositories;
//namespace DD.Persistence.Repository.Repositories;
/// <summary>
/// Репозиторий для хранения разных наборов данных временных рядов.
/// idDiscriminator - идентифицирует конкретный набор данных, прим.: циклы измерения АСИБР, или отчет о DrillTest.
/// idDiscriminator формируют клиенты и только им известно что они обозначают.
/// Так как данные приходят редко, то их прореживания для построения графиков не предусмотрено.
/// </summary>
public class TimestampedSetRepository : ITimestampedSetRepository
{
private readonly DbContext db;
///// <summary>
///// Репозиторий для хранения разных наборов данных временных рядов.
///// idDiscriminator - идентифицирует конкретный набор данных, прим.: циклы измерения АСИБР, или отчет о DrillTest.
///// idDiscriminator формируют клиенты и только им известно что они обозначают.
///// Так как данные приходят редко, то их прореживания для построения графиков не предусмотрено.
///// </summary>
//public class TimestampedSetRepository : ITimestampedSetRepository
//{
// private readonly DbContext db;
public TimestampedSetRepository(DbContext db)
{
this.db = db;
}
// public TimestampedSetRepository(DbContext db)
// {
// this.db = db;
// }
public Task<int> AddRange(Guid idDiscriminator, IEnumerable<TimestampedSetDto> sets, CancellationToken token)
{
var entities = sets.Select(set => new TimestampedSet(idDiscriminator, set.Timestamp.ToUniversalTime(), set.Set));
var dbSet = db.Set<TimestampedSet>();
dbSet.AddRange(entities);
return db.SaveChangesAsync(token);
}
// public Task<int> AddRange(Guid idDiscriminator, IEnumerable<TimestampedSetDto> sets, CancellationToken token)
// {
// var entities = sets.Select(set => new TimestampedSet(idDiscriminator, set.Timestamp.ToUniversalTime(), set.Set));
// var dbSet = db.Set<TimestampedSet>();
// dbSet.AddRange(entities);
// return db.SaveChangesAsync(token);
// }
public async Task<IEnumerable<TimestampedSetDto>> Get(Guid idDiscriminator, DateTimeOffset? geTimestamp, IEnumerable<string>? columnNames, int skip, int take, CancellationToken token)
{
var dbSet = db.Set<TimestampedSet>();
var query = dbSet.Where(entity => entity.IdDiscriminator == idDiscriminator);
// public async Task<IEnumerable<TimestampedSetDto>> Get(Guid idDiscriminator, DateTimeOffset? geTimestamp, IEnumerable<string>? columnNames, int skip, int take, CancellationToken token)
// {
// var dbSet = db.Set<TimestampedSet>();
// var query = dbSet.Where(entity => entity.IdDiscriminator == idDiscriminator);
if (geTimestamp.HasValue)
query = ApplyGeTimestamp(query, geTimestamp.Value);
// if (geTimestamp.HasValue)
// query = ApplyGeTimestamp(query, geTimestamp.Value);
query = query
.OrderBy(item => item.Timestamp)
.Skip(skip)
.Take(take);
// query = query
// .OrderBy(item => item.Timestamp)
// .Skip(skip)
// .Take(take);
var data = await Materialize(query, token);
// var data = await Materialize(query, token);
if (columnNames is not null && columnNames.Any())
data = ReduceSetColumnsByNames(data, columnNames);
// if (columnNames is not null && columnNames.Any())
// data = ReduceSetColumnsByNames(data, columnNames);
return data;
}
// return data;
// }
public async Task<IEnumerable<TimestampedSetDto>> GetLast(Guid idDiscriminator, IEnumerable<string>? columnNames, int take, CancellationToken token)
{
var dbSet = db.Set<TimestampedSet>();
var query = dbSet.Where(entity => entity.IdDiscriminator == idDiscriminator);
// public async Task<IEnumerable<TimestampedSetDto>> GetLast(Guid idDiscriminator, IEnumerable<string>? columnNames, int take, CancellationToken token)
// {
// var dbSet = db.Set<TimestampedSet>();
// var query = dbSet.Where(entity => entity.IdDiscriminator == idDiscriminator);
query = query.OrderByDescending(entity => entity.Timestamp)
.Take(take)
.OrderBy(entity => entity.Timestamp);
// query = query.OrderByDescending(entity => entity.Timestamp)
// .Take(take)
// .OrderBy(entity => entity.Timestamp);
var data = await Materialize(query, token);
// var data = await Materialize(query, token);
if (columnNames is not null && columnNames.Any())
data = ReduceSetColumnsByNames(data, columnNames);
// if (columnNames is not null && columnNames.Any())
// data = ReduceSetColumnsByNames(data, columnNames);
return data;
}
// return data;
// }
public Task<int> Count(Guid idDiscriminator, CancellationToken token)
{
var dbSet = db.Set<TimestampedSet>();
var query = dbSet.Where(entity => entity.IdDiscriminator == idDiscriminator);
return query.CountAsync(token);
}
// public Task<int> Count(Guid idDiscriminator, CancellationToken token)
// {
// var dbSet = db.Set<TimestampedSet>();
// var query = dbSet.Where(entity => entity.IdDiscriminator == idDiscriminator);
// return query.CountAsync(token);
// }
public async Task<DatesRangeDto?> GetDatesRange(Guid idDiscriminator, CancellationToken token)
{
var query = db.Set<TimestampedSet>()
.GroupBy(entity => entity.IdDiscriminator)
.Select(group => new
{
Min = group.Min(entity => entity.Timestamp),
Max = group.Max(entity => entity.Timestamp),
});
// public async Task<DatesRangeDto?> GetDatesRange(Guid idDiscriminator, CancellationToken token)
// {
// var query = db.Set<TimestampedSet>()
// .GroupBy(entity => entity.IdDiscriminator)
// .Select(group => new
// {
// Min = group.Min(entity => entity.Timestamp),
// Max = group.Max(entity => entity.Timestamp),
// });
var item = await query.FirstOrDefaultAsync(token);
if (item is null)
return null;
// var item = await query.FirstOrDefaultAsync(token);
// if (item is null)
// return null;
return new DatesRangeDto
{
From = item.Min,
To = item.Max,
};
}
// return new DatesRangeDto
// {
// From = item.Min,
// To = item.Max,
// };
// }
private static async Task<IEnumerable<TimestampedSetDto>> Materialize(IQueryable<TimestampedSet> query, CancellationToken token)
{
var dtoQuery = query.Select(entity => new TimestampedSetDto(entity.Timestamp, entity.Set));
var dtos = await dtoQuery.ToArrayAsync(token);
return dtos;
}
// private static async Task<IEnumerable<TimestampedSetDto>> Materialize(IQueryable<TimestampedSet> query, CancellationToken token)
// {
// var dtoQuery = query.Select(entity => new TimestampedSetDto(entity.Timestamp, entity.Set));
// var dtos = await dtoQuery.ToArrayAsync(token);
// return dtos;
// }
private static IQueryable<TimestampedSet> ApplyGeTimestamp(IQueryable<TimestampedSet> query, DateTimeOffset geTimestamp)
{
var geTimestampUtc = geTimestamp.ToUniversalTime();
return query.Where(entity => entity.Timestamp >= geTimestampUtc);
}
// private static IQueryable<TimestampedSet> ApplyGeTimestamp(IQueryable<TimestampedSet> query, DateTimeOffset geTimestamp)
// {
// var geTimestampUtc = geTimestamp.ToUniversalTime();
// return query.Where(entity => entity.Timestamp >= geTimestampUtc);
// }
private static IEnumerable<TimestampedSetDto> ReduceSetColumnsByNames(IEnumerable<TimestampedSetDto> query, IEnumerable<string> columnNames)
{
var newQuery = query
.Select(entity => new TimestampedSetDto(
entity.Timestamp,
entity.Set
.Where(prop => columnNames.Contains(prop.Key))
.ToDictionary(prop => prop.Key, prop => prop.Value)
));
return newQuery;
}
}
// private static IEnumerable<TimestampedSetDto> ReduceSetColumnsByNames(IEnumerable<TimestampedSetDto> query, IEnumerable<string> columnNames)
// {
// var newQuery = query
// .Select(entity => new TimestampedSetDto(
// entity.Timestamp,
// entity.Set
// .Where(prop => columnNames.Contains(prop.Key))
// .ToDictionary(prop => prop.Key, prop => prop.Value)
// ));
// return newQuery;
// }
//}

View File

@ -1,9 +1,11 @@
using DD.Persistence.Database.Entity;
using DD.Persistence.Models;
using DD.Persistence.Models.Common;
using DD.Persistence.ModelsAbstractions;
using DD.Persistence.Repositories;
using Mapster;
using Microsoft.EntityFrameworkCore;
using System.Text.Json;
namespace DD.Persistence.Repository.Repositories;
public class TimestampedValuesRepository<TDto> : ITimestampedValuesRepository<TDto>
@ -18,20 +20,28 @@ public class TimestampedValuesRepository<TDto> : ITimestampedValuesRepository<TD
protected virtual IQueryable<TimestampedValues> GetQueryReadOnly() => this.db.Set<TimestampedValues>();
public virtual async Task<DatesRangeDto?> GetDatesRange(CancellationToken token)
public virtual async Task<DatesRangeDto?> GetDatesRange(Guid discriminatorId, CancellationToken token)
{
var query = GetQueryReadOnly();
var minDate = await query.MinAsync(o => o.Timestamp, token);
var maxDate = await query.MaxAsync(o => o.Timestamp, token);
var query = GetQueryReadOnly()
.GroupBy(entity => entity.DiscriminatorId)
.Select(group => new
{
Min = group.Min(entity => entity.Timestamp),
Max = group.Max(entity => entity.Timestamp),
});
var item = await query.FirstOrDefaultAsync(token);
if (item is null)
return null;
return new DatesRangeDto
{
From = minDate,
To = maxDate
From = item.Min,
To = item.Max,
};
}
public virtual async Task<IEnumerable<TDto>> GetGtDate(DateTimeOffset date, CancellationToken token)
public virtual async Task<IEnumerable<TDto>> GetGtDate(Guid discriminatorId, DateTimeOffset date, CancellationToken token)
{
var query = GetQueryReadOnly().Where(e => e.Timestamp > date);
var entities = await query.ToArrayAsync(token);
@ -41,9 +51,13 @@ public class TimestampedValuesRepository<TDto> : ITimestampedValuesRepository<TD
return dtos;
}
public virtual async Task<int> AddRange(IEnumerable<TDto> dtos, CancellationToken token)
public virtual async Task<int> AddRange(Guid discriminatorId, IEnumerable<TDto> dtos, CancellationToken token)
{
var entities = dtos.Select(d => d.Adapt<TimestampedValues>());
var entities = dtos
.Select(d => d.Adapt<TimestampedValues>())
.ToList();
entities.ForEach(d => d.DiscriminatorId = discriminatorId);
await db.Set<TimestampedValues>().AddRangeAsync(entities, token);
var result = await db.SaveChangesAsync(token);
@ -78,12 +92,13 @@ public class TimestampedValuesRepository<TDto> : ITimestampedValuesRepository<TD
}
public async virtual Task<IEnumerable<TDto>> GetResampledData(
Guid discriminatorId,
DateTimeOffset dateBegin,
double intervalSec = 600d,
int approxPointsCount = 1024,
CancellationToken token = default)
{
var dtos = await GetGtDate(dateBegin, token);
var dtos = await GetGtDate(discriminatorId, dateBegin, token);
var dateEnd = dateBegin.AddSeconds(intervalSec);
dtos = dtos
@ -96,4 +111,77 @@ public class TimestampedValuesRepository<TDto> : ITimestampedValuesRepository<TD
return dtos;
}
public async Task<IEnumerable<TimestampedValuesDto>> Get(Guid idDiscriminator, DateTimeOffset? geTimestamp, IEnumerable<string>? columnNames, int skip, int take, CancellationToken token)
{
var dbSet = db.Set<TimestampedValues>();
var query = dbSet.Where(entity => entity.DiscriminatorId == idDiscriminator);
if (geTimestamp.HasValue)
query = ApplyGeTimestamp(query, geTimestamp.Value);
query = query
.OrderBy(item => item.Timestamp)
.Skip(skip)
.Take(take);
var data = await Materialize(query, token);
if (columnNames is not null && columnNames.Any())
data = ReduceSetColumnsByNames(data, columnNames);
return data;
}
public async Task<IEnumerable<TimestampedValuesDto>> GetLast(Guid idDiscriminator, IEnumerable<string>? columnNames, int take, CancellationToken token)
{
var dbSet = db.Set<TimestampedValues>();
var query = dbSet.Where(entity => entity.DiscriminatorId == idDiscriminator);
query = query.OrderByDescending(entity => entity.Timestamp)
.Take(take)
.OrderBy(entity => entity.Timestamp);
var data = await Materialize(query, token);
if (columnNames is not null && columnNames.Any())
data = ReduceSetColumnsByNames(data, columnNames);
return data;
}
public Task<int> Count(Guid idDiscriminator, CancellationToken token)
{
var dbSet = db.Set<TimestampedValues>();
var query = dbSet.Where(entity => entity.DiscriminatorId == idDiscriminator);
return query.CountAsync(token);
}
private static async Task<IEnumerable<TimestampedValuesDto>> Materialize(IQueryable<TimestampedValues> query, CancellationToken token)
{
var dtoQuery = query.Select(entity => new TimestampedValuesDto() { Timestamp = entity.Timestamp, Values = entity.Values });
var dtos = await dtoQuery.ToArrayAsync(token);
return dtos;
}
private static IQueryable<TimestampedValues> ApplyGeTimestamp(IQueryable<TimestampedValues> query, DateTimeOffset geTimestamp)
{
var geTimestampUtc = geTimestamp.ToUniversalTime();
return query.Where(entity => entity.Timestamp >= geTimestampUtc);
}
private static IEnumerable<TimestampedValuesDto> ReduceSetColumnsByNames(IEnumerable<TimestampedValuesDto> query, IEnumerable<string> columnNames)
{
var newQuery = query
.Select(entity => new TimestampedValuesDto()
{
Timestamp = entity.Timestamp,
Values = entity.Values?
.Where(prop => columnNames.Contains(
JsonSerializer.Deserialize<Dictionary<string, object>>(prop.ToString()!)?
.FirstOrDefault().Key
)).ToArray()
});
return newQuery;
}
}

View File

@ -1,9 +1,11 @@
namespace DD.Persistence.Models;
using DD.Persistence.ModelsAbstractions;
namespace DD.Persistence.Models;
/// <summary>
/// Набор данных с отметкой времени
/// </summary>
public class TimestampedSetDto
public class TimestampedValuesDto : ITimestampAbstractDto
{
/// <summary>
/// Временная отметка
@ -13,5 +15,5 @@ public class TimestampedSetDto
/// <summary>
/// Набор данных
/// </summary>
public required object[] Set { get; set; }
public object[]? Values { get; set; }
}

View File

@ -1,60 +0,0 @@
using DD.Persistence.Models;
using DD.Persistence.Models.Common;
namespace DD.Persistence.Repositories;
/// <summary>
/// Репозиторий для хранения разных наборов данных рядов.
/// idDiscriminator - идентифицирует конкретный набор данных, прим.: циклы измерения АСИБР, или отчет о DrillTest.
/// idDiscriminator формируют клиенты и только им известно что они обозначают.
/// Так как данные приходят редко, то их прореживания для построения графиков не предусмотрено.
/// </summary>
public interface ITimestampedSetRepository
{
/// <summary>
/// Количество записей по указанному набору в БД. Для пагинации.
/// </summary>
/// <param name="idDiscriminator">Дискриминатор (идентификатор) набора</param>
/// <param name="token"></param>
/// <returns></returns>
Task<int> Count(Guid idDiscriminator, CancellationToken token);
/// <summary>
/// Получение данных с фильтрацией. Значение фильтра null - отключен
/// </summary>
/// <param name="idDiscriminator">Дискриминатор (идентификатор) набора</param>
/// <param name="geTimestamp">Фильтр позднее даты</param>
/// <param name="columnNames">Фильтр свойств набора. Можно запросить только некоторые свойства из набора</param>
/// <param name="skip"></param>
/// <param name="take"></param>
/// <param name="token"></param>
/// <returns></returns>
Task<IEnumerable<TimestampedSetDto>> Get(Guid idDiscriminator, DateTimeOffset? geTimestamp, IEnumerable<string>? columnNames, int skip, int take, CancellationToken token);
/// <summary>
/// Диапазон дат за которые есть данные
/// </summary>
/// <param name="idDiscriminator">Дискриминатор (идентификатор) набора</param>
/// <param name="token"></param>
/// <returns></returns>
Task<DatesRangeDto?> GetDatesRange(Guid idDiscriminator, CancellationToken token);
/// <summary>
/// Получить последние данные
/// </summary>
/// <param name="idDiscriminator">Дискриминатор (идентификатор) набора</param>
/// <param name="columnNames">Фильтр свойств набора. Можно запросить только некоторые свойства из набора</param>
/// <param name="take"></param>
/// <param name="token"></param>
/// <returns></returns>
Task<IEnumerable<TimestampedSetDto>> GetLast(Guid idDiscriminator, IEnumerable<string>? columnNames, int take, CancellationToken token);
/// <summary>
/// Добавление новых данных
/// </summary>
/// <param name="idDiscriminator">Дискриминатор (идентификатор) набора</param>
/// <param name="sets"></param>
/// <param name="token"></param>
/// <returns></returns>
Task<int> AddRange(Guid idDiscriminator, IEnumerable<TimestampedSetDto> sets, CancellationToken token);
}

View File

@ -1,5 +1,6 @@
using DD.Persistence.Models;
using DD.Persistence.ModelsAbstractions;
using DD.Persistence.RepositoriesAbstractions;
namespace DD.Persistence.Repositories;
@ -7,14 +8,45 @@ namespace DD.Persistence.Repositories;
/// Интерфейс по работе с временными данными
/// </summary>
/// <typeparam name="TDto"></typeparam>
public interface ITimestampedValuesRepository<TDto> : ISyncRepository<TDto>, ITimeSeriesBaseRepository
public interface ITimestampedValuesRepository<TDto> : ISyncRepository<TDto>, ITimeSeriesBaseRepository<TDto>
where TDto : class, ITimestampAbstractDto, new()
{
/// <summary>
/// Добавление записей
/// </summary>
/// <param name="idDiscriminator"></param>
/// <param name="dtos"></param>
/// <param name="token"></param>
/// <returns></returns>
Task<int> AddRange(IEnumerable<TimestampedSetDto> dtos, CancellationToken token);
Task<int> AddRange(Guid idDiscriminator, IEnumerable<TDto> dtos, CancellationToken token);
/// <summary>
/// Количество записей по указанному набору в БД. Для пагинации.
/// </summary>
/// <param name="idDiscriminator">Дискриминатор (идентификатор) набора</param>
/// <param name="token"></param>
/// <returns></returns>
Task<int> Count(Guid idDiscriminator, CancellationToken token);
/// <summary>
/// Получить последние данные
/// </summary>
/// <param name="idDiscriminator">Дискриминатор (идентификатор) набора</param>
/// <param name="columnNames">Фильтр свойств набора. Можно запросить только некоторые свойства из набора</param>
/// <param name="take"></param>
/// <param name="token"></param>
/// <returns></returns>
Task<IEnumerable<TimestampedValuesDto>> GetLast(Guid idDiscriminator, IEnumerable<string>? columnNames, int take, CancellationToken token);
/// <summary>
/// Получение данных с фильтрацией. Значение фильтра null - отключен
/// </summary>
/// <param name="idDiscriminator">Дискриминатор (идентификатор) набора</param>
/// <param name="geTimestamp">Фильтр позднее даты</param>
/// <param name="columnNames">Фильтр свойств набора. Можно запросить только некоторые свойства из набора</param>
/// <param name="skip"></param>
/// <param name="take"></param>
/// <param name="token"></param>
/// <returns></returns>
Task<IEnumerable<TimestampedValuesDto>> Get(Guid idDiscriminator, DateTimeOffset? geTimestamp, IEnumerable<string>? columnNames, int skip, int take, CancellationToken token);
}

View File

@ -1,6 +1,6 @@
using DD.Persistence.Models.Common;
namespace DD.Persistence.Repositories;
namespace DD.Persistence.RepositoriesAbstractions;
/// <summary>
/// Интерфейс по работе с данными
@ -11,15 +11,18 @@ public interface ISyncRepository<TDto>
/// <summary>
/// Получить данные, начиная с определенной даты
/// </summary>
/// <param name="discriminatorId"></param>
/// <param name="dateBegin">дата начала</param>
/// <param name="token"></param> /// <returns></returns>
Task<IEnumerable<TDto>> GetGtDate(DateTimeOffset dateBegin, CancellationToken token);
/// <param name="token"></param>
/// <returns></returns>
Task<IEnumerable<TDto>> GetGtDate(Guid discriminatorId, DateTimeOffset dateBegin, CancellationToken token);
/// <summary>
/// Получить диапазон дат, для которых есть данные в репозитории
/// </summary>
/// <param name="discriminatorId"></param>
/// <param name="token"></param>
/// <returns></returns>
Task<DatesRangeDto?> GetDatesRange(CancellationToken token);
Task<DatesRangeDto?> GetDatesRange(Guid discriminatorId, CancellationToken token);
}

View File

@ -1,19 +1,23 @@
using DD.Persistence.Models;
namespace DD.Persistence.Repositories;
namespace DD.Persistence.RepositoriesAbstractions;
/// <summary>
/// Интерфейс по работе с прореженными данными
/// </summary>
public interface ITimeSeriesBaseRepository
public interface ITimeSeriesBaseRepository<TDto>
{
/// <summary>
/// Получить список объектов с прореживанием
/// </summary>
/// <param name="discriminatorId"></param>
/// <param name="dateBegin">дата начала</param>
/// <param name="intervalSec"></param>
/// <param name="approxPointsCount"></param>
/// <param name="token"></param>
/// <returns></returns>
Task<IEnumerable<TimestampedSetDto>> GetResampledData(
Task<IEnumerable<TDto>> GetResampledData(
Guid discriminatorId,
DateTimeOffset dateBegin,
double intervalSec = 600d,
int approxPointsCount = 1024,