Рефакторинг импорта ГГД

This commit is contained in:
parent ac578bce38
commit 14615517d6
7 changed files with 72 additions and 85 deletions

View File

@ -25,7 +25,7 @@ public class RowDto
/// <summary> /// <summary>
/// Описание категории /// Описание категории
/// </summary> /// </summary>
public string CategoryInfo { get; set; } = null!; public string? CategoryInfo { get; set; }
/// <summary> /// <summary>
/// Начальная глубина операции /// Начальная глубина операции

View File

@ -1,3 +1,4 @@
using System.Collections.Generic;
using System.ComponentModel.DataAnnotations; using System.ComponentModel.DataAnnotations;
namespace AsbCloudApp.Data.WellOperationImport; namespace AsbCloudApp.Data.WellOperationImport;
@ -5,19 +6,29 @@ namespace AsbCloudApp.Data.WellOperationImport;
/// <summary> /// <summary>
/// Опции для настройки парсинга документа /// Опции для настройки парсинга документа
/// </summary> /// </summary>
public class WellOperationParserOptionsDto public class WellOperationParserOptionsDto : IValidatableObject
{ {
/// <summary> /// <summary>
/// Название листа /// Название листа
/// </summary> /// </summary>
[StringLength(250, MinimumLength =1, ErrorMessage = "Название листа должно быть задано")] public string? SheetName { get; set; }
public string SheetName { get; set; } = null!;
/// <summary> /// <summary>
/// Id шаблона /// Тип операции
/// 0 - плановая операция
/// 1 - фактическая операция
/// </summary>
[Required]
[Range(0, 1, ErrorMessage = "Тип операции недопустим. Допустимые: 0, 1")]
public int IdType { get; set; }
/// <summary>
/// Тип шаблона
/// 0 - Дефолтный шаблон /// 0 - Дефолтный шаблон
/// 1 - Газпром хантос /// 1 - Газпром хантос
/// </summary> /// </summary>
[Required]
[Range(0, 1, ErrorMessage = "Тип шаблона недопустим. Допустимые: 0, 1")]
public int IdTemplate { get; set; } public int IdTemplate { get; set; }
/// <summary> /// <summary>
@ -29,4 +40,11 @@ public class WellOperationParserOptionsDto
/// Конечная строка /// Конечная строка
/// </summary> /// </summary>
public int? EndRow { get; set; } public int? EndRow { get; set; }
/// <inheritdoc/>
public IEnumerable<ValidationResult> Validate(ValidationContext validationContext)
{
if (IdTemplate != 0 && string.IsNullOrWhiteSpace(SheetName))
yield return new ValidationResult("Название листа должно быть задано", new[] { nameof(SheetName) });
}
} }

View File

@ -14,13 +14,11 @@ public interface IWellOperationImportService
/// Загрузить из excel список операций /// Загрузить из excel список операций
/// </summary> /// </summary>
/// <param name="idWell"></param> /// <param name="idWell"></param>
/// <param name="idType"></param>
/// <param name="stream"></param> /// <param name="stream"></param>
/// <param name="idUser"></param> /// <param name="idUser"></param>
/// <param name="deleteWellOperationsBeforeImport"></param> /// <param name="deleteBeforeImport"></param>
/// <param name="cancellationToken"></param> /// <param name="cancellationToken"></param>
/// <param name="options"></param> /// <param name="options"></param>
Task ImportAsync(int idWell, int idUser, int idType, Stream stream, WellOperationParserOptionsDto options, Task ImportAsync(int idWell, int idUser, Stream stream, WellOperationParserOptionsDto options, bool deleteBeforeImport,
bool deleteWellOperationsBeforeImport,
CancellationToken cancellationToken); CancellationToken cancellationToken);
} }

View File

@ -3,7 +3,6 @@ using System.Collections.Generic;
using System.IO; using System.IO;
using System.Linq; using System.Linq;
using AsbCloudApp.Data.WellOperationImport; using AsbCloudApp.Data.WellOperationImport;
using AsbCloudApp.Exceptions;
using AsbCloudApp.Services.WellOperationImport; using AsbCloudApp.Services.WellOperationImport;
using AsbCloudDb.Model; using AsbCloudDb.Model;
using AsbCloudInfrastructure.Services.WellOperationImport.Constants; using AsbCloudInfrastructure.Services.WellOperationImport.Constants;
@ -25,11 +24,12 @@ public class WellOperationDefaultExcelParser : IWellOperationExcelParser
private static IEnumerable<RowDto> ParseWorkbook(IXLWorkbook workbook, WellOperationParserOptionsDto options) private static IEnumerable<RowDto> ParseWorkbook(IXLWorkbook workbook, WellOperationParserOptionsDto options)
{ {
if (string.IsNullOrWhiteSpace(options.SheetName)) var sheetName = options.IdType == WellOperation.IdOperationTypePlan
throw new ArgumentInvalidException(nameof(options.SheetName), "Не указано название листа"); ? DefaultTemplateInfo.SheetNamePlan
: DefaultTemplateInfo.SheetNameFact;
var sheet = workbook.Worksheets.FirstOrDefault(ws => var sheet = workbook.Worksheets.FirstOrDefault(ws =>
string.Equals(ws.Name, options.SheetName, StringComparison.CurrentCultureIgnoreCase)) string.Equals(ws.Name, sheetName, StringComparison.CurrentCultureIgnoreCase))
?? throw new FileFormatException($"Книга excel не содержит листа '{options.SheetName}'"); ?? throw new FileFormatException($"Книга excel не содержит листа '{options.SheetName}'");
return ParseSheet(sheet); return ParseSheet(sheet);
@ -81,12 +81,12 @@ public class WellOperationDefaultExcelParser : IWellOperationExcelParser
Number = xlRow.RowNumber(), Number = xlRow.RowNumber(),
Section = xlRow.Cell(DefaultTemplateInfo.ColumnSection).GetCellValue<string>(), Section = xlRow.Cell(DefaultTemplateInfo.ColumnSection).GetCellValue<string>(),
Category = xlRow.Cell(DefaultTemplateInfo.ColumnCategory).GetCellValue<string>(), Category = xlRow.Cell(DefaultTemplateInfo.ColumnCategory).GetCellValue<string>(),
CategoryInfo = xlRow.Cell(DefaultTemplateInfo.ColumnCategoryInfo).GetCellValue<string>(), CategoryInfo = xlRow.Cell(DefaultTemplateInfo.ColumnCategoryInfo).GetCellValue<string?>(),
DepthStart = xlRow.Cell(DefaultTemplateInfo.ColumnDepthStart).GetCellValue<double>(), DepthStart = xlRow.Cell(DefaultTemplateInfo.ColumnDepthStart).GetCellValue<double>(),
DepthEnd = xlRow.Cell(DefaultTemplateInfo.ColumnDepthEnd).GetCellValue<double>(), DepthEnd = xlRow.Cell(DefaultTemplateInfo.ColumnDepthEnd).GetCellValue<double>(),
Date = xlRow.Cell(DefaultTemplateInfo.ColumnDate).GetCellValue<DateTime>(), Date = xlRow.Cell(DefaultTemplateInfo.ColumnDate).GetCellValue<DateTime>(),
Duration = xlRow.Cell(DefaultTemplateInfo.ColumnDuration).GetCellValue<double>(), Duration = xlRow.Cell(DefaultTemplateInfo.ColumnDuration).GetCellValue<double>(),
Comment = xlRow.Cell(DefaultTemplateInfo.ColumnComment).GetCellValue<string>() Comment = xlRow.Cell(DefaultTemplateInfo.ColumnComment).GetCellValue<string?>()
}; };
} }
} }

View File

@ -1,6 +1,5 @@
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Globalization;
using System.IO; using System.IO;
using System.Linq; using System.Linq;
using System.Reflection; using System.Reflection;
@ -20,7 +19,7 @@ public class WellOperationGazpromKhantosExcelParser : IWellOperationExcelParser
{ {
public int RowNumber { get; set; } public int RowNumber { get; set; }
public string CategoryInfo { get; set; } = null!; public string? CategoryInfo { get; set; }
public double SectionDiameter { get; set; } public double SectionDiameter { get; set; }
@ -31,18 +30,12 @@ public class WellOperationGazpromKhantosExcelParser : IWellOperationExcelParser
public DateTime Date { get; set; } public DateTime Date { get; set; }
} }
private readonly CosineSimilarity cosineSimilarity; private readonly CosineSimilarity cosineSimilarity = new();
private readonly Dictionary<string, string> operationDict = InitDict("Operations.txt", '='); private readonly Dictionary<string, string> operationDict = InitDict("Operations.txt", '=');
private readonly Dictionary<string, string> sectionDict = InitDict("Sections.txt", '='); private readonly Dictionary<string, string> sectionDict = InitDict("Sections.txt", '=');
private readonly Dictionary<string, string> operationAttributesDict = InitDict("OperationAttributes.txt", '='); private readonly Dictionary<string, string> operationAttributesDict = InitDict("OperationAttributes.txt", '=');
public WellOperationGazpromKhantosExcelParser()
{
cosineSimilarity = new CosineSimilarity();
}
public int IdTemplate => Templates.IdGazpromKhantosTemplate; public int IdTemplate => Templates.IdGazpromKhantosTemplate;
public IEnumerable<int> IdTypes => new[] { WellOperation.IdOperationTypePlan }; public IEnumerable<int> IdTypes => new[] { WellOperation.IdOperationTypePlan };
@ -56,9 +49,6 @@ public class WellOperationGazpromKhantosExcelParser : IWellOperationExcelParser
private IEnumerable<RowDto> ParseWorkBook(IXLWorkbook workbook, WellOperationParserOptionsDto options) private IEnumerable<RowDto> ParseWorkBook(IXLWorkbook workbook, WellOperationParserOptionsDto options)
{ {
if (string.IsNullOrWhiteSpace(options.SheetName))
throw new ArgumentInvalidException(nameof(options.SheetName), "Не указано название листа");
if (options.StartRow is null or < 1 or > 1048576) if (options.StartRow is null or < 1 or > 1048576)
throw new ArgumentInvalidException(nameof(options.StartRow), "Некорректное значение начальной строки"); throw new ArgumentInvalidException(nameof(options.StartRow), "Некорректное значение начальной строки");
@ -97,7 +87,7 @@ public class WellOperationGazpromKhantosExcelParser : IWellOperationExcelParser
operations.Add(new Operation operations.Add(new Operation
{ {
RowNumber = xlRow.RowNumber(), RowNumber = xlRow.RowNumber(),
CategoryInfo = xlRow.Cell(operationAttributes[OperationAttributes.CategoryInfo]).GetCellValue<string>(), CategoryInfo = xlRow.Cell(operationAttributes[OperationAttributes.CategoryInfo]).GetCellValue<string?>(),
SectionDiameter =xlRow.Cell(operationAttributes[OperationAttributes.SectionDiameter]).GetCellValue<double>(), SectionDiameter =xlRow.Cell(operationAttributes[OperationAttributes.SectionDiameter]).GetCellValue<double>(),
Depth = xlRow.Cell(operationAttributes[OperationAttributes.Depth]).GetCellValue<double>(), Depth = xlRow.Cell(operationAttributes[OperationAttributes.Depth]).GetCellValue<double>(),
Duration = xlRow.Cell(operationAttributes[OperationAttributes.Duration]).GetCellValue<double>(), Duration = xlRow.Cell(operationAttributes[OperationAttributes.Duration]).GetCellValue<double>(),
@ -198,8 +188,11 @@ public class WellOperationGazpromKhantosExcelParser : IWellOperationExcelParser
return operationAttributes is not null && operationAttributes.Count == countOperationAttributes ? operationAttributes : null; return operationAttributes is not null && operationAttributes.Count == countOperationAttributes ? operationAttributes : null;
} }
private string? GetValueDictionary(IDictionary<string, string> dict, string cellValue, double? minSimilarity) private string? GetValueDictionary(IDictionary<string, string> dict, string? cellValue, double? minSimilarity)
{ {
if (string.IsNullOrWhiteSpace(cellValue))
return null;
var similarValues = new List<(double similarity, string value)>(); var similarValues = new List<(double similarity, string value)>();
var profile1 = cosineSimilarity.GetProfile(cellValue); var profile1 = cosineSimilarity.GetProfile(cellValue);

View File

@ -10,8 +10,6 @@ using AsbCloudApp.Exceptions;
using AsbCloudApp.Repositories; using AsbCloudApp.Repositories;
using AsbCloudApp.Requests; using AsbCloudApp.Requests;
using AsbCloudApp.Services.WellOperationImport; using AsbCloudApp.Services.WellOperationImport;
using AsbCloudDb.Model;
using AsbCloudInfrastructure.Services.WellOperationImport.Constants;
namespace AsbCloudInfrastructure.Services.WellOperationImport; namespace AsbCloudInfrastructure.Services.WellOperationImport;
@ -31,35 +29,25 @@ public class WellOperationImportService : IWellOperationImportService
this.wellOperationRepository = wellOperationRepository; this.wellOperationRepository = wellOperationRepository;
} }
public async Task ImportAsync(int idWell, int idUser, int idType, Stream stream, WellOperationParserOptionsDto options, public async Task ImportAsync(int idWell, int idUser, Stream stream, WellOperationParserOptionsDto options, bool deleteBeforeImport,
bool deleteWellOperationsBeforeImport, CancellationToken cancellationToken) CancellationToken cancellationToken)
{ {
var excelParser = excelParsers.FirstOrDefault(p => p.IdTemplate == options.IdTemplate && p.IdTypes.Contains(idType)) var excelParser = excelParsers.FirstOrDefault(p =>
p.IdTemplate == options.IdTemplate &&
p.IdTypes.Contains(options.IdType))
?? throw new ArgumentInvalidException(nameof(options.IdTemplate), "Невозможно импортировать файл"); ?? throw new ArgumentInvalidException(nameof(options.IdTemplate), "Невозможно импортировать файл");
if (idType != WellOperation.IdOperationTypePlan && idType != WellOperation.IdOperationTypeFact) IEnumerable<RowDto> rows;
throw new ArgumentInvalidException(nameof(idType), "Операции не существует");
RowDto[] rows;
var validationErrors = new List<string>(); var validationErrors = new List<string>();
var sections = wellOperationRepository.GetSectionTypes(); var sections = wellOperationRepository.GetSectionTypes();
var categories = wellOperationRepository.GetCategories(false); var categories = wellOperationRepository.GetCategories(false);
switch (options.IdTemplate) rows = options.IdTemplate switch
{ {
case 0: 0 => excelParser.Parse(stream, options),
options.SheetName = idType == WellOperation.IdOperationTypePlan _ => excelParser.Parse(stream, options)
? DefaultTemplateInfo.SheetNamePlan };
: DefaultTemplateInfo.SheetNameFact;
rows = excelParser.Parse(stream, options).ToArray();
break;
default:
if (string.IsNullOrWhiteSpace(options.SheetName))
throw new FileFormatException("Не указано название листа");
rows = excelParser.Parse(stream, options).ToArray();
break;
}
var operations = new List<WellOperationDto>(); var operations = new List<WellOperationDto>();
@ -80,16 +68,20 @@ public class WellOperationImportService : IWellOperationImportService
throw new FileFormatException($"Лист '{options.SheetName}'. В строке '{row.Number}' не удалось определить операцию"); throw new FileFormatException($"Лист '{options.SheetName}'. В строке '{row.Number}' не удалось определить операцию");
if (row.DepthStart is not (>= 0d and <= 20_000d)) if (row.DepthStart is not (>= 0d and <= 20_000d))
throw new FileFormatException($"Лист '{options.SheetName}'. Строка '{row.Number}' некорректная глубина на начало операции"); throw new FileFormatException(
$"Лист '{options.SheetName}'. Строка '{row.Number}' некорректная глубина на начало операции");
if (row.DepthEnd is not (>= 0d and <= 20_000d)) if (row.DepthEnd is not (>= 0d and <= 20_000d))
throw new FileFormatException($"Лист '{options.SheetName}'. Строка '{row.Number}' некорректная глубина на конец операции"); throw new FileFormatException(
$"Лист '{options.SheetName}'. Строка '{row.Number}' некорректная глубина на конец операции");
if (row.Date < dateLimitMin && row.Date > dateLimitMax) if (row.Date < dateLimitMin && row.Date > dateLimitMax)
throw new FileFormatException($"Лист '{options.SheetName}'. Строка '{row.Number}' неправильно получена дата начала операции"); throw new FileFormatException(
$"Лист '{options.SheetName}'. Строка '{row.Number}' неправильно получена дата начала операции");
if (operations.LastOrDefault()?.DateStart > row.Date) if (operations.LastOrDefault()?.DateStart > row.Date)
throw new FileFormatException($"Лист '{options.SheetName}' строка '{row.Number}' дата позднее даты предыдущей операции"); throw new FileFormatException(
$"Лист '{options.SheetName}' строка '{row.Number}' дата позднее даты предыдущей операции");
if (row.Duration is not (>= 0d and <= 240d)) if (row.Duration is not (>= 0d and <= 240d))
throw new FileFormatException($"Лист '{options.SheetName}'. Строка '{row.Number}' некорректная длительность операции"); throw new FileFormatException($"Лист '{options.SheetName}'. Строка '{row.Number}' некорректная длительность операции");
@ -98,7 +90,7 @@ public class WellOperationImportService : IWellOperationImportService
{ {
IdWell = idWell, IdWell = idWell,
IdUser = idUser, IdUser = idUser,
IdType = idType, IdType = options.IdType,
IdWellSectionType = section.Id, IdWellSectionType = section.Id,
IdCategory = category.Id, IdCategory = category.Id,
CategoryInfo = row.CategoryInfo, CategoryInfo = row.CategoryInfo,
@ -123,7 +115,7 @@ public class WellOperationImportService : IWellOperationImportService
if (!operations.Any()) if (!operations.Any())
return; return;
if (deleteWellOperationsBeforeImport) if (deleteBeforeImport)
{ {
var existingOperations = await wellOperationRepository.GetAsync(new WellOperationRequest var existingOperations = await wellOperationRepository.GetAsync(new WellOperationRequest
{ {

View File

@ -290,25 +290,17 @@ namespace AsbCloudWebApi.Controllers
/// Импорт плановых операций из excel (xlsx) файла /// Импорт плановых операций из excel (xlsx) файла
/// </summary> /// </summary>
/// <param name="idWell">id скважины</param> /// <param name="idWell">id скважины</param>
/// <param name="idType">Тип операции</param> /// <param name="options">Параметры для парсинга файла</param>
/// <param name="startRow">Начальная строка</param>
/// <param name="endRow">Конечная строка</param>
/// <param name="files">Коллекция из одного файла xlsx</param> /// <param name="files">Коллекция из одного файла xlsx</param>
/// <param name="options">Удалить операции перед импортом = 1, если файл валидный</param> /// <param name="deleteBeforeImport">Удалить операции перед импортом = 1, если файл валидный</param>
/// <param name="sheetName">Название листа</param> /// <param name="token"></param>
/// <param name="token">Токен отмены задачи </param>
/// <param name="idTemplate">Шаблон файла. 0 - стандартный, 1 - Газпромнефть Хантос</param>
/// <returns></returns> /// <returns></returns>
[HttpPost("import/{options}")] [HttpPost("import/{deleteBeforeImport}")]
[Permission] [Permission]
public async Task<IActionResult> ImportAsync(int idWell, public async Task<IActionResult> ImportAsync(int idWell,
[Required] int idType, [FromQuery] WellOperationParserOptionsDto options,
string? sheetName,
[Required] int idTemplate,
int? startRow,
int? endRow,
[FromForm] IFormFileCollection files, [FromForm] IFormFileCollection files,
int options, [Range(0, 1, ErrorMessage = "Недопустимое значение. Допустимые: 0, 1")] int deleteBeforeImport,
CancellationToken token) CancellationToken token)
{ {
var idCompany = User.GetCompanyId(); var idCompany = User.GetCompanyId();
@ -338,13 +330,7 @@ namespace AsbCloudWebApi.Controllers
try try
{ {
await wellOperationImportService.ImportAsync(idWell, idUser.Value, idType, stream, new WellOperationParserOptionsDto await wellOperationImportService.ImportAsync(idWell, idUser.Value, stream, options, (deleteBeforeImport & 1) > 0, token);
{
SheetName = sheetName,
IdTemplate = idTemplate,
StartRow = startRow,
EndRow = endRow
}, (options & 1) > 0, token);
} }
catch (FileFormatException ex) catch (FileFormatException ex)
{ {