Сервисы парсинга

1. Добавил парсинг Газпромовских файлов
2. Сделал рефакторинг существующего импорта
This commit is contained in:
parent 59c96aa9e9
commit ff02a29115
14 changed files with 766 additions and 387 deletions

View File

@ -1,32 +0,0 @@
using System.IO;
namespace AsbCloudApp.Services
{
/// <summary>
/// сервис импорта/экспорта операций по скважине вводимых вручную
/// </summary>
public interface IWellOperationImportService
{
/// <summary>
/// скачать в excel
/// </summary>
/// <param name="idWell"></param>
/// <returns></returns>
Stream Export(int idWell);
/// <summary>
/// скачать шаблон для заполнения
/// </summary>
/// <returns></returns>
Stream GetExcelTemplateStream();
/// <summary>
/// закгрузить из excel список операций
/// </summary>
/// <param name="idWell"></param>
/// <param name="stream"></param>
/// <param name="idUser"></param>
/// <param name="deleteWellOperationsBeforeImport">Очистить старые перед импортом (если файл проходит валидацию)</param>
void Import(int idWell, Stream stream, int idUser, bool deleteWellOperationsBeforeImport = false);
}
}

View File

@ -0,0 +1,29 @@
using System.Collections.Generic;
using System.IO;
using AsbCloudApp.Data.WellOperationImport;
namespace AsbCloudApp.Services.WellOperationImport;
/// <summary>
/// Парсинг операций из excel файла
/// </summary>
public interface IWellOperationExcelParser
{
/// <summary>
/// Id шаблона
/// </summary>
int IdTemplate { get; }
/// <summary>
/// Типы операций, которые можно получить из файла
/// </summary>
IEnumerable<int> IdTypes { get; }
/// <summary>
/// Метод парсинга документа
/// </summary>
/// <param name="stream"></param>
/// <param name="options"></param>
/// <returns></returns>
IEnumerable<RowDto> Parse(Stream stream, WellOperationParserOptionsDto options);
}

View File

@ -0,0 +1,19 @@
using System.IO;
using System.Threading;
using System.Threading.Tasks;
namespace AsbCloudApp.Services.WellOperationImport;
/// <summary>
/// Экспорт ГГД
/// </summary>
public interface IWellOperationExportService
{
/// <summary>
/// Скачать в excel
/// </summary>
/// <param name="idWell"></param>
/// <param name="cancellationToken"></param>
/// <returns></returns>
Task<Stream> ExportAsync(int idWell, CancellationToken cancellationToken);
}

View File

@ -0,0 +1,26 @@
using System.IO;
using System.Threading;
using System.Threading.Tasks;
using AsbCloudApp.Data.WellOperationImport;
namespace AsbCloudApp.Services.WellOperationImport;
/// <summary>
/// Импорт ГГД
/// </summary>
public interface IWellOperationImportService
{
/// <summary>
/// Загрузить из excel список операций
/// </summary>
/// <param name="idWell"></param>
/// <param name="idType"></param>
/// <param name="stream"></param>
/// <param name="idUser"></param>
/// <param name="deleteWellOperationsBeforeImport"></param>
/// <param name="cancellationToken"></param>
/// <param name="options"></param>
Task ImportAsync(int idWell, int idUser, int idType, Stream stream, WellOperationParserOptionsDto options,
bool deleteWellOperationsBeforeImport,
CancellationToken cancellationToken);
}

View File

@ -0,0 +1,15 @@
using System.IO;
namespace AsbCloudApp.Services.WellOperationImport;
/// <summary>
/// Сервис для получения шаблонов ГГД
/// </summary>
public interface IWellOperationImportTemplateService
{
/// <summary>
/// Скачать шаблон для заполнения
/// </summary>
/// <returns></returns>
Stream GetExcelTemplateStream();
}

View File

@ -25,7 +25,10 @@ using Microsoft.Extensions.DependencyInjection;
using System; using System;
using AsbCloudApp.Services.AutoGeneratedDailyReports; using AsbCloudApp.Services.AutoGeneratedDailyReports;
using AsbCloudApp.Services.Notifications; using AsbCloudApp.Services.Notifications;
using AsbCloudApp.Services.WellOperationImport;
using AsbCloudInfrastructure.Services.AutoGeneratedDailyReports; using AsbCloudInfrastructure.Services.AutoGeneratedDailyReports;
using AsbCloudInfrastructure.Services.WellOperationImport;
using AsbCloudInfrastructure.Services.WellOperationImport.FileParser;
namespace AsbCloudInfrastructure namespace AsbCloudInfrastructure
{ {
@ -129,7 +132,6 @@ namespace AsbCloudInfrastructure
services.AddTransient<ITelemetryUserService, TelemetryUserService>(); services.AddTransient<ITelemetryUserService, TelemetryUserService>();
services.AddTransient<ITimezoneService, TimezoneService>(); services.AddTransient<ITimezoneService, TimezoneService>();
services.AddTransient<IWellService, WellService>(); services.AddTransient<IWellService, WellService>();
services.AddTransient<IWellOperationImportService, WellOperationImportService>();
services.AddTransient<IProcessMapPlanImportService, ProcessMapPlanImportService>(); services.AddTransient<IProcessMapPlanImportService, ProcessMapPlanImportService>();
services.AddTransient<IPlannedTrajectoryImportService, PlannedTrajectoryImportService>(); services.AddTransient<IPlannedTrajectoryImportService, PlannedTrajectoryImportService>();
services.AddTransient<IWellOperationRepository, WellOperationRepository>(); services.AddTransient<IWellOperationRepository, WellOperationRepository>();
@ -224,6 +226,13 @@ namespace AsbCloudInfrastructure
services.AddTransient<IWellboreService, WellboreService>(); services.AddTransient<IWellboreService, WellboreService>();
services.AddTransient<IWellOperationExportService, WellOperationExportService>();
services.AddTransient<IWellOperationImportService, WellOperationImportService>();
services.AddTransient<IWellOperationImportTemplateService, WellOperationImportTemplateService>();
services.AddTransient<IWellOperationExcelParser, WellOperationDefaultExcelParser>();
services.AddTransient<IWellOperationExcelParser, WellOperationGazpromKhantosExcelParser>();
return services; return services;
} }

View File

@ -343,6 +343,7 @@ public class ProcessMapPlanImportService : IProcessMapPlanImportService
_ => "Ручной", _ => "Ручной",
}; };
//TODO: вынести в метод расширения
private static T GetCellValue<T>(IXLRow row, int columnNumber) private static T GetCellValue<T>(IXLRow row, int columnNumber)
{ {
try try

View File

@ -0,0 +1,106 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using AsbCloudApp.Data.WellOperationImport;
using AsbCloudApp.Exceptions;
using AsbCloudApp.Services.WellOperationImport;
using AsbCloudDb.Model;
using AsbCloudInfrastructure.Services.WellOperationImport.Constants;
using ClosedXML.Excel;
namespace AsbCloudInfrastructure.Services.WellOperationImport.FileParser;
public class WellOperationDefaultExcelParser : IWellOperationExcelParser
{
public int IdTemplate => Templates.IdDefaultTemplate;
public IEnumerable<int> IdTypes => new[] { WellOperation.IdOperationTypePlan, WellOperation.IdOperationTypeFact };
public IEnumerable<RowDto> Parse(Stream stream, WellOperationParserOptionsDto options)
{
using var workbook = new XLWorkbook(stream, XLEventTracking.Disabled);
return ParseWorkbook(workbook, options);
}
private static IEnumerable<RowDto> ParseWorkbook(IXLWorkbook workbook, WellOperationParserOptionsDto options)
{
if (string.IsNullOrWhiteSpace(options.SheetName))
throw new ArgumentInvalidException("Не указано название листа", nameof(options.SheetName));
var sheet = workbook.Worksheets.FirstOrDefault(ws =>
string.Equals(ws.Name, options.SheetName, StringComparison.CurrentCultureIgnoreCase))
?? throw new FileFormatException($"Книга excel не содержит листа '{options.SheetName}'");
return ParseSheet(sheet);
}
private static IEnumerable<RowDto> ParseSheet(IXLWorksheet sheet)
{
if (sheet.RangeUsed().RangeAddress.LastAddress.ColumnNumber < 7)
throw new FileFormatException($"Лист {sheet.Name} содержит меньшее количество столбцов.");
var count = sheet.RowsUsed().Count() - DefaultTemplateInfo.HeaderRowsCount;
switch (count)
{
case > 1024:
throw new FileFormatException($"Лист {sheet.Name} содержит слишком большое количество операций.");
case <= 0:
return Enumerable.Empty<RowDto>();
}
var rows = new RowDto[count];
var cellValuesErrors = new List<string>();
for (int i = 0; i < rows.Length; i++)
{
try
{
var xlRow = sheet.Row(1 + i + DefaultTemplateInfo.HeaderRowsCount);
rows[i] = ParseRow(xlRow);
}
catch (FileFormatException ex)
{
cellValuesErrors.Add(ex.Message);
}
}
if (cellValuesErrors.Any())
throw new FileFormatException(string.Join("\r\n", cellValuesErrors));
return rows;
}
private static RowDto ParseRow(IXLRow xlRow)
{
return new RowDto
{
Number = xlRow.RowNumber(),
Section = GetCellValue<string>(xlRow.Cell(DefaultTemplateInfo.ColumnSection)),
Category = GetCellValue<string>(xlRow.Cell(DefaultTemplateInfo.ColumnCategory)),
CategoryInfo = GetCellValue<string>(xlRow.Cell(DefaultTemplateInfo.ColumnCategoryInfo)),
DepthStart = GetCellValue<double>(xlRow.Cell(DefaultTemplateInfo.ColumnDepthStart)),
DepthEnd = GetCellValue<double>(xlRow.Cell(DefaultTemplateInfo.ColumnDepthEnd)),
Date = GetCellValue<DateTime>(xlRow.Cell(DefaultTemplateInfo.ColumnDate)),
Duration = GetCellValue<double>(xlRow.Cell(DefaultTemplateInfo.ColumnDuration)),
Comment = GetCellValue<string>(xlRow.Cell(DefaultTemplateInfo.ColumnComment))
};
}
//TODO: вынести в метод расширения
private static T GetCellValue<T>(IXLCell cell)
{
try
{
return (T)Convert.ChangeType(cell.Value, typeof(T));
}
catch
{
throw new FileFormatException(
$"Лист '{cell.Worksheet.Name}'. Ячейка: ({cell.Address.RowNumber},{cell.Address.ColumnNumber}) содержит некорректное значение");
}
}
}

View File

@ -0,0 +1,254 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Reflection;
using AsbCloudApp.Data.WellOperationImport;
using AsbCloudApp.Exceptions;
using AsbCloudApp.Services.WellOperationImport;
using AsbCloudDb.Model;
using AsbCloudInfrastructure.Services.WellOperationImport.Constants;
using AsbCloudInfrastructure.Services.WellOperationImport.FileParser.StringSimilarity;
using ClosedXML.Excel;
namespace AsbCloudInfrastructure.Services.WellOperationImport.FileParser;
public class WellOperationGazpromKhantosExcelParser : IWellOperationExcelParser
{
private class Operation
{
public int RowNumber { get; set; }
public string CategoryInfo { get; set; } = null!;
public double SectionDiameter { get; set; }
public double Depth { get; set; }
public double Duration { get; set; }
public DateTime Date { get; set; }
}
private readonly CosineSimilarity cosineSimilarity;
private readonly Dictionary<string, string> operationDict = InitDict("Operations.txt", '=');
private readonly Dictionary<string, string> sectionDict = InitDict("Sections.txt", '=');
private readonly Dictionary<string, string> operationAttributesDict = InitDict("OperationAttributes.txt", '=');
public WellOperationGazpromKhantosExcelParser()
{
cosineSimilarity = new CosineSimilarity();
}
public int IdTemplate => Templates.IdGazpromKhantosTemplate;
public IEnumerable<int> IdTypes => new[] { WellOperation.IdOperationTypePlan };
public IEnumerable<RowDto> Parse(Stream stream, WellOperationParserOptionsDto options)
{
using var workbook = new XLWorkbook(stream, XLEventTracking.Disabled);
return ParseWorkBook(workbook, options);
}
private IEnumerable<RowDto> ParseWorkBook(IXLWorkbook workbook, WellOperationParserOptionsDto options)
{
if (string.IsNullOrWhiteSpace(options.SheetName))
throw new ArgumentInvalidException("Не указано название листа", nameof(options.SheetName));
if (options.StartRow is null or < 1 or > 1048576)
throw new ArgumentInvalidException("Некорректное значение начальной строки", nameof(options.StartRow));
if (options.EndRow is null or < 1 or > 1048576)
throw new ArgumentInvalidException("Некорректное значение конечной строки", nameof(options.EndRow));
if (options.EndRow < options.StartRow)
throw new ArgumentInvalidException("Конечный номер строки не может быть больше начального", nameof(options.EndRow));
var sheet = workbook.Worksheets.FirstOrDefault(ws =>
string.Equals(ws.Name, options.SheetName, StringComparison.CurrentCultureIgnoreCase))
?? throw new FileFormatException($"Книга excel не содержит листа '{options.SheetName}'");
return ParseSheet(sheet, options.StartRow.Value, options.EndRow.Value);
}
private IEnumerable<RowDto> ParseSheet(IXLWorksheet sheet, int startRow, int endRow)
{
var operationAttributes = GetOperationAttributes(sheet.RowsUsed());
if (operationAttributes is null)
return Enumerable.Empty<RowDto>();
var rowsCount = endRow - startRow + 1;
var operations = new List<Operation>();
var cellValuesErrors = new List<string>();
for (int i = 0; i < rowsCount; i++)
{
var xlRow = sheet.Row(startRow + i);
try
{
operations.Add(new Operation
{
RowNumber = xlRow.RowNumber(),
CategoryInfo = GetCellValue<string>(xlRow.Cell(operationAttributes[OperationAttributes.CategoryInfo])),
SectionDiameter = GetCellValue<double>(xlRow.Cell(operationAttributes[OperationAttributes.SectionDiameter])),
Depth = GetCellValue<double>(xlRow.Cell(operationAttributes[OperationAttributes.Depth])),
Duration = GetCellValue<double>(xlRow.Cell(operationAttributes[OperationAttributes.Duration])),
Date = GetCellValue<DateTime>(xlRow.Cell(operationAttributes[OperationAttributes.Date]))
});
}
catch (FileFormatException ex)
{
cellValuesErrors.Add(ex.Message);
}
}
if (cellValuesErrors.Any())
throw new FileFormatException(string.Join("\r\n", cellValuesErrors));
return BuildRows();
IEnumerable<(double Diameter, string Name)> BuildSections()
{
var groupedOperations = operations.GroupBy(o => o.SectionDiameter)
.Select(s => new
{
Diameter = s.Key,
CategoryInfo = string.Concat(s.Select(o => o.CategoryInfo))
});
var repeatedSections = new[] { "xвостовик" };
var sections = new List<(double diameter, string section)>();
foreach (var groupedOperation in groupedOperations)
{
var sectionNamesSet = new HashSet<string>(sections.Select(s => s.section));
sections.Add(new ValueTuple<double, string>(groupedOperation.Diameter, sectionDict.FirstOrDefault(item =>
groupedOperation.CategoryInfo.Contains(item.Key) &&
(!sectionNamesSet.Contains(item.Value) || repeatedSections.Contains(item.Value.ToLowerInvariant()))).Value));
}
return sections;
}
IEnumerable<RowDto> BuildRows()
{
if (!operations.Any())
return Enumerable.Empty<RowDto>();
var rows = new List<RowDto>();
for (int i = 0; i < operations.Count; i++)
{
var currentOperation = operations[i];
var nextOperation = i + 1 < operations.Count ? operations[i + 1] : currentOperation;
rows.Add(new RowDto
{
Number = currentOperation.RowNumber,
Section = BuildSections().FirstOrDefault(s => Math.Abs(s.Diameter - currentOperation.SectionDiameter) < 0.1).Name,
Category = GetValueDictionary(operationDict, currentOperation.CategoryInfo, 0.3),
CategoryInfo = currentOperation.CategoryInfo,
DepthStart = currentOperation.Depth,
DepthEnd = nextOperation.Depth,
Duration = currentOperation.Duration,
Date = currentOperation.Date.AddHours(-currentOperation.Duration)
});
}
return rows;
}
}
private IDictionary<string, int>? GetOperationAttributes(IXLRows xlRows)
{
const int countOperationAttributes = 5;
IDictionary<string, int>? operationAttributes = null;
foreach (var xlRow in xlRows)
{
operationAttributes = new Dictionary<string, int>();
var cells = xlRow.CellsUsed().ToArray();
foreach (var cell in cells)
{
var operationAttribute = GetValueDictionary(operationAttributesDict, GetCellValue<string>(cell), 0.7);
if (operationAttribute is null || operationAttributes.Any(a => a.Key == operationAttribute))
continue;
operationAttributes.Add(operationAttribute, cell.Address.ColumnNumber);
}
if (operationAttributes.Count >= countOperationAttributes)
break;
}
return operationAttributes is not null && operationAttributes.Count == countOperationAttributes ? operationAttributes : null;
}
private string? GetValueDictionary(IDictionary<string, string> dict, string cellValue, double? minSimilarity)
{
var similarValues = new List<(double similarity, string value)>();
var profile1 = cosineSimilarity.GetProfile(cellValue);
foreach (var item in dict)
{
var profile2 = cosineSimilarity.GetProfile(item.Key);
var similarity = cosineSimilarity.Similarity(profile1, profile2);
similarValues.Add((similarity, item.Value));
}
var mostSimilarValue = similarValues.MaxBy(v => v.similarity);
return minSimilarity.HasValue && mostSimilarValue.similarity >= minSimilarity ? mostSimilarValue.value : null;
}
private static Dictionary<string, string> InitDict(string fileName, char separator)
{
var resourceName = Assembly.GetExecutingAssembly()
.GetManifestResourceNames()
.FirstOrDefault(n => n.EndsWith(fileName))!;
var stream = Assembly.GetExecutingAssembly()
.GetManifestResourceStream(resourceName)!;
using var reader = new StreamReader(stream);
return reader.ReadToEnd().Split('\r')
.Where(s => !string.IsNullOrWhiteSpace(s))
.Select(line => line.Split(separator))
.ToDictionary(parts => parts[0].Trim(), parts => parts[1].Trim());
}
//TODO: вынести в метод расширения
private static T GetCellValue<T>(IXLCell cell)
{
try
{
if (typeof(T) != typeof(DateTime))
return (T)Convert.ChangeType(cell.GetFormattedString(), typeof(T), CultureInfo.InvariantCulture);
return (T)(object)DateTime.FromOADate((double)cell.Value);
}
catch
{
throw new FileFormatException(
$"Лист '{cell.Worksheet.Name}'. Ячейка: ({cell.Address.RowNumber},{cell.Address.ColumnNumber}) содержит некорректное значение");
}
}
}

View File

@ -0,0 +1,105 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using AsbCloudApp.Data;
using AsbCloudApp.Repositories;
using AsbCloudApp.Requests;
using AsbCloudApp.Services;
using AsbCloudApp.Services.WellOperationImport;
using AsbCloudInfrastructure.Services.WellOperationImport.Constants;
using ClosedXML.Excel;
namespace AsbCloudInfrastructure.Services.WellOperationImport;
public class WellOperationExportService : IWellOperationExportService
{
private readonly IWellOperationRepository wellOperationRepository;
private readonly IWellService wellService;
private readonly IWellOperationImportTemplateService wellOperationImportTemplateService;
public WellOperationExportService(IWellOperationRepository wellOperationRepository,
IWellService wellService,
IWellOperationImportTemplateService wellOperationImportTemplateService)
{
this.wellOperationRepository = wellOperationRepository;
this.wellService = wellService;
this.wellOperationImportTemplateService = wellOperationImportTemplateService;
}
public async Task<Stream> ExportAsync(int idWell, CancellationToken cancellationToken)
{
var operations = await wellOperationRepository.GetAsync(new WellOperationRequest()
{
IdWell = idWell
}, cancellationToken);
var timezone = wellService.GetTimezone(idWell);
return await MakeExcelFileStreamAsync(operations, timezone.Hours, cancellationToken);
}
private async Task<Stream> MakeExcelFileStreamAsync(IEnumerable<WellOperationDto> operations, double timezoneOffset,
CancellationToken cancellationToken)
{
using Stream ecxelTemplateStream = wellOperationImportTemplateService.GetExcelTemplateStream();
using var workbook = new XLWorkbook(ecxelTemplateStream, XLEventTracking.Disabled);
await AddOperationsToWorkbook(workbook, operations, timezoneOffset, cancellationToken);
var memoryStream = new MemoryStream();
workbook.SaveAs(memoryStream, new SaveOptions { });
memoryStream.Seek(0, SeekOrigin.Begin);
return memoryStream;
}
private async Task AddOperationsToWorkbook(XLWorkbook workbook, IEnumerable<WellOperationDto> operations, double timezoneOffset,
CancellationToken cancellationToken)
{
var planOperations = operations.Where(o => o.IdType == 0);
if (planOperations.Any())
{
var sheetPlan = workbook.Worksheets.FirstOrDefault(ws => ws.Name == DefaultTemplateInfo.SheetNamePlan);
if (sheetPlan is not null)
await AddOperationsToSheetAsync(sheetPlan, planOperations, timezoneOffset, cancellationToken);
}
var factOperations = operations.Where(o => o.IdType == 1);
if (factOperations.Any())
{
var sheetFact = workbook.Worksheets.FirstOrDefault(ws => ws.Name == DefaultTemplateInfo.SheetNameFact);
if (sheetFact is not null)
await AddOperationsToSheetAsync(sheetFact, factOperations, timezoneOffset, cancellationToken);
}
}
private async Task AddOperationsToSheetAsync(IXLWorksheet sheet, IEnumerable<WellOperationDto> operations, double timezoneOffset,
CancellationToken cancellationToken)
{
var operationsToArray = operations.ToArray();
var sections = wellOperationRepository.GetSectionTypes();
var categories = wellOperationRepository.GetCategories(false);
for (int i = 0; i < operationsToArray.Length; i++)
{
var row = sheet.Row(1 + i + DefaultTemplateInfo.HeaderRowsCount);
AddOperationToRow(row, operationsToArray[i], sections, categories, timezoneOffset);
}
}
private static void AddOperationToRow(IXLRow row, WellOperationDto operation, IEnumerable<WellSectionTypeDto> sections,
IEnumerable<WellOperationCategoryDto> categories, double timezoneOffset)
{
row.Cell(DefaultTemplateInfo.ColumnSection).Value = sections.First(s => s.Id == operation.IdWellSectionType).Caption;
row.Cell(DefaultTemplateInfo.ColumnCategory).Value = categories.First(o => o.Id == operation.IdCategory).Name;
row.Cell(DefaultTemplateInfo.ColumnCategoryInfo).Value = operation.CategoryInfo;
row.Cell(DefaultTemplateInfo.ColumnDepthStart).Value = operation.DepthStart;
row.Cell(DefaultTemplateInfo.ColumnDepthEnd).Value = operation.DepthEnd;
row.Cell(DefaultTemplateInfo.ColumnDate).Value = new DateTimeOffset(operation.DateStart).ToRemoteDateTime(timezoneOffset);
row.Cell(DefaultTemplateInfo.ColumnDuration).Value = operation.DurationHours;
row.Cell(DefaultTemplateInfo.ColumnComment).Value = operation.Comment;
}
}

View File

@ -0,0 +1,141 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using AsbCloudApp.Data;
using AsbCloudApp.Data.WellOperationImport;
using AsbCloudApp.Exceptions;
using AsbCloudApp.Repositories;
using AsbCloudApp.Requests;
using AsbCloudApp.Services.WellOperationImport;
using AsbCloudDb.Model;
using AsbCloudInfrastructure.Services.WellOperationImport.Constants;
namespace AsbCloudInfrastructure.Services.WellOperationImport;
public class WellOperationImportService : IWellOperationImportService
{
private readonly IEnumerable<IWellOperationExcelParser> excelParsers;
private readonly IWellOperationRepository wellOperationRepository;
private static readonly DateTime dateLimitMin = new(2001, 1, 1, 0, 0, 0);
private static readonly DateTime dateLimitMax = new(2099, 1, 1, 0, 0, 0);
private static readonly TimeSpan drillingDurationLimitMax = TimeSpan.FromDays(366);
public WellOperationImportService(IEnumerable<IWellOperationExcelParser> excelParsers,
IWellOperationRepository wellOperationRepository)
{
this.excelParsers = excelParsers;
this.wellOperationRepository = wellOperationRepository;
}
public async Task ImportAsync(int idWell, int idUser, int idType, Stream stream, WellOperationParserOptionsDto options,
bool deleteWellOperationsBeforeImport, CancellationToken cancellationToken)
{
var excelParser = excelParsers.FirstOrDefault(p => p.IdTemplate == options.IdTemplate &&
p.IdTypes.Contains(idType));
if (excelParser is null)
throw new ArgumentInvalidException("Невозможно импортировать файл", nameof(options.IdTemplate));
if (idType != WellOperation.IdOperationTypePlan && idType != WellOperation.IdOperationTypeFact)
throw new ArgumentInvalidException("Операции не существует", nameof(idType));
RowDto[] rows;
var validationErrors = new List<string>();
var sections = wellOperationRepository.GetSectionTypes();
var categories = wellOperationRepository.GetCategories(false);
switch (options.IdTemplate)
{
case 0:
options.SheetName = idType == WellOperation.IdOperationTypePlan
? DefaultTemplateInfo.SheetNamePlan
: DefaultTemplateInfo.SheetNameFact;
rows = excelParser.Parse(stream, options).ToArray();
break;
default:
if (string.IsNullOrWhiteSpace(options.SheetName))
throw new FileFormatException("Не указано название листа");
rows = excelParser.Parse(stream, options).ToArray();
break;
}
var operations = new List<WellOperationDto>();
foreach (var row in rows)
{
try
{
var section = sections.FirstOrDefault(s =>
string.Equals(s.Caption, row.Section, StringComparison.CurrentCultureIgnoreCase));
if (section is null)
throw new FileFormatException($"Лист '{options.SheetName}'. В строке '{row.Number}' не удалось определить секцию");
var category = categories.FirstOrDefault(c =>
string.Equals(c.Name, row.Category, StringComparison.CurrentCultureIgnoreCase));
if (category is null)
throw new FileFormatException($"Лист '{options.SheetName}'. В строке '{row.Number}' не удалось определить операцию");
if (row.DepthStart is not (>= 0d and <= 20_000d))
throw new FileFormatException($"Лист '{options.SheetName}'. Строка '{row.Number}' некорректная глубина на начало операции");
if (row.DepthEnd is not (>= 0d and <= 20_000d))
throw new FileFormatException($"Лист '{options.SheetName}'. Строка '{row.Number}' некорректная глубина на конец операции");
if (row.Date < dateLimitMin && row.Date > dateLimitMax)
throw new FileFormatException($"Лист '{options.SheetName}'. Строка '{row.Number}' неправильно получена дата начала операции");
if (operations.LastOrDefault()?.DateStart > row.Date)
throw new FileFormatException($"Лист '{options.SheetName}' строка '{row.Number}' дата позднее даты предыдущей операции");
if (row.Duration is not (>= 0d and <= 240d))
throw new FileFormatException($"Лист '{options.SheetName}'. Строка '{row.Number}' некорректная длительность операции");
operations.Add(new WellOperationDto
{
IdWell = idWell,
IdUser = idUser,
IdType = idType,
IdWellSectionType = section.Id,
IdCategory = category.Id,
CategoryInfo = row.CategoryInfo,
DepthStart = row.DepthStart,
DepthEnd = row.DepthEnd,
DateStart = row.Date,
DurationHours = row.Duration
});
}
catch (FileFormatException ex)
{
validationErrors.Add(ex.Message);
}
}
if (operations.Any() && operations.Min(o => o.DateStart) - operations.Max(o => o.DateStart) > drillingDurationLimitMax)
validationErrors.Add($"Лист {options.SheetName} содержит диапазон дат больше {drillingDurationLimitMax}");
if (validationErrors.Any())
throw new FileFormatException(string.Join("\r\n", validationErrors));
if(!operations.Any())
return;
if (deleteWellOperationsBeforeImport)
{
var existingOperations = await wellOperationRepository.GetAsync(new WellOperationRequest
{
IdWell = idWell
}, cancellationToken);
await wellOperationRepository.DeleteAsync(existingOperations.Select(o => o.Id), cancellationToken);
}
await wellOperationRepository.InsertRangeAsync(operations, cancellationToken);
}
}

View File

@ -0,0 +1,21 @@
using System.IO;
using System.Linq;
using System.Reflection;
using AsbCloudApp.Services.WellOperationImport;
namespace AsbCloudInfrastructure.Services.WellOperationImport;
public class WellOperationImportTemplateService : IWellOperationImportTemplateService
{
public Stream GetExcelTemplateStream()
{
var resourceName = Assembly.GetExecutingAssembly()
.GetManifestResourceNames()
.FirstOrDefault(n => n.EndsWith("WellOperationImportTemplate.xlsx"))!;
var stream = Assembly.GetExecutingAssembly()
.GetManifestResourceStream(resourceName)!;
return stream;
}
}

View File

@ -1,341 +0,0 @@
using AsbCloudApp.Data;
using AsbCloudApp.Services;
using AsbCloudDb.Model;
using ClosedXML.Excel;
using Mapster;
using Microsoft.EntityFrameworkCore;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
namespace AsbCloudInfrastructure.Services.WellOperationService
{
/*
* password for WellOperationImportTemplate.xlsx is ASB2020!
*/
public class WellOperationImportService : IWellOperationImportService
{
private const string sheetNamePlan = "План";
private const string sheetNameFact = "Факт";
private const int headerRowsCount = 1;
private const int columnSection = 1;
private const int columnCategory = 2;
private const int columnCategoryInfo = 3;
private const int columnDepthStart = 4;
private const int columnDepthEnd = 5;
private const int columnDate = 6;
private const int columnDuration = 7;
private const int columnComment = 8;
private static readonly DateTime dateLimitMin = new DateTime(2001, 1, 1, 0, 0, 0);
private static readonly DateTime dateLimitMax = new DateTime(2099, 1, 1, 0, 0, 0);
private static readonly TimeSpan drillingDurationLimitMax = TimeSpan.FromDays(366);
private readonly IAsbCloudDbContext db;
private readonly IWellService wellService;
private List<WellOperationCategory> categories = null!;
public List<WellOperationCategory> Categories
{
get
{
if (categories is null)
{
categories = db.WellOperationCategories
.Where(c => c.Id >= 5000)
.AsNoTracking()
.ToList();
}
return categories;
}
}
private List<WellSectionType> sections = null!;
public List<WellSectionType> Sections
{
get
{
if (sections is null)
sections = db.WellSectionTypes
.AsNoTracking()
.ToList();
return sections;
}
}
// TODO: use WellOperationRepository instead of DB
public WellOperationImportService(IAsbCloudDbContext db, IWellService wellService)
{
this.db = db;
this.wellService = wellService;
}
public void Import(int idWell, Stream stream, int idUser, bool deleteWellOperationsBeforeImport = false)
{
using var workbook = new XLWorkbook(stream, XLEventTracking.Disabled);
var operations = ParseFileStream(stream);
foreach (var operation in operations)
{
operation.IdWell = idWell;
operation.IdUser = idUser;
}
SaveOperations(idWell, operations, deleteWellOperationsBeforeImport);
}
public Stream Export(int idWell)
{
var operations = db.WellOperations
.Include(o => o.WellSectionType)
.Include(o => o.OperationCategory)
.Where(o => o.IdWell == idWell)
.OrderBy(o => o.DateStart)
.AsNoTracking()
.ToList();
var timezone = wellService.GetTimezone(idWell);
return MakeExelFileStream(operations, timezone.Hours);
}
public Stream GetExcelTemplateStream()
{
var resourceName = System.Reflection.Assembly.GetExecutingAssembly()
.GetManifestResourceNames()
.FirstOrDefault(n => n.EndsWith("WellOperationImportTemplate.xlsx"))!;
var stream = System.Reflection.Assembly.GetExecutingAssembly()
.GetManifestResourceStream(resourceName)!;
return stream;
}
private Stream MakeExelFileStream(IEnumerable<WellOperation> operations, double timezoneOffset)
{
using Stream ecxelTemplateStream = GetExcelTemplateStream();
using var workbook = new XLWorkbook(ecxelTemplateStream, XLEventTracking.Disabled);
AddOperationsToWorkbook(workbook, operations, timezoneOffset);
MemoryStream memoryStream = new MemoryStream();
workbook.SaveAs(memoryStream, new SaveOptions { });
memoryStream.Seek(0, SeekOrigin.Begin);
return memoryStream;
}
private static void AddOperationsToWorkbook(XLWorkbook workbook, IEnumerable<WellOperation> operations, double timezoneOffset)
{
var planOperations = operations.Where(o => o.IdType == 0);
if (planOperations.Any())
{
var sheetPlan = workbook.Worksheets.FirstOrDefault(ws => ws.Name == sheetNamePlan);
if (sheetPlan is not null)
AddOperationsToSheet(sheetPlan, planOperations, timezoneOffset);
}
var factOperations = operations.Where(o => o.IdType == 1);
if (factOperations.Any())
{
var sheetFact = workbook.Worksheets.FirstOrDefault(ws => ws.Name == sheetNameFact);
if (sheetFact is not null)
AddOperationsToSheet(sheetFact, factOperations, timezoneOffset);
}
}
private static void AddOperationsToSheet(IXLWorksheet sheet, IEnumerable<WellOperation> operations, double timezoneOffset)
{
var operationsList = operations.ToList();
for (int i = 0; i < operationsList.Count; i++)
{
var row = sheet.Row(1 + i + headerRowsCount);
AddOperationToRow(row, operationsList[i], timezoneOffset);
}
}
private static void AddOperationToRow(IXLRow row, WellOperation operation, double timezoneOffset)
{
row.Cell(columnSection).Value = operation.WellSectionType?.Caption;
row.Cell(columnCategory).Value = operation.OperationCategory?.Name;
row.Cell(columnCategoryInfo).Value = operation.CategoryInfo;
row.Cell(columnDepthStart).Value = operation.DepthStart;
row.Cell(columnDepthEnd).Value = operation.DepthEnd;
row.Cell(columnDate).Value = operation.DateStart.ToRemoteDateTime(timezoneOffset);
row.Cell(columnDuration).Value = operation.DurationHours;
row.Cell(columnComment).Value = operation.Comment;
}
private void SaveOperations(int idWell, IEnumerable<WellOperationDto> operations, bool deleteWellOperationsBeforeImport = false)
{
var timezone = wellService.GetTimezone(idWell);
var transaction = db.Database.BeginTransaction();
try
{
if (deleteWellOperationsBeforeImport)
db.WellOperations.RemoveRange(db.WellOperations.Where(o => o.IdWell == idWell));
var entities = operations.Select(o =>
{
var entity = o.Adapt<WellOperation>();
entity.IdWell = idWell;
entity.DateStart = o.DateStart.ToUtcDateTimeOffset(timezone.Hours);
entity.LastUpdateDate = DateTimeOffset.UtcNow;
return entity;
});
db.WellOperations.AddRange(entities);
db.SaveChanges();
transaction.Commit();
}
catch
{
transaction.Rollback();
throw;
}
}
private IEnumerable<WellOperationDto> ParseFileStream(Stream stream)
{
using var workbook = new XLWorkbook(stream, XLEventTracking.Disabled);
return ParseWorkbook(workbook);
}
private IEnumerable<WellOperationDto> ParseWorkbook(IXLWorkbook workbook)
{
var sheetPlan = workbook.Worksheets.FirstOrDefault(ws => ws.Name == sheetNamePlan);
if (sheetPlan is null)
throw new FileFormatException($"Книга excel не содержит листа {sheetNamePlan}.");
var sheetFact = workbook.Worksheets.FirstOrDefault(ws => ws.Name == sheetNameFact);
if (sheetFact is null)
throw new FileFormatException($"Книга excel не содержит листа {sheetNameFact}.");
//sheetPlan.RangeUsed().RangeAddress.LastAddress.ColumnNumber
var wellOperations = new List<WellOperationDto>();
var wellOperationsPlan = ParseSheet(sheetPlan, 0);
wellOperations.AddRange(wellOperationsPlan);
var wellOperationsFact = ParseSheet(sheetFact, 1);
wellOperations.AddRange(wellOperationsFact);
return wellOperations;
}
private IEnumerable<WellOperationDto> ParseSheet(IXLWorksheet sheet, int idType)
{
if (sheet.RangeUsed().RangeAddress.LastAddress.ColumnNumber < 7)
throw new FileFormatException($"Лист {sheet.Name} содержит меньшее количество столбцов.");
var count = sheet.RowsUsed().Count() - headerRowsCount;
if (count > 1024)
throw new FileFormatException($"Лист {sheet.Name} содержит слишком большое количество операций.");
if (count <= 0)
return new List<WellOperationDto>();
var operations = new List<WellOperationDto>(count);
var parseErrors = new List<string>();
DateTime lastOperationDateStart = new DateTime();
for (int i = 0; i < count; i++)
{
var row = sheet.Row(1 + i + headerRowsCount);
try
{
var operation = ParseRow(row, idType);
operations.Add(operation);
if (lastOperationDateStart > operation.DateStart)
parseErrors.Add($"Лист {sheet.Name} строка {row.RowNumber()} дата позднее даты предыдущей операции.");
lastOperationDateStart = operation.DateStart;
}
catch (FileFormatException ex)
{
parseErrors.Add(ex.Message);
}
};
if (parseErrors.Any())
throw new FileFormatException(string.Join("\r\n", parseErrors));
else
{
if (operations.Any())
if (operations.Min(o => o.DateStart) - operations.Max(o => o.DateStart) > drillingDurationLimitMax)
parseErrors.Add($"Лист {sheet.Name} содержит диапазон дат больше {drillingDurationLimitMax}");
}
return operations;
}
private WellOperationDto ParseRow(IXLRow row, int idType)
{
var vSection = row.Cell(columnSection).Value;
var vCategory = row.Cell(columnCategory).Value;
var vCategoryInfo = row.Cell(columnCategoryInfo).Value;
var vDepthStart = row.Cell(columnDepthStart).Value;
var vDepthEnd = row.Cell(columnDepthEnd).Value;
var vDate = row.Cell(columnDate).Value;
var vDuration = row.Cell(columnDuration).Value;
var vComment = row.Cell(columnComment).Value;
var operation = new WellOperationDto { IdType = idType };
if (vSection is string sectionName)
{
var section = Sections.Find(c => c.Caption.ToLower() == sectionName.ToLower());
if (section is null)
throw new FileFormatException($"Лист {row.Worksheet.Name}. Строка {row.RowNumber()} указана некорректная секция");
operation.IdWellSectionType = section.Id;
operation.WellSectionTypeName = section.Caption;
}
else
throw new FileFormatException($"Лист {row.Worksheet.Name}. Строка {row.RowNumber()} не указана секция");
if (vCategory is string categoryName)
{
var category = Categories.Find(c => c.Name.ToLower() == categoryName.ToLower());
if (category is null)
throw new FileFormatException($"Лист {row.Worksheet.Name}. Строка {row.RowNumber()} указана некорректная операция ({categoryName})");
operation.IdCategory = category.Id;
operation.CategoryName = category.Name;
}
else
throw new FileFormatException($"Лист {row.Worksheet.Name}. Строка {row.RowNumber()} не указана операция");
if (vCategoryInfo is not null)
operation.CategoryInfo = vCategoryInfo.ToString();
if (vDepthStart is double depthStart && depthStart >= 0d && depthStart <= 20_000d)
operation.DepthStart = depthStart;
else
throw new FileFormatException($"Лист {row.Worksheet.Name}. Строка {row.RowNumber()} не указана глубина на начало операции");
if (vDepthEnd is double depthEnd && depthEnd >= 0d && depthEnd <= 20_000d)
operation.DepthEnd = depthEnd;
else
throw new FileFormatException($"Лист {row.Worksheet.Name}. Строка {row.RowNumber()} не указана глубина при завершении операции");
if (vDate is DateTime date && date > dateLimitMin && date < dateLimitMax)
operation.DateStart = date;
else
throw new FileFormatException($"Лист {row.Worksheet.Name}. Строка {row.RowNumber()} неправильно указана дата/время начала операции");
if (vDuration is double duration && duration >= 0d && duration <= 240d)
operation.DurationHours = duration;
else
throw new FileFormatException($"Лист {row.Worksheet.Name}. Строка {row.RowNumber()} не указана длительность операции");
if (vComment is not null)
operation.Comment = vComment.ToString();
return operation;
}
}
}

View File

@ -11,6 +11,8 @@ using System.ComponentModel.DataAnnotations;
using System.IO; using System.IO;
using System.Threading; using System.Threading;
using System.Threading.Tasks; using System.Threading.Tasks;
using AsbCloudApp.Data.WellOperationImport;
using AsbCloudApp.Services.WellOperationImport;
namespace AsbCloudWebApi.Controllers namespace AsbCloudWebApi.Controllers
{ {
@ -25,12 +27,20 @@ namespace AsbCloudWebApi.Controllers
{ {
private readonly IWellOperationRepository operationRepository; private readonly IWellOperationRepository operationRepository;
private readonly IWellService wellService; private readonly IWellService wellService;
private readonly IWellOperationExportService wellOperationExportService;
private readonly IWellOperationImportTemplateService wellOperationImportTemplateService;
private readonly IWellOperationImportService wellOperationImportService; private readonly IWellOperationImportService wellOperationImportService;
public WellOperationController(IWellOperationRepository operationService, IWellService wellService, IWellOperationImportService wellOperationImportService) public WellOperationController(IWellOperationRepository operationService,
IWellService wellService,
IWellOperationImportTemplateService wellOperationImportTemplateService,
IWellOperationExportService wellOperationExportService,
IWellOperationImportService wellOperationImportService)
{ {
this.operationRepository = operationService; this.operationRepository = operationService;
this.wellService = wellService; this.wellService = wellService;
this.wellOperationImportTemplateService = wellOperationImportTemplateService;
this.wellOperationExportService = wellOperationExportService;
this.wellOperationImportService = wellOperationImportService; this.wellOperationImportService = wellOperationImportService;
} }
@ -266,41 +276,57 @@ namespace AsbCloudWebApi.Controllers
/// <summary> /// <summary>
/// Импортирует операции из excel (xlsx) файла /// Импорт плановых операций из excel (xlsx) файла
/// </summary> /// </summary>
/// <param name="idWell">id скважины</param> /// <param name="idWell">id скважины</param>
/// <param name="idType">Тип операции</param>
/// <param name="startRow">Начальная строка</param>
/// <param name="endRow">Конечная строка</param>
/// <param name="files">Коллекция из одного файла xlsx</param> /// <param name="files">Коллекция из одного файла xlsx</param>
/// <param name="options">Удалить операции перед импортом = 1, если фал валидный</param> /// <param name="options">Удалить операции перед импортом = 1, если файл валидный</param>
/// <param name="token"> Токен отмены задачи </param> /// <param name="sheetName">Название листа</param>
/// <param name="token">Токен отмены задачи </param>
/// <param name="idTemplate">Шаблон файла. 0 - стандартный, 1 - Газпромнефть Хантос</param>
/// <returns></returns> /// <returns></returns>
[HttpPost("import/{options}")] [HttpPost("import/{options}")]
[Permission] [Permission]
public async Task<IActionResult> ImportAsync(int idWell, public async Task<IActionResult> ImportAsync(int idWell,
[Required] int idType,
string? sheetName,
[Required] int idTemplate,
int? startRow,
int? endRow,
[FromForm] IFormFileCollection files, [FromForm] IFormFileCollection files,
int options, int options,
CancellationToken token) CancellationToken token)
{ {
int? idCompany = User.GetCompanyId(); var idCompany = User.GetCompanyId();
int? idUser = User.GetUserId(); var idUser = User.GetUserId();
if (idCompany is null || idUser is null) if (idCompany is null || idUser is null)
return Forbid(); return Forbid();
if (!await wellService.IsCompanyInvolvedInWellAsync((int)idCompany, if (!await wellService.IsCompanyInvolvedInWellAsync((int)idCompany, idWell, token))
idWell, token).ConfigureAwait(false))
return Forbid(); return Forbid();
if (files.Count < 1) if (files.Count < 1)
return BadRequest("нет файла"); return BadRequest("Нет файла");
var file = files[0]; var file = files[0];
if (Path.GetExtension(file.FileName).ToLower() != ".xlsx") if (Path.GetExtension(file.FileName).ToLower() != ".xlsx")
return BadRequest("Требуется xlsx файл."); return BadRequest("Требуется xlsx файл.");
using Stream stream = file.OpenReadStream(); using Stream stream = file.OpenReadStream();
try try
{ {
wellOperationImportService.Import(idWell, stream, idUser.Value, (options & 1) > 0); await wellOperationImportService.ImportAsync(idWell, idUser.Value, idType, stream, new WellOperationParserOptionsDto
{
SheetName = sheetName,
IdTemplate = idTemplate,
StartRow = startRow,
EndRow = endRow
}, (options & 1) > 0, token);
} }
catch (FileFormatException ex) catch (FileFormatException ex)
{ {
@ -331,7 +357,7 @@ namespace AsbCloudWebApi.Controllers
idWell, token).ConfigureAwait(false)) idWell, token).ConfigureAwait(false))
return Forbid(); return Forbid();
var stream = wellOperationImportService.Export(idWell); var stream = await wellOperationExportService.ExportAsync(idWell, token);
var fileName = await wellService.GetWellCaptionByIdAsync(idWell, token) + "_operations.xlsx"; var fileName = await wellService.GetWellCaptionByIdAsync(idWell, token) + "_operations.xlsx";
return File(stream, "application/octet-stream", fileName); return File(stream, "application/octet-stream", fileName);
} }
@ -371,7 +397,7 @@ namespace AsbCloudWebApi.Controllers
[ProducesResponseType(typeof(PhysicalFileResult), (int)System.Net.HttpStatusCode.OK, "application/octet-stream")] [ProducesResponseType(typeof(PhysicalFileResult), (int)System.Net.HttpStatusCode.OK, "application/octet-stream")]
public IActionResult GetTemplate() public IActionResult GetTemplate()
{ {
var stream = wellOperationImportService.GetExcelTemplateStream(); var stream = wellOperationImportTemplateService.GetExcelTemplateStream();
var fileName = "ЕЦП_шаблон_файла_операций.xlsx"; var fileName = "ЕЦП_шаблон_файла_операций.xlsx";
return File(stream, "application/octet-stream", fileName); return File(stream, "application/octet-stream", fileName);
} }