diff --git a/AsbCloudApp/Services/IWellOperationImportService.cs b/AsbCloudApp/Services/IWellOperationImportService.cs deleted file mode 100644 index f8c5fac9..00000000 --- a/AsbCloudApp/Services/IWellOperationImportService.cs +++ /dev/null @@ -1,32 +0,0 @@ -using System.IO; - -namespace AsbCloudApp.Services -{ - /// - /// сервис импорта/экспорта операций по скважине вводимых вручную - /// - public interface IWellOperationImportService - { - /// - /// скачать в excel - /// - /// - /// - Stream Export(int idWell); - - /// - /// скачать шаблон для заполнения - /// - /// - Stream GetExcelTemplateStream(); - - /// - /// закгрузить из excel список операций - /// - /// - /// - /// - /// Очистить старые перед импортом (если файл проходит валидацию) - void Import(int idWell, Stream stream, int idUser, bool deleteWellOperationsBeforeImport = false); - } -} \ No newline at end of file diff --git a/AsbCloudApp/Services/WellOperationImport/IWellOperationExcelParser.cs b/AsbCloudApp/Services/WellOperationImport/IWellOperationExcelParser.cs new file mode 100644 index 00000000..c8c8042a --- /dev/null +++ b/AsbCloudApp/Services/WellOperationImport/IWellOperationExcelParser.cs @@ -0,0 +1,29 @@ +using System.Collections.Generic; +using System.IO; +using AsbCloudApp.Data.WellOperationImport; + +namespace AsbCloudApp.Services.WellOperationImport; + +/// +/// Парсинг операций из excel файла +/// +public interface IWellOperationExcelParser +{ + /// + /// Id шаблона + /// + int IdTemplate { get; } + + /// + /// Типы операций, которые можно получить из файла + /// + IEnumerable IdTypes { get; } + + /// + /// Метод парсинга документа + /// + /// + /// + /// + IEnumerable Parse(Stream stream, WellOperationParserOptionsDto options); +} \ No newline at end of file diff --git a/AsbCloudApp/Services/WellOperationImport/IWellOperationExportService.cs b/AsbCloudApp/Services/WellOperationImport/IWellOperationExportService.cs new file mode 100644 index 00000000..9b0a96ac --- /dev/null +++ b/AsbCloudApp/Services/WellOperationImport/IWellOperationExportService.cs @@ -0,0 +1,19 @@ +using System.IO; +using System.Threading; +using System.Threading.Tasks; + +namespace AsbCloudApp.Services.WellOperationImport; + +/// +/// Экспорт ГГД +/// +public interface IWellOperationExportService +{ + /// + /// Скачать в excel + /// + /// + /// + /// + Task ExportAsync(int idWell, CancellationToken cancellationToken); +} \ No newline at end of file diff --git a/AsbCloudApp/Services/WellOperationImport/IWellOperationImportService.cs b/AsbCloudApp/Services/WellOperationImport/IWellOperationImportService.cs new file mode 100644 index 00000000..1da68cf8 --- /dev/null +++ b/AsbCloudApp/Services/WellOperationImport/IWellOperationImportService.cs @@ -0,0 +1,26 @@ +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using AsbCloudApp.Data.WellOperationImport; + +namespace AsbCloudApp.Services.WellOperationImport; + +/// +/// Импорт ГГД +/// +public interface IWellOperationImportService +{ + /// + /// Загрузить из excel список операций + /// + /// + /// + /// + /// + /// + /// + /// + Task ImportAsync(int idWell, int idUser, int idType, Stream stream, WellOperationParserOptionsDto options, + bool deleteWellOperationsBeforeImport, + CancellationToken cancellationToken); +} \ No newline at end of file diff --git a/AsbCloudApp/Services/WellOperationImport/IWellOperationImportTemplateService.cs b/AsbCloudApp/Services/WellOperationImport/IWellOperationImportTemplateService.cs new file mode 100644 index 00000000..817f3ae5 --- /dev/null +++ b/AsbCloudApp/Services/WellOperationImport/IWellOperationImportTemplateService.cs @@ -0,0 +1,15 @@ +using System.IO; + +namespace AsbCloudApp.Services.WellOperationImport; + +/// +/// Сервис для получения шаблонов ГГД +/// +public interface IWellOperationImportTemplateService +{ + /// + /// Скачать шаблон для заполнения + /// + /// + Stream GetExcelTemplateStream(); +} \ No newline at end of file diff --git a/AsbCloudInfrastructure/DependencyInjection.cs b/AsbCloudInfrastructure/DependencyInjection.cs index bb894d63..5b065d4b 100644 --- a/AsbCloudInfrastructure/DependencyInjection.cs +++ b/AsbCloudInfrastructure/DependencyInjection.cs @@ -25,7 +25,10 @@ using Microsoft.Extensions.DependencyInjection; using System; using AsbCloudApp.Services.AutoGeneratedDailyReports; using AsbCloudApp.Services.Notifications; +using AsbCloudApp.Services.WellOperationImport; using AsbCloudInfrastructure.Services.AutoGeneratedDailyReports; +using AsbCloudInfrastructure.Services.WellOperationImport; +using AsbCloudInfrastructure.Services.WellOperationImport.FileParser; namespace AsbCloudInfrastructure { @@ -129,7 +132,6 @@ namespace AsbCloudInfrastructure services.AddTransient(); services.AddTransient(); services.AddTransient(); - services.AddTransient(); services.AddTransient(); services.AddTransient(); services.AddTransient(); @@ -224,6 +226,13 @@ namespace AsbCloudInfrastructure services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + + services.AddTransient(); + services.AddTransient(); + return services; } diff --git a/AsbCloudInfrastructure/Services/ProcessMap/ProcessMapPlanImportService.cs b/AsbCloudInfrastructure/Services/ProcessMap/ProcessMapPlanImportService.cs index 21f118a8..e8185a34 100644 --- a/AsbCloudInfrastructure/Services/ProcessMap/ProcessMapPlanImportService.cs +++ b/AsbCloudInfrastructure/Services/ProcessMap/ProcessMapPlanImportService.cs @@ -342,7 +342,8 @@ public class ProcessMapPlanImportService : IProcessMapPlanImportService 2 => "Слайд", _ => "Ручной", }; - + + //TODO: вынести в метод расширения private static T GetCellValue(IXLRow row, int columnNumber) { try diff --git a/AsbCloudInfrastructure/Services/WellOperationImport/FileParser/WellOperationDefaultExcelParser.cs b/AsbCloudInfrastructure/Services/WellOperationImport/FileParser/WellOperationDefaultExcelParser.cs new file mode 100644 index 00000000..7d79e243 --- /dev/null +++ b/AsbCloudInfrastructure/Services/WellOperationImport/FileParser/WellOperationDefaultExcelParser.cs @@ -0,0 +1,106 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using AsbCloudApp.Data.WellOperationImport; +using AsbCloudApp.Exceptions; +using AsbCloudApp.Services.WellOperationImport; +using AsbCloudDb.Model; +using AsbCloudInfrastructure.Services.WellOperationImport.Constants; +using ClosedXML.Excel; + +namespace AsbCloudInfrastructure.Services.WellOperationImport.FileParser; + +public class WellOperationDefaultExcelParser : IWellOperationExcelParser +{ + public int IdTemplate => Templates.IdDefaultTemplate; + public IEnumerable IdTypes => new[] { WellOperation.IdOperationTypePlan, WellOperation.IdOperationTypeFact }; + + public IEnumerable Parse(Stream stream, WellOperationParserOptionsDto options) + { + using var workbook = new XLWorkbook(stream, XLEventTracking.Disabled); + + return ParseWorkbook(workbook, options); + } + + private static IEnumerable ParseWorkbook(IXLWorkbook workbook, WellOperationParserOptionsDto options) + { + if (string.IsNullOrWhiteSpace(options.SheetName)) + throw new ArgumentInvalidException("Не указано название листа", nameof(options.SheetName)); + + var sheet = workbook.Worksheets.FirstOrDefault(ws => + string.Equals(ws.Name, options.SheetName, StringComparison.CurrentCultureIgnoreCase)) + ?? throw new FileFormatException($"Книга excel не содержит листа '{options.SheetName}'"); + + return ParseSheet(sheet); + } + + private static IEnumerable ParseSheet(IXLWorksheet sheet) + { + if (sheet.RangeUsed().RangeAddress.LastAddress.ColumnNumber < 7) + throw new FileFormatException($"Лист {sheet.Name} содержит меньшее количество столбцов."); + + var count = sheet.RowsUsed().Count() - DefaultTemplateInfo.HeaderRowsCount; + + switch (count) + { + case > 1024: + throw new FileFormatException($"Лист {sheet.Name} содержит слишком большое количество операций."); + case <= 0: + return Enumerable.Empty(); + } + + var rows = new RowDto[count]; + + var cellValuesErrors = new List(); + + for (int i = 0; i < rows.Length; i++) + { + try + { + var xlRow = sheet.Row(1 + i + DefaultTemplateInfo.HeaderRowsCount); + + rows[i] = ParseRow(xlRow); + } + catch (FileFormatException ex) + { + cellValuesErrors.Add(ex.Message); + } + } + + if (cellValuesErrors.Any()) + throw new FileFormatException(string.Join("\r\n", cellValuesErrors)); + + return rows; + } + + private static RowDto ParseRow(IXLRow xlRow) + { + return new RowDto + { + Number = xlRow.RowNumber(), + Section = GetCellValue(xlRow.Cell(DefaultTemplateInfo.ColumnSection)), + Category = GetCellValue(xlRow.Cell(DefaultTemplateInfo.ColumnCategory)), + CategoryInfo = GetCellValue(xlRow.Cell(DefaultTemplateInfo.ColumnCategoryInfo)), + DepthStart = GetCellValue(xlRow.Cell(DefaultTemplateInfo.ColumnDepthStart)), + DepthEnd = GetCellValue(xlRow.Cell(DefaultTemplateInfo.ColumnDepthEnd)), + Date = GetCellValue(xlRow.Cell(DefaultTemplateInfo.ColumnDate)), + Duration = GetCellValue(xlRow.Cell(DefaultTemplateInfo.ColumnDuration)), + Comment = GetCellValue(xlRow.Cell(DefaultTemplateInfo.ColumnComment)) + }; + } + + //TODO: вынести в метод расширения + private static T GetCellValue(IXLCell cell) + { + try + { + return (T)Convert.ChangeType(cell.Value, typeof(T)); + } + catch + { + throw new FileFormatException( + $"Лист '{cell.Worksheet.Name}'. Ячейка: ({cell.Address.RowNumber},{cell.Address.ColumnNumber}) содержит некорректное значение"); + } + } +} \ No newline at end of file diff --git a/AsbCloudInfrastructure/Services/WellOperationImport/FileParser/WellOperationGazpromKhantosExcelParser.cs b/AsbCloudInfrastructure/Services/WellOperationImport/FileParser/WellOperationGazpromKhantosExcelParser.cs new file mode 100644 index 00000000..4de99d77 --- /dev/null +++ b/AsbCloudInfrastructure/Services/WellOperationImport/FileParser/WellOperationGazpromKhantosExcelParser.cs @@ -0,0 +1,254 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Reflection; +using AsbCloudApp.Data.WellOperationImport; +using AsbCloudApp.Exceptions; +using AsbCloudApp.Services.WellOperationImport; +using AsbCloudDb.Model; +using AsbCloudInfrastructure.Services.WellOperationImport.Constants; +using AsbCloudInfrastructure.Services.WellOperationImport.FileParser.StringSimilarity; +using ClosedXML.Excel; + +namespace AsbCloudInfrastructure.Services.WellOperationImport.FileParser; + +public class WellOperationGazpromKhantosExcelParser : IWellOperationExcelParser +{ + private class Operation + { + public int RowNumber { get; set; } + + public string CategoryInfo { get; set; } = null!; + + public double SectionDiameter { get; set; } + + public double Depth { get; set; } + + public double Duration { get; set; } + + public DateTime Date { get; set; } + } + + private readonly CosineSimilarity cosineSimilarity; + + private readonly Dictionary operationDict = InitDict("Operations.txt", '='); + private readonly Dictionary sectionDict = InitDict("Sections.txt", '='); + private readonly Dictionary operationAttributesDict = InitDict("OperationAttributes.txt", '='); + + + public WellOperationGazpromKhantosExcelParser() + { + cosineSimilarity = new CosineSimilarity(); + } + + public int IdTemplate => Templates.IdGazpromKhantosTemplate; + + public IEnumerable IdTypes => new[] { WellOperation.IdOperationTypePlan }; + + public IEnumerable Parse(Stream stream, WellOperationParserOptionsDto options) + { + using var workbook = new XLWorkbook(stream, XLEventTracking.Disabled); + + return ParseWorkBook(workbook, options); + } + + private IEnumerable ParseWorkBook(IXLWorkbook workbook, WellOperationParserOptionsDto options) + { + if (string.IsNullOrWhiteSpace(options.SheetName)) + throw new ArgumentInvalidException("Не указано название листа", nameof(options.SheetName)); + + if (options.StartRow is null or < 1 or > 1048576) + throw new ArgumentInvalidException("Некорректное значение начальной строки", nameof(options.StartRow)); + + if (options.EndRow is null or < 1 or > 1048576) + throw new ArgumentInvalidException("Некорректное значение конечной строки", nameof(options.EndRow)); + + if (options.EndRow < options.StartRow) + throw new ArgumentInvalidException("Конечный номер строки не может быть больше начального", nameof(options.EndRow)); + + var sheet = workbook.Worksheets.FirstOrDefault(ws => + string.Equals(ws.Name, options.SheetName, StringComparison.CurrentCultureIgnoreCase)) + ?? throw new FileFormatException($"Книга excel не содержит листа '{options.SheetName}'"); + + return ParseSheet(sheet, options.StartRow.Value, options.EndRow.Value); + } + + private IEnumerable ParseSheet(IXLWorksheet sheet, int startRow, int endRow) + { + var operationAttributes = GetOperationAttributes(sheet.RowsUsed()); + + if (operationAttributes is null) + return Enumerable.Empty(); + + var rowsCount = endRow - startRow + 1; + + var operations = new List(); + + var cellValuesErrors = new List(); + + for (int i = 0; i < rowsCount; i++) + { + var xlRow = sheet.Row(startRow + i); + + try + { + operations.Add(new Operation + { + RowNumber = xlRow.RowNumber(), + CategoryInfo = GetCellValue(xlRow.Cell(operationAttributes[OperationAttributes.CategoryInfo])), + SectionDiameter = GetCellValue(xlRow.Cell(operationAttributes[OperationAttributes.SectionDiameter])), + Depth = GetCellValue(xlRow.Cell(operationAttributes[OperationAttributes.Depth])), + Duration = GetCellValue(xlRow.Cell(operationAttributes[OperationAttributes.Duration])), + Date = GetCellValue(xlRow.Cell(operationAttributes[OperationAttributes.Date])) + }); + } + catch (FileFormatException ex) + { + cellValuesErrors.Add(ex.Message); + } + } + + if (cellValuesErrors.Any()) + throw new FileFormatException(string.Join("\r\n", cellValuesErrors)); + + return BuildRows(); + + IEnumerable<(double Diameter, string Name)> BuildSections() + { + var groupedOperations = operations.GroupBy(o => o.SectionDiameter) + .Select(s => new + { + Diameter = s.Key, + CategoryInfo = string.Concat(s.Select(o => o.CategoryInfo)) + }); + + var repeatedSections = new[] { "xвостовик" }; + + var sections = new List<(double diameter, string section)>(); + + foreach (var groupedOperation in groupedOperations) + { + var sectionNamesSet = new HashSet(sections.Select(s => s.section)); + + sections.Add(new ValueTuple(groupedOperation.Diameter, sectionDict.FirstOrDefault(item => + groupedOperation.CategoryInfo.Contains(item.Key) && + (!sectionNamesSet.Contains(item.Value) || repeatedSections.Contains(item.Value.ToLowerInvariant()))).Value)); + } + + return sections; + } + + IEnumerable BuildRows() + { + if (!operations.Any()) + return Enumerable.Empty(); + + var rows = new List(); + + for (int i = 0; i < operations.Count; i++) + { + var currentOperation = operations[i]; + var nextOperation = i + 1 < operations.Count ? operations[i + 1] : currentOperation; + + rows.Add(new RowDto + { + Number = currentOperation.RowNumber, + Section = BuildSections().FirstOrDefault(s => Math.Abs(s.Diameter - currentOperation.SectionDiameter) < 0.1).Name, + Category = GetValueDictionary(operationDict, currentOperation.CategoryInfo, 0.3), + CategoryInfo = currentOperation.CategoryInfo, + DepthStart = currentOperation.Depth, + DepthEnd = nextOperation.Depth, + Duration = currentOperation.Duration, + Date = currentOperation.Date.AddHours(-currentOperation.Duration) + }); + } + + return rows; + } + } + + private IDictionary? GetOperationAttributes(IXLRows xlRows) + { + const int countOperationAttributes = 5; + + IDictionary? operationAttributes = null; + + foreach (var xlRow in xlRows) + { + operationAttributes = new Dictionary(); + + var cells = xlRow.CellsUsed().ToArray(); + + foreach (var cell in cells) + { + var operationAttribute = GetValueDictionary(operationAttributesDict, GetCellValue(cell), 0.7); + + if (operationAttribute is null || operationAttributes.Any(a => a.Key == operationAttribute)) + continue; + + operationAttributes.Add(operationAttribute, cell.Address.ColumnNumber); + } + + if (operationAttributes.Count >= countOperationAttributes) + break; + } + + return operationAttributes is not null && operationAttributes.Count == countOperationAttributes ? operationAttributes : null; + } + + private string? GetValueDictionary(IDictionary dict, string cellValue, double? minSimilarity) + { + var similarValues = new List<(double similarity, string value)>(); + + var profile1 = cosineSimilarity.GetProfile(cellValue); + + foreach (var item in dict) + { + var profile2 = cosineSimilarity.GetProfile(item.Key); + + var similarity = cosineSimilarity.Similarity(profile1, profile2); + + similarValues.Add((similarity, item.Value)); + } + + var mostSimilarValue = similarValues.MaxBy(v => v.similarity); + + return minSimilarity.HasValue && mostSimilarValue.similarity >= minSimilarity ? mostSimilarValue.value : null; + } + + private static Dictionary InitDict(string fileName, char separator) + { + var resourceName = Assembly.GetExecutingAssembly() + .GetManifestResourceNames() + .FirstOrDefault(n => n.EndsWith(fileName))!; + + var stream = Assembly.GetExecutingAssembly() + .GetManifestResourceStream(resourceName)!; + + using var reader = new StreamReader(stream); + + return reader.ReadToEnd().Split('\r') + .Where(s => !string.IsNullOrWhiteSpace(s)) + .Select(line => line.Split(separator)) + .ToDictionary(parts => parts[0].Trim(), parts => parts[1].Trim()); + } + + //TODO: вынести в метод расширения + private static T GetCellValue(IXLCell cell) + { + try + { + if (typeof(T) != typeof(DateTime)) + return (T)Convert.ChangeType(cell.GetFormattedString(), typeof(T), CultureInfo.InvariantCulture); + + return (T)(object)DateTime.FromOADate((double)cell.Value); + } + catch + { + throw new FileFormatException( + $"Лист '{cell.Worksheet.Name}'. Ячейка: ({cell.Address.RowNumber},{cell.Address.ColumnNumber}) содержит некорректное значение"); + } + } +} \ No newline at end of file diff --git a/AsbCloudInfrastructure/Services/WellOperationImport/WellOperationExportService.cs b/AsbCloudInfrastructure/Services/WellOperationImport/WellOperationExportService.cs new file mode 100644 index 00000000..df49b7aa --- /dev/null +++ b/AsbCloudInfrastructure/Services/WellOperationImport/WellOperationExportService.cs @@ -0,0 +1,105 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using AsbCloudApp.Data; +using AsbCloudApp.Repositories; +using AsbCloudApp.Requests; +using AsbCloudApp.Services; +using AsbCloudApp.Services.WellOperationImport; +using AsbCloudInfrastructure.Services.WellOperationImport.Constants; +using ClosedXML.Excel; + +namespace AsbCloudInfrastructure.Services.WellOperationImport; + +public class WellOperationExportService : IWellOperationExportService +{ + private readonly IWellOperationRepository wellOperationRepository; + private readonly IWellService wellService; + private readonly IWellOperationImportTemplateService wellOperationImportTemplateService; + + public WellOperationExportService(IWellOperationRepository wellOperationRepository, + IWellService wellService, + IWellOperationImportTemplateService wellOperationImportTemplateService) + { + this.wellOperationRepository = wellOperationRepository; + this.wellService = wellService; + this.wellOperationImportTemplateService = wellOperationImportTemplateService; + } + + public async Task ExportAsync(int idWell, CancellationToken cancellationToken) + { + var operations = await wellOperationRepository.GetAsync(new WellOperationRequest() + { + IdWell = idWell + }, cancellationToken); + + var timezone = wellService.GetTimezone(idWell); + + return await MakeExcelFileStreamAsync(operations, timezone.Hours, cancellationToken); + } + + private async Task MakeExcelFileStreamAsync(IEnumerable operations, double timezoneOffset, + CancellationToken cancellationToken) + { + using Stream ecxelTemplateStream = wellOperationImportTemplateService.GetExcelTemplateStream(); + + using var workbook = new XLWorkbook(ecxelTemplateStream, XLEventTracking.Disabled); + await AddOperationsToWorkbook(workbook, operations, timezoneOffset, cancellationToken); + + var memoryStream = new MemoryStream(); + workbook.SaveAs(memoryStream, new SaveOptions { }); + memoryStream.Seek(0, SeekOrigin.Begin); + return memoryStream; + } + + private async Task AddOperationsToWorkbook(XLWorkbook workbook, IEnumerable operations, double timezoneOffset, + CancellationToken cancellationToken) + { + var planOperations = operations.Where(o => o.IdType == 0); + if (planOperations.Any()) + { + var sheetPlan = workbook.Worksheets.FirstOrDefault(ws => ws.Name == DefaultTemplateInfo.SheetNamePlan); + if (sheetPlan is not null) + await AddOperationsToSheetAsync(sheetPlan, planOperations, timezoneOffset, cancellationToken); + } + + var factOperations = operations.Where(o => o.IdType == 1); + if (factOperations.Any()) + { + var sheetFact = workbook.Worksheets.FirstOrDefault(ws => ws.Name == DefaultTemplateInfo.SheetNameFact); + if (sheetFact is not null) + await AddOperationsToSheetAsync(sheetFact, factOperations, timezoneOffset, cancellationToken); + } + } + + private async Task AddOperationsToSheetAsync(IXLWorksheet sheet, IEnumerable operations, double timezoneOffset, + CancellationToken cancellationToken) + { + var operationsToArray = operations.ToArray(); + + var sections = wellOperationRepository.GetSectionTypes(); + var categories = wellOperationRepository.GetCategories(false); + + for (int i = 0; i < operationsToArray.Length; i++) + { + var row = sheet.Row(1 + i + DefaultTemplateInfo.HeaderRowsCount); + AddOperationToRow(row, operationsToArray[i], sections, categories, timezoneOffset); + } + } + + private static void AddOperationToRow(IXLRow row, WellOperationDto operation, IEnumerable sections, + IEnumerable categories, double timezoneOffset) + { + row.Cell(DefaultTemplateInfo.ColumnSection).Value = sections.First(s => s.Id == operation.IdWellSectionType).Caption; + row.Cell(DefaultTemplateInfo.ColumnCategory).Value = categories.First(o => o.Id == operation.IdCategory).Name; + row.Cell(DefaultTemplateInfo.ColumnCategoryInfo).Value = operation.CategoryInfo; + row.Cell(DefaultTemplateInfo.ColumnDepthStart).Value = operation.DepthStart; + row.Cell(DefaultTemplateInfo.ColumnDepthEnd).Value = operation.DepthEnd; + row.Cell(DefaultTemplateInfo.ColumnDate).Value = new DateTimeOffset(operation.DateStart).ToRemoteDateTime(timezoneOffset); + row.Cell(DefaultTemplateInfo.ColumnDuration).Value = operation.DurationHours; + row.Cell(DefaultTemplateInfo.ColumnComment).Value = operation.Comment; + } +} \ No newline at end of file diff --git a/AsbCloudInfrastructure/Services/WellOperationImport/WellOperationImportService.cs b/AsbCloudInfrastructure/Services/WellOperationImport/WellOperationImportService.cs new file mode 100644 index 00000000..c57a3531 --- /dev/null +++ b/AsbCloudInfrastructure/Services/WellOperationImport/WellOperationImportService.cs @@ -0,0 +1,141 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using AsbCloudApp.Data; +using AsbCloudApp.Data.WellOperationImport; +using AsbCloudApp.Exceptions; +using AsbCloudApp.Repositories; +using AsbCloudApp.Requests; +using AsbCloudApp.Services.WellOperationImport; +using AsbCloudDb.Model; +using AsbCloudInfrastructure.Services.WellOperationImport.Constants; + +namespace AsbCloudInfrastructure.Services.WellOperationImport; + +public class WellOperationImportService : IWellOperationImportService +{ + private readonly IEnumerable excelParsers; + private readonly IWellOperationRepository wellOperationRepository; + + private static readonly DateTime dateLimitMin = new(2001, 1, 1, 0, 0, 0); + private static readonly DateTime dateLimitMax = new(2099, 1, 1, 0, 0, 0); + private static readonly TimeSpan drillingDurationLimitMax = TimeSpan.FromDays(366); + + public WellOperationImportService(IEnumerable excelParsers, + IWellOperationRepository wellOperationRepository) + { + this.excelParsers = excelParsers; + this.wellOperationRepository = wellOperationRepository; + } + + public async Task ImportAsync(int idWell, int idUser, int idType, Stream stream, WellOperationParserOptionsDto options, + bool deleteWellOperationsBeforeImport, CancellationToken cancellationToken) + { + var excelParser = excelParsers.FirstOrDefault(p => p.IdTemplate == options.IdTemplate && + p.IdTypes.Contains(idType)); + + if (excelParser is null) + throw new ArgumentInvalidException("Невозможно импортировать файл", nameof(options.IdTemplate)); + + if (idType != WellOperation.IdOperationTypePlan && idType != WellOperation.IdOperationTypeFact) + throw new ArgumentInvalidException("Операции не существует", nameof(idType)); + + RowDto[] rows; + var validationErrors = new List(); + + var sections = wellOperationRepository.GetSectionTypes(); + var categories = wellOperationRepository.GetCategories(false); + + switch (options.IdTemplate) + { + case 0: + options.SheetName = idType == WellOperation.IdOperationTypePlan + ? DefaultTemplateInfo.SheetNamePlan + : DefaultTemplateInfo.SheetNameFact; + rows = excelParser.Parse(stream, options).ToArray(); + break; + default: + if (string.IsNullOrWhiteSpace(options.SheetName)) + throw new FileFormatException("Не указано название листа"); + rows = excelParser.Parse(stream, options).ToArray(); + break; + } + + var operations = new List(); + + foreach (var row in rows) + { + try + { + var section = sections.FirstOrDefault(s => + string.Equals(s.Caption, row.Section, StringComparison.CurrentCultureIgnoreCase)); + + if (section is null) + throw new FileFormatException($"Лист '{options.SheetName}'. В строке '{row.Number}' не удалось определить секцию"); + + var category = categories.FirstOrDefault(c => + string.Equals(c.Name, row.Category, StringComparison.CurrentCultureIgnoreCase)); + + if (category is null) + throw new FileFormatException($"Лист '{options.SheetName}'. В строке '{row.Number}' не удалось определить операцию"); + + if (row.DepthStart is not (>= 0d and <= 20_000d)) + throw new FileFormatException($"Лист '{options.SheetName}'. Строка '{row.Number}' некорректная глубина на начало операции"); + + if (row.DepthEnd is not (>= 0d and <= 20_000d)) + throw new FileFormatException($"Лист '{options.SheetName}'. Строка '{row.Number}' некорректная глубина на конец операции"); + + if (row.Date < dateLimitMin && row.Date > dateLimitMax) + throw new FileFormatException($"Лист '{options.SheetName}'. Строка '{row.Number}' неправильно получена дата начала операции"); + + if (operations.LastOrDefault()?.DateStart > row.Date) + throw new FileFormatException($"Лист '{options.SheetName}' строка '{row.Number}' дата позднее даты предыдущей операции"); + + if (row.Duration is not (>= 0d and <= 240d)) + throw new FileFormatException($"Лист '{options.SheetName}'. Строка '{row.Number}' некорректная длительность операции"); + + operations.Add(new WellOperationDto + { + IdWell = idWell, + IdUser = idUser, + IdType = idType, + IdWellSectionType = section.Id, + IdCategory = category.Id, + CategoryInfo = row.CategoryInfo, + DepthStart = row.DepthStart, + DepthEnd = row.DepthEnd, + DateStart = row.Date, + DurationHours = row.Duration + }); + } + catch (FileFormatException ex) + { + validationErrors.Add(ex.Message); + } + } + + if (operations.Any() && operations.Min(o => o.DateStart) - operations.Max(o => o.DateStart) > drillingDurationLimitMax) + validationErrors.Add($"Лист {options.SheetName} содержит диапазон дат больше {drillingDurationLimitMax}"); + + if (validationErrors.Any()) + throw new FileFormatException(string.Join("\r\n", validationErrors)); + + if(!operations.Any()) + return; + + if (deleteWellOperationsBeforeImport) + { + var existingOperations = await wellOperationRepository.GetAsync(new WellOperationRequest + { + IdWell = idWell + }, cancellationToken); + + await wellOperationRepository.DeleteAsync(existingOperations.Select(o => o.Id), cancellationToken); + } + + await wellOperationRepository.InsertRangeAsync(operations, cancellationToken); + } +} \ No newline at end of file diff --git a/AsbCloudInfrastructure/Services/WellOperationImport/WellOperationImportTemplateService.cs b/AsbCloudInfrastructure/Services/WellOperationImport/WellOperationImportTemplateService.cs new file mode 100644 index 00000000..edb71c14 --- /dev/null +++ b/AsbCloudInfrastructure/Services/WellOperationImport/WellOperationImportTemplateService.cs @@ -0,0 +1,21 @@ +using System.IO; +using System.Linq; +using System.Reflection; +using AsbCloudApp.Services.WellOperationImport; + +namespace AsbCloudInfrastructure.Services.WellOperationImport; + +public class WellOperationImportTemplateService : IWellOperationImportTemplateService +{ + public Stream GetExcelTemplateStream() + { + var resourceName = Assembly.GetExecutingAssembly() + .GetManifestResourceNames() + .FirstOrDefault(n => n.EndsWith("WellOperationImportTemplate.xlsx"))!; + + var stream = Assembly.GetExecutingAssembly() + .GetManifestResourceStream(resourceName)!; + + return stream; + } +} \ No newline at end of file diff --git a/AsbCloudInfrastructure/Services/WellOperationService/WellOperationImportService.cs b/AsbCloudInfrastructure/Services/WellOperationService/WellOperationImportService.cs deleted file mode 100644 index 949aaf71..00000000 --- a/AsbCloudInfrastructure/Services/WellOperationService/WellOperationImportService.cs +++ /dev/null @@ -1,341 +0,0 @@ -using AsbCloudApp.Data; -using AsbCloudApp.Services; -using AsbCloudDb.Model; -using ClosedXML.Excel; -using Mapster; -using Microsoft.EntityFrameworkCore; -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; - -namespace AsbCloudInfrastructure.Services.WellOperationService -{ - - /* - * password for WellOperationImportTemplate.xlsx is ASB2020! - */ - - public class WellOperationImportService : IWellOperationImportService - { - private const string sheetNamePlan = "План"; - private const string sheetNameFact = "Факт"; - - private const int headerRowsCount = 1; - private const int columnSection = 1; - private const int columnCategory = 2; - private const int columnCategoryInfo = 3; - private const int columnDepthStart = 4; - private const int columnDepthEnd = 5; - private const int columnDate = 6; - private const int columnDuration = 7; - private const int columnComment = 8; - - private static readonly DateTime dateLimitMin = new DateTime(2001, 1, 1, 0, 0, 0); - private static readonly DateTime dateLimitMax = new DateTime(2099, 1, 1, 0, 0, 0); - private static readonly TimeSpan drillingDurationLimitMax = TimeSpan.FromDays(366); - - private readonly IAsbCloudDbContext db; - private readonly IWellService wellService; - private List categories = null!; - public List Categories - { - get - { - if (categories is null) - { - categories = db.WellOperationCategories - .Where(c => c.Id >= 5000) - .AsNoTracking() - .ToList(); - } - - return categories; - } - } - - private List sections = null!; - public List Sections - { - get - { - if (sections is null) - sections = db.WellSectionTypes - .AsNoTracking() - .ToList(); - return sections; - } - } - - // TODO: use WellOperationRepository instead of DB - public WellOperationImportService(IAsbCloudDbContext db, IWellService wellService) - { - this.db = db; - this.wellService = wellService; - } - - public void Import(int idWell, Stream stream, int idUser, bool deleteWellOperationsBeforeImport = false) - { - using var workbook = new XLWorkbook(stream, XLEventTracking.Disabled); - var operations = ParseFileStream(stream); - foreach (var operation in operations) - { - operation.IdWell = idWell; - operation.IdUser = idUser; - } - - SaveOperations(idWell, operations, deleteWellOperationsBeforeImport); - } - - public Stream Export(int idWell) - { - var operations = db.WellOperations - .Include(o => o.WellSectionType) - .Include(o => o.OperationCategory) - .Where(o => o.IdWell == idWell) - .OrderBy(o => o.DateStart) - .AsNoTracking() - .ToList(); - - var timezone = wellService.GetTimezone(idWell); - - return MakeExelFileStream(operations, timezone.Hours); - } - - public Stream GetExcelTemplateStream() - { - var resourceName = System.Reflection.Assembly.GetExecutingAssembly() - .GetManifestResourceNames() - .FirstOrDefault(n => n.EndsWith("WellOperationImportTemplate.xlsx"))!; - - var stream = System.Reflection.Assembly.GetExecutingAssembly() - .GetManifestResourceStream(resourceName)!; - return stream; - } - - private Stream MakeExelFileStream(IEnumerable operations, double timezoneOffset) - { - using Stream ecxelTemplateStream = GetExcelTemplateStream(); - - using var workbook = new XLWorkbook(ecxelTemplateStream, XLEventTracking.Disabled); - AddOperationsToWorkbook(workbook, operations, timezoneOffset); - - MemoryStream memoryStream = new MemoryStream(); - workbook.SaveAs(memoryStream, new SaveOptions { }); - memoryStream.Seek(0, SeekOrigin.Begin); - return memoryStream; - } - - private static void AddOperationsToWorkbook(XLWorkbook workbook, IEnumerable operations, double timezoneOffset) - { - var planOperations = operations.Where(o => o.IdType == 0); - if (planOperations.Any()) - { - var sheetPlan = workbook.Worksheets.FirstOrDefault(ws => ws.Name == sheetNamePlan); - if (sheetPlan is not null) - AddOperationsToSheet(sheetPlan, planOperations, timezoneOffset); - } - - var factOperations = operations.Where(o => o.IdType == 1); - if (factOperations.Any()) - { - var sheetFact = workbook.Worksheets.FirstOrDefault(ws => ws.Name == sheetNameFact); - if (sheetFact is not null) - AddOperationsToSheet(sheetFact, factOperations, timezoneOffset); - } - } - - private static void AddOperationsToSheet(IXLWorksheet sheet, IEnumerable operations, double timezoneOffset) - { - var operationsList = operations.ToList(); - for (int i = 0; i < operationsList.Count; i++) - { - var row = sheet.Row(1 + i + headerRowsCount); - AddOperationToRow(row, operationsList[i], timezoneOffset); - } - } - - private static void AddOperationToRow(IXLRow row, WellOperation operation, double timezoneOffset) - { - row.Cell(columnSection).Value = operation.WellSectionType?.Caption; - row.Cell(columnCategory).Value = operation.OperationCategory?.Name; - row.Cell(columnCategoryInfo).Value = operation.CategoryInfo; - row.Cell(columnDepthStart).Value = operation.DepthStart; - row.Cell(columnDepthEnd).Value = operation.DepthEnd; - row.Cell(columnDate).Value = operation.DateStart.ToRemoteDateTime(timezoneOffset); - row.Cell(columnDuration).Value = operation.DurationHours; - row.Cell(columnComment).Value = operation.Comment; - } - - private void SaveOperations(int idWell, IEnumerable operations, bool deleteWellOperationsBeforeImport = false) - { - var timezone = wellService.GetTimezone(idWell); - - var transaction = db.Database.BeginTransaction(); - try - { - if (deleteWellOperationsBeforeImport) - db.WellOperations.RemoveRange(db.WellOperations.Where(o => o.IdWell == idWell)); - var entities = operations.Select(o => - { - var entity = o.Adapt(); - entity.IdWell = idWell; - entity.DateStart = o.DateStart.ToUtcDateTimeOffset(timezone.Hours); - entity.LastUpdateDate = DateTimeOffset.UtcNow; - return entity; - }); - db.WellOperations.AddRange(entities); - db.SaveChanges(); - transaction.Commit(); - } - catch - { - transaction.Rollback(); - throw; - } - } - - private IEnumerable ParseFileStream(Stream stream) - { - using var workbook = new XLWorkbook(stream, XLEventTracking.Disabled); - return ParseWorkbook(workbook); - } - - private IEnumerable ParseWorkbook(IXLWorkbook workbook) - { - var sheetPlan = workbook.Worksheets.FirstOrDefault(ws => ws.Name == sheetNamePlan); - if (sheetPlan is null) - throw new FileFormatException($"Книга excel не содержит листа {sheetNamePlan}."); - - var sheetFact = workbook.Worksheets.FirstOrDefault(ws => ws.Name == sheetNameFact); - if (sheetFact is null) - throw new FileFormatException($"Книга excel не содержит листа {sheetNameFact}."); - - //sheetPlan.RangeUsed().RangeAddress.LastAddress.ColumnNumber - var wellOperations = new List(); - - var wellOperationsPlan = ParseSheet(sheetPlan, 0); - wellOperations.AddRange(wellOperationsPlan); - - var wellOperationsFact = ParseSheet(sheetFact, 1); - wellOperations.AddRange(wellOperationsFact); - - return wellOperations; - } - - private IEnumerable ParseSheet(IXLWorksheet sheet, int idType) - { - - if (sheet.RangeUsed().RangeAddress.LastAddress.ColumnNumber < 7) - throw new FileFormatException($"Лист {sheet.Name} содержит меньшее количество столбцов."); - - var count = sheet.RowsUsed().Count() - headerRowsCount; - - if (count > 1024) - throw new FileFormatException($"Лист {sheet.Name} содержит слишком большое количество операций."); - - if (count <= 0) - return new List(); - - var operations = new List(count); - var parseErrors = new List(); - DateTime lastOperationDateStart = new DateTime(); - for (int i = 0; i < count; i++) - { - var row = sheet.Row(1 + i + headerRowsCount); - try - { - var operation = ParseRow(row, idType); - operations.Add(operation); - - if (lastOperationDateStart > operation.DateStart) - parseErrors.Add($"Лист {sheet.Name} строка {row.RowNumber()} дата позднее даты предыдущей операции."); - - lastOperationDateStart = operation.DateStart; - } - catch (FileFormatException ex) - { - parseErrors.Add(ex.Message); - } - }; - - if (parseErrors.Any()) - throw new FileFormatException(string.Join("\r\n", parseErrors)); - else - { - if (operations.Any()) - if (operations.Min(o => o.DateStart) - operations.Max(o => o.DateStart) > drillingDurationLimitMax) - parseErrors.Add($"Лист {sheet.Name} содержит диапазон дат больше {drillingDurationLimitMax}"); - } - - return operations; - } - - private WellOperationDto ParseRow(IXLRow row, int idType) - { - var vSection = row.Cell(columnSection).Value; - var vCategory = row.Cell(columnCategory).Value; - var vCategoryInfo = row.Cell(columnCategoryInfo).Value; - var vDepthStart = row.Cell(columnDepthStart).Value; - var vDepthEnd = row.Cell(columnDepthEnd).Value; - var vDate = row.Cell(columnDate).Value; - var vDuration = row.Cell(columnDuration).Value; - var vComment = row.Cell(columnComment).Value; - - var operation = new WellOperationDto { IdType = idType }; - - if (vSection is string sectionName) - { - var section = Sections.Find(c => c.Caption.ToLower() == sectionName.ToLower()); - if (section is null) - throw new FileFormatException($"Лист {row.Worksheet.Name}. Строка {row.RowNumber()} указана некорректная секция"); - - operation.IdWellSectionType = section.Id; - operation.WellSectionTypeName = section.Caption; - } - else - throw new FileFormatException($"Лист {row.Worksheet.Name}. Строка {row.RowNumber()} не указана секция"); - - if (vCategory is string categoryName) - { - var category = Categories.Find(c => c.Name.ToLower() == categoryName.ToLower()); - if (category is null) - throw new FileFormatException($"Лист {row.Worksheet.Name}. Строка {row.RowNumber()} указана некорректная операция ({categoryName})"); - - operation.IdCategory = category.Id; - operation.CategoryName = category.Name; - } - else - throw new FileFormatException($"Лист {row.Worksheet.Name}. Строка {row.RowNumber()} не указана операция"); - - if (vCategoryInfo is not null) - operation.CategoryInfo = vCategoryInfo.ToString(); - - if (vDepthStart is double depthStart && depthStart >= 0d && depthStart <= 20_000d) - operation.DepthStart = depthStart; - else - throw new FileFormatException($"Лист {row.Worksheet.Name}. Строка {row.RowNumber()} не указана глубина на начало операции"); - - if (vDepthEnd is double depthEnd && depthEnd >= 0d && depthEnd <= 20_000d) - operation.DepthEnd = depthEnd; - else - throw new FileFormatException($"Лист {row.Worksheet.Name}. Строка {row.RowNumber()} не указана глубина при завершении операции"); - - if (vDate is DateTime date && date > dateLimitMin && date < dateLimitMax) - operation.DateStart = date; - else - throw new FileFormatException($"Лист {row.Worksheet.Name}. Строка {row.RowNumber()} неправильно указана дата/время начала операции"); - - if (vDuration is double duration && duration >= 0d && duration <= 240d) - operation.DurationHours = duration; - else - throw new FileFormatException($"Лист {row.Worksheet.Name}. Строка {row.RowNumber()} не указана длительность операции"); - - if (vComment is not null) - operation.Comment = vComment.ToString(); - - return operation; - } - } - -} diff --git a/AsbCloudWebApi/Controllers/WellOperationController.cs b/AsbCloudWebApi/Controllers/WellOperationController.cs index 0ab6f0c1..c6809ac4 100644 --- a/AsbCloudWebApi/Controllers/WellOperationController.cs +++ b/AsbCloudWebApi/Controllers/WellOperationController.cs @@ -11,6 +11,8 @@ using System.ComponentModel.DataAnnotations; using System.IO; using System.Threading; using System.Threading.Tasks; +using AsbCloudApp.Data.WellOperationImport; +using AsbCloudApp.Services.WellOperationImport; namespace AsbCloudWebApi.Controllers { @@ -25,12 +27,20 @@ namespace AsbCloudWebApi.Controllers { private readonly IWellOperationRepository operationRepository; private readonly IWellService wellService; + private readonly IWellOperationExportService wellOperationExportService; + private readonly IWellOperationImportTemplateService wellOperationImportTemplateService; private readonly IWellOperationImportService wellOperationImportService; - public WellOperationController(IWellOperationRepository operationService, IWellService wellService, IWellOperationImportService wellOperationImportService) + public WellOperationController(IWellOperationRepository operationService, + IWellService wellService, + IWellOperationImportTemplateService wellOperationImportTemplateService, + IWellOperationExportService wellOperationExportService, + IWellOperationImportService wellOperationImportService) { this.operationRepository = operationService; this.wellService = wellService; + this.wellOperationImportTemplateService = wellOperationImportTemplateService; + this.wellOperationExportService = wellOperationExportService; this.wellOperationImportService = wellOperationImportService; } @@ -266,41 +276,57 @@ namespace AsbCloudWebApi.Controllers /// - /// Импортирует операции из excel (xlsx) файла + /// Импорт плановых операций из excel (xlsx) файла /// /// id скважины + /// Тип операции + /// Начальная строка + /// Конечная строка /// Коллекция из одного файла xlsx - /// Удалить операции перед импортом = 1, если фал валидный - /// Токен отмены задачи + /// Удалить операции перед импортом = 1, если файл валидный + /// Название листа + /// Токен отмены задачи + /// Шаблон файла. 0 - стандартный, 1 - Газпромнефть Хантос /// [HttpPost("import/{options}")] [Permission] public async Task ImportAsync(int idWell, + [Required] int idType, + string? sheetName, + [Required] int idTemplate, + int? startRow, + int? endRow, [FromForm] IFormFileCollection files, int options, CancellationToken token) { - int? idCompany = User.GetCompanyId(); - int? idUser = User.GetUserId(); + var idCompany = User.GetCompanyId(); + var idUser = User.GetUserId(); if (idCompany is null || idUser is null) return Forbid(); - if (!await wellService.IsCompanyInvolvedInWellAsync((int)idCompany, - idWell, token).ConfigureAwait(false)) + if (!await wellService.IsCompanyInvolvedInWellAsync((int)idCompany, idWell, token)) return Forbid(); if (files.Count < 1) - return BadRequest("нет файла"); + return BadRequest("Нет файла"); var file = files[0]; if (Path.GetExtension(file.FileName).ToLower() != ".xlsx") return BadRequest("Требуется xlsx файл."); + using Stream stream = file.OpenReadStream(); try { - wellOperationImportService.Import(idWell, stream, idUser.Value, (options & 1) > 0); + await wellOperationImportService.ImportAsync(idWell, idUser.Value, idType, stream, new WellOperationParserOptionsDto + { + SheetName = sheetName, + IdTemplate = idTemplate, + StartRow = startRow, + EndRow = endRow + }, (options & 1) > 0, token); } catch (FileFormatException ex) { @@ -331,7 +357,7 @@ namespace AsbCloudWebApi.Controllers idWell, token).ConfigureAwait(false)) return Forbid(); - var stream = wellOperationImportService.Export(idWell); + var stream = await wellOperationExportService.ExportAsync(idWell, token); var fileName = await wellService.GetWellCaptionByIdAsync(idWell, token) + "_operations.xlsx"; return File(stream, "application/octet-stream", fileName); } @@ -371,7 +397,7 @@ namespace AsbCloudWebApi.Controllers [ProducesResponseType(typeof(PhysicalFileResult), (int)System.Net.HttpStatusCode.OK, "application/octet-stream")] public IActionResult GetTemplate() { - var stream = wellOperationImportService.GetExcelTemplateStream(); + var stream = wellOperationImportTemplateService.GetExcelTemplateStream(); var fileName = "ЕЦП_шаблон_файла_операций.xlsx"; return File(stream, "application/octet-stream", fileName); }