From 2b61669e7ac9d1ba410df9a60edef7ae8d86db54 Mon Sep 17 00:00:00 2001 From: Ilya Kozhemyakin Date: Thu, 15 May 2025 01:37:27 +0400 Subject: [PATCH] =?UTF-8?q?=D0=98=D0=B7=D0=BC=D0=B5=D0=BD=D0=B5=D0=BD?= =?UTF-8?q?=D0=B8=D1=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Pf2eDataScrapper/.env.example | 3 + Pf2eDataScrapper/App/ProcessorAttribute.cs | 2 +- Pf2eDataScrapper/App/ProcessorCollection.cs | 6 + Pf2eDataScrapper/ApplicationSettings.cs | 10 + .../EntityTreeViewerControl.cs | 1 + .../EntityTreeViewerControlRow.cs | 1 + .../EntityTreeViewer/StructureBuilder.cs | 2 +- .../Controls/ProcessorCollectionControl.cs | 2 +- Pf2eDataScrapper/Helpers/ProcessProgress.cs | 1 - Pf2eDataScrapper/ImportApplication.cs | 5 + .../Implementations/Aonprd/AonprdImporter.cs | 1 - .../Aonprd/Helpers/AonprdBuilderGlobals.cs | 13 +- .../Aonprd/Helpers/LegacyEntityHandler.cs | 105 --------- .../Processors/AbstractEntityProcessor.cs | 210 +++++------------- .../Processors/AbstractSubclassProcessor.cs | 5 +- .../Processors/Common/ExternalProcessor.cs | 6 +- .../Processors/Common/LegacyProcessor.cs | 48 ++-- .../Processors/Common/SourceProcessor.cs | 9 +- .../Subclasses/ApparitionProcessor.cs | 3 +- .../Processors/Types/ActionProcessor.cs | 5 +- .../Processors/Types/AncestryProcessor.cs | 5 +- .../Processors/Types/ArchetypeProcessor.cs | 5 +- .../Processors/Types/BackgroundProcessor.cs | 5 +- .../Aonprd/Processors/Types/ClassProcessor.cs | 5 +- .../Aonprd/Processors/Types/FeatProcessor.cs | 5 +- .../Processors/Types/LanguageProcessor.cs | 5 +- .../Processors/Types/RitualProcessor.cs | 5 +- .../Processors/Types/Rules/RuleProcessor.cs | 8 +- .../Aonprd/Processors/Types/SpellProcessor.cs | 5 +- .../Aonprd/Processors/Types/TraitProcessor.cs | 16 +- .../Processors/ProcessorExecutor.cs | 49 ++-- Pf2eModel/EfCleanHelper.cs | 4 +- ...signer.cs => 20250514120355_1.Designer.cs} | 2 +- ...0250514102154_1.cs => 20250514120355_1.cs} | 0 Pf2eModel/Model/Base/Entity.cs | 2 +- Pf2eModel/Pf2eDbContext.cs | 1 + 36 files changed, 200 insertions(+), 360 deletions(-) delete mode 100644 Pf2eDataScrapper/Importers/Implementations/Aonprd/Helpers/LegacyEntityHandler.cs rename Pf2eModel/Migrations/{20250514102154_1.Designer.cs => 20250514120355_1.Designer.cs} (99%) rename Pf2eModel/Migrations/{20250514102154_1.cs => 20250514120355_1.cs} (100%) diff --git a/Pf2eDataScrapper/.env.example b/Pf2eDataScrapper/.env.example index d7a97be..77f1d48 100644 --- a/Pf2eDataScrapper/.env.example +++ b/Pf2eDataScrapper/.env.example @@ -6,3 +6,6 @@ OUTPUT_DIR=D:\path\to\output\directory # Строка подключения к БД DATABASE_CONNECTION=Host=localhost;Port=5433;Database=usersdb2;Username=postgres;Password=password + +# Максимальное количество потоков. +MAX_THREADS=60 \ No newline at end of file diff --git a/Pf2eDataScrapper/App/ProcessorAttribute.cs b/Pf2eDataScrapper/App/ProcessorAttribute.cs index aa02c8d..3497ce6 100644 --- a/Pf2eDataScrapper/App/ProcessorAttribute.cs +++ b/Pf2eDataScrapper/App/ProcessorAttribute.cs @@ -30,7 +30,7 @@ public class ProcessorAttribute(string name) : Attribute /// /// Разрешить выполнять процессов в параллельном режиме /// - public bool Parallel { get; init; } = true; + public bool Parallel { get; init; } = false; /// /// Обработчик набора элементов diff --git a/Pf2eDataScrapper/App/ProcessorCollection.cs b/Pf2eDataScrapper/App/ProcessorCollection.cs index bd7b61c..74d435c 100644 --- a/Pf2eDataScrapper/App/ProcessorCollection.cs +++ b/Pf2eDataScrapper/App/ProcessorCollection.cs @@ -90,6 +90,7 @@ internal class ProcessorCollection InnerType = innerType, Compendium = compendium, Active = processorAttribute.Active, + Parallel = processorAttribute.Parallel, }; } @@ -241,4 +242,9 @@ internal class ProcessorCollectionItem /// null - обработать все типы. /// public ProcessorCompendiumEnum? Compendium { get; init; } + + /// + /// Разрешить выполнять процессов в параллельном режиме + /// + public bool Parallel { get; init; } = true; } diff --git a/Pf2eDataScrapper/ApplicationSettings.cs b/Pf2eDataScrapper/ApplicationSettings.cs index 6e8da60..f5b4938 100644 --- a/Pf2eDataScrapper/ApplicationSettings.cs +++ b/Pf2eDataScrapper/ApplicationSettings.cs @@ -19,6 +19,7 @@ internal class ApplicationSettings DataDirPath = EnvReader.GetStringValue("DATA_DIR"); OutputDirPath = EnvReader.GetStringValue("OUTPUT_DIR"); DatabaseConnection = EnvReader.GetStringValue("DATABASE_CONNECTION"); + MaxThreads = Convert.ToInt32(EnvReader.GetStringValue("MAX_THREADS")); } #region Настройки из .env @@ -62,6 +63,15 @@ internal class ApplicationSettings [Description("Редактирование через .env-файл.")] public string DatabaseConnection { get; } + /// + /// Максимальное количество потоков. + /// + [Browsable(true)] + [ReadOnly(true)] + [DisplayName("Максимальное количество потоков.")] + [Description("Редактирование через .env-файл.")] + public int MaxThreads { get; } + #endregion #region Основные настройки diff --git a/Pf2eDataScrapper/Forms/Controls/EntityTreeViewer/EntityTreeViewerControl.cs b/Pf2eDataScrapper/Forms/Controls/EntityTreeViewer/EntityTreeViewerControl.cs index daef077..e41f95c 100644 --- a/Pf2eDataScrapper/Forms/Controls/EntityTreeViewer/EntityTreeViewerControl.cs +++ b/Pf2eDataScrapper/Forms/Controls/EntityTreeViewer/EntityTreeViewerControl.cs @@ -1,4 +1,5 @@ using System.ComponentModel; +using Pf2eDataScrapper.Forms.Controls.EntityTreeViewer; namespace Pf2eViewer.EntityTreeViewer; diff --git a/Pf2eDataScrapper/Forms/Controls/EntityTreeViewer/EntityTreeViewerControlRow.cs b/Pf2eDataScrapper/Forms/Controls/EntityTreeViewer/EntityTreeViewerControlRow.cs index 1baecfb..601ea84 100644 --- a/Pf2eDataScrapper/Forms/Controls/EntityTreeViewer/EntityTreeViewerControlRow.cs +++ b/Pf2eDataScrapper/Forms/Controls/EntityTreeViewer/EntityTreeViewerControlRow.cs @@ -1,4 +1,5 @@ using FontAwesome.Sharp; +using Pf2eDataScrapper.Forms.Controls.EntityTreeViewer; using Font = System.Drawing.Font; using FontStyle = System.Drawing.FontStyle; using Image = System.Drawing.Image; diff --git a/Pf2eDataScrapper/Forms/Controls/EntityTreeViewer/StructureBuilder.cs b/Pf2eDataScrapper/Forms/Controls/EntityTreeViewer/StructureBuilder.cs index 0dfea73..d1681c9 100644 --- a/Pf2eDataScrapper/Forms/Controls/EntityTreeViewer/StructureBuilder.cs +++ b/Pf2eDataScrapper/Forms/Controls/EntityTreeViewer/StructureBuilder.cs @@ -3,7 +3,7 @@ using System.Reflection; using LinqToDB.Extensions; using Pf2eModel.Model.Common.I18n; -namespace Pf2eViewer.EntityTreeViewer; +namespace Pf2eDataScrapper.Forms.Controls.EntityTreeViewer; /// /// Класс для построения структуры данных класса. diff --git a/Pf2eDataScrapper/Forms/Controls/ProcessorCollectionControl.cs b/Pf2eDataScrapper/Forms/Controls/ProcessorCollectionControl.cs index 37f32d7..07b2a5c 100644 --- a/Pf2eDataScrapper/Forms/Controls/ProcessorCollectionControl.cs +++ b/Pf2eDataScrapper/Forms/Controls/ProcessorCollectionControl.cs @@ -54,7 +54,7 @@ internal partial class ProcessorCollectionControl : UserControl FlowDirection = FlowDirection.LeftToRight, WrapContents = true, BorderStyle = BorderStyle.FixedSingle, - Margin = new Padding(10, 0, 10, 15), + Margin = new(10, 0, 10, 15), }; RootFlowLayoutPanel.Controls.Add(GroupCheckBoxes[group]); diff --git a/Pf2eDataScrapper/Helpers/ProcessProgress.cs b/Pf2eDataScrapper/Helpers/ProcessProgress.cs index 168fbb9..bb3d6b7 100644 --- a/Pf2eDataScrapper/Helpers/ProcessProgress.cs +++ b/Pf2eDataScrapper/Helpers/ProcessProgress.cs @@ -125,7 +125,6 @@ internal class ProcessProgress : AbstractLogger /// public void End() { - _text = ""; _current++; Update?.Invoke(); } diff --git a/Pf2eDataScrapper/ImportApplication.cs b/Pf2eDataScrapper/ImportApplication.cs index 23d7dbc..3400960 100644 --- a/Pf2eDataScrapper/ImportApplication.cs +++ b/Pf2eDataScrapper/ImportApplication.cs @@ -177,7 +177,12 @@ internal sealed class ImportApplication : ApplicationContext public static void RegisterServices(IServiceCollection services) { // Контекст базы данных + services.AddEntityFrameworkNpgsql(); services.AddDbContext(); + //services.AddDbContextPool(options => + //{ + // options.Us + //}); // Основная программа services.AddSingleton(); diff --git a/Pf2eDataScrapper/Importers/Implementations/Aonprd/AonprdImporter.cs b/Pf2eDataScrapper/Importers/Implementations/Aonprd/AonprdImporter.cs index e6b7095..2bbf2a3 100644 --- a/Pf2eDataScrapper/Importers/Implementations/Aonprd/AonprdImporter.cs +++ b/Pf2eDataScrapper/Importers/Implementations/Aonprd/AonprdImporter.cs @@ -30,7 +30,6 @@ internal class AonprdImporter : AbstractImporter services.AddSingleton(); services.AddSingleton(); - services.AddSingleton(); services.AddSingleton(); services.AddTransient(); diff --git a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Helpers/AonprdBuilderGlobals.cs b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Helpers/AonprdBuilderGlobals.cs index 9fe0ceb..f8891ca 100644 --- a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Helpers/AonprdBuilderGlobals.cs +++ b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Helpers/AonprdBuilderGlobals.cs @@ -4,14 +4,23 @@ namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Helpers; /// /// Класс-помощник, хранящий необходимые глобальные данные. -/// Реализует шаблон Singleton. /// internal class AonprdBuilderGlobals { /// /// Цель внешних ссылок Архивов Нефисов. /// - public ExternalTarget AonprdTarget = null!; + public ExternalTarget AonprdTarget { get; set; } = null!; + + /// + /// Словарь идентификаторов источников. + /// + public Dictionary SourceIds { get; set; } = []; + + /// + /// Словарь идентификаторов признаков. + /// + public Dictionary TraitIds { get; set; } = []; /// /// Конструктор класса . diff --git a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Helpers/LegacyEntityHandler.cs b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Helpers/LegacyEntityHandler.cs deleted file mode 100644 index 24b378e..0000000 --- a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Helpers/LegacyEntityHandler.cs +++ /dev/null @@ -1,105 +0,0 @@ -using Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors; -using Pf2eModel.Model.Base; -using Pf2eModel.Model.Base.Source; -using Pf2eModel.Model.Common.I18n; - -namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Helpers; - -/// -/// Обработчик сущностей, помеченных как устаревшие. -/// -internal class LegacyEntityHandler -{ - /// - /// Идентификатор устаревшей сущности. - /// - public string LegacyId { get; private set; } - - /// - /// Название устаревшей сущности. - /// - public string LegacyName { get; private set; } = ""; - - /// - /// Коллекция источников устревшей сущности. - /// - public List LegacySources { get; private set; } = []; - - /// - /// Флаг, указывающий, что сведения об устаревшей сущности готовы к добавлению в обновлённые. - /// - public bool LegacyPrepared { get; private set; } = false; - - /// - /// Список обновлённых сущностей, что отметели устаревшую сущность как предшественника. - /// - public List RemasterEntities { get; } = []; - - /// - /// Конструктор класса , если сущность сама отмечена как устаревшая. - /// - /// Обработчик устаревшей сущности. - public LegacyEntityHandler(AbstractEntityProcessor processor) - { - LegacyId = processor.Input.GetRequired("id"); - Prepare(processor); - } - - /// - /// Конструктор класса , если обновлённая сущность указывает на ещё недобавленную устаревшую сущность. - /// - /// Идентификатор устаревшей сущности. - /// Обновлённая сущность. - public LegacyEntityHandler(string legacyId, Entity remasterEntity) - { - LegacyId = legacyId; - LegacyPrepared = false; - RemasterEntities.Add(remasterEntity); - } - - /// - /// Подготовить данные об устравшей сущности. - /// - /// Обработчик устаревшей сущности. - public void Prepare(AbstractEntityProcessor processor) - { - if (LegacyPrepared) - { - return; - } - - LegacyName = processor.Input.GetRequired("name"); - LegacySources = processor.GetSources(); - LegacyPrepared = true; - - foreach (Entity remasterEntity in RemasterEntities) - { - Apply(remasterEntity); - } - } - - /// - /// Применить данные устаревшей сущности к обновлённой. - /// - /// Обновлённая сущность. - public void Apply(Entity remasterEntity) - { - if (!remasterEntity.Name.Contains(LegacyName)) - { - remasterEntity.Name[LocaleType.LegacyEnglish] = LegacyName; - } - - foreach (EntitySource legacySource in LegacySources) - { - EntitySource entitySource = new() - { - Entity = remasterEntity, - Source = legacySource.Source, - Suffix = legacySource.Suffix, - IsRemaster = false, - }; - - remasterEntity.Sources.Add(entitySource); - } - } -} diff --git a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/AbstractEntityProcessor.cs b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/AbstractEntityProcessor.cs index 19f47c3..1395ede 100644 --- a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/AbstractEntityProcessor.cs +++ b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/AbstractEntityProcessor.cs @@ -1,8 +1,4 @@ -using System.Collections.Concurrent; -using System.Linq; -using System.Security.Cryptography; -using System.Text; -using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore; using Newtonsoft.Json; using Newtonsoft.Json.Linq; using Pf2eDataScrapper.App; @@ -31,13 +27,12 @@ namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors; /// internal abstract class AbstractEntityProcessor( ProcessorInput input, - Pf2eDbContext context, StringHelper stringHelper, DatabaseHelper helper, AonprdBuilderGlobals globals ) : AbstractInputProcessor(input) { - public Pf2eDbContext Context { get; } = context; + public Pf2eDbContext Context { get; } = new(); public StringHelper StringHelper { get; } = stringHelper; public DatabaseHelper Helper { get; } = helper; public AonprdBuilderGlobals Globals { get; } = globals; @@ -60,19 +55,22 @@ internal abstract class AbstractEntityProcessor( /// /// Добавление основной сущности в контекст базы данных. /// - public abstract void Add(); + protected abstract void Add(); /// public override void Process() { - ProcessRemaster(); + if (Input.GetOptional("remaster_id") is not null) + { + throw new LegacyItemException(Input.GetOptional("name", "")); + } AddEntity(); AddSource(); AddExternal(); AddTraits(); - ProcessLegacy(); + AddLegacy(); } /// @@ -116,6 +114,23 @@ internal abstract class AbstractEntityProcessor( protected virtual string GetDescription() => Input.GetRequired("markdown") + AddedDescription(AddedDescriptionPaths()); + /// + /// Генерация JSON-строки из объектов, располагаюхися по путям, указанным в . + /// + /// Коллекция путей, из которых необходимо достатить объекты. + /// JSON-строка. + protected string GenerateJsonString(params ICollection paths) + { + Dictionary data = []; + + foreach (string key in paths) + { + data.Add(key, Input.Input.SelectToken(key)); + } + + return JsonConvert.SerializeObject(data, new JsonSerializerSettings()); + } + #endregion #region "Генерация сущности" @@ -146,8 +161,29 @@ internal abstract class AbstractEntityProcessor( /// protected virtual void AddSource() { - foreach (EntitySource entitySource in GetSources()) + string[] sourceNames = Input + .GetRequired("source") + .Select(token => token.ToString()) + .ToArray(); + + if (sourceNames.Length == 0) { + throw new( + $"Источники для сущности \"{Entity.Name[LocaleType.English]}\" " + + $"(Aonprd id \"{Input.GetRequired("id")}\") не найдены." + ); + } + + foreach (string sourceName in sourceNames) + { + EntitySource entitySource = new() + { + Entity = Entity, + SourceId = Globals.SourceIds[sourceName], + Suffix = StringHelper.ResolveSource(Input.GetRequired("source_markdown")), + IsRemaster = Input.GetOptional("legacy_id") is not null, + }; + Entity.Sources.Add(entitySource); } } @@ -160,7 +196,7 @@ internal abstract class AbstractEntityProcessor( EntityExternal external = new() { Entity = Entity, - Target = Globals.AonprdTarget, + TargetId = Globals.AonprdTarget.Id, ExternalId = Input.GetRequired("id"), Suffix = Input.GetRequired("url"), Md5 = Input.Md5(), @@ -182,26 +218,11 @@ internal abstract class AbstractEntityProcessor( return; } - IEnumerable names = traitsJson.ToArray().Select(jToken => jToken.ToString()); - List traitEntities = Context - .Traits.Select(t => t.Entity) - // ReSharper disable once EntityFramework.UnsupportedServerSideFunctionCall - .Where(e => Pf2eDbContext.LocaleSearchArray(e.Name, names)) - .ToList(); + List names = traitsJson.ToArray().Select(jToken => jToken.ToString()).ToList(); - List traits = Context - .Traits.Include(t => t.Entity) - .Where(t => traitEntities.Contains(t.Entity)) - .ToList(); + List traitIds = names.Select(name => Globals.TraitIds[name]).ToList(); - foreach (string name in names) - { - if (!traits.Any(t => t.Entity.Name.Contains(name))) - { - //TODO - //throw new Exception($"Trait '{name}' is not found!!!"); - } - } + List traits = Context.Traits.Where(t => traitIds.Contains(t.Id)).ToList(); foreach (Trait trait in traits) { @@ -209,138 +230,29 @@ internal abstract class AbstractEntityProcessor( } } - #endregion - /// - /// Обработка сущности, отмеченной как устаревшей. - /// - /// Обработка происходит в классе + /// Добавление устаревших сущностей. /// - protected virtual void ProcessLegacy() + protected virtual void AddLegacy() { JArray? legacyIdArray = Input.GetOptional("legacy_id"); - if (legacyIdArray is null) + if (legacyIdArray is null || legacyIdArray.Count == 0) { return; } string[] legacyIds = legacyIdArray.ToArray().Select(t => t.ToString()).ToArray(); + List legacyEntities = Context + .LegacyEntityExternals.Where(lee => legacyIds.Contains(lee.ExternalId)) + .Select(lee => lee.LegacyEntity) + .ToList(); - if (legacyIds.Length == 0) + foreach (LegacyEntity legacyEntity in legacyEntities) { - return; - } - - foreach (string legacyId in legacyIds) - { - LegacyEntityHandler? handler = LegacyHandlers.FirstOrDefault(h => - h.LegacyId == legacyId - ); - - if (handler is not null) - { - handler.Apply(Entity); - } - else - { - handler = new(legacyId, Entity); - LegacyHandlers.Add(handler); - } + Entity.LegacyEntities.Add(legacyEntity); } } - protected virtual void ProcessRemaster() - { - JArray? remasterIdArray = Input.GetOptional("remaster_id"); - - if (remasterIdArray is null) - { - return; - } - - if (remasterIdArray.Count > 1) - { - throw new("Несколько отметок ремастер-сущности на устаревшие!"); - } - - string[] remasterIds = remasterIdArray.ToArray().Select(t => t.ToString()).ToArray(); - - foreach (string remasterId in remasterIds) - { - LegacyEntityHandler? handler = null; - //LegacyHandlers.FirstOrDefault(h => - // h.RemasterEntities.Contains(remasterId) - //); - - if (handler is null) - { - handler = new(this); - LegacyHandlers.Add(handler); - } - else { } - } - - throw new LegacyItemException(Input.GetRequired("name")); - } - - /// - /// Получение списка источников сущности. - /// - /// - public List GetSources() - { - string[] sourceNames = Input - .GetRequired("source") - .Select(token => token.ToString()) - .ToArray(); - - if (sourceNames.Length == 0) - { - throw new( - $"Source for entity with id \"{Entity.Name[LocaleType.English]}\" " - + $"(Aonprd id \"{Input.GetRequired("id")}\") is not found" - ); - } - - List entitySources = []; - - foreach (string sourceName in sourceNames) - { - Source source = - Context.Sources.FirstOrDefault(s => s.Name == sourceName) - ?? throw new($"Source \"{sourceName}\" did not found"); - - EntitySource entitySource = new() - { - Entity = Entity, - Source = source, - Suffix = StringHelper.ResolveSource(Input.GetRequired("source_markdown")), - IsRemaster = Input.GetOptional("legacy_id") is not null, - }; - - entitySources.Add(entitySource); - } - - return entitySources; - } - - /// - /// Генерация JSON-строки из объектов, располагаюхися по путям, указанным в . - /// - /// Коллекция путей, из которых необходимо достатить объекты. - /// JSON-строка. - public string GenerateJsonString(params ICollection paths) - { - Dictionary data = []; - - foreach (string key in paths) - { - data.Add(key, Input.Input.SelectToken(key)); - } - - return JsonConvert.SerializeObject(data, new JsonSerializerSettings()); - } - - protected static readonly ConcurrentBag LegacyHandlers = []; + #endregion } diff --git a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/AbstractSubclassProcessor.cs b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/AbstractSubclassProcessor.cs index 8b3a1aa..5d3eb47 100644 --- a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/AbstractSubclassProcessor.cs +++ b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/AbstractSubclassProcessor.cs @@ -18,11 +18,10 @@ namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors; /// internal abstract class AbstractSubclassProcessor( ProcessorInput input, - Pf2eDbContext context, StringHelper stringHelper, DatabaseHelper helper, AonprdBuilderGlobals globals -) : AbstractEntityProcessor(input, context, stringHelper, helper, globals) +) : AbstractEntityProcessor(input, stringHelper, helper, globals) { /// public override EntityType Type => EntityType.Subclass; @@ -41,7 +40,7 @@ internal abstract class AbstractSubclassProcessor( public abstract string GroupName { get; } /// - public override void Add() => Context.Subclasses.Add(Object); + protected override void Add() => Context.Subclasses.Add(Object); /// public override void Process() diff --git a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Common/ExternalProcessor.cs b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Common/ExternalProcessor.cs index 7b8f8ee..ca297c4 100644 --- a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Common/ExternalProcessor.cs +++ b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Common/ExternalProcessor.cs @@ -12,10 +12,9 @@ namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Common; Group = ProcessorGroupEnum.AonprdPreparation )] [ProcessorDependencies(["aonprd:repository"])] -internal class ExternalProcessor(Pf2eDbContext context, AonprdBuilderGlobals globals) - : AbstractProcessor +internal class ExternalProcessor(AonprdBuilderGlobals globals) : AbstractProcessor { - public Pf2eDbContext Context { get; } = context; + public Pf2eDbContext Context { get; } = new(); public AonprdBuilderGlobals Globals { get; } = globals; public ExternalTarget AonprdTarget = null!; @@ -33,6 +32,7 @@ internal class ExternalProcessor(Pf2eDbContext context, AonprdBuilderGlobals glo public override void Save() { Context.ExternalTargets.Add(AonprdTarget); + Context.SaveChanges(); Globals.AonprdTarget = AonprdTarget; } diff --git a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Common/LegacyProcessor.cs b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Common/LegacyProcessor.cs index f0c311d..692f8ae 100644 --- a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Common/LegacyProcessor.cs +++ b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Common/LegacyProcessor.cs @@ -5,7 +5,6 @@ using Pf2eDataScrapper.Importers.Implementations.Aonprd.Helpers; using Pf2eDataScrapper.Processors; using Pf2eModel; using Pf2eModel.Model.Base.Legacy; -using Pf2eModel.Model.Base.Source; using Pf2eModel.Model.Common.I18n; namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Common; @@ -15,17 +14,17 @@ namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Common; InnerType = null, CompendiumType = ProcessorCompendiumEnum.Aonprd, Description = "Создание ссылок на устаревшие сущности", - Group = ProcessorGroupEnum.AonprdPreparation + Group = ProcessorGroupEnum.AonprdPreparation, + Parallel = true )] [ProcessorDependencies(["aonprd:source", "aonprd:external"])] internal class LegacyProcessor( ProcessorInput input, - Pf2eDbContext context, AonprdBuilderGlobals globals, StringHelper stringHelper ) : AbstractInputProcessor(input) { - public Pf2eDbContext Context { get; } = context; + public Pf2eDbContext Context { get; } = new(); public AonprdBuilderGlobals Globals { get; } = globals; public StringHelper StringHelper { get; } = stringHelper; @@ -33,8 +32,6 @@ internal class LegacyProcessor( public override void Process() { - JArray? remasterIdArray = Input.GetOptional("remaster_id"); - if (Input.GetOptional("remaster_id") is null) { throw new SilentSkipItemException(); @@ -53,18 +50,6 @@ internal class LegacyProcessor( /// Генерация источников сущности. /// protected virtual void AddSource() - { - foreach (LegacyEntitySource entitySource in GetSources()) - { - LegacyEntity.Sources.Add(entitySource); - } - } - - /// - /// Получение списка источников сущности. - /// - /// - public List GetSources() { string[] sourceNames = Input .GetRequired("source") @@ -78,27 +63,25 @@ internal class LegacyProcessor( + $"(Aonprd id \"{Input.GetRequired("id")}\") is not found" ); } - - List entitySources = []; - + /* + Dictionary sourceIds = Context + .Sources.AsNoTracking() + .Where(s => sourceNames.Contains(s.Name)) + .ToDictionary(s => s.Name, s => s.Id); + */ foreach (string sourceName in sourceNames) { - Source source = - Context.Sources.FirstOrDefault(s => s.Name == sourceName) - ?? throw new($"Source \"{sourceName}\" did not found"); - LegacyEntitySource entitySource = new() { LegacyEntity = LegacyEntity, - Source = source, + SourceId = Globals.SourceIds[sourceName], Suffix = StringHelper.ResolveSource(Input.GetRequired("source_markdown")), IsRemaster = Input.GetOptional("legacy_id") is not null, }; - entitySources.Add(entitySource); + Context.LegacyEntitySources.Add(entitySource); + LegacyEntity.Sources.Add(entitySource); } - - return entitySources; } /// @@ -109,7 +92,7 @@ internal class LegacyProcessor( LegacyEntityExternal external = new() { LegacyEntity = LegacyEntity, - Target = Globals.AonprdTarget, + TargetId = Globals.AonprdTarget.Id, ExternalId = Input.GetRequired("id"), Suffix = Input.GetRequired("url"), Md5 = Input.Md5(), @@ -121,5 +104,10 @@ internal class LegacyProcessor( public override void Save() { Context.LegacyEntities.Add(LegacyEntity); + + Context.SaveChanges(false); + Context.Dispose(); } + + private static readonly Lock Lock = new(); } diff --git a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Common/SourceProcessor.cs b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Common/SourceProcessor.cs index 570cf42..88b29fc 100644 --- a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Common/SourceProcessor.cs +++ b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Common/SourceProcessor.cs @@ -1,4 +1,5 @@ using Pf2eDataScrapper.App; +using Pf2eDataScrapper.Importers.Implementations.Aonprd.Helpers; using Pf2eDataScrapper.Processors; using Pf2eModel; using Pf2eModel.Model.Base.Source; @@ -14,10 +15,11 @@ namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Common; Group = ProcessorGroupEnum.AonprdPreparation )] [ProcessorDependencies(["aonprd:compendium"])] -internal class SourceProcessor(ProcessorInput input, Pf2eDbContext context) +internal class SourceProcessor(ProcessorInput input, AonprdBuilderGlobals globals) : AbstractInputProcessor(input) { - public Pf2eDbContext Context { get; } = context; + public AonprdBuilderGlobals Globals { get; } = globals; + public Pf2eDbContext Context { get; } = new(); /// /// Источник сущности. @@ -39,5 +41,8 @@ internal class SourceProcessor(ProcessorInput input, Pf2eDbContext context) public override void Save() { Context.Sources.Add(Source); + Context.SaveChanges(); + + Globals.SourceIds.Add(Source.Name, Source.Id); } } diff --git a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Subclasses/ApparitionProcessor.cs b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Subclasses/ApparitionProcessor.cs index 47d2cc9..a8632e7 100644 --- a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Subclasses/ApparitionProcessor.cs +++ b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Subclasses/ApparitionProcessor.cs @@ -24,11 +24,10 @@ namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Subclasse )] internal class ApparitionProcessor( ProcessorInput input, - Pf2eDbContext context, StringHelper stringHelper, DatabaseHelper helper, AonprdBuilderGlobals globals -) : AbstractSubclassProcessor(input, context, stringHelper, helper, globals) +) : AbstractSubclassProcessor(input, stringHelper, helper, globals) { /// public override Subclass Object { get; } = new(); diff --git a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/ActionProcessor.cs b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/ActionProcessor.cs index 2cbc2fc..88204dd 100644 --- a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/ActionProcessor.cs +++ b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/ActionProcessor.cs @@ -18,11 +18,10 @@ namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Types; [ProcessorDependencies(["aonprd:trait", "aonprd:source"])] internal class ActionProcessor( ProcessorInput input, - Pf2eDbContext context, StringHelper stringHelper, DatabaseHelper helper, AonprdBuilderGlobals globals -) : AbstractEntityProcessor(input, context, stringHelper, helper, globals) +) : AbstractEntityProcessor(input, stringHelper, helper, globals) { /// public override EntityType Type => EntityType.Action; @@ -31,7 +30,7 @@ internal class ActionProcessor( public override Pf2eModel.Model.Structure.Action Object { get; } = new(); /// - public override void Add() => Context.Actions.Add(Object); + protected override void Add() => Context.Actions.Add(Object); /// public override void Process() diff --git a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/AncestryProcessor.cs b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/AncestryProcessor.cs index 3631619..9891f4b 100644 --- a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/AncestryProcessor.cs +++ b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/AncestryProcessor.cs @@ -20,11 +20,10 @@ namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Types; [ProcessorDependencies(["aonprd:trait", "aonprd:source", "aonprd:language"])] internal class AncestryProcessor( ProcessorInput input, - Pf2eDbContext context, StringHelper stringHelper, DatabaseHelper helper, AonprdBuilderGlobals globals -) : AbstractEntityProcessor(input, context, stringHelper, helper, globals) +) : AbstractEntityProcessor(input, stringHelper, helper, globals) { /// public override EntityType Type => EntityType.Ancestry; @@ -33,7 +32,7 @@ internal class AncestryProcessor( public override Ancestry Object { get; } = new(); /// - public override void Add() => Context.Ancestries.Add(Object); + protected override void Add() => Context.Ancestries.Add(Object); /// public override void Process() diff --git a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/ArchetypeProcessor.cs b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/ArchetypeProcessor.cs index e8d9b62..afa6f1b 100644 --- a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/ArchetypeProcessor.cs +++ b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/ArchetypeProcessor.cs @@ -19,11 +19,10 @@ namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Types; [ProcessorDependencies(["aonprd:trait", "aonprd:source"])] internal class ArchetypeProcessor( ProcessorInput input, - Pf2eDbContext context, StringHelper stringHelper, DatabaseHelper helper, AonprdBuilderGlobals globals -) : AbstractEntityProcessor(input, context, stringHelper, helper, globals) +) : AbstractEntityProcessor(input, stringHelper, helper, globals) { /// public override EntityType Type => EntityType.Archetype; @@ -32,7 +31,7 @@ internal class ArchetypeProcessor( public override Archetype Object { get; } = new(); /// - public override void Add() => Context.Archetypes.Add(Object); + protected override void Add() => Context.Archetypes.Add(Object); /// public override void Process() diff --git a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/BackgroundProcessor.cs b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/BackgroundProcessor.cs index 2d857d6..d8313e8 100644 --- a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/BackgroundProcessor.cs +++ b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/BackgroundProcessor.cs @@ -21,11 +21,10 @@ namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Types; [ProcessorDependencies(["aonprd:trait", "aonprd:source"])] internal class BackgroundProcessor( ProcessorInput input, - Pf2eDbContext context, StringHelper stringHelper, DatabaseHelper helper, AonprdBuilderGlobals globals -) : AbstractEntityProcessor(input, context, stringHelper, helper, globals) +) : AbstractEntityProcessor(input, stringHelper, helper, globals) { /// public override EntityType Type => EntityType.Background; @@ -34,7 +33,7 @@ internal class BackgroundProcessor( public override Background Object { get; } = new(); /// - public override void Add() => Context.Backgrounds.Add(Object); + protected override void Add() => Context.Backgrounds.Add(Object); /// public override void Process() diff --git a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/ClassProcessor.cs b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/ClassProcessor.cs index 869bcd2..a6c50bf 100644 --- a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/ClassProcessor.cs +++ b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/ClassProcessor.cs @@ -25,11 +25,10 @@ namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Types; [ProcessorDependencies(["aonprd:trait", "aonprd:source", "aonprd:skill"])] internal partial class ClassProcessor( ProcessorInput input, - Pf2eDbContext context, StringHelper stringHelper, DatabaseHelper helper, AonprdBuilderGlobals globals -) : AbstractEntityProcessor(input, context, stringHelper, helper, globals) +) : AbstractEntityProcessor(input, stringHelper, helper, globals) { /// public override EntityType Type => EntityType.Class; @@ -38,7 +37,7 @@ internal partial class ClassProcessor( public override Class Object { get; } = new(); /// - public override void Add() => Context.Classes.Add(Object); + protected override void Add() => Context.Classes.Add(Object); /// public override void Process() diff --git a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/FeatProcessor.cs b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/FeatProcessor.cs index e1e01da..146883f 100644 --- a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/FeatProcessor.cs +++ b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/FeatProcessor.cs @@ -20,11 +20,10 @@ namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Types; [ProcessorDependencies(["aonprd:trait", "aonprd:source", "aonprd:archetype"])] internal class FeatProcessor( ProcessorInput input, - Pf2eDbContext context, StringHelper stringHelper, DatabaseHelper helper, AonprdBuilderGlobals globals -) : AbstractEntityProcessor(input, context, stringHelper, helper, globals) +) : AbstractEntityProcessor(input, stringHelper, helper, globals) { /// public override EntityType Type => EntityType.Feat; @@ -33,7 +32,7 @@ internal class FeatProcessor( public override Feat Object { get; } = new(); /// - public override void Add() => Context.Feats.Add(Object); + protected override void Add() => Context.Feats.Add(Object); /// public override void Process() diff --git a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/LanguageProcessor.cs b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/LanguageProcessor.cs index 9ded0f6..eccfc72 100644 --- a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/LanguageProcessor.cs +++ b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/LanguageProcessor.cs @@ -19,11 +19,10 @@ namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Types; [ProcessorDependencies(["aonprd:trait", "aonprd:source"])] internal class LanguageProcessor( ProcessorInput input, - Pf2eDbContext context, StringHelper stringHelper, DatabaseHelper helper, AonprdBuilderGlobals globals -) : AbstractEntityProcessor(input, context, stringHelper, helper, globals) +) : AbstractEntityProcessor(input, stringHelper, helper, globals) { /// public override EntityType Type => EntityType.Language; @@ -32,5 +31,5 @@ internal class LanguageProcessor( public override Language Object { get; } = new(); /// - public override void Add() => Context.Languages.Add(Object); + protected override void Add() => Context.Languages.Add(Object); } diff --git a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/RitualProcessor.cs b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/RitualProcessor.cs index 1a73ccd..617e8d2 100644 --- a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/RitualProcessor.cs +++ b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/RitualProcessor.cs @@ -23,11 +23,10 @@ namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Types; [ProcessorDependencies(["aonprd:trait", "aonprd:source", "aonprd:skill"])] internal partial class RitualProcessor( ProcessorInput input, - Pf2eDbContext context, StringHelper stringHelper, DatabaseHelper helper, AonprdBuilderGlobals globals -) : AbstractEntityProcessor(input, context, stringHelper, helper, globals) +) : AbstractEntityProcessor(input, stringHelper, helper, globals) { /// public override EntityType Type => EntityType.Ritual; @@ -36,7 +35,7 @@ internal partial class RitualProcessor( public override Ritual Object { get; } = new(); /// - public override void Add() => Context.Rituals.Add(Object); + protected override void Add() => Context.Rituals.Add(Object); /// public override void Process() diff --git a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/Rules/RuleProcessor.cs b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/Rules/RuleProcessor.cs index 6534154..f62bf09 100644 --- a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/Rules/RuleProcessor.cs +++ b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/Rules/RuleProcessor.cs @@ -5,7 +5,6 @@ using Pf2eDataScrapper.Importers.Implementations.Aonprd.DataBuilder.Helpers; using Pf2eDataScrapper.Importers.Implementations.Aonprd.Helpers; using Pf2eDataScrapper.Processors; using Pf2eModel; -using Pf2eModel.Model.Base.Legacy; using Pf2eModel.Model.Base.Source; using Pf2eModel.Model.Base.Targets; using Pf2eModel.Model.Common.I18n; @@ -18,17 +17,16 @@ namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Types.Rul "aonprd:rules", InnerType = "rules", CompendiumType = ProcessorCompendiumEnum.Aonprd, - Description = "Создание цели внешних ссылок", + Description = "Правила", Group = ProcessorGroupEnum.AonprdEntities )] [ProcessorDependencies(["aonprd:trait", "aonprd:source", "aonprd:rules-toc"])] internal class RuleProcessor( ProcessorInput input, - Pf2eDbContext context, StringHelper stringHelper, DatabaseHelper helper, AonprdBuilderGlobals globals -) : AbstractEntityProcessor(input, context, stringHelper, helper, globals) +) : AbstractEntityProcessor(input, stringHelper, helper, globals) { /// public override EntityType Type => EntityType.Rule; @@ -37,7 +35,7 @@ internal class RuleProcessor( public override Rule Object { get; } = new(); /// - public override void Add() => Context.Rules.Add(Object); + protected override void Add() => Context.Rules.Add(Object); /// /// Структура генерации содержания правил. diff --git a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/SpellProcessor.cs b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/SpellProcessor.cs index 9a4aaf9..dcfa38b 100644 --- a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/SpellProcessor.cs +++ b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/SpellProcessor.cs @@ -22,11 +22,10 @@ namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Types; [ProcessorDependencies(["aonprd:trait", "aonprd:source"])] internal class SpellProcessor( ProcessorInput input, - Pf2eDbContext context, StringHelper stringHelper, DatabaseHelper helper, AonprdBuilderGlobals globals -) : AbstractEntityProcessor(input, context, stringHelper, helper, globals) +) : AbstractEntityProcessor(input, stringHelper, helper, globals) { /// public override EntityType Type => EntityType.Spell; @@ -35,7 +34,7 @@ internal class SpellProcessor( public override Spell Object { get; } = new(); /// - public override void Add() => Context.Spells.Add(Object); + protected override void Add() => Context.Spells.Add(Object); /// public override void Process() diff --git a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/TraitProcessor.cs b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/TraitProcessor.cs index 80e55db..7e4d907 100644 --- a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/TraitProcessor.cs +++ b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/TraitProcessor.cs @@ -4,6 +4,8 @@ using Pf2eDataScrapper.Importers.Implementations.Aonprd.DataBuilder.Helpers; using Pf2eDataScrapper.Importers.Implementations.Aonprd.Helpers; using Pf2eDataScrapper.Processors; using Pf2eModel; +using Pf2eModel.Model.Base.Legacy; +using Pf2eModel.Model.Common.I18n; using Pf2eModel.Model.Enum; using Pf2eModel.Model.Structure; @@ -23,7 +25,7 @@ internal class TraitProcessor( StringHelper stringHelper, DatabaseHelper helper, AonprdBuilderGlobals globals -) : AbstractEntityProcessor(input, context, stringHelper, helper, globals) +) : AbstractEntityProcessor(input, stringHelper, helper, globals) { /// public override EntityType Type => EntityType.Trait; @@ -32,7 +34,17 @@ internal class TraitProcessor( public override Trait Object { get; } = new(); /// - public override void Add() => Context.Traits.Add(Object); + protected override void Add() + { + Context.Traits.Add(Object); + + Globals.TraitIds[Entity.Name[LocaleType.English]] = Entity.Id; + + foreach (LegacyEntity legacyEntity in Entity.LegacyEntities) + { + Globals.TraitIds[legacyEntity.Name[LocaleType.English]] = Entity.Id; + } + } /// public override void Process() diff --git a/Pf2eDataScrapper/Processors/ProcessorExecutor.cs b/Pf2eDataScrapper/Processors/ProcessorExecutor.cs index 9d74fa0..78e431a 100644 --- a/Pf2eDataScrapper/Processors/ProcessorExecutor.cs +++ b/Pf2eDataScrapper/Processors/ProcessorExecutor.cs @@ -1,4 +1,5 @@ using System.ComponentModel; +using System.Diagnostics; using Microsoft.Extensions.DependencyInjection; using Newtonsoft.Json.Linq; using Pf2eDataScrapper.App; @@ -16,15 +17,16 @@ internal sealed class ProcessorExecutor( ProcessProgress progress, Pf2eDbContext context, AonprdCompendium aonprdCompendium, - FoundryEnCompendium foundryEnCompendium + FoundryEnCompendium foundryEnCompendium, + ApplicationSettings settings ) { public IServiceProvider ServiceProvider { get; } = serviceProvider; public ProcessorCollection Collection { get; } = collection; public ProcessProgress Progress { get; } = progress; - public Pf2eDbContext Context { get; } = context; public AonprdCompendium AonprdCompendium { get; } = aonprdCompendium; public FoundryEnCompendium FoundryEnCompendium { get; } = foundryEnCompendium; + public ApplicationSettings Settings { get; } = settings; public void Start() { @@ -48,31 +50,36 @@ internal sealed class ProcessorExecutor( worker.DoWork += (_, _) => { - foreach (IProcessor processor in IterateProcessors(item)) - { - try - { - processor.Process(); - processor.Save(); - } - catch (SkipItemException exception) - { - Progress.Information(exception.Message); - } - catch (SilentSkipItemException) - { - // - } - } - Context.SaveChanges(); + Parallel.ForEach( + IterateProcessors(item), + new() { MaxDegreeOfParallelism = item.Parallel ? Settings.MaxThreads : 1 }, + RunProcessor + ); }; - worker.RunWorkerCompleted += Continue; + worker.RunWorkerCompleted += (_, _) => Continue(); worker.RunWorkerAsync(); } - private void Continue(object? sender, RunWorkerCompletedEventArgs e) + private void RunProcessor(IProcessor processor, ParallelLoopState state, long arg3) + { + try + { + processor.Process(); + processor.Save(); + } + catch (SkipItemException exception) + { + Progress.Information(exception.Message); + } + catch (SilentSkipItemException) + { + // + } + } + + private void Continue() { Progress.Processors.End(); _currentIndex++; diff --git a/Pf2eModel/EfCleanHelper.cs b/Pf2eModel/EfCleanHelper.cs index 1cb9cee..f26bc7e 100644 --- a/Pf2eModel/EfCleanHelper.cs +++ b/Pf2eModel/EfCleanHelper.cs @@ -69,7 +69,7 @@ public static class EfCleanHelper public static string Truncate(this DbSet dbSet) where T : class { - string cmd = $"TRUNCATE TABLE {TableName(dbSet)} CASCADE;"; + string cmd = $"TRUNCATE TABLE {TableName(dbSet)} RESTART IDENTITY CASCADE;"; DbContext context = dbSet.GetService().Context; context.Database.ExecuteSqlRaw(cmd); return cmd; @@ -87,7 +87,7 @@ public static class EfCleanHelper foreach (IEntityType entityType in tables) { - string cmd = $"TRUNCATE TABLE {GetName(entityType)} CASCADE;"; + string cmd = $"TRUNCATE TABLE {GetName(entityType)} RESTART IDENTITY CASCADE;"; context.Database.ExecuteSqlRaw(cmd); cmdList.Add(cmd); } diff --git a/Pf2eModel/Migrations/20250514102154_1.Designer.cs b/Pf2eModel/Migrations/20250514120355_1.Designer.cs similarity index 99% rename from Pf2eModel/Migrations/20250514102154_1.Designer.cs rename to Pf2eModel/Migrations/20250514120355_1.Designer.cs index fc193fc..f1f0fcb 100644 --- a/Pf2eModel/Migrations/20250514102154_1.Designer.cs +++ b/Pf2eModel/Migrations/20250514120355_1.Designer.cs @@ -13,7 +13,7 @@ using Pf2eModel.Model.Enum; namespace Pf2eModel.Migrations { [DbContext(typeof(Pf2eDbContext))] - [Migration("20250514102154_1")] + [Migration("20250514120355_1")] partial class _1 { /// diff --git a/Pf2eModel/Migrations/20250514102154_1.cs b/Pf2eModel/Migrations/20250514120355_1.cs similarity index 100% rename from Pf2eModel/Migrations/20250514102154_1.cs rename to Pf2eModel/Migrations/20250514120355_1.cs diff --git a/Pf2eModel/Model/Base/Entity.cs b/Pf2eModel/Model/Base/Entity.cs index 89cff39..ecc28fc 100644 --- a/Pf2eModel/Model/Base/Entity.cs +++ b/Pf2eModel/Model/Base/Entity.cs @@ -104,5 +104,5 @@ public class Entity /// /// Коллекция устаревших сущностей. /// - public IEnumerable LegacyEntities { get; set; } = []; + public ICollection LegacyEntities { get; set; } = []; } diff --git a/Pf2eModel/Pf2eDbContext.cs b/Pf2eModel/Pf2eDbContext.cs index 2ecf8c7..0d49ace 100644 --- a/Pf2eModel/Pf2eDbContext.cs +++ b/Pf2eModel/Pf2eDbContext.cs @@ -136,6 +136,7 @@ public partial class Pf2eDbContext : DbContext // optionsBuilder.LogTo(Console.WriteLine); optionsBuilder.UseSnakeCaseNamingConvention(); + optionsBuilder.EnableSensitiveDataLogging(); optionsBuilder.UseNpgsql( databaseConnection,