diff --git a/Pf2eDataScrapper/App/ProcessorCollection.cs b/Pf2eDataScrapper/App/ProcessorCollection.cs index 7dbae40..c73748c 100644 --- a/Pf2eDataScrapper/App/ProcessorCollection.cs +++ b/Pf2eDataScrapper/App/ProcessorCollection.cs @@ -48,7 +48,7 @@ internal class ProcessorCollection /// private ProcessorCollectionItem CreateItem(Type type) { - if (!type.IsSubclassOf(typeof(IProcessor))) + if (!type.IsAssignableTo(typeof(IProcessor))) { throw new( $"'ProcessorAttribute' attribute should only be applied to children of 'IProcessor' class. '{type.FullName}' is not." @@ -61,15 +61,6 @@ internal class ProcessorCollection bool isCollection = processorAttribute.InnerType is not null; - ConstructorInfo? constructor = isCollection - ? type.GetConstructor([]) - : type.GetConstructor([typeof(ProcessorInput)])!; - - if (constructor is null) - { - throw new($"Не удаётся определить конструктор класса {type}"); - } - return new() { Name = processorAttribute.Name, @@ -79,7 +70,6 @@ internal class ProcessorCollection Dependencies = dependenciesAttribute?.Dependencies ?? [], IsCollectionProcessor = isCollection, InnerType = processorAttribute.InnerType, - Constructor = constructor, }; } @@ -111,7 +101,7 @@ internal class ProcessorCollection { foreach (ProcessorCollectionItem item in Items) { - if (item.Priority > 0) + if (item.Priority >= 0) { continue; } @@ -125,7 +115,10 @@ internal class ProcessorCollection continue; } - item.Priority = dependencyPriorities.Any() ? dependencyPriorities.Max() + 1 : 1; + item.Priority = Math.Min( + dependencyPriorities.Any() ? dependencyPriorities.Max() + 1 : 1, + (int)item.Group * 100 + ); priorities.Add(item.Name, item.Priority); } } @@ -215,9 +208,4 @@ internal class ProcessorCollectionItem /// Флаг, указывающий внутренний тип коллекции обрабатываемых элементов. /// public string? InnerType { get; init; } - - /// - /// Конструктор обработчика. - /// - public required ConstructorInfo Constructor { get; init; } } diff --git a/Pf2eDataScrapper/App/ProcessorGroupEnum.cs b/Pf2eDataScrapper/App/ProcessorGroupEnum.cs index 09d40ac..85b918f 100644 --- a/Pf2eDataScrapper/App/ProcessorGroupEnum.cs +++ b/Pf2eDataScrapper/App/ProcessorGroupEnum.cs @@ -18,4 +18,7 @@ public enum ProcessorGroupEnum [Description("Foundry EN: Подготовка")] FoundryEnPreparation, + + [Description("Foundry EN: Основные сущности")] + FoundryEnEntities, } diff --git a/Pf2eDataScrapper/ApplicationSettings.cs b/Pf2eDataScrapper/ApplicationSettings.cs index 48a80a7..4dc914d 100644 --- a/Pf2eDataScrapper/ApplicationSettings.cs +++ b/Pf2eDataScrapper/ApplicationSettings.cs @@ -15,6 +15,10 @@ internal class ApplicationSettings public ApplicationSettings() { DotEnv.Load(); + + DataDirPath = EnvReader.GetStringValue("DATA_DIR"); + OutputDirPath = EnvReader.GetStringValue("OUTPUT_DIR"); + DatabaseConnection = EnvReader.GetStringValue("DATABASE_CONNECTION"); } #region Настройки из .env @@ -26,7 +30,7 @@ internal class ApplicationSettings [ReadOnly(true)] [DisplayName("Путь к папке с входными данными.")] [Description("Редактирование через .env-файл.")] - public string DataDirPath { get; } = EnvReader.GetStringValue("DATA_DIR"); + public string DataDirPath { get; } /// /// Папка в входными данными. @@ -41,7 +45,7 @@ internal class ApplicationSettings [ReadOnly(true)] [DisplayName("Путь к папке для сохранения результатов.")] [Description("Редактирование через .env-файл.")] - public string OutputDirPath { get; } = EnvReader.GetStringValue("OUTPUT_DIR"); + public string OutputDirPath { get; } /// /// Папка для сохранения результатов. @@ -56,7 +60,7 @@ internal class ApplicationSettings [ReadOnly(true)] [DisplayName("Строка подключения к базе данных.")] [Description("Редактирование через .env-файл.")] - public string DatabaseConnection { get; } = EnvReader.GetStringValue("DATABASE_CONNECTION"); + public string DatabaseConnection { get; } #endregion @@ -74,7 +78,7 @@ internal class ApplicationSettings /// Список обработчиков, которые должны быть выполнены. /// Пустой список означает, что должны быть выполнены все обработчики. /// - public ICollection ProcessorNames { get; } = new List(); + public ICollection ProcessorNames { get; } = []; #endregion } diff --git a/Pf2eDataScrapper/Forms/StartForm.Designer.cs b/Pf2eDataScrapper/Forms/StartForm.Designer.cs index 31eb6f5..88ccf32 100644 --- a/Pf2eDataScrapper/Forms/StartForm.Designer.cs +++ b/Pf2eDataScrapper/Forms/StartForm.Designer.cs @@ -135,6 +135,7 @@ partial class StartForm StartButton.TabIndex = 0; StartButton.Text = "Запуск"; StartButton.UseVisualStyleBackColor = true; + StartButton.Click += StartButton_Click; // // StartForm // diff --git a/Pf2eDataScrapper/Forms/StartForm.cs b/Pf2eDataScrapper/Forms/StartForm.cs index 58811ff..ac9466e 100644 --- a/Pf2eDataScrapper/Forms/StartForm.cs +++ b/Pf2eDataScrapper/Forms/StartForm.cs @@ -1,5 +1,6 @@ using System.ComponentModel; using System.Reflection; +using Newtonsoft.Json.Linq; using Pf2eDataScrapper.App; namespace Pf2eDataScrapper.Forms; @@ -32,19 +33,14 @@ internal partial class StartForm : Form ProcessorsListView.Groups.Clear(); - foreach (FieldInfo field in typeof(ProcessorGroupEnum).GetFields()) + foreach (ProcessorGroupEnum group in Enum.GetValues()) { - ProcessorGroupEnum groupEnum = (ProcessorGroupEnum)field.GetValue(null)!; - - if (!items.TryGetValue(groupEnum, out List? list)) + if (!items.TryGetValue(group, out List? list)) { continue; } - DescriptionAttribute? attribute = (DescriptionAttribute?) - Attribute.GetCustomAttribute(field, typeof(DescriptionAttribute)); - - string name = attribute?.Description ?? field.Name; + string name = Enum.GetDescription(group); ListViewGroup listViewGroup = new(name); ProcessorsListView.Groups.Add(listViewGroup); @@ -61,4 +57,17 @@ internal partial class StartForm : Form } } } + + #region "Кнопка старта" + + private void StartButton_Click(object sender, EventArgs e) + { + StartButtonPressed?.Invoke(); + } + + public delegate void StartButtonPressedDelegate(); + + public event StartButtonPressedDelegate? StartButtonPressed; + + #endregion } diff --git a/Pf2eDataScrapper/Helpers/DatabaseHelper.cs b/Pf2eDataScrapper/Helpers/DatabaseHelper.cs index 7b0960e..85bebfa 100644 --- a/Pf2eDataScrapper/Helpers/DatabaseHelper.cs +++ b/Pf2eDataScrapper/Helpers/DatabaseHelper.cs @@ -8,6 +8,16 @@ namespace Pf2eDataScrapper.Helpers; /// public class DatabaseHelper { + /// + /// Context для доступа к базе данных. + /// + public Pf2eDbContext Context { get; } + + /// + /// Сервис для работы с URL-Slug. Используется для преобразования названий в URL-friendly формат. + /// + public SlugHelper Slugger { get; } + /// /// Последний найденный идентификатор сущности. /// Используется для генерации новых ID сущностей. @@ -19,18 +29,10 @@ public class DatabaseHelper /// public int NextEntityId => ++_id; - /// - /// Сервис для работы с URL-Slug. Используется для преобразования названий в URL-friendly формат. - /// - public readonly SlugHelper Slugger = new(); - - /// - /// Context для доступа к базе данных. - /// - public readonly Pf2eDbContext Context = new(); - - private DatabaseHelper() + private DatabaseHelper(Pf2eDbContext context, SlugHelper slugHelper) { + Context = context; + Slugger = slugHelper; _id = Context.Entities.Any() ? Context.Entities.Max(e => e.Id) : 0; } @@ -41,24 +43,4 @@ public class DatabaseHelper { return Slugger.GenerateSlug(name); } - - /// - /// Возвращает singleton-инстанцию класса . - /// - public static readonly DatabaseHelper Instance = new(); - - /// - /// Контекст для доступа к базе данных. - /// - public static Pf2eDbContext DbContext => Instance.Context; - - /// - /// Обновляет контекст и сохраняет изменения в базе данных. - /// - public static void ResetContext() - { - Instance.Context.SaveChanges(); - // Instance.Context.Dispose(); - // Instance.Context = new(); - } } diff --git a/Pf2eDataScrapper/Processors/ProcessProgress.cs b/Pf2eDataScrapper/Helpers/ProcessProgress.cs similarity index 100% rename from Pf2eDataScrapper/Processors/ProcessProgress.cs rename to Pf2eDataScrapper/Helpers/ProcessProgress.cs diff --git a/Pf2eDataScrapper/ImportApplication.cs b/Pf2eDataScrapper/ImportApplication.cs index 16b8671..a7e5e09 100644 --- a/Pf2eDataScrapper/ImportApplication.cs +++ b/Pf2eDataScrapper/ImportApplication.cs @@ -1,45 +1,37 @@ using Microsoft.Extensions.DependencyInjection; using Pf2eDataScrapper.App; using Pf2eDataScrapper.Forms; +using Pf2eDataScrapper.Helpers; using Pf2eDataScrapper.Importers; using Pf2eDataScrapper.Importers.Implementations.Aonprd; +using Pf2eDataScrapper.Importers.Implementations.FoundryEn; using Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry; using Pf2eDataScrapper.Processors; using Pf2eModel; +using Slugify; namespace Pf2eDataScrapper; internal sealed class ImportApplication : ApplicationContext { - public ImportApplication() + public ImportApplication( + IServiceProvider serviceProvider, + ApplicationSettings settings, + ProcessorCollection processors + ) { - InitializeServices(); + ServiceProvider = serviceProvider; + Settings = settings; + Processors = processors; + CreateStartForm(); } - #region "Внедрение зависимостей (DI)" - - public readonly IServiceCollection Services = new ServiceCollection(); - - public IServiceProvider ServiceProvider { get; private set; } = null!; - - private void InitializeServices() - { - Services.AddSingleton(this); - Services.AddSingleton(Settings); - Services.AddSingleton(Processors); - - Services.AddDbContext(); - - ServiceProvider = Services.BuildServiceProvider(); - } - - #endregion - #region "Настройки" - public readonly ApplicationSettings Settings = new(); - public readonly ProcessorCollection Processors = new(); + public IServiceProvider ServiceProvider { get; } + public ApplicationSettings Settings { get; } + public ProcessorCollection Processors { get; } #endregion @@ -55,6 +47,7 @@ internal sealed class ImportApplication : ApplicationContext { _startForm = ActivatorUtilities.CreateInstance(ServiceProvider); _startForm.StartPosition = FormStartPosition.CenterParent; + _startForm.StartButtonPressed += StartProcess; _startForm.Show(); } @@ -62,64 +55,39 @@ internal sealed class ImportApplication : ApplicationContext { _startForm!.Close(); - Processors.SortItems(); + ProcessorExecutor executor = ActivatorUtilities.CreateInstance( + ServiceProvider + ); - ProcessProgress progress = new ProcessProgress(); - - foreach (ProcessorCollectionItem item in Processors.Items) - { - if (item.IsCollectionProcessor) - { - RunCollection(item, progress); - } - else - { - RunSingle(item, progress); - } - } + executor.Start(); } - private void RunSingle(ProcessorCollectionItem item, ProcessProgress progress) + #endregion + + #region "Внедрение зависимостей" + + public static void RegisterServices(IServiceCollection services) { - progress.Local.Reset(1); - progress.Local.Start(""); + // Контекст базы данных + services.AddDbContext(); - IProcessor processor = (IProcessor) - ActivatorUtilities.CreateInstance(ServiceProvider, item.Type); + // Основная программа + services.AddSingleton(); - processor.Process(); - processor.Save(); + // Настройки программы + services.AddSingleton(); - progress.Local.Complete(); - } + // Набор процессоров + services.AddSingleton(); - private void RunCollection(ProcessorCollectionItem item, ProcessProgress progress) - { - AbstractImporter importer = item.Group switch - { - ProcessorGroupEnum.AonprdPreparation - or ProcessorGroupEnum.AonprdEntities - or ProcessorGroupEnum.AonprdSubclasses => new AonprdImporter(), - ProcessorGroupEnum.FoundryEnPreparation => new Pf2EFoundryImporter(), - _ => throw new("Can't recognize importer"), - }; + // Помощник идентификации базы данных + services.AddSingleton(); - progress.Local.Reset(importer.CountInputs(item.InnerType!)); + // Помощник создание URL-совместимых названий + services.AddTransient(); - foreach (ProcessorInput input in importer.ResolveInputs(item.InnerType!)) - { - progress.Local.Start(input.Name); - - IProcessor processor = (IProcessor) - ActivatorUtilities.CreateInstance(ServiceProvider, item.Type); - - processor.Process(); - processor.Save(); - - progress.Local.End(); - } - - progress.Local.Complete(); + // Счётчик прогресса + services.AddSingleton(); } #endregion diff --git a/Pf2eDataScrapper/Importers/AbstractCompendium.cs b/Pf2eDataScrapper/Importers/AbstractCompendium.cs new file mode 100644 index 0000000..96818ef --- /dev/null +++ b/Pf2eDataScrapper/Importers/AbstractCompendium.cs @@ -0,0 +1,42 @@ +using Newtonsoft.Json.Linq; +using Pf2eDataScrapper.Importers.Repositories; + +namespace Pf2eDataScrapper.Importers; + +/// +/// Компендиум данных. +/// +internal abstract class AbstractCompendium +{ + /// + /// Репозиторий данных. + /// + public abstract AbstractImporterRepository Repository { get; } + + /// + /// Путь к хранению исходных данных. + /// + public string Dir => Repository.RepositoryDirPath; + + /// + /// Возвращает количество элементов в компендиуме. + /// + /// Количество элементов. + public abstract int Count(); + + /// + /// Возвращает количество элементов для указанного типа данных. + /// + /// Тип данных. + /// Количество элементов. + public abstract int Count(string type); + + /// + /// Получение итератора сущностей. Если указан тип - то только указанного типа. + /// + /// Тип сущностей, которые нужно получить. + /// Итератор сущностей. + public abstract IEnumerable> Iterate( + string? type = null + ); +} diff --git a/Pf2eDataScrapper/Importers/AbstractCompendiumItem.cs b/Pf2eDataScrapper/Importers/AbstractCompendiumItem.cs new file mode 100644 index 0000000..eeca1a2 --- /dev/null +++ b/Pf2eDataScrapper/Importers/AbstractCompendiumItem.cs @@ -0,0 +1,23 @@ +namespace Pf2eDataScrapper.Importers; + +/// +/// Абстрактная ссылка на сущность. +/// +internal abstract class AbstractCompendiumItem +{ + /// + /// Идентификатор сущности. + /// + public abstract string Id { get; } + + /// + /// Тип сущности. + /// + public abstract string Type { get; } + + /// + /// Строковое представление имени сущности. + /// + /// Строковое представление. + public override string ToString() => Id; +} diff --git a/Pf2eDataScrapper/Importers/AbstractImporter.cs b/Pf2eDataScrapper/Importers/AbstractImporter.cs index ae2cc29..020ec30 100644 --- a/Pf2eDataScrapper/Importers/AbstractImporter.cs +++ b/Pf2eDataScrapper/Importers/AbstractImporter.cs @@ -1,5 +1,4 @@ using Pf2eDataScrapper.Importers.Repositories; -using Pf2eDataScrapper.Processors; namespace Pf2eDataScrapper.Importers; @@ -23,46 +22,13 @@ internal abstract class AbstractImporter /// public abstract AbstractImporterRepository Repository { get; } + /// + /// Компендиум данных импортёра. + /// + public abstract AbstractCompendium Compendium { get; } + /// /// Путь к директории с данными для импортёра. /// public string Dir => Path.Combine(EnvSettings.DataDirPath, Slug); - - /// - /// Перечисление процессоров, которые будут использоваться для обработки данных. - /// При пустом массиве будут обработаны все доступные процессоры. - /// - public string[] ProcessorNames { get; set; } = []; - - /// - /// Метод подготовки данных для импорта. - /// - [Obsolete] - public abstract void Prepare(); - - /// - /// Импорт данных - /// - [Obsolete] - public abstract void Import(); - - /// - /// Построение схем данных. - /// - [Obsolete] - public abstract void BuildSchemas(); - - /// - /// Получение количества записей, принаделажащих типу . - /// - /// Тип записей. - /// Количество записей. - public abstract int CountInputs(string type); - - /// - /// Итератор ввода для записей, принаделажащих типу . - /// - /// Тип записей. - /// Итератор. - public abstract IEnumerable ResolveInputs(string type); } diff --git a/Pf2eDataScrapper/Importers/Implementations/Aonprd/AonprdImporter.cs b/Pf2eDataScrapper/Importers/Implementations/Aonprd/AonprdImporter.cs index 114573e..b4b6063 100644 --- a/Pf2eDataScrapper/Importers/Implementations/Aonprd/AonprdImporter.cs +++ b/Pf2eDataScrapper/Importers/Implementations/Aonprd/AonprdImporter.cs @@ -1,15 +1,15 @@ -using Newtonsoft.Json.Linq; +using Microsoft.Extensions.DependencyInjection; using Pf2eDataScrapper.Importers.Implementations.Aonprd.Compendium; -using Pf2eDataScrapper.Importers.Implementations.Aonprd.DataBuilder; -using Pf2eDataScrapper.Importers.Repositories; -using Pf2eDataScrapper.Processors; +using Pf2eDataScrapper.Importers.Implementations.Aonprd.DataBuilder.Helpers; +using Pf2eDataScrapper.Importers.Implementations.Aonprd.Helpers; namespace Pf2eDataScrapper.Importers.Implementations.Aonprd; /// /// Класс обработки данных с сайта Архивов Нефеса. /// -internal class AonprdImporter : AbstractImporter +internal class AonprdImporter(AonprdRepository repository, AonprdCompendium compendium) + : AbstractImporter { /// /// Название внешней ссылки. @@ -23,59 +23,24 @@ internal class AonprdImporter : AbstractImporter public override string Slug => "aonprd"; /// - public override AbstractImporterRepository Repository { get; } + public override AonprdRepository Repository { get; } = repository; - /// - /// Инструмент для работы с данными сайта Архивов Нефеса. - /// - public AonprdLoader Loader { get; } + /// + public override AonprdCompendium Compendium { get; } = compendium; - /// - /// Конструктор класса . - /// - public AonprdImporter() + #region "Внедрение зависимостей" + + public static void RegisterServices(IServiceCollection services) { - Repository = new AonprdRepository(this, "https://elasticsearch.aonprd.com"); - Loader = new(this); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddTransient(); } - /// - public override void Prepare() { } - - /// - public override void BuildSchemas() { } - - /// - public override void Import() { } - - /// - public override int CountInputs(string type) - { - return Loader.Compendium.Count(type); - } - - /// - public override IEnumerable ResolveInputs(string type) - { - foreach (string file in Loader.Compendium.Files(type)) - { - using StreamReader fileReader = new(Path.Combine(Dir, $"{file}.json")); - JArray array = JArray.Parse(fileReader.ReadToEnd()); - - foreach (JToken item in array) - { - string? id = item.SelectToken("id")?.ToString(); - - if (id is null) - { - continue; - } - - if (AonprdReference.ResolveType(id) == type) - { - yield return new AonprdProcessorInput(id, (JObject)item); - } - } - } - } + #endregion } diff --git a/Pf2eDataScrapper/Importers/Implementations/Aonprd/AonprdLoader.cs b/Pf2eDataScrapper/Importers/Implementations/Aonprd/AonprdLoader.cs deleted file mode 100644 index d97b8ec..0000000 --- a/Pf2eDataScrapper/Importers/Implementations/Aonprd/AonprdLoader.cs +++ /dev/null @@ -1,105 +0,0 @@ -using Newtonsoft.Json.Linq; -using Pf2eDataScrapper.Importers.Implementations.Aonprd.Compendium; - -namespace Pf2eDataScrapper.Importers.Implementations.Aonprd; - -/// -/// Класс загрузки данных с сайта Архивов Нефеса. -/// -internal class AonprdLoader(AonprdImporter importer) -{ - /// - /// Референс на импортер, используемый для загрузки данных. - /// - public AonprdImporter Importer { get; } = importer; - - /// - /// Компендиум данных с сайта Архивов Нефеса. - /// - public AonprdCompendium Compendium { get; } = new(); - - /// - /// Число элементов в компендарии. - /// - public int ItemCount => Compendium.Count(); - - /// - /// Подготовка данных перед загрузкой. - /// - public void Prepare() - { - using StreamReader indexFile = new(Path.Combine(Importer.Dir, "index.json")); - JObject indexJson = JObject.Parse(indexFile.ReadToEnd()); - - foreach ((string file, JToken? value) in indexJson) - { - JArray items = (JArray)value!; - - foreach (JToken item in items) - { - Compendium.AddReference(item.ToString(), file); - } - } - } - - /// - /// Возвращает коллекцию всех доступных загружаемых элементов. - /// - /// Коллекция всех загружаемых элементов. - public IEnumerable Entries() - { - return Entries([.. Compendium.Types.Keys]); - } - - /// - /// Возвращает коллекцию доступных загружаемых элементов. - /// - /// Тип, которые нужно включить в результат. - /// Коллекция загружаемых элементов, соответствующих предоставленному типу. - public IEnumerable Entries(string types) - { - return Entries([types]); - } - - /// - /// Возвращает коллекцию доступных загружаемых элементов. - /// - /// Типы, которые нужно включить в результат. - /// Коллекция загружаемых элементов, соответствующих предоставленным типам. - public IEnumerable Entries(string[] types) - { - foreach (string file in Compendium.Files(types)) - { - using StreamReader fileReader = new(Path.Combine(Importer.Dir, $"{file}.json")); - JArray array = JArray.Parse(fileReader.ReadToEnd()); - - foreach (JToken item in array) - { - string? id = item.SelectToken("id")?.ToString(); - - if (id is null) - { - continue; - } - - LoaderEntry entry = new(id, file, (JObject)item); - - if (types.Contains(entry.Type)) - { - yield return entry; - } - } - } - } - - /// - /// Класс, представляющий загружаемый элемент. - /// - public class LoaderEntry(string id, string file, JObject itemJson) : AonprdReference(id, file) - { - /// - /// Возвращает JSON-объект, содерждающий данные о элементе. - /// - public JObject Json { get; } = itemJson; - } -} diff --git a/Pf2eDataScrapper/Importers/Implementations/Aonprd/AonprdRepository.cs b/Pf2eDataScrapper/Importers/Implementations/Aonprd/AonprdRepository.cs index 0a21ba3..4fd6785 100644 --- a/Pf2eDataScrapper/Importers/Implementations/Aonprd/AonprdRepository.cs +++ b/Pf2eDataScrapper/Importers/Implementations/Aonprd/AonprdRepository.cs @@ -1,5 +1,4 @@ -using Konsole; -using Newtonsoft.Json.Linq; +using Newtonsoft.Json.Linq; using Pf2eDataScrapper.Importers.Repositories; using Pf2eDataScrapper.Processors; @@ -8,13 +7,12 @@ namespace Pf2eDataScrapper.Importers.Implementations.Aonprd; /// /// Основной класс для работы с репозиторием данных Aonprd. /// -internal class AonprdRepository(AbstractImporter importer, string searchUrl) - : AbstractImporterRepository(importer) +internal class AonprdRepository(AonprdImporter importer) : AbstractImporterRepository(importer) { /// /// URL для доступа к API Aonprd. /// - public readonly string SearchUrl = searchUrl; + public readonly string SearchUrl = "https://elasticsearch.aonprd.com"; /// /// Текущая ревизия данных. Значение обновляется при обновлении репозитория. @@ -96,7 +94,7 @@ internal class AonprdRepository(AbstractImporter importer, string searchUrl) Revision = Convert.ToInt32( page.SelectToken("_index")?.ToObject()?.Replace("aon", "") - ?? throw new Exception("Can't get revision") + ?? throw new("Не удаётся получить ревизию Архивов Нефиса.") ); string revisionFile = Path.Combine(RepositoryDirPath, "revision.txt"); diff --git a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Compendium/AonprdCompendium.cs b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Compendium/AonprdCompendium.cs index 4b5242c..3980c48 100644 --- a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Compendium/AonprdCompendium.cs +++ b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Compendium/AonprdCompendium.cs @@ -1,67 +1,106 @@ -namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Compendium; +using Newtonsoft.Json.Linq; + +namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Compendium; /// /// Класс хранения данных компендиума. /// -internal class AonprdCompendium +internal sealed class AonprdCompendium(AonprdRepository repository) : AbstractCompendium { - /// - /// Словарь данных компендиума. - /// - public readonly Dictionary Types = []; + public override AonprdRepository Repository { get; } = repository; /// - /// Добавить ссылку на сущность с указанием файла, в котором она хранится. + /// Загрузка данных компендиума. /// - /// Идентификатор сущности. - /// Имя файла. - public void AddReference(string id, string file) + public void Load() { - AonprdReference reference = new(id, file); + using StreamReader indexFile = new( + Path.Combine(Repository.RepositoryDirPath, "index.json") + ); + JObject indexJson = JObject.Parse(indexFile.ReadToEnd()); - if (Types.TryGetValue(reference.Type, out AonprdType? type)) + foreach ((string file, JToken? value) in indexJson) { - type.Add(reference); - } - else - { - Types.Add(reference.Type, new(reference)); + JArray items = (JArray)value!; + + foreach (JToken item in items) + { + AonprdCompendiumItem compendiumItem = new(item.ToString(), file); + + Items.Add(compendiumItem); + } } } + /// + /// Коллекция всех записей компендиума. + /// + public List Items { get; } = []; + /// /// Возвращает количество элементов в компендиуме. /// /// Количество элементов. - public int Count() => Types.Values.Sum(t => t.Count); + public override int Count() => Items.Count; /// /// Возвращает количество элементов для указанного типа данных. /// /// Тип данных. /// Количество элементов. - public int Count(string type) => Types.Values.First(t => t.Type == type).Count; + public override int Count(string type) => Items.Count(i => i.Type == type); /// - /// Возвращает количество элементов для указанных типов данных. + /// Возвращает список всех файлов. /// - /// Список типов данных. - /// Количество элементов. - public int Count(IEnumerable types) => - Types.Values.Where(t => types.Contains(t.Type)).Sum(t => t.Count); + /// Список искомых файлов. + private IEnumerable Files() => Items.Select(t => t.File).Distinct(); /// - /// Возвращает список файлов для выбора сущностей указанного типа данных. + /// Возвращает список файлов, в которых есть сущности указанного типа данных. /// /// Тип данных. /// Список искомых файлов. - public IEnumerable Files(string type) => Types.Values.First(t => t.Type == type).Files; + private IEnumerable Files(string type) => + Items.Where(t => t.Type == type).Select(t => t.File).Distinct(); /// - /// Возвращает список файлов для выбора сущностей указанных типов данных. + /// Получение итератора сущностей. Если указан тип - то только указанного типа. /// - /// Список типов данных. - /// Список искомых файлов. - public IEnumerable Files(IEnumerable types) => - Types.Values.Where(t => types.Contains(t.Type)).SelectMany(t => t.Files).Distinct(); + /// Тип сущностей, которые нужно получить. + /// Итератор сущностей. + public override IEnumerable> Iterate(string? type = null) + { + foreach (string file in type is null ? Files() : Files(type)) + { + Dictionary fileReferences = Items + .Where(i => i.File == file) + .ToDictionary(i => i.Id); + + using StreamReader fileReader = new( + Path.Combine(Repository.RepositoryDirPath, $"{file}.json") + ); + JArray array = JArray.Parse(fileReader.ReadToEnd()); + + foreach (JToken item in array) + { + string? id = item.SelectToken("id")?.ToString(); + + if ( + id is null + || !fileReferences.TryGetValue(id, out AonprdCompendiumItem? reference) + ) + { + continue; + } + + if (type is not null && reference.Type != type) + { + continue; + } + + yield return new(reference, (JObject)item); + } + } + } } diff --git a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Compendium/AonprdReference.cs b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Compendium/AonprdCompendiumItem.cs similarity index 84% rename from Pf2eDataScrapper/Importers/Implementations/Aonprd/Compendium/AonprdReference.cs rename to Pf2eDataScrapper/Importers/Implementations/Aonprd/Compendium/AonprdCompendiumItem.cs index c1fcd51..695929b 100644 --- a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Compendium/AonprdReference.cs +++ b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Compendium/AonprdCompendiumItem.cs @@ -5,22 +5,22 @@ /// /// Идентификатор сущности. /// Имя файла. -internal class AonprdReference(string id, string file) +internal class AonprdCompendiumItem(string id, string file) : AbstractCompendiumItem { /// /// Идентификатор сущности. /// - public readonly string Id = id; + public override string Id { get; } = id; /// /// Тип сущности. /// - public readonly string Type = ResolveType(id); + public override string Type { get; } = ResolveType(id); /// /// Имя файла, в котором хранится сущность. /// - public readonly string File = file; + public string File { get; } = file; /// /// Определение типа сущности как префикса идентификатора. diff --git a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Compendium/AonprdType.cs b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Compendium/AonprdType.cs deleted file mode 100644 index e55f1b8..0000000 --- a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Compendium/AonprdType.cs +++ /dev/null @@ -1,41 +0,0 @@ -namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Compendium; - -/// -/// Класс типа элементов с Архива Нефиса. -/// -internal class AonprdType(string type) -{ - /// - /// Идентификатор типа данных. - /// - public readonly string Type = type; - - /// - /// Список ссылок на сущности, связанных с этим типом. - /// - public readonly List Items = []; - - public AonprdType(AonprdReference reference) - : this(reference.Type) - { - Items.Add(reference); - } - - /// - /// Добавляет новую ссылку в список. - /// - public void Add(AonprdReference reference) - { - Items.Add(reference); - } - - /// - /// Возвращает уникальные файлы, использованные в ссылках. - /// - public List Files => Items.Select(x => x.File).Distinct().ToList(); - - /// - /// Число элементов в списке ссылок. - /// - public int Count => Items.Count; -} diff --git a/Pf2eDataScrapper/Importers/Implementations/Aonprd/AonprdProcessorInput.cs b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Helpers/AonprdProcessorInputExtension.cs similarity index 83% rename from Pf2eDataScrapper/Importers/Implementations/Aonprd/AonprdProcessorInput.cs rename to Pf2eDataScrapper/Importers/Implementations/Aonprd/Helpers/AonprdProcessorInputExtension.cs index 31b6bcd..ed5f530 100644 --- a/Pf2eDataScrapper/Importers/Implementations/Aonprd/AonprdProcessorInput.cs +++ b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Helpers/AonprdProcessorInputExtension.cs @@ -1,17 +1,11 @@ -using Newtonsoft.Json.Linq; -using Pf2eDataScrapper.Importers.Implementations.Aonprd.Helpers; -using Pf2eDataScrapper.Processors; +using Pf2eDataScrapper.Processors; -namespace Pf2eDataScrapper.Importers.Implementations.Aonprd; +namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Helpers; -/// -/// Класс для работы с JSON-объектом при обработки данных с Архивов Нефиса. -/// -/// JSON-объект, который будет использоваться для доступа к данным. -internal class AonprdProcessorInput(string name, JObject input) : ProcessorInput(name, input) +internal static class AonprdProcessorInputExtension { - /// - public override T Enum(string value) + public static T Enum(this ProcessorInput input, string value) + where T : Enum { try { @@ -19,7 +13,7 @@ internal class AonprdProcessorInput(string name, JObject input) : ProcessorInput } catch { - return base.Enum(value); + return input.Enum(value); } } @@ -28,7 +22,7 @@ internal class AonprdProcessorInput(string name, JObject input) : ProcessorInput /// /// Входная строка с данными. /// Структура, хранящая информацию о количетсве действий. - public ActionStruct ResolveActionString(string input) + public static ActionStruct ResolveActionString(this ProcessorInput _, string input) { ActionStruct result = new(); diff --git a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Helpers/LegacyEntityHandler.cs b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Helpers/LegacyEntityHandler.cs index 240b5ff..24b378e 100644 --- a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Helpers/LegacyEntityHandler.cs +++ b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Helpers/LegacyEntityHandler.cs @@ -1,5 +1,4 @@ -using Pf2eDataScrapper.Importers.Implementations.Aonprd.DataBuilder.Processors; -using Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors; +using Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors; using Pf2eModel.Model.Base; using Pf2eModel.Model.Base.Source; using Pf2eModel.Model.Common.I18n; diff --git a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/AbstractEntityProcessor.cs b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/AbstractEntityProcessor.cs index c0ad18f..442eec0 100644 --- a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/AbstractEntityProcessor.cs +++ b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/AbstractEntityProcessor.cs @@ -27,7 +27,7 @@ namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors; /// Объект обработки вводимого JSON. /// internal abstract class AbstractEntityProcessor( - AonprdProcessorInput input, + ProcessorInput input, Pf2eDbContext context, StringHelper stringHelper, DatabaseHelper helper, diff --git a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/AbstractSubclassProcessor.cs b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/AbstractSubclassProcessor.cs index 03e0bc6..8b3a1aa 100644 --- a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/AbstractSubclassProcessor.cs +++ b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/AbstractSubclassProcessor.cs @@ -2,6 +2,7 @@ using Pf2eDataScrapper.Helpers; using Pf2eDataScrapper.Importers.Implementations.Aonprd.DataBuilder.Helpers; using Pf2eDataScrapper.Importers.Implementations.Aonprd.Helpers; +using Pf2eDataScrapper.Processors; using Pf2eModel; using Pf2eModel.Model.Base; using Pf2eModel.Model.Common.I18n; @@ -16,7 +17,7 @@ namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors; /// для которых указан атрибут . /// internal abstract class AbstractSubclassProcessor( - AonprdProcessorInput input, + ProcessorInput input, Pf2eDbContext context, StringHelper stringHelper, DatabaseHelper helper, diff --git a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Common/AonprdSchemaBuildProcessor.cs b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Common/AonprdSchemaBuildProcessor.cs deleted file mode 100644 index 8d09197..0000000 --- a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Common/AonprdSchemaBuildProcessor.cs +++ /dev/null @@ -1,170 +0,0 @@ -using Pf2eDataScrapper.App; -using Pf2eDataScrapper.Processors; -using Pf2eDataScrapper.SchemaBuilder; - -namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Common; - -[Processor( - "aonprd:schema", - Description = "Построение схемы данных", - Group = ProcessorGroupEnum.AonprdPreparation -)] -[ProcessorDependencies(["aonprd:repository"])] -internal class AonprdSchemaBuildProcessor( - AonprdImporter importer, - ProcessProgress progress, - ApplicationSettings settings -) : IProcessor -{ - public AonprdImporter Importer { get; } = importer; - public ProcessProgress Progress { get; } = progress; - public ApplicationSettings Settings { get; } = settings; - - /// - /// Загузчик данных с сайта Архивов Нефеса. - /// - public AonprdLoader Loader => Importer.Loader; - - /// - /// Дополнительные схемы данных. - /// - public readonly Dictionary Schemas = new() - { - ["_subclass"] = new() { Path = "_subclass" }, - }; - - /// - /// Построение схемы данных. - /// - public void Process() - { - Progress.Local.Reset(Loader.ItemCount); - - foreach (AonprdLoader.LoaderEntry entry in Loader.Entries()) - { - Progress.Local.Start(entry.Type); - GetSchema(entry).AddValue(entry.Json); - - if (_subclassSchemaTypes.Contains(entry.Type)) - { - Schemas["_subclass"].AddValue(entry.Json); - } - - Progress.Local.End(); - } - - Progress.Local.Complete(); - } - - /// - /// Сохранение постренных схем данных. - /// - public void Save() - { - string targetDir = Settings.OutputDir; - - targetDir = Path.Combine(targetDir, Loader.Importer.Slug); - - foreach (KeyValuePair kvp in Schemas) - { - SaveSchema(targetDir, kvp.Key, kvp.Value); - } - } - - /// - /// Список типов, являющихся подклассами. - /// - private readonly string[] _subclassSchemaTypes = - [ - "research-field", - "apparition", - "practice", - "instinct", - "muse", - "cause", - "tenet", - "doctrine", - "druidic-order", - "methodology", - "element", - "hybrid-study", - "mystery", - "conscious-mind", - "subconscious-mind", - "hunters-edge", - "racket", - "bloodline", - "style", - "implement", - "lesson", - "patron", - "arcane-school", - "arcane-thesis", - "way", - "innovation", - "ikon", - "epithet", - ]; - - /// - /// Выбор целевого элемента схемы данных для ввода. - /// - /// Входные данные. - /// Целевой элемент схемы данных. - private SchemaNode GetSchema(AonprdLoader.LoaderEntry entry) - { - string key = entry.Type; - - if (Schemas.TryGetValue(key, out SchemaNode? value)) - { - return value; - } - - value = new() - { - // Resolver = Resolver, - Path = key, - }; - Schemas[key] = value; - - return value; - } - - /// - /// Сохранение отдельной схемы данных. - /// - /// Целевой каталог. - /// Название схемы. - /// Сохраняемая схема. - protected void SaveSchema(string dir, string name, SchemaNode schema) - { - char[] invalids = Path.GetInvalidFileNameChars(); - string fileName = string.Join( - "_", - schema.Path.Split(invalids, StringSplitOptions.RemoveEmptyEntries) - ) - .TrimEnd('.'); - string path = Path.Combine(dir, fileName + ".yaml"); - - string? dirPath = Path.GetDirectoryName(path); - if (dirPath is not null && !Directory.Exists(dirPath)) - { - Directory.CreateDirectory(dirPath); - } - - using StreamWriter writer = new(path); - - schema.Beautify(); - - writer.WriteLine( - SchemaNode.FormatYaml( - new Dictionary() - { - ["name"] = name, - ["values"] = schema.Values.ToString(), - }, - schema.Types[0].GetYamlData() - ) - ); - } -} diff --git a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Common/AonprdRepositoryUpdateProcessor.cs b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Common/RepositoryUpdateProcessor.cs similarity index 51% rename from Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Common/AonprdRepositoryUpdateProcessor.cs rename to Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Common/RepositoryUpdateProcessor.cs index d4c921d..3117e5d 100644 --- a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Common/AonprdRepositoryUpdateProcessor.cs +++ b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Common/RepositoryUpdateProcessor.cs @@ -1,4 +1,5 @@ using Pf2eDataScrapper.App; +using Pf2eDataScrapper.Importers.Implementations.Aonprd.Compendium; using Pf2eDataScrapper.Processors; namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Common; @@ -8,19 +9,23 @@ namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Common; Description = "Обновление репозитория", Group = ProcessorGroupEnum.AonprdPreparation )] -internal class AonprdRepositoryUpdateProcessor(AonprdImporter importer, ProcessProgress progress) - : IProcessor +internal class RepositoryUpdateProcessor( + AonprdRepository repository, + AonprdCompendium compendium, + ProcessProgress progress +) : IProcessor { - private AonprdImporter Importer { get; } = importer; + public AonprdRepository Repository { get; } = repository; + public AonprdCompendium Compendium { get; } = compendium; public ProcessProgress Progress { get; } = progress; public void Process() { - Importer.Repository.Update(Progress); + Repository.Update(Progress); } public void Save() { - Importer.Loader.Prepare(); + Compendium.Load(); } } diff --git a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Common/SchemaBuildProcessor.cs b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Common/SchemaBuildProcessor.cs new file mode 100644 index 0000000..1b1820d --- /dev/null +++ b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Common/SchemaBuildProcessor.cs @@ -0,0 +1,92 @@ +using Pf2eDataScrapper.App; +using Pf2eDataScrapper.Importers.Implementations.Aonprd.Compendium; +using Pf2eDataScrapper.Processors; +using Pf2eDataScrapper.SchemaBuilder; + +namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Common; + +[Processor( + "aonprd:schema", + Description = "Построение схемы данных", + Group = ProcessorGroupEnum.AonprdPreparation +)] +[ProcessorDependencies(["aonprd:repository"])] +internal class SchemaBuildProcessor( + AonprdImporter importer, + AonprdCompendium compendium, + ProcessProgress progress, + ApplicationSettings settings +) : AbstractSchemaBuildProcessor +{ + /// + protected AonprdImporter Importer { get; } = importer; + + /// + protected override AonprdCompendium Compendium { get; } = compendium; + + /// + protected override ProcessProgress Progress { get; } = progress; + + /// + protected ApplicationSettings Settings { get; } = settings; + + /// + protected override string SavePath => Path.Combine(Settings.OutputDirPath, Importer.Slug); + + /// + protected override AonprdTypeResolver Resolver { get; } = new(); + + /// + /// Список типов, являющихся подклассами. + /// + private readonly string[] _subclassSchemaTypes = + [ + "research-field", + "apparition", + "practice", + "instinct", + "muse", + "cause", + "tenet", + "doctrine", + "druidic-order", + "methodology", + "element", + "hybrid-study", + "mystery", + "conscious-mind", + "subconscious-mind", + "hunters-edge", + "racket", + "bloodline", + "style", + "implement", + "lesson", + "patron", + "arcane-school", + "arcane-thesis", + "way", + "innovation", + "ikon", + "epithet", + ]; + + protected override ICollection GetSchemaNodes(AbstractCompendiumItem item) + { + List nodes = [item.Type]; + + // Нода подкласса + if (_subclassSchemaTypes.Contains(item.Type)) + { + nodes.Add("_subclass"); + } + + return nodes; + } +} + +/// +internal class AonprdTypeResolver : TypeResolver +{ + // +} diff --git a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Common/SkillProcessor.cs b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Common/SkillProcessor.cs index dcf3081..869aefc 100644 --- a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Common/SkillProcessor.cs +++ b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Common/SkillProcessor.cs @@ -13,7 +13,7 @@ namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Common; Description = "Создание цели внешних ссылок", Group = ProcessorGroupEnum.AonprdPreparation )] -internal class SkillProcessor(AonprdProcessorInput input, Pf2eDbContext context) +internal class SkillProcessor(ProcessorInput input, Pf2eDbContext context) : AbstractInputProcessor(input) { public Pf2eDbContext Context { get; } = context; diff --git a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Common/SourceProcessor.cs b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Common/SourceProcessor.cs index 3cea58e..80d7b26 100644 --- a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Common/SourceProcessor.cs +++ b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Common/SourceProcessor.cs @@ -13,7 +13,7 @@ namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Common; Group = ProcessorGroupEnum.AonprdPreparation )] [ProcessorDependencies(["aonprd:repository"])] -internal class SourceProcessor(AonprdProcessorInput input, Pf2eDbContext context) +internal class SourceProcessor(ProcessorInput input, Pf2eDbContext context) : AbstractInputProcessor(input) { public Pf2eDbContext Context { get; } = context; diff --git a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Subclasses/ApparitionProcessor.cs b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Subclasses/ApparitionProcessor.cs index c66bc42..4be5018 100644 --- a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Subclasses/ApparitionProcessor.cs +++ b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Subclasses/ApparitionProcessor.cs @@ -2,6 +2,7 @@ using Pf2eDataScrapper.Helpers; using Pf2eDataScrapper.Importers.Implementations.Aonprd.DataBuilder.Helpers; using Pf2eDataScrapper.Importers.Implementations.Aonprd.Helpers; +using Pf2eDataScrapper.Processors; using Pf2eModel; using Pf2eModel.Model.Base; using Pf2eModel.Model.Common; @@ -21,7 +22,7 @@ namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Subclasse ["aonprd:trait", "aonprd:source", "aonprd:class", "aonprd:spell", "aonprd:skill"] )] internal class ApparitionProcessor( - AonprdProcessorInput input, + ProcessorInput input, Pf2eDbContext context, StringHelper stringHelper, DatabaseHelper helper, diff --git a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/ActionProcessor.cs b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/ActionProcessor.cs index 86eac1d..ba385ac 100644 --- a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/ActionProcessor.cs +++ b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/ActionProcessor.cs @@ -2,6 +2,7 @@ using Pf2eDataScrapper.Helpers; using Pf2eDataScrapper.Importers.Implementations.Aonprd.DataBuilder.Helpers; using Pf2eDataScrapper.Importers.Implementations.Aonprd.Helpers; +using Pf2eDataScrapper.Processors; using Pf2eModel; using Pf2eModel.Model.Enum; @@ -15,7 +16,7 @@ namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Types; )] [ProcessorDependencies(["aonprd:trait", "aonprd:source"])] internal class ActionProcessor( - AonprdProcessorInput input, + ProcessorInput input, Pf2eDbContext context, StringHelper stringHelper, DatabaseHelper helper, diff --git a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/AncestryProcessor.cs b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/AncestryProcessor.cs index 841ab9a..526f4a3 100644 --- a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/AncestryProcessor.cs +++ b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/AncestryProcessor.cs @@ -3,6 +3,7 @@ using Pf2eDataScrapper.App; using Pf2eDataScrapper.Helpers; using Pf2eDataScrapper.Importers.Implementations.Aonprd.DataBuilder.Helpers; using Pf2eDataScrapper.Importers.Implementations.Aonprd.Helpers; +using Pf2eDataScrapper.Processors; using Pf2eModel; using Pf2eModel.Model.Enum; using Pf2eModel.Model.Structure; @@ -17,7 +18,7 @@ namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Types; )] [ProcessorDependencies(["aonprd:trait", "aonprd:source", "aonprd:language"])] internal class AncestryProcessor( - AonprdProcessorInput input, + ProcessorInput input, Pf2eDbContext context, StringHelper stringHelper, DatabaseHelper helper, diff --git a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/ArchetypeProcessor.cs b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/ArchetypeProcessor.cs index dcab3e8..17d848e 100644 --- a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/ArchetypeProcessor.cs +++ b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/ArchetypeProcessor.cs @@ -2,6 +2,7 @@ using Pf2eDataScrapper.Helpers; using Pf2eDataScrapper.Importers.Implementations.Aonprd.DataBuilder.Helpers; using Pf2eDataScrapper.Importers.Implementations.Aonprd.Helpers; +using Pf2eDataScrapper.Processors; using Pf2eModel; using Pf2eModel.Model.Enum; using Pf2eModel.Model.Structure; @@ -16,7 +17,7 @@ namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Types; )] [ProcessorDependencies(["aonprd:trait", "aonprd:source"])] internal class ArchetypeProcessor( - AonprdProcessorInput input, + ProcessorInput input, Pf2eDbContext context, StringHelper stringHelper, DatabaseHelper helper, diff --git a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/BackgroundProcessor.cs b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/BackgroundProcessor.cs index 82a266b..b720049 100644 --- a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/BackgroundProcessor.cs +++ b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/BackgroundProcessor.cs @@ -3,6 +3,7 @@ using Pf2eDataScrapper.App; using Pf2eDataScrapper.Helpers; using Pf2eDataScrapper.Importers.Implementations.Aonprd.DataBuilder.Helpers; using Pf2eDataScrapper.Importers.Implementations.Aonprd.Helpers; +using Pf2eDataScrapper.Processors; using Pf2eModel; using Pf2eModel.Model.Enum; using Pf2eModel.Model.Structure; @@ -18,7 +19,7 @@ namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Types; )] [ProcessorDependencies(["aonprd:trait", "aonprd:source"])] internal class BackgroundProcessor( - AonprdProcessorInput input, + ProcessorInput input, Pf2eDbContext context, StringHelper stringHelper, DatabaseHelper helper, diff --git a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/ClassProcessor.cs b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/ClassProcessor.cs index 86d2fe8..72fb8af 100644 --- a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/ClassProcessor.cs +++ b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/ClassProcessor.cs @@ -4,6 +4,7 @@ using Pf2eDataScrapper.App; using Pf2eDataScrapper.Helpers; using Pf2eDataScrapper.Importers.Implementations.Aonprd.DataBuilder.Helpers; using Pf2eDataScrapper.Importers.Implementations.Aonprd.Helpers; +using Pf2eDataScrapper.Processors; using Pf2eModel; using Pf2eModel.Model.Common; using Pf2eModel.Model.Common.I18n; @@ -22,7 +23,7 @@ namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Types; )] [ProcessorDependencies(["aonprd:trait", "aonprd:source", "aonprd:skill"])] internal partial class ClassProcessor( - AonprdProcessorInput input, + ProcessorInput input, Pf2eDbContext context, StringHelper stringHelper, DatabaseHelper helper, diff --git a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/FeatProcessor.cs b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/FeatProcessor.cs index d281bb4..b6ef9d6 100644 --- a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/FeatProcessor.cs +++ b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/FeatProcessor.cs @@ -3,6 +3,7 @@ using Pf2eDataScrapper.App; using Pf2eDataScrapper.Helpers; using Pf2eDataScrapper.Importers.Implementations.Aonprd.DataBuilder.Helpers; using Pf2eDataScrapper.Importers.Implementations.Aonprd.Helpers; +using Pf2eDataScrapper.Processors; using Pf2eModel; using Pf2eModel.Model.Enum; using Pf2eModel.Model.Structure; @@ -17,7 +18,7 @@ namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Types; )] [ProcessorDependencies(["aonprd:trait", "aonprd:source", "aonprd:archetype"])] internal class FeatProcessor( - AonprdProcessorInput input, + ProcessorInput input, Pf2eDbContext context, StringHelper stringHelper, DatabaseHelper helper, @@ -44,7 +45,7 @@ internal class FeatProcessor( ? FeatCategory.Class : FeatCategory.Uncategorized; - AonprdProcessorInput.ActionStruct action = Input.ResolveActionString( + AonprdProcessorInputExtension.ActionStruct action = Input.ResolveActionString( Input.GetOptional("actions") ?? "" ); Object.Actions = action.Actions; diff --git a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/LanguageProcessor.cs b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/LanguageProcessor.cs index d9a723a..8ddf42f 100644 --- a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/LanguageProcessor.cs +++ b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/LanguageProcessor.cs @@ -2,6 +2,7 @@ using Pf2eDataScrapper.Helpers; using Pf2eDataScrapper.Importers.Implementations.Aonprd.DataBuilder.Helpers; using Pf2eDataScrapper.Importers.Implementations.Aonprd.Helpers; +using Pf2eDataScrapper.Processors; using Pf2eModel; using Pf2eModel.Model.Enum; using Pf2eModel.Model.Structure; @@ -16,7 +17,7 @@ namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Types; )] [ProcessorDependencies(["aonprd:trait", "aonprd:source"])] internal class LanguageProcessor( - AonprdProcessorInput input, + ProcessorInput input, Pf2eDbContext context, StringHelper stringHelper, DatabaseHelper helper, diff --git a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/RitualProcessor.cs b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/RitualProcessor.cs index 7e9b611..0a45e36 100644 --- a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/RitualProcessor.cs +++ b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/RitualProcessor.cs @@ -4,6 +4,7 @@ using Pf2eDataScrapper.App; using Pf2eDataScrapper.Helpers; using Pf2eDataScrapper.Importers.Implementations.Aonprd.DataBuilder.Helpers; using Pf2eDataScrapper.Importers.Implementations.Aonprd.Helpers; +using Pf2eDataScrapper.Processors; using Pf2eModel; using Pf2eModel.Model.Common.I18n; using Pf2eModel.Model.Enum; @@ -20,7 +21,7 @@ namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Types; )] [ProcessorDependencies(["aonprd:trait", "aonprd:source", "aonprd:skill"])] internal partial class RitualProcessor( - AonprdProcessorInput input, + ProcessorInput input, Pf2eDbContext context, StringHelper stringHelper, DatabaseHelper helper, diff --git a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/Rules/RuleProcessor.cs b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/Rules/RuleProcessor.cs index b3f4e53..aeb25fe 100644 --- a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/Rules/RuleProcessor.cs +++ b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/Rules/RuleProcessor.cs @@ -3,6 +3,7 @@ using Pf2eDataScrapper.App; using Pf2eDataScrapper.Helpers; using Pf2eDataScrapper.Importers.Implementations.Aonprd.DataBuilder.Helpers; using Pf2eDataScrapper.Importers.Implementations.Aonprd.Helpers; +using Pf2eDataScrapper.Processors; using Pf2eModel; using Pf2eModel.Model.Base.Source; using Pf2eModel.Model.Base.Targets; @@ -20,7 +21,7 @@ namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Types.Rul )] [ProcessorDependencies(["aonprd:trait", "aonprd:source", "aonprd:rules-toc"])] internal class RuleProcessor( - AonprdProcessorInput input, + ProcessorInput input, Pf2eDbContext context, StringHelper stringHelper, DatabaseHelper helper, diff --git a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/Rules/RulesOrderProcessor.cs b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/Rules/RulesOrderProcessor.cs index 7a1d6f0..200bae7 100644 --- a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/Rules/RulesOrderProcessor.cs +++ b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/Rules/RulesOrderProcessor.cs @@ -5,17 +5,24 @@ // так что ставить порядок ВСЕХ правил в архиве не получится, только тех, что есть в содержании. // Ну а пока просто расставляем родительские и дочерние элементы. -using System.Data; +using Pf2eDataScrapper.App; using Pf2eDataScrapper.Importers.Implementations.Aonprd.Helpers; using Pf2eDataScrapper.Processors; -using Pf2eModel.Model.Structure; +using Pf2eModel; using Rule = Pf2eModel.Model.Structure.Rule; -namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.DataBuilder.Processors.Types.Rules; +namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Types.Rules; -[AonprdProcessor("rules-order", null, ["rules"], false)] -internal class RulesOrderProcessor(AonprdProcessorInput input) : AbstractProcessor(input) +[Processor( + "aonprd:rules-order", + Description = "Создание цели внешних ссылок", + Group = ProcessorGroupEnum.AonprdEntities +)] +[ProcessorDependencies(["aonprd:rules"])] +internal class RulesOrderProcessor(Pf2eDbContext context) : IProcessor { + public Pf2eDbContext Context { get; } = context; + /// /// Структура генерации содержания правил. /// @@ -27,7 +34,7 @@ internal class RulesOrderProcessor(AonprdProcessorInput input) : AbstractProcess private readonly List _orderItems = []; /// - public override void Process() + public void Process() { Toc.SaveFile(); @@ -149,7 +156,7 @@ internal class RulesOrderProcessor(AonprdProcessorInput input) : AbstractProcess } /// - public override void Save() + public void Save() { Context.SaveChanges(); } diff --git a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/Rules/RulesTocProcessor.cs b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/Rules/RulesTocProcessor.cs index 9ecc12d..c062fa2 100644 --- a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/Rules/RulesTocProcessor.cs +++ b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/Rules/RulesTocProcessor.cs @@ -2,15 +2,24 @@ using System.Web; using AngleSharp; using AngleSharp.Dom; +using Pf2eDataScrapper.App; using Pf2eDataScrapper.Importers.Implementations.Aonprd.Helpers; using Pf2eDataScrapper.Processors; +using Pf2eModel; using Pf2eModel.Model.Base.Source; -namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.DataBuilder.Processors.Types.Rules; +namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Types.Rules; -[AonprdProcessor("rules-toc", null, ["source"], false)] -internal partial class RulesTocProcessor(AonprdProcessorInput input) : AbstractProcessor(input) +[Processor( + "aonprd:rules-toc", + Description = "Создание цели внешних ссылок", + Group = ProcessorGroupEnum.AonprdEntities +)] +[ProcessorDependencies(["aonprd:source"])] +internal partial class RulesTocProcessor(Pf2eDbContext context) : IProcessor { + public Pf2eDbContext Context { get; } = context; + /// /// Селектор HTML-элемента, в котором находится содержание. /// @@ -27,7 +36,7 @@ internal partial class RulesTocProcessor(AonprdProcessorInput input) : AbstractP public readonly RuleTocCollection Toc = RuleTocCollection.Instance; /// - public override void Process() + public void Process() { IDocument document = DownloadPage(); @@ -121,7 +130,7 @@ internal partial class RulesTocProcessor(AonprdProcessorInput input) : AbstractP } /// - public override void Save() { } + public void Save() { } /// /// Получить скомпилированное регулярное выражение для получения HTML-ссылки. diff --git a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/SpellProcessor.cs b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/SpellProcessor.cs index 27573de..3a427b2 100644 --- a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/SpellProcessor.cs +++ b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/SpellProcessor.cs @@ -3,6 +3,7 @@ using Pf2eDataScrapper.App; using Pf2eDataScrapper.Helpers; using Pf2eDataScrapper.Importers.Implementations.Aonprd.DataBuilder.Helpers; using Pf2eDataScrapper.Importers.Implementations.Aonprd.Helpers; +using Pf2eDataScrapper.Processors; using Pf2eModel; using Pf2eModel.Model.Common.I18n; using Pf2eModel.Model.Enum; @@ -19,7 +20,7 @@ namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Types; )] [ProcessorDependencies(["aonprd:trait", "aonprd:source"])] internal class SpellProcessor( - AonprdProcessorInput input, + ProcessorInput input, Pf2eDbContext context, StringHelper stringHelper, DatabaseHelper helper, diff --git a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/TraitProcessor.cs b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/TraitProcessor.cs index 9ac249e..9809b7f 100644 --- a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/TraitProcessor.cs +++ b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/TraitProcessor.cs @@ -2,6 +2,7 @@ using Pf2eDataScrapper.Helpers; using Pf2eDataScrapper.Importers.Implementations.Aonprd.DataBuilder.Helpers; using Pf2eDataScrapper.Importers.Implementations.Aonprd.Helpers; +using Pf2eDataScrapper.Processors; using Pf2eModel; using Pf2eModel.Model.Enum; using Pf2eModel.Model.Structure; @@ -16,7 +17,7 @@ namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Types; )] [ProcessorDependencies(["aonprd:external"])] internal class TraitProcessor( - AonprdProcessorInput input, + ProcessorInput input, Pf2eDbContext context, StringHelper stringHelper, DatabaseHelper helper, diff --git a/Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/Compendium/CompendiumDocumentType.cs b/Pf2eDataScrapper/Importers/Implementations/FoundryEn/Compendium/CompendiumDocumentType.cs similarity index 75% rename from Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/Compendium/CompendiumDocumentType.cs rename to Pf2eDataScrapper/Importers/Implementations/FoundryEn/Compendium/CompendiumDocumentType.cs index 72928df..3a7acdc 100644 --- a/Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/Compendium/CompendiumDocumentType.cs +++ b/Pf2eDataScrapper/Importers/Implementations/FoundryEn/Compendium/CompendiumDocumentType.cs @@ -1,4 +1,4 @@ -namespace Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.Compendium; +namespace Pf2eDataScrapper.Importers.Implementations.FoundryEn.Compendium; /// /// Тип документов компендиума. diff --git a/Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/Compendium/CompendiumFolder.cs b/Pf2eDataScrapper/Importers/Implementations/FoundryEn/Compendium/CompendiumFolder.cs similarity index 84% rename from Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/Compendium/CompendiumFolder.cs rename to Pf2eDataScrapper/Importers/Implementations/FoundryEn/Compendium/CompendiumFolder.cs index e51fe04..49f1f15 100644 --- a/Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/Compendium/CompendiumFolder.cs +++ b/Pf2eDataScrapper/Importers/Implementations/FoundryEn/Compendium/CompendiumFolder.cs @@ -1,6 +1,6 @@ using Newtonsoft.Json.Linq; -namespace Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.Compendium; +namespace Pf2eDataScrapper.Importers.Implementations.FoundryEn.Compendium; /// /// Папка компендиума. @@ -10,32 +10,32 @@ internal class CompendiumFolder /// /// Идентификатор папки. /// - public readonly string Id; + public string Id { get; } /// /// Название папки. /// - public readonly string Name; + public string Name { get; } /// /// Путь к папке. /// - public readonly string? Path; + public string? Path { get; } /// /// Указатель, что папка содержится в других папках. /// - public readonly bool IsContained; - - /// - /// Родительская папка. - /// - public CompendiumFolder? Parent; + public bool IsContained { get; } /// /// Тип документов в папке. /// - public readonly CompendiumDocumentType Type; + public CompendiumDocumentType Type { get; } + + /// + /// Родительская папка. + /// + public CompendiumFolder? Parent { get; set; } = null; /// /// Конструктор папки компендиума из JSON-объекта. @@ -60,8 +60,7 @@ internal class CompendiumFolder IsContained = false; } - Type = (CompendiumDocumentType) - Enum.Parse(typeof(CompendiumDocumentType), json["type"]!.ToString()); + Type = Enum.Parse(json["type"]!.ToString()); } /// diff --git a/Pf2eDataScrapper/Importers/Implementations/FoundryEn/Compendium/CompendiumFolderCollection.cs b/Pf2eDataScrapper/Importers/Implementations/FoundryEn/Compendium/CompendiumFolderCollection.cs new file mode 100644 index 0000000..69d0bf0 --- /dev/null +++ b/Pf2eDataScrapper/Importers/Implementations/FoundryEn/Compendium/CompendiumFolderCollection.cs @@ -0,0 +1,31 @@ +namespace Pf2eDataScrapper.Importers.Implementations.FoundryEn.Compendium; + +/// +/// Коллекция папок компендиума +/// +internal class CompendiumFolderCollection +{ + /// + /// Список загруженных папок. + /// + public List Items { get; } = []; + + /// + /// Добавление информации о папке. + /// + /// Добавляемая папка. + public void Add(CompendiumFolder folder) + { + Items.Add(folder); + } + + /// + /// Получение папки по идентификатору + /// + /// Искомый идентификатор + /// Искомая папка + public CompendiumFolder? GetById(string id) + { + return Items.FirstOrDefault(x => x.Id == id); + } +} diff --git a/Pf2eDataScrapper/Importers/Implementations/FoundryEn/Compendium/FoundryEnCompendiumItem.cs b/Pf2eDataScrapper/Importers/Implementations/FoundryEn/Compendium/FoundryEnCompendiumItem.cs new file mode 100644 index 0000000..0e4fa95 --- /dev/null +++ b/Pf2eDataScrapper/Importers/Implementations/FoundryEn/Compendium/FoundryEnCompendiumItem.cs @@ -0,0 +1,25 @@ +namespace Pf2eDataScrapper.Importers.Implementations.FoundryEn.Compendium; + +/// +/// Ссылка на сущность с указанием файла, в котором она хранится. +/// +/// Идентификатор сущности. +/// Тип сущности. +/// Имя файла, в котором хранится сущность. +internal class FoundryEnCompendiumItem(string id, string type, string file) : AbstractCompendiumItem +{ + /// + /// Идентификатор сущности. + /// + public override string Id { get; } = id; + + /// + /// Тип сущности. + /// + public override string Type { get; } = type; + + /// + /// Имя файла, в котором хранится сущность. + /// + public string File { get; } = file; +} diff --git a/Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/DataBuilder/AbstractProcessor.cs b/Pf2eDataScrapper/Importers/Implementations/FoundryEn/DataBuilder/FoundryEnAbstractProcessor.cs similarity index 91% rename from Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/DataBuilder/AbstractProcessor.cs rename to Pf2eDataScrapper/Importers/Implementations/FoundryEn/DataBuilder/FoundryEnAbstractProcessor.cs index 709ae64..95d8c2a 100644 --- a/Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/DataBuilder/AbstractProcessor.cs +++ b/Pf2eDataScrapper/Importers/Implementations/FoundryEn/DataBuilder/FoundryEnAbstractProcessor.cs @@ -13,15 +13,17 @@ using Pf2eModel.Model.Common.I18n; using Pf2eModel.Model.Enum; using Pf2eModel.Model.Structure; -namespace Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.DataBuilder; +namespace Pf2eDataScrapper.Importers.Implementations.FoundryEn.DataBuilder; -internal abstract class AbstractProcessor(ProcessorInput input) +internal abstract class FoundryEnAbstractProcessor( + DatabaseHelper databaseHelper, + Pf2eDbContext context, + ProcessorInput input +) : AbstractInputProcessor(input) { - protected DatabaseHelper Helper = DatabaseHelper.Instance; + protected DatabaseHelper Helper { get; } = databaseHelper; - public readonly Pf2eDbContext Context = DatabaseHelper.DbContext; - - public readonly ProcessorInput Input = input; + public Pf2eDbContext Context { get; } = context; public Entity Entity { get; set; } = new(); @@ -29,7 +31,7 @@ internal abstract class AbstractProcessor(ProcessorInput input) public abstract IEntity Object { get; } - public virtual void Process() + public override void Process() { Entity.Id = Helper.NextEntityId; Entity.Type = Type; @@ -95,7 +97,7 @@ internal abstract class AbstractProcessor(ProcessorInput input) } } - public void Save() + public override void Save() { Context.SaveChanges(); } diff --git a/Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/DataBuilder/Processors/ActionProcessor.cs b/Pf2eDataScrapper/Importers/Implementations/FoundryEn/DataBuilder/Processors/ActionProcessor.cs similarity index 62% rename from Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/DataBuilder/Processors/ActionProcessor.cs rename to Pf2eDataScrapper/Importers/Implementations/FoundryEn/DataBuilder/Processors/ActionProcessor.cs index 5e0d2b5..0438961 100644 --- a/Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/DataBuilder/Processors/ActionProcessor.cs +++ b/Pf2eDataScrapper/Importers/Implementations/FoundryEn/DataBuilder/Processors/ActionProcessor.cs @@ -1,11 +1,25 @@ using Newtonsoft.Json.Linq; +using Pf2eDataScrapper.App; +using Pf2eDataScrapper.Helpers; +using Pf2eDataScrapper.Importers.Implementations.FoundryEn.DataBuilder; using Pf2eDataScrapper.Processors; +using Pf2eModel; using Pf2eModel.Model.Enum; namespace Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.DataBuilder.Processors; -[Pf2eTypeProcessor("Item.action")] -internal class ActionProcessor(ProcessorInput input) : AbstractProcessor(input) +[Processor( + "foundry-en:action", + InnerType = "action", + Description = "Создание цели внешних ссылок", + Group = ProcessorGroupEnum.FoundryEnEntities +)] +[ProcessorDependencies(["foundry-en:compendium"])] +internal class ActionProcessor( + DatabaseHelper databaseHelper, + Pf2eDbContext context, + ProcessorInput input +) : FoundryEnAbstractProcessor(databaseHelper, context, input) { public override EntityType Type => EntityType.Action; diff --git a/Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/DataBuilder/Processors/ConditionProcessor.cs b/Pf2eDataScrapper/Importers/Implementations/FoundryEn/DataBuilder/Processors/ConditionProcessor.cs similarity index 61% rename from Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/DataBuilder/Processors/ConditionProcessor.cs rename to Pf2eDataScrapper/Importers/Implementations/FoundryEn/DataBuilder/Processors/ConditionProcessor.cs index acb7ca1..ce0accd 100644 --- a/Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/DataBuilder/Processors/ConditionProcessor.cs +++ b/Pf2eDataScrapper/Importers/Implementations/FoundryEn/DataBuilder/Processors/ConditionProcessor.cs @@ -1,12 +1,26 @@ using Newtonsoft.Json.Linq; +using Pf2eDataScrapper.App; +using Pf2eDataScrapper.Helpers; +using Pf2eDataScrapper.Importers.Implementations.FoundryEn.DataBuilder; using Pf2eDataScrapper.Processors; +using Pf2eModel; using Pf2eModel.Model.Enum; using Pf2eModel.Model.Structure; namespace Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.DataBuilder.Processors; -[Pf2eTypeProcessor("Item.condition")] -internal class ConditionProcessor(ProcessorInput input) : AbstractProcessor(input) +[Processor( + "foundry-en:condition", + InnerType = "Item.condition", + Description = "Создание цели внешних ссылок", + Group = ProcessorGroupEnum.FoundryEnEntities +)] +[ProcessorDependencies(["foundry-en:compendium"])] +internal class ConditionProcessor( + DatabaseHelper databaseHelper, + Pf2eDbContext context, + ProcessorInput input +) : FoundryEnAbstractProcessor(databaseHelper, context, input) { public override EntityType Type => EntityType.Condition; diff --git a/Pf2eDataScrapper/Importers/Implementations/FoundryEn/FoundryEnCompendium.cs b/Pf2eDataScrapper/Importers/Implementations/FoundryEn/FoundryEnCompendium.cs new file mode 100644 index 0000000..bf99a45 --- /dev/null +++ b/Pf2eDataScrapper/Importers/Implementations/FoundryEn/FoundryEnCompendium.cs @@ -0,0 +1,63 @@ +using Newtonsoft.Json.Linq; +using Pf2eDataScrapper.Importers.Implementations.FoundryEn.Compendium; + +namespace Pf2eDataScrapper.Importers.Implementations.FoundryEn; + +/// +/// Загрузчик данных компендиума. +/// +/// Репозиторий данных. +internal sealed class FoundryEnCompendium( + FoundryEnRepository repository, + CompendiumFolderCollection folders +) : AbstractCompendium +{ + /// + /// Репозиторий данных. + /// + public override FoundryEnRepository Repository { get; } = repository; + + /// + /// Набор папок компендиума + /// + public CompendiumFolderCollection Folders { get; } = folders; + + /// + /// Коллекция всех записей компендиума. + /// + public List Items { get; } = []; + + /// + /// Возвращает количество элементов в компендиуме. + /// + /// Количество элементов. + public override int Count() => Items.Count; + + /// + /// Возвращает количество элементов для указанного типа данных. + /// + /// Тип данных. + /// Количество элементов. + public override int Count(string type) => Items.Count(i => i.Type == type); + + /// + /// Получение итератора сущностей. Если указан тип - то только указанного типа. + /// + /// Тип сущностей, которые нужно получить. + /// Итератор сущностей. + public override IEnumerable> Iterate(string? type = null) + { + IEnumerable items = type is null + ? Items + : Items.Where(i => i.Type == type); + + foreach (FoundryEnCompendiumItem item in items) + { + using StreamReader fileReader = new( + Path.Combine(Repository.RepositoryDirPath, item.File) + ); + + yield return new(item, JObject.Parse(fileReader.ReadToEnd())); + } + } +} diff --git a/Pf2eDataScrapper/Importers/Implementations/FoundryEn/FoundryEnImporter.cs b/Pf2eDataScrapper/Importers/Implementations/FoundryEn/FoundryEnImporter.cs new file mode 100644 index 0000000..dc2bce3 --- /dev/null +++ b/Pf2eDataScrapper/Importers/Implementations/FoundryEn/FoundryEnImporter.cs @@ -0,0 +1,57 @@ +using Microsoft.Extensions.DependencyInjection; +using Pf2eDataScrapper.Importers.Implementations.FoundryEn.Compendium; +using Pf2eDataScrapper.Importers.Implementations.FoundryEn.DataBuilder; +using Pf2eDataScrapper.Importers.Repositories; + +namespace Pf2eDataScrapper.Importers.Implementations.FoundryEn; + +/// +/// Импортёр для модуля Pf2e FoundryVTT. +/// +internal class FoundryEnImporter(FoundryEnRepository repository, FoundryEnCompendium compendium) + : AbstractImporter +{ + /// + /// Название внешней ссылки. + /// + public const string TargetName = "Pf2e FoundryVTT module"; + + /// + /// Отображаемое имя импортёра. + /// + public override string Name => TargetName; + + /// + /// Короткое имя импортёра. + /// + public override string Slug => "foundry-en"; + + public override FoundryEnRepository Repository { get; } = repository; + + /// + /// Загрузчик данных компендиума. + /// + public override FoundryEnCompendium Compendium { get; } = compendium; + + #region "Внедрение зависимостей" + + public static void RegisterServices(IServiceCollection services) + { + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + + services.AddSingleton(); + } + + #endregion +} + +/// +/// Git-репозиторий данных FoundryVTT-модуля "pf2e". +/// +/// +internal class FoundryEnRepository(FoundryEnImporter importer) : GitRepository(importer, GitUrl) +{ + public const string GitUrl = "https://github.com/foundryvtt/pf2e.git"; +} diff --git a/Pf2eDataScrapper/Importers/Implementations/FoundryEn/Processors/Preparation/CompendiumFolderProcessor.cs b/Pf2eDataScrapper/Importers/Implementations/FoundryEn/Processors/Preparation/CompendiumFolderProcessor.cs new file mode 100644 index 0000000..a53b9a0 --- /dev/null +++ b/Pf2eDataScrapper/Importers/Implementations/FoundryEn/Processors/Preparation/CompendiumFolderProcessor.cs @@ -0,0 +1,123 @@ +using Newtonsoft.Json.Linq; +using Pf2eDataScrapper.App; +using Pf2eDataScrapper.Importers.Implementations.FoundryEn.Compendium; +using Pf2eDataScrapper.Processors; + +namespace Pf2eDataScrapper.Importers.Implementations.FoundryEn.Processors.Preparation; + +[Processor( + "foundry-en:compendium-folders", + Description = "Загрузка каталогов компендиума", + Group = ProcessorGroupEnum.FoundryEnPreparation +)] +[ProcessorDependencies(["foundry-en:repository"])] +internal class CompendiumFolderProcessor( + FoundryEnRepository repository, + CompendiumFolderCollection folders +) : IProcessor +{ + private FoundryEnRepository Repository { get; } = repository; + private CompendiumFolderCollection Folders { get; } = folders; + + private string Dir => Repository.RepositoryDirPath; + private int NextFolderId { get; set; } = 0; + + public void Process() + { + LoadPacks(); + ScanFolders(); + } + + /// + /// Загрузка основных папок. + /// + private void LoadPacks() + { + using StreamReader systemFile = new(Path.Combine(Dir, "static", "system.json")); + JObject systemData = JObject.Parse(systemFile.ReadToEnd()); + + foreach (JToken pack in systemData["packs"]!) + { + CompendiumFolder folder = new((JObject)pack); + + Folders.Add(folder); + } + + if (systemData.TryGetValue("packFolders", out JToken? packFolders)) + { + ProcessPackFolders((JArray)packFolders, null); + } + } + + /// + /// Обработка папок компендиума в поле `packFolders`. + /// + /// + /// + private void ProcessPackFolders(JArray folders, CompendiumFolder? parent) + { + foreach (JToken token in folders) + { + JObject obj = (JObject)token; + + string id = $"folder-{NextFolderId++}"; + string name = obj["name"]!.ToString(); + + CompendiumFolder folder = new(id, name) { Parent = parent }; + Folders.Add(folder); + + if (obj.TryGetValue("packs", out JToken? packs)) + { + foreach (JToken jToken in packs) + { + string packId = jToken.ToString(); + CompendiumFolder? child = Folders.GetById(packId); + + if (child is not null) + { + child.Parent = folder; + } + } + } + + if (obj.TryGetValue("folders", out JToken? innerFolders)) + { + ProcessPackFolders((JArray)innerFolders, folder); + } + } + } + + private void ScanFolders() + { + // Поиск _folders.json + foreach (CompendiumFolder parent in Folders.Items) + { + if (parent.Path is null) + { + continue; + } + + string dirPath = Path.Combine(Dir, parent.Path); + + foreach ( + string foldersFilePath in Directory.GetFiles( + dirPath, + "_folders.json", + SearchOption.AllDirectories + ) + ) + { + using StreamReader file = new(foldersFilePath); + JArray data = JArray.Parse(file.ReadToEnd()); + + foreach (JToken pack in data) + { + CompendiumFolder folder = new((JObject)pack) { Parent = parent }; + Folders.Add(folder); + } + } + } + } + + public void Save() { } +} diff --git a/Pf2eDataScrapper/Importers/Implementations/FoundryEn/Processors/Preparation/CompendiumItemsProcessor.cs b/Pf2eDataScrapper/Importers/Implementations/FoundryEn/Processors/Preparation/CompendiumItemsProcessor.cs new file mode 100644 index 0000000..8afe640 --- /dev/null +++ b/Pf2eDataScrapper/Importers/Implementations/FoundryEn/Processors/Preparation/CompendiumItemsProcessor.cs @@ -0,0 +1,104 @@ +using Newtonsoft.Json.Linq; +using Pf2eDataScrapper.App; +using Pf2eDataScrapper.Importers.Implementations.FoundryEn.Compendium; +using Pf2eDataScrapper.Processors; + +namespace Pf2eDataScrapper.Importers.Implementations.FoundryEn.Processors.Preparation; + +[Processor( + "foundry-en:compendium", + Description = "Загрузка компендиума", + Group = ProcessorGroupEnum.FoundryEnPreparation +)] +[ProcessorDependencies(["foundry-en:compendium-folders"])] +internal class CompendiumItemsProcessor( + FoundryEnRepository repository, + FoundryEnCompendium compendium, + ProcessProgress progress +) : IProcessor +{ + private FoundryEnRepository Repository { get; } = repository; + private FoundryEnCompendium Compendium { get; } = compendium; + public ProcessProgress Progress { get; } = progress; + + private string Dir => Repository.RepositoryDirPath; + private List RootFolders { get; } = + compendium.Folders.Items.FindAll(f => f.Path != null); + + public void Process() + { + Progress.Local.Reset(CountItems()); + + foreach (CompendiumFolder folder in RootFolders) + { + string dirPath = Path.Combine(Dir, folder.Path!); + + foreach ( + string filePath in Directory.GetFiles( + dirPath, + "*.json", + SearchOption.AllDirectories + ) + ) + { + string relativePath = Path.GetRelativePath(filePath, Dir); + Progress.Local.Start(relativePath); + + FoundryEnCompendiumItem? item = ProcessFile(relativePath, folder); + + if (item is not null) + { + Compendium.Items.Add(item); + } + + Progress.Local.End(); + } + } + + Progress.Local.Complete(); + } + + private FoundryEnCompendiumItem? ProcessFile(string relativePath, CompendiumFolder folder) + { + FileInfo file = new(Path.Combine(Dir, relativePath)); + + if (file.Name == "_folders.json") + { + return null; + } + + using StreamReader reader = new(file.FullName); + JObject itemJson = JObject.Parse(reader.ReadToEnd()); + + CompendiumFolder? itemFolder = folder; + + JToken? itemJsonFolder = itemJson.SelectToken("folder"); + if (itemJsonFolder is not null) + { + itemFolder = Compendium.Folders.GetById(itemJsonFolder.ToString()); + } + + if (itemFolder is null) + { + throw new( + $"Не удаётся определить папку, к которой относится элемент {file.Name} (путь: {file.FullName})" + ); + } + + string? type = itemJson.SelectToken("type")?.ToString(); + + return new(file.Name, itemFolder.Type + (type is not null ? $".{type}" : ""), relativePath); + } + + private int CountItems() + { + return RootFolders.Sum(folder => + { + string dirPath = Path.Combine(Dir, folder.Path!); + return Directory.GetFiles(dirPath, "*.json", SearchOption.AllDirectories).Length + - Directory.GetFiles(dirPath, "_folders.json", SearchOption.AllDirectories).Length; + }); + } + + public void Save() { } +} diff --git a/Pf2eDataScrapper/Importers/Implementations/FoundryEn/Processors/Preparation/RepositoryUpdateProcessor.cs b/Pf2eDataScrapper/Importers/Implementations/FoundryEn/Processors/Preparation/RepositoryUpdateProcessor.cs new file mode 100644 index 0000000..ccfb96f --- /dev/null +++ b/Pf2eDataScrapper/Importers/Implementations/FoundryEn/Processors/Preparation/RepositoryUpdateProcessor.cs @@ -0,0 +1,23 @@ +using Pf2eDataScrapper.App; +using Pf2eDataScrapper.Processors; + +namespace Pf2eDataScrapper.Importers.Implementations.FoundryEn.Processors.Preparation; + +[Processor( + "foundry-en:repository", + Description = "Обновление репозитория", + Group = ProcessorGroupEnum.FoundryEnPreparation +)] +internal class RepositoryUpdateProcessor(FoundryEnRepository repository, ProcessProgress progress) + : IProcessor +{ + public FoundryEnRepository Repository { get; } = repository; + public ProcessProgress Progress { get; } = progress; + + public void Process() + { + Repository.Update(Progress); + } + + public void Save() { } +} diff --git a/Pf2eDataScrapper/Importers/Implementations/FoundryEn/Processors/Preparation/SchemaBuildProcessor.cs b/Pf2eDataScrapper/Importers/Implementations/FoundryEn/Processors/Preparation/SchemaBuildProcessor.cs new file mode 100644 index 0000000..34b2cfc --- /dev/null +++ b/Pf2eDataScrapper/Importers/Implementations/FoundryEn/Processors/Preparation/SchemaBuildProcessor.cs @@ -0,0 +1,99 @@ +using Pf2eDataScrapper.App; +using Pf2eDataScrapper.Importers.Implementations.FoundryEn.Compendium; +using Pf2eDataScrapper.Processors; +using Pf2eDataScrapper.SchemaBuilder; + +namespace Pf2eDataScrapper.Importers.Implementations.FoundryEn.Processors.Preparation; + +[Processor( + "foundry-en:schema", + Description = "Построение схемы данных", + Group = ProcessorGroupEnum.AonprdPreparation +)] +[ProcessorDependencies(["foundry-en:compendium"])] +internal class SchemaBuildProcessor( + FoundryEnImporter importer, + FoundryEnRepository repository, + FoundryEnCompendium compendium, + ProcessProgress progress, + ApplicationSettings settings +) : AbstractSchemaBuildProcessor +{ + /// + protected FoundryEnImporter Importer { get; } = importer; + + /// + protected FoundryEnRepository Repository { get; } = repository; + + /// + protected override FoundryEnCompendium Compendium { get; } = compendium; + + /// + protected override ProcessProgress Progress { get; } = progress; + + /// + protected ApplicationSettings Settings { get; } = settings; + + /// + protected override string SavePath => Path.Combine(Settings.OutputDirPath, Importer.Slug); + + /// + /// Определитель типов данных. + /// + protected override SchemaTypeResolver Resolver { get; } = new(); + + /// + /// Список типов, являющихся экипировкой. + /// + private readonly string[] _equipmentTypes = + [ + "Item.equipment", + "Item.armor", + "Item.backpack", + "Item.consumable", + "Item.kit", + "Item.shield", + "Item.treasure", + "Item.weapon", + ]; + + /// + /// Выбор целевых элементом схемы данных для типа элемента. + /// + /// Тип входного элемента. + /// Список целевых элементов схемы данных. + private List GetSchemaNodes(string type) + { + List nodes = []; + + // Общая нода + if (!Schemas.TryGetValue(type, out SchemaNode? typeNode)) + { + typeNode = new() { Resolver = Resolver, Path = type }; + Schemas[type] = typeNode; + } + + nodes.Add(typeNode); + + // Нода подкласса + if (_equipmentTypes.Contains(type)) + { + nodes.Add(Schemas["Equipment"]); + } + + return nodes; + } + + protected override ICollection GetSchemaNodes(AbstractCompendiumItem item) + { + List nodes = [item.Type]; + + // Нода подкласса + if (_equipmentTypes.Contains(item.Type)) + { + nodes.Add("Equipment"); + } + + return nodes; + } +} diff --git a/Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/SchemaBuilder/CompendiumSchemaTypeResolver.cs b/Pf2eDataScrapper/Importers/Implementations/FoundryEn/Processors/Preparation/SchemaTypeResolver.cs similarity index 93% rename from Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/SchemaBuilder/CompendiumSchemaTypeResolver.cs rename to Pf2eDataScrapper/Importers/Implementations/FoundryEn/Processors/Preparation/SchemaTypeResolver.cs index b77f11b..e741c24 100644 --- a/Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/SchemaBuilder/CompendiumSchemaTypeResolver.cs +++ b/Pf2eDataScrapper/Importers/Implementations/FoundryEn/Processors/Preparation/SchemaTypeResolver.cs @@ -2,9 +2,9 @@ using Pf2eDataScrapper.SchemaBuilder; using Pf2eDataScrapper.SchemaBuilder.Types; -namespace Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.SchemaBuilder; +namespace Pf2eDataScrapper.Importers.Implementations.FoundryEn.Processors.Preparation; -internal class CompendiumSchemaTypeResolver : TypeResolver +internal class SchemaTypeResolver : TypeResolver { public class FoundryRulesSchemaType(SchemaNode parent) : IgnoreSchemaType(parent) { diff --git a/Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/Compendium/CompendiumFolderCollection.cs b/Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/Compendium/CompendiumFolderCollection.cs deleted file mode 100644 index 04c0381..0000000 --- a/Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/Compendium/CompendiumFolderCollection.cs +++ /dev/null @@ -1,26 +0,0 @@ -namespace Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.Compendium; - -/// -/// Коллекция папок компендиума -/// -internal class CompendiumFolderCollection : List -{ - /// - /// Получение папки по идентификатору - /// - /// Искомый идентификатор - /// Искомая папка - public CompendiumFolder? GetById(string id) - { - return this.FirstOrDefault(x => x.Id == id); - } - - /// - /// Фильтрация требуемых папок компендиума. - /// - /// - public CompendiumFolderCollection Filter() - { - return [.. this.FindAll(x => x.Type == CompendiumDocumentType.Item)]; - } -} diff --git a/Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/CompendiumLoader.cs b/Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/CompendiumLoader.cs deleted file mode 100644 index 7010191..0000000 --- a/Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/CompendiumLoader.cs +++ /dev/null @@ -1,178 +0,0 @@ -using Newtonsoft.Json.Linq; -using Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.Compendium; - -namespace Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry; - -/// -/// Загрузчик данных компендиума -/// -/// Импортёр -internal class CompendiumLoader(Pf2EFoundryImporter importer) -{ - /// - /// Импортёр - /// - public readonly Pf2EFoundryImporter Importer = importer; - - /// - /// Набор папок компендиума - /// - public CompendiumFolderCollection Collection = []; - - /// - /// Счётчик папок - /// - private int _folderCount; - - /// - /// Общее количество элементов в компендиуме - /// - public int ItemCount { get; protected set; } - - /// - /// Обработка папок компендиума в поле `packFolders`. - /// - /// - /// - protected void ProcessPackFolders(JArray folders, CompendiumFolder? parent) - { - foreach (JToken token in folders) - { - JObject obj = (JObject)token; - - string id = $"folder-{_folderCount++}"; - string name = obj["name"]!.ToString(); - - CompendiumFolder folder = new(id, name) { Parent = parent }; - Collection.Add(folder); - - if (obj.TryGetValue("packs", out JToken? packs)) - { - foreach (JToken jToken in packs) - { - string packId = jToken.ToString(); - CompendiumFolder? child = Collection.GetById(packId); - - if (child is not null) - { - child.Parent = folder; - } - } - } - - if (obj.TryGetValue("folders", out JToken? innerFolders)) - { - ProcessPackFolders((JArray)innerFolders, folder); - } - } - - Collection = Collection.Filter(); - } - - /// - /// Подготовка набора папок компендиума - /// - public void Prepare() - { - using StreamReader systemFile = new(Path.Combine(Importer.Dir, "static", "system.json")); - JObject systemData = JObject.Parse(systemFile.ReadToEnd()); - - foreach (JToken pack in systemData["packs"]!) - { - CompendiumFolder folder = new((JObject)pack); - - Collection.Add(folder); - } - - if (systemData.TryGetValue("packFolders", out JToken? packFolders)) - { - ProcessPackFolders((JArray)packFolders, null); - } - - // Поиск _folders.json - foreach (CompendiumFolder parent in Collection.ToList()) - { - if (parent.Path is null) - { - continue; - } - - string dirPath = Path.Combine(Importer.Dir, parent.Path); - - ItemCount += Directory.GetFiles(dirPath, "*.json", SearchOption.AllDirectories).Length; - - foreach ( - string foldersFilePath in Directory.GetFiles( - dirPath, - "_folders.json", - SearchOption.AllDirectories - ) - ) - { - ItemCount--; - using StreamReader file = new(foldersFilePath); - JArray data = JArray.Parse(file.ReadToEnd()); - - foreach (JToken pack in data) - { - CompendiumFolder folder = new((JObject)pack) { Parent = parent }; - Collection.Add(folder); - } - } - } - - // foreach (var folder in Collection) - // { - // Console.WriteLine(folder); - // } - - Collection = Collection.Filter(); - - Console.WriteLine($"Total items: {ItemCount}"); - } - - public IEnumerable Enrties() - { - foreach (CompendiumFolder folder in Collection.FindAll(f => f.Path is not null)) - { - string dirPath = Path.Combine(Importer.Dir, folder.Path!); - - foreach ( - string filePath in Directory.GetFiles( - dirPath, - "*.json", - SearchOption.AllDirectories - ) - ) - { - FileInfo file = new(filePath); - - if (file.Name == "_folders.json") - { - continue; - } - - using StreamReader reader = new(file.FullName); - JObject itemJson = JObject.Parse(reader.ReadToEnd()); - - CompendiumFolder? itemFolder = folder; - if (itemJson.TryGetValue("folder", out JToken? itemJsonFolder)) - { - itemFolder = Collection.GetById(itemJsonFolder.ToString()); - } - - yield return new LoaderEnrty(file, itemJson, itemFolder ?? folder); - } - } - } - - public class LoaderEnrty(FileInfo file, JObject itemJson, CompendiumFolder folder) - { - public FileInfo File { get; } = file; - public JObject Json { get; } = itemJson; - public CompendiumFolder Folder { get; } = folder; - public string Type { get; } = - itemJson["type"]?.ToString() - ?? throw new Exception($"Expected to have `type` field in file {file.FullName}"); - } -} diff --git a/Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/DataBuilder/PF2eFoundryDataBuilder.cs b/Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/DataBuilder/PF2eFoundryDataBuilder.cs deleted file mode 100644 index 16ca5e1..0000000 --- a/Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/DataBuilder/PF2eFoundryDataBuilder.cs +++ /dev/null @@ -1,88 +0,0 @@ -using System.Reflection; -using Konsole; -using Newtonsoft.Json.Linq; -using Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.Compendium; -using Pf2eDataScrapper.Processors; -using Pf2eModel; - -namespace Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.DataBuilder; - -internal class PF2eFoundryDataBuilder -{ - /// - /// Загрузчик данных. - /// - public readonly CompendiumLoader Loader; - - /// - /// Коллекция папок компендиума. - /// - public CompendiumFolderCollection Collection => Loader.Collection; - - public Dictionary Processors = []; - - public PF2eFoundryDataBuilder(CompendiumLoader loader) - { - Loader = loader; - PrepareProcessors(); - } - - public void PrepareProcessors() - { - foreach (Type type in Assembly.GetExecutingAssembly().GetTypes()) - { - if (type.GetCustomAttributes(typeof(Pf2eTypeProcessor), true).Length <= 0) - { - continue; - } - Pf2eTypeProcessor? attribute = type.GetCustomAttribute(); - Processors.Add(attribute!.Type, type.GetConstructor([typeof(ProcessorInput)])!); - } - } - - public AbstractProcessor? GetProcessor(string type, JObject input) - { - return Processors.TryGetValue(type, out ConstructorInfo? constructor) - ? (AbstractProcessor)constructor.Invoke([new ProcessorInput(input)]) - : null; - } - - public void TruncateTables() - { - Console.WriteLine("Truncating all old data..."); - - using Pf2eDbContext context = new(); - string cmd = context.Entities.Truncate() + "\n" + context.TruncateAll(); - - Console.WriteLine(cmd); - - Console.WriteLine("Truncation completed!"); - } - - public void Build() - { - Console.WriteLine("Building data..."); - - ProgressBar bar = new(Loader.ItemCount); - int i = 0; - - foreach (CompendiumLoader.LoaderEnrty entry in Loader.Enrties()) - { - bar.Refresh(i++, $"{entry.Folder.Name}/{entry.File.Name}"); - - AbstractProcessor? processor = GetProcessor( - $"{entry.Folder.Type}.{entry.Type}", - entry.Json - ); - if (processor is null) - { - continue; - } - - processor.Process(); - processor.Save(); - } - - bar.Refresh(Loader.ItemCount, "Done!"); - } -} diff --git a/Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/DataBuilder/Pf2eTypeProcessor.cs b/Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/DataBuilder/Pf2eTypeProcessor.cs deleted file mode 100644 index 7e5e37a..0000000 --- a/Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/DataBuilder/Pf2eTypeProcessor.cs +++ /dev/null @@ -1,9 +0,0 @@ -namespace Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.DataBuilder; - -[AttributeUsage(AttributeTargets.Class)] -internal class Pf2eTypeProcessor(string type, int priority = 0) : Attribute -{ - public readonly string Type = type; - - public readonly int Priority = priority; -} diff --git a/Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/Pf2eFoundryImporter.cs b/Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/Pf2eFoundryImporter.cs deleted file mode 100644 index b98dd8b..0000000 --- a/Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/Pf2eFoundryImporter.cs +++ /dev/null @@ -1,67 +0,0 @@ -using Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.DataBuilder; -using Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.SchemaBuilder; -using Pf2eDataScrapper.Importers.Repositories; - -namespace Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry; - -/// -/// Импортёр для модуля Pf2e FoundryVTT. -/// -internal class Pf2EFoundryImporter : AbstractImporter -{ - /// - /// Отображаемое имя импортёра. - /// - public override string Name => "Pf2e FoundryVTT module"; - - /// - /// Короткое имя импортёра. - /// - public override string Slug => "pf2e"; - - public override AbstractImporterRepository Repository => - new GitRepository(this, "https://github.com/foundryvtt/pf2e.git"); - - /// - /// Загрузчик данных компендиума. - /// - public CompendiumLoader Loader { get; } - - /// - /// Конструктор импортёра. - /// - public Pf2EFoundryImporter() - { - Loader = new CompendiumLoader(this); - } - - /// - /// Метод подготовки данных для импорта. - /// - public override void Prepare() - { - Loader.Prepare(); - } - - /// - /// Импорт данных. - /// - public override void Import() - { - PF2eFoundryDataBuilder builder = new(Loader); - - builder.TruncateTables(); - builder.Build(); - } - - /// - /// Построение схем данных - /// - public override void BuildSchemas() - { - CompendiumSchemaBuilder builder = new(Loader); - - builder.BuildSchemas(); - builder.Save(EnvSettings.OutputDir); - } -} diff --git a/Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/SchemaBuilder/CompendiumSchemaBuilder.cs b/Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/SchemaBuilder/CompendiumSchemaBuilder.cs deleted file mode 100644 index 01392f8..0000000 --- a/Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/SchemaBuilder/CompendiumSchemaBuilder.cs +++ /dev/null @@ -1,197 +0,0 @@ -using Konsole; -using Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.Compendium; -using Pf2eDataScrapper.SchemaBuilder; - -namespace Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.SchemaBuilder; - -/// -/// Класс построения схемы компендиумов. -/// -/// Загрузчик данных. -internal class CompendiumSchemaBuilder(CompendiumLoader loader) -{ - /// - /// Загрузчик данных. - /// - public CompendiumLoader Loader = loader; - - /// - /// Коллекция папок компендиума. - /// - public CompendiumFolderCollection Collection = loader.Collection; - - /// - /// Объект для определения типов полей в компендиумах PF2E. - /// - public CompendiumSchemaTypeResolver Resolver = new(); - - /// - /// Схемы данных по папкам. - /// - public Dictionary PackSchemas = []; - - /// - /// Схемы данных по типам. - /// - public Dictionary TypeSchemas = []; - - private readonly string[] _equipmentTypes = - [ - "equipment", - "armor", - "backpack", - "consumable", - "kit", - "shield", - "treasure", - "weapon", - ]; - - private SchemaNode GetPackSchema(CompendiumFolder folder) - { - if (!PackSchemas.TryGetValue(folder, out SchemaNode? value)) - { - value = new SchemaNode() { Resolver = Resolver, Path = folder.Name }; - PackSchemas[folder] = value; - } - - return value; - } - - private SchemaNode GetTypeSchema(string key) - { - if (!TypeSchemas.TryGetValue(key, out SchemaNode? value)) - { - value = new SchemaNode() { Resolver = Resolver, Path = key }; - TypeSchemas[key] = value; - } - - return value; - } - - /// - /// Получение набора схем, к которым относится проверяемый объект. - /// - /// Папка компендиума, в которой находится объект. - /// Указанный тип объекта. - /// - protected List GetItemSchemas(CompendiumFolder folder, string? itemType) - { - string key = folder.Type.ToString() + (itemType is not null ? $".{itemType}" : ""); - - return _equipmentTypes.Contains(itemType) - ? [GetPackSchema(folder), GetTypeSchema(key), GetTypeSchema("Equipment")] - : [GetPackSchema(folder), GetTypeSchema(key)]; - } - - /// - /// Построение схем данных. - /// - public void BuildSchemas() - { - Console.WriteLine("Building schemas..."); - - ProgressBar bar = new(Loader.ItemCount); - int i = 0; - - foreach (CompendiumLoader.LoaderEnrty entry in Loader.Enrties()) - { - bar.Refresh(i++, entry.Folder.Name); - - List schemas = GetItemSchemas(entry.Folder, entry.Type); - - foreach (SchemaNode schema in schemas) - { - schema.AddValue(entry.Json); - } - } - - bar.Refresh(Loader.ItemCount, "Done!"); - } - - /// - /// Сохранение данных. - /// - /// Папка для сохранения. - public void Save(string targetDir) - { - Console.WriteLine("Dumping schemas..."); - - ProgressBar bar = new(PackSchemas.Count + TypeSchemas.Count); - int i = 0; - - targetDir = Path.Combine(targetDir, Loader.Importer.Slug); - - if (!Directory.Exists(targetDir)) - { - Directory.CreateDirectory(targetDir); - } - Array.ForEach(Directory.GetFiles(@targetDir), File.Delete); - - // Packs - string packsDir = Path.Combine(targetDir, "packs"); - Directory.CreateDirectory(packsDir); - - foreach (KeyValuePair pair in PackSchemas) - { - bar.Refresh(i++, pair.Key.Name); - SaveSchema(packsDir, pair.Key.Name, pair.Value); - } - - // Types - string typesDir = Path.Combine(targetDir, "types"); - Directory.CreateDirectory(typesDir); - foreach (KeyValuePair pair in TypeSchemas) - { - bar.Refresh(i++, pair.Key); - SaveSchema(typesDir, pair.Key, pair.Value); - } - - Console.WriteLine("Done!"); - } - - /// - /// Сохранение конкретной схемы. - /// - /// Корневая папка для сохранения этой категории схем. - /// Название сохраняемой схемы. - /// Сохраняемая схема. - protected void SaveSchema(string dir, string name, SchemaNode schema) - { - char[] invalids = Path.GetInvalidFileNameChars(); - string fileName = string.Join( - "_", - schema.Path.Split(invalids, StringSplitOptions.RemoveEmptyEntries) - ) - .TrimEnd('.'); - - string path = Path.Combine(dir, fileName + ".yaml"); - - string? dirPath = Path.GetDirectoryName(path); - - if (dirPath is not null && !Directory.Exists(dirPath)) - { - Directory.CreateDirectory(dirPath); - } - - using StreamWriter writer = new(path); - - schema.Beautify(); - - writer.WriteLine( - SchemaNode.FormatYaml( - new Dictionary() - { - ["name"] = name, - ["values"] = schema.Values.ToString(), - }, - schema.Types[0].GetYamlData() - ) - ); - - //writer.WriteLine(name); - //writer.WriteLine($"Different values: {schema.Values}"); - //writer.WriteLine("====================================="); - //writer.Write(schema.ToString()); - } -} diff --git a/Pf2eDataScrapper/Processors/DatabaseTruncateProcessor.cs b/Pf2eDataScrapper/Processors/General/DatabaseTruncateProcessor.cs similarity index 89% rename from Pf2eDataScrapper/Processors/DatabaseTruncateProcessor.cs rename to Pf2eDataScrapper/Processors/General/DatabaseTruncateProcessor.cs index 9967454..ce8a65a 100644 --- a/Pf2eDataScrapper/Processors/DatabaseTruncateProcessor.cs +++ b/Pf2eDataScrapper/Processors/General/DatabaseTruncateProcessor.cs @@ -1,7 +1,7 @@ using Pf2eDataScrapper.App; using Pf2eModel; -namespace Pf2eDataScrapper.Processors; +namespace Pf2eDataScrapper.Processors.General; [Processor("truncate", Description = "Отчистка таблиц БД", Group = ProcessorGroupEnum.General)] internal class DatabaseTruncateProcessor : IProcessor diff --git a/Pf2eDataScrapper/Processors/ProcessorExecutor.cs b/Pf2eDataScrapper/Processors/ProcessorExecutor.cs new file mode 100644 index 0000000..bdb313a --- /dev/null +++ b/Pf2eDataScrapper/Processors/ProcessorExecutor.cs @@ -0,0 +1,106 @@ +using System; +using System.ComponentModel; +using System.Diagnostics; +using Microsoft.Extensions.DependencyInjection; +using Newtonsoft.Json.Linq; +using Pf2eDataScrapper.App; +using Pf2eDataScrapper.Importers; +using Pf2eDataScrapper.Importers.Implementations.Aonprd; +using Pf2eDataScrapper.Importers.Implementations.FoundryEn; + +namespace Pf2eDataScrapper.Processors; + +internal sealed class ProcessorExecutor( + IServiceProvider serviceProvider, + ProcessorCollection collection, + ProcessProgress progress +) +{ + public IServiceProvider ServiceProvider { get; } = serviceProvider; + public ProcessorCollection Collection { get; } = collection; + public ProcessProgress Progress { get; } = progress; + + public void Start() + { + Collection.SortItems(); + + Progress.Processors.Reset(Collection.Items.Count); + + foreach (ProcessorCollectionItem item in Collection.Items) + { + Progress.Processors.Start(item.Description); + + Process(item); + + Progress.Processors.End(); + } + + Progress.Processors.Complete(); + } + + private void Process(ProcessorCollectionItem item) + { + Thread thread = new(() => + { + foreach (IProcessor processor in IterateProcessors(item)) + { + processor.Process(); + processor.Save(); + } + }); + } + + private IEnumerable IterateProcessors(ProcessorCollectionItem item) + { + if (!item.IsCollectionProcessor) + { + Progress.Local.Reset(1); + Progress.Local.Start(item.Description); + yield return (IProcessor)ActivatorUtilities.CreateInstance(ServiceProvider, item.Type); + Progress.Local.Complete(); + yield break; + } + + if (item.Group == ProcessorGroupEnum.General) + { + throw new( + $"Обработчик {item.Description} помечен как общий (поле Group == General) и коллекционный (поле InnerType), что запрещено." + ); + } + + Type importerType = item.Group switch + { + ProcessorGroupEnum.AonprdPreparation + or ProcessorGroupEnum.AonprdEntities + or ProcessorGroupEnum.AonprdSubclasses => typeof(AonprdImporter), + ProcessorGroupEnum.FoundryEnPreparation or ProcessorGroupEnum.FoundryEnEntities => + typeof(FoundryEnImporter), + _ => throw new($"Не могу выбрать импортёр для группы {item.Group}"), + }; + + AbstractImporter importer = (AbstractImporter) + ActivatorUtilities.CreateInstance(ServiceProvider, importerType); + + Progress.Local.Reset(importer.Compendium.Count(item.InnerType!)); + + foreach ( + (AbstractCompendiumItem compendiumItem, JObject json) in importer.Compendium.Iterate( + item.InnerType! + ) + ) + { + Progress.Local.Start(compendiumItem.Id); + + yield return (IProcessor) + ActivatorUtilities.CreateInstance( + ServiceProvider, + item.Type, + new ProcessorInput(compendiumItem.Id, json) + ); + + Progress.Local.End(); + } + + Progress.Local.Complete(); + } +} diff --git a/Pf2eDataScrapper/Program.cs b/Pf2eDataScrapper/Program.cs index 328179e..c848b0e 100644 --- a/Pf2eDataScrapper/Program.cs +++ b/Pf2eDataScrapper/Program.cs @@ -1,4 +1,8 @@ -namespace Pf2eDataScrapper; +using Microsoft.Extensions.DependencyInjection; +using Pf2eDataScrapper.Importers.Implementations.Aonprd; +using Pf2eDataScrapper.Importers.Implementations.FoundryEn; + +namespace Pf2eDataScrapper; internal static class Program { @@ -6,7 +10,7 @@ internal static class Program /// The main entry point for the application. /// [STAThread] - static void Main() + public static void Main() { // To customize application configuration such as set high DPI settings or default font, // see https://aka.ms/applicationconfiguration. @@ -18,6 +22,22 @@ internal static class Program Application.SetColorMode(SystemColorMode.System); #pragma warning restore WFO5001 - Application.Run(new ImportApplication()); + IServiceCollection services = RegisterServices(); + + IServiceProvider serviceProvider = services.BuildServiceProvider(); + + Application.Run(ActivatorUtilities.CreateInstance(serviceProvider)); + } + + private static IServiceCollection RegisterServices() + { + IServiceCollection services = new ServiceCollection(); + + ImportApplication.RegisterServices(services); + + AonprdImporter.RegisterServices(services); + FoundryEnImporter.RegisterServices(services); + + return services; } } diff --git a/Pf2eDataScrapper/SchemaBuilder/AbstractSchemaBuildProcessor.cs b/Pf2eDataScrapper/SchemaBuilder/AbstractSchemaBuildProcessor.cs new file mode 100644 index 0000000..8e449d4 --- /dev/null +++ b/Pf2eDataScrapper/SchemaBuilder/AbstractSchemaBuildProcessor.cs @@ -0,0 +1,117 @@ +using Newtonsoft.Json.Linq; +using Pf2eDataScrapper.Importers; +using Pf2eDataScrapper.Processors; + +namespace Pf2eDataScrapper.SchemaBuilder; + +internal abstract class AbstractSchemaBuildProcessor : IProcessor +{ + /// + /// Связаный компендиум + /// + protected abstract AbstractCompendium Compendium { get; } + + /// + protected abstract ProcessProgress Progress { get; } + + /// + /// Определитель типов данных. + /// + protected abstract TypeResolver Resolver { get; } + + /// + /// Путь к каталогу для сохранения вывода. + /// + protected abstract string SavePath { get; } + + /// + /// Набор схем данных. + /// + protected Dictionary Schemas { get; } = []; + + /// + /// Получить список названий элементов структуры данных на основе вводимого элемента. + /// + /// Вводимый элемент + /// Коллекция названий элементов структуры данных/ + protected abstract ICollection GetSchemaNodes(AbstractCompendiumItem item); + + /// + /// Построение схемы данных. + /// + public void Process() + { + Progress.Local.Reset(Compendium.Count()); + + foreach ((AbstractCompendiumItem reference, JObject json) in Compendium.Iterate()) + { + Progress.Local.Start(reference.ToString()!); + + foreach (string nodeName in GetSchemaNodes(reference)) + { + if (!Schemas.TryGetValue(nodeName, out SchemaNode? node)) + { + node = new() { Path = nodeName, Resolver = Resolver }; + Schemas.Add(nodeName, node); + } + + node.AddValue(json); + } + + Progress.Local.End(); + } + + Progress.Local.Complete(); + } + + /// + /// Сохранение постренных схем данных. + /// + public void Save() + { + string targetDir = SavePath; + + foreach (KeyValuePair kvp in Schemas) + { + SaveSchema(targetDir, kvp.Key, kvp.Value); + } + } + + /// + /// Сохранение отдельной схемы данных. + /// + /// Целевой каталог. + /// Название схемы. + /// Сохраняемая схема. + protected void SaveSchema(string dir, string name, SchemaNode schema) + { + char[] invalids = Path.GetInvalidFileNameChars(); + string fileName = string.Join( + "_", + schema.Path.Split(invalids, StringSplitOptions.RemoveEmptyEntries) + ) + .TrimEnd('.'); + string path = Path.Combine(dir, fileName + ".yaml"); + + string? dirPath = Path.GetDirectoryName(path); + if (dirPath is not null && !Directory.Exists(dirPath)) + { + Directory.CreateDirectory(dirPath); + } + + using StreamWriter writer = new(path); + + schema.Beautify(); + + writer.WriteLine( + SchemaNode.FormatYaml( + new Dictionary() + { + ["name"] = name, + ["values"] = schema.Values.ToString(), + }, + schema.Types[0].GetYamlData() + ) + ); + } +}