Глобальное обновление #1
@@ -48,7 +48,7 @@ internal class ProcessorCollection
|
||||
/// <returns></returns>
|
||||
private ProcessorCollectionItem CreateItem(Type type)
|
||||
{
|
||||
if (!type.IsSubclassOf(typeof(IProcessor)))
|
||||
if (!type.IsAssignableTo(typeof(IProcessor)))
|
||||
{
|
||||
throw new(
|
||||
$"'ProcessorAttribute' attribute should only be applied to children of 'IProcessor' class. '{type.FullName}' is not."
|
||||
@@ -61,15 +61,6 @@ internal class ProcessorCollection
|
||||
|
||||
bool isCollection = processorAttribute.InnerType is not null;
|
||||
|
||||
ConstructorInfo? constructor = isCollection
|
||||
? type.GetConstructor([])
|
||||
: type.GetConstructor([typeof(ProcessorInput)])!;
|
||||
|
||||
if (constructor is null)
|
||||
{
|
||||
throw new($"Не удаётся определить конструктор класса {type}");
|
||||
}
|
||||
|
||||
return new()
|
||||
{
|
||||
Name = processorAttribute.Name,
|
||||
@@ -79,7 +70,6 @@ internal class ProcessorCollection
|
||||
Dependencies = dependenciesAttribute?.Dependencies ?? [],
|
||||
IsCollectionProcessor = isCollection,
|
||||
InnerType = processorAttribute.InnerType,
|
||||
Constructor = constructor,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -111,7 +101,7 @@ internal class ProcessorCollection
|
||||
{
|
||||
foreach (ProcessorCollectionItem item in Items)
|
||||
{
|
||||
if (item.Priority > 0)
|
||||
if (item.Priority >= 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
@@ -125,7 +115,10 @@ internal class ProcessorCollection
|
||||
continue;
|
||||
}
|
||||
|
||||
item.Priority = dependencyPriorities.Any() ? dependencyPriorities.Max() + 1 : 1;
|
||||
item.Priority = Math.Min(
|
||||
dependencyPriorities.Any() ? dependencyPriorities.Max() + 1 : 1,
|
||||
(int)item.Group * 100
|
||||
);
|
||||
priorities.Add(item.Name, item.Priority);
|
||||
}
|
||||
}
|
||||
@@ -215,9 +208,4 @@ internal class ProcessorCollectionItem
|
||||
/// Флаг, указывающий внутренний тип коллекции обрабатываемых элементов.
|
||||
/// </summary>
|
||||
public string? InnerType { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Конструктор обработчика.
|
||||
/// </summary>
|
||||
public required ConstructorInfo Constructor { get; init; }
|
||||
}
|
||||
|
||||
@@ -18,4 +18,7 @@ public enum ProcessorGroupEnum
|
||||
|
||||
[Description("Foundry EN: Подготовка")]
|
||||
FoundryEnPreparation,
|
||||
|
||||
[Description("Foundry EN: Основные сущности")]
|
||||
FoundryEnEntities,
|
||||
}
|
||||
|
||||
@@ -15,6 +15,10 @@ internal class ApplicationSettings
|
||||
public ApplicationSettings()
|
||||
{
|
||||
DotEnv.Load();
|
||||
|
||||
DataDirPath = EnvReader.GetStringValue("DATA_DIR");
|
||||
OutputDirPath = EnvReader.GetStringValue("OUTPUT_DIR");
|
||||
DatabaseConnection = EnvReader.GetStringValue("DATABASE_CONNECTION");
|
||||
}
|
||||
|
||||
#region Настройки из .env
|
||||
@@ -26,7 +30,7 @@ internal class ApplicationSettings
|
||||
[ReadOnly(true)]
|
||||
[DisplayName("Путь к папке с входными данными.")]
|
||||
[Description("Редактирование через .env-файл.")]
|
||||
public string DataDirPath { get; } = EnvReader.GetStringValue("DATA_DIR");
|
||||
public string DataDirPath { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Папка в входными данными.
|
||||
@@ -41,7 +45,7 @@ internal class ApplicationSettings
|
||||
[ReadOnly(true)]
|
||||
[DisplayName("Путь к папке для сохранения результатов.")]
|
||||
[Description("Редактирование через .env-файл.")]
|
||||
public string OutputDirPath { get; } = EnvReader.GetStringValue("OUTPUT_DIR");
|
||||
public string OutputDirPath { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Папка для сохранения результатов.
|
||||
@@ -56,7 +60,7 @@ internal class ApplicationSettings
|
||||
[ReadOnly(true)]
|
||||
[DisplayName("Строка подключения к базе данных.")]
|
||||
[Description("Редактирование через .env-файл.")]
|
||||
public string DatabaseConnection { get; } = EnvReader.GetStringValue("DATABASE_CONNECTION");
|
||||
public string DatabaseConnection { get; }
|
||||
|
||||
#endregion
|
||||
|
||||
@@ -74,7 +78,7 @@ internal class ApplicationSettings
|
||||
/// Список обработчиков, которые должны быть выполнены.
|
||||
/// Пустой список означает, что должны быть выполнены все обработчики.
|
||||
/// </summary>
|
||||
public ICollection<string> ProcessorNames { get; } = new List<string>();
|
||||
public ICollection<string> ProcessorNames { get; } = [];
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
+1
@@ -135,6 +135,7 @@ partial class StartForm
|
||||
StartButton.TabIndex = 0;
|
||||
StartButton.Text = "Запуск";
|
||||
StartButton.UseVisualStyleBackColor = true;
|
||||
StartButton.Click += StartButton_Click;
|
||||
//
|
||||
// StartForm
|
||||
//
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
using System.ComponentModel;
|
||||
using System.Reflection;
|
||||
using Newtonsoft.Json.Linq;
|
||||
using Pf2eDataScrapper.App;
|
||||
|
||||
namespace Pf2eDataScrapper.Forms;
|
||||
@@ -32,19 +33,14 @@ internal partial class StartForm : Form
|
||||
|
||||
ProcessorsListView.Groups.Clear();
|
||||
|
||||
foreach (FieldInfo field in typeof(ProcessorGroupEnum).GetFields())
|
||||
foreach (ProcessorGroupEnum group in Enum.GetValues<ProcessorGroupEnum>())
|
||||
{
|
||||
ProcessorGroupEnum groupEnum = (ProcessorGroupEnum)field.GetValue(null)!;
|
||||
|
||||
if (!items.TryGetValue(groupEnum, out List<ProcessorCollectionItem>? list))
|
||||
if (!items.TryGetValue(group, out List<ProcessorCollectionItem>? list))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
DescriptionAttribute? attribute = (DescriptionAttribute?)
|
||||
Attribute.GetCustomAttribute(field, typeof(DescriptionAttribute));
|
||||
|
||||
string name = attribute?.Description ?? field.Name;
|
||||
string name = Enum.GetDescription(group);
|
||||
|
||||
ListViewGroup listViewGroup = new(name);
|
||||
ProcessorsListView.Groups.Add(listViewGroup);
|
||||
@@ -61,4 +57,17 @@ internal partial class StartForm : Form
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#region "Кнопка старта"
|
||||
|
||||
private void StartButton_Click(object sender, EventArgs e)
|
||||
{
|
||||
StartButtonPressed?.Invoke();
|
||||
}
|
||||
|
||||
public delegate void StartButtonPressedDelegate();
|
||||
|
||||
public event StartButtonPressedDelegate? StartButtonPressed;
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
@@ -8,6 +8,16 @@ namespace Pf2eDataScrapper.Helpers;
|
||||
/// </summary>
|
||||
public class DatabaseHelper
|
||||
{
|
||||
/// <summary>
|
||||
/// Context для доступа к базе данных.
|
||||
/// </summary>
|
||||
public Pf2eDbContext Context { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Сервис для работы с URL-Slug. Используется для преобразования названий в URL-friendly формат.
|
||||
/// </summary>
|
||||
public SlugHelper Slugger { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Последний найденный идентификатор сущности.
|
||||
/// Используется для генерации новых ID сущностей.
|
||||
@@ -19,18 +29,10 @@ public class DatabaseHelper
|
||||
/// </summary>
|
||||
public int NextEntityId => ++_id;
|
||||
|
||||
/// <summary>
|
||||
/// Сервис для работы с URL-Slug. Используется для преобразования названий в URL-friendly формат.
|
||||
/// </summary>
|
||||
public readonly SlugHelper Slugger = new();
|
||||
|
||||
/// <summary>
|
||||
/// Context для доступа к базе данных.
|
||||
/// </summary>
|
||||
public readonly Pf2eDbContext Context = new();
|
||||
|
||||
private DatabaseHelper()
|
||||
private DatabaseHelper(Pf2eDbContext context, SlugHelper slugHelper)
|
||||
{
|
||||
Context = context;
|
||||
Slugger = slugHelper;
|
||||
_id = Context.Entities.Any() ? Context.Entities.Max(e => e.Id) : 0;
|
||||
}
|
||||
|
||||
@@ -41,24 +43,4 @@ public class DatabaseHelper
|
||||
{
|
||||
return Slugger.GenerateSlug(name);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Возвращает singleton-инстанцию класса <see cref="DatabaseHelper"/>.
|
||||
/// </summary>
|
||||
public static readonly DatabaseHelper Instance = new();
|
||||
|
||||
/// <summary>
|
||||
/// Контекст для доступа к базе данных.
|
||||
/// </summary>
|
||||
public static Pf2eDbContext DbContext => Instance.Context;
|
||||
|
||||
/// <summary>
|
||||
/// Обновляет контекст и сохраняет изменения в базе данных.
|
||||
/// </summary>
|
||||
public static void ResetContext()
|
||||
{
|
||||
Instance.Context.SaveChanges();
|
||||
// Instance.Context.Dispose();
|
||||
// Instance.Context = new();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,45 +1,37 @@
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Pf2eDataScrapper.App;
|
||||
using Pf2eDataScrapper.Forms;
|
||||
using Pf2eDataScrapper.Helpers;
|
||||
using Pf2eDataScrapper.Importers;
|
||||
using Pf2eDataScrapper.Importers.Implementations.Aonprd;
|
||||
using Pf2eDataScrapper.Importers.Implementations.FoundryEn;
|
||||
using Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry;
|
||||
using Pf2eDataScrapper.Processors;
|
||||
using Pf2eModel;
|
||||
using Slugify;
|
||||
|
||||
namespace Pf2eDataScrapper;
|
||||
|
||||
internal sealed class ImportApplication : ApplicationContext
|
||||
{
|
||||
public ImportApplication()
|
||||
public ImportApplication(
|
||||
IServiceProvider serviceProvider,
|
||||
ApplicationSettings settings,
|
||||
ProcessorCollection processors
|
||||
)
|
||||
{
|
||||
InitializeServices();
|
||||
ServiceProvider = serviceProvider;
|
||||
Settings = settings;
|
||||
Processors = processors;
|
||||
|
||||
CreateStartForm();
|
||||
}
|
||||
|
||||
#region "Внедрение зависимостей (DI)"
|
||||
|
||||
public readonly IServiceCollection Services = new ServiceCollection();
|
||||
|
||||
public IServiceProvider ServiceProvider { get; private set; } = null!;
|
||||
|
||||
private void InitializeServices()
|
||||
{
|
||||
Services.AddSingleton(this);
|
||||
Services.AddSingleton(Settings);
|
||||
Services.AddSingleton(Processors);
|
||||
|
||||
Services.AddDbContext<Pf2eDbContext>();
|
||||
|
||||
ServiceProvider = Services.BuildServiceProvider();
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region "Настройки"
|
||||
|
||||
public readonly ApplicationSettings Settings = new();
|
||||
public readonly ProcessorCollection Processors = new();
|
||||
public IServiceProvider ServiceProvider { get; }
|
||||
public ApplicationSettings Settings { get; }
|
||||
public ProcessorCollection Processors { get; }
|
||||
|
||||
#endregion
|
||||
|
||||
@@ -55,6 +47,7 @@ internal sealed class ImportApplication : ApplicationContext
|
||||
{
|
||||
_startForm = ActivatorUtilities.CreateInstance<StartForm>(ServiceProvider);
|
||||
_startForm.StartPosition = FormStartPosition.CenterParent;
|
||||
_startForm.StartButtonPressed += StartProcess;
|
||||
_startForm.Show();
|
||||
}
|
||||
|
||||
@@ -62,64 +55,39 @@ internal sealed class ImportApplication : ApplicationContext
|
||||
{
|
||||
_startForm!.Close();
|
||||
|
||||
Processors.SortItems();
|
||||
ProcessorExecutor executor = ActivatorUtilities.CreateInstance<ProcessorExecutor>(
|
||||
ServiceProvider
|
||||
);
|
||||
|
||||
ProcessProgress progress = new ProcessProgress();
|
||||
|
||||
foreach (ProcessorCollectionItem item in Processors.Items)
|
||||
{
|
||||
if (item.IsCollectionProcessor)
|
||||
{
|
||||
RunCollection(item, progress);
|
||||
}
|
||||
else
|
||||
{
|
||||
RunSingle(item, progress);
|
||||
}
|
||||
}
|
||||
executor.Start();
|
||||
}
|
||||
|
||||
private void RunSingle(ProcessorCollectionItem item, ProcessProgress progress)
|
||||
#endregion
|
||||
|
||||
#region "Внедрение зависимостей"
|
||||
|
||||
public static void RegisterServices(IServiceCollection services)
|
||||
{
|
||||
progress.Local.Reset(1);
|
||||
progress.Local.Start("");
|
||||
// Контекст базы данных
|
||||
services.AddDbContext<Pf2eDbContext>();
|
||||
|
||||
IProcessor processor = (IProcessor)
|
||||
ActivatorUtilities.CreateInstance(ServiceProvider, item.Type);
|
||||
// Основная программа
|
||||
services.AddSingleton<ImportApplication>();
|
||||
|
||||
processor.Process();
|
||||
processor.Save();
|
||||
// Настройки программы
|
||||
services.AddSingleton<ApplicationSettings>();
|
||||
|
||||
progress.Local.Complete();
|
||||
}
|
||||
// Набор процессоров
|
||||
services.AddSingleton<ProcessorCollection>();
|
||||
|
||||
private void RunCollection(ProcessorCollectionItem item, ProcessProgress progress)
|
||||
{
|
||||
AbstractImporter importer = item.Group switch
|
||||
{
|
||||
ProcessorGroupEnum.AonprdPreparation
|
||||
or ProcessorGroupEnum.AonprdEntities
|
||||
or ProcessorGroupEnum.AonprdSubclasses => new AonprdImporter(),
|
||||
ProcessorGroupEnum.FoundryEnPreparation => new Pf2EFoundryImporter(),
|
||||
_ => throw new("Can't recognize importer"),
|
||||
};
|
||||
// Помощник идентификации базы данных
|
||||
services.AddSingleton<DatabaseHelper>();
|
||||
|
||||
progress.Local.Reset(importer.CountInputs(item.InnerType!));
|
||||
// Помощник создание URL-совместимых названий
|
||||
services.AddTransient<SlugHelper>();
|
||||
|
||||
foreach (ProcessorInput input in importer.ResolveInputs(item.InnerType!))
|
||||
{
|
||||
progress.Local.Start(input.Name);
|
||||
|
||||
IProcessor processor = (IProcessor)
|
||||
ActivatorUtilities.CreateInstance(ServiceProvider, item.Type);
|
||||
|
||||
processor.Process();
|
||||
processor.Save();
|
||||
|
||||
progress.Local.End();
|
||||
}
|
||||
|
||||
progress.Local.Complete();
|
||||
// Счётчик прогресса
|
||||
services.AddSingleton<ProcessProgress>();
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
using Newtonsoft.Json.Linq;
|
||||
using Pf2eDataScrapper.Importers.Repositories;
|
||||
|
||||
namespace Pf2eDataScrapper.Importers;
|
||||
|
||||
/// <summary>
|
||||
/// Компендиум данных.
|
||||
/// </summary>
|
||||
internal abstract class AbstractCompendium
|
||||
{
|
||||
/// <summary>
|
||||
/// Репозиторий данных.
|
||||
/// </summary>
|
||||
public abstract AbstractImporterRepository Repository { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Путь к хранению исходных данных.
|
||||
/// </summary>
|
||||
public string Dir => Repository.RepositoryDirPath;
|
||||
|
||||
/// <summary>
|
||||
/// Возвращает количество элементов в компендиуме.
|
||||
/// </summary>
|
||||
/// <returns>Количество элементов.</returns>
|
||||
public abstract int Count();
|
||||
|
||||
/// <summary>
|
||||
/// Возвращает количество элементов для указанного типа данных.
|
||||
/// </summary>
|
||||
/// <param name="type">Тип данных.</param>
|
||||
/// <returns>Количество элементов.</returns>
|
||||
public abstract int Count(string type);
|
||||
|
||||
/// <summary>
|
||||
/// Получение итератора сущностей. Если указан тип <paramref name="type"/> - то только указанного типа.
|
||||
/// </summary>
|
||||
/// <param name="type">Тип сущностей, которые нужно получить.</param>
|
||||
/// <returns>Итератор сущностей.</returns>
|
||||
public abstract IEnumerable<Tuple<AbstractCompendiumItem, JObject>> Iterate(
|
||||
string? type = null
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
namespace Pf2eDataScrapper.Importers;
|
||||
|
||||
/// <summary>
|
||||
/// Абстрактная ссылка на сущность.
|
||||
/// </summary>
|
||||
internal abstract class AbstractCompendiumItem
|
||||
{
|
||||
/// <summary>
|
||||
/// Идентификатор сущности.
|
||||
/// </summary>
|
||||
public abstract string Id { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Тип сущности.
|
||||
/// </summary>
|
||||
public abstract string Type { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Строковое представление имени сущности.
|
||||
/// </summary>
|
||||
/// <returns>Строковое представление.</returns>
|
||||
public override string ToString() => Id;
|
||||
}
|
||||
@@ -1,5 +1,4 @@
|
||||
using Pf2eDataScrapper.Importers.Repositories;
|
||||
using Pf2eDataScrapper.Processors;
|
||||
|
||||
namespace Pf2eDataScrapper.Importers;
|
||||
|
||||
@@ -23,46 +22,13 @@ internal abstract class AbstractImporter
|
||||
/// </summary>
|
||||
public abstract AbstractImporterRepository Repository { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Компендиум данных импортёра.
|
||||
/// </summary>
|
||||
public abstract AbstractCompendium Compendium { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Путь к директории с данными для импортёра.
|
||||
/// </summary>
|
||||
public string Dir => Path.Combine(EnvSettings.DataDirPath, Slug);
|
||||
|
||||
/// <summary>
|
||||
/// Перечисление процессоров, которые будут использоваться для обработки данных.
|
||||
/// При пустом массиве будут обработаны все доступные процессоры.
|
||||
/// </summary>
|
||||
public string[] ProcessorNames { get; set; } = [];
|
||||
|
||||
/// <summary>
|
||||
/// Метод подготовки данных для импорта.
|
||||
/// </summary>
|
||||
[Obsolete]
|
||||
public abstract void Prepare();
|
||||
|
||||
/// <summary>
|
||||
/// Импорт данных
|
||||
/// </summary>
|
||||
[Obsolete]
|
||||
public abstract void Import();
|
||||
|
||||
/// <summary>
|
||||
/// Построение схем данных.
|
||||
/// </summary>
|
||||
[Obsolete]
|
||||
public abstract void BuildSchemas();
|
||||
|
||||
/// <summary>
|
||||
/// Получение количества записей, принаделажащих типу <paramref name="type"/>.
|
||||
/// </summary>
|
||||
/// <param name="type">Тип записей.</param>
|
||||
/// <returns>Количество записей.</returns>
|
||||
public abstract int CountInputs(string type);
|
||||
|
||||
/// <summary>
|
||||
/// Итератор ввода для записей, принаделажащих типу <paramref name="type"/>.
|
||||
/// </summary>
|
||||
/// <param name="type">Тип записей.</param>
|
||||
/// <returns>Итератор.</returns>
|
||||
public abstract IEnumerable<ProcessorInput> ResolveInputs(string type);
|
||||
}
|
||||
|
||||
@@ -1,15 +1,15 @@
|
||||
using Newtonsoft.Json.Linq;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Pf2eDataScrapper.Importers.Implementations.Aonprd.Compendium;
|
||||
using Pf2eDataScrapper.Importers.Implementations.Aonprd.DataBuilder;
|
||||
using Pf2eDataScrapper.Importers.Repositories;
|
||||
using Pf2eDataScrapper.Processors;
|
||||
using Pf2eDataScrapper.Importers.Implementations.Aonprd.DataBuilder.Helpers;
|
||||
using Pf2eDataScrapper.Importers.Implementations.Aonprd.Helpers;
|
||||
|
||||
namespace Pf2eDataScrapper.Importers.Implementations.Aonprd;
|
||||
|
||||
/// <summary>
|
||||
/// Класс обработки данных с сайта Архивов Нефеса.
|
||||
/// </summary>
|
||||
internal class AonprdImporter : AbstractImporter
|
||||
internal class AonprdImporter(AonprdRepository repository, AonprdCompendium compendium)
|
||||
: AbstractImporter
|
||||
{
|
||||
/// <summary>
|
||||
/// Название внешней ссылки.
|
||||
@@ -23,59 +23,24 @@ internal class AonprdImporter : AbstractImporter
|
||||
public override string Slug => "aonprd";
|
||||
|
||||
/// <inheritdoc/>
|
||||
public override AbstractImporterRepository Repository { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Инструмент для работы с данными сайта Архивов Нефеса.
|
||||
/// </summary>
|
||||
public AonprdLoader Loader { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Конструктор класса <see cref="AonprdImporter"/>.
|
||||
/// </summary>
|
||||
public AonprdImporter()
|
||||
{
|
||||
Repository = new AonprdRepository(this, "https://elasticsearch.aonprd.com");
|
||||
Loader = new(this);
|
||||
}
|
||||
public override AonprdRepository Repository { get; } = repository;
|
||||
|
||||
/// <inheritdoc/>
|
||||
public override void Prepare() { }
|
||||
public override AonprdCompendium Compendium { get; } = compendium;
|
||||
|
||||
/// <inheritdoc/>
|
||||
public override void BuildSchemas() { }
|
||||
#region "Внедрение зависимостей"
|
||||
|
||||
/// <inheritdoc/>
|
||||
public override void Import() { }
|
||||
|
||||
/// <inheritdoc/>
|
||||
public override int CountInputs(string type)
|
||||
public static void RegisterServices(IServiceCollection services)
|
||||
{
|
||||
return Loader.Compendium.Count(type);
|
||||
services.AddSingleton<AonprdImporter>();
|
||||
services.AddSingleton<AonprdRepository>();
|
||||
services.AddSingleton<AonprdCompendium>();
|
||||
|
||||
services.AddSingleton<AonprdBuilderGlobals>();
|
||||
services.AddSingleton<LegacyEntityHandler>();
|
||||
services.AddSingleton<RuleTocCollection>();
|
||||
services.AddTransient<StringHelper>();
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public override IEnumerable<ProcessorInput> ResolveInputs(string type)
|
||||
{
|
||||
foreach (string file in Loader.Compendium.Files(type))
|
||||
{
|
||||
using StreamReader fileReader = new(Path.Combine(Dir, $"{file}.json"));
|
||||
JArray array = JArray.Parse(fileReader.ReadToEnd());
|
||||
|
||||
foreach (JToken item in array)
|
||||
{
|
||||
string? id = item.SelectToken("id")?.ToString();
|
||||
|
||||
if (id is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (AonprdReference.ResolveType(id) == type)
|
||||
{
|
||||
yield return new AonprdProcessorInput(id, (JObject)item);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
#endregion
|
||||
}
|
||||
|
||||
@@ -1,105 +0,0 @@
|
||||
using Newtonsoft.Json.Linq;
|
||||
using Pf2eDataScrapper.Importers.Implementations.Aonprd.Compendium;
|
||||
|
||||
namespace Pf2eDataScrapper.Importers.Implementations.Aonprd;
|
||||
|
||||
/// <summary>
|
||||
/// Класс загрузки данных с сайта Архивов Нефеса.
|
||||
/// </summary>
|
||||
internal class AonprdLoader(AonprdImporter importer)
|
||||
{
|
||||
/// <summary>
|
||||
/// Референс на импортер, используемый для загрузки данных.
|
||||
/// </summary>
|
||||
public AonprdImporter Importer { get; } = importer;
|
||||
|
||||
/// <summary>
|
||||
/// Компендиум данных с сайта Архивов Нефеса.
|
||||
/// </summary>
|
||||
public AonprdCompendium Compendium { get; } = new();
|
||||
|
||||
/// <summary>
|
||||
/// Число элементов в компендарии.
|
||||
/// </summary>
|
||||
public int ItemCount => Compendium.Count();
|
||||
|
||||
/// <summary>
|
||||
/// Подготовка данных перед загрузкой.
|
||||
/// </summary>
|
||||
public void Prepare()
|
||||
{
|
||||
using StreamReader indexFile = new(Path.Combine(Importer.Dir, "index.json"));
|
||||
JObject indexJson = JObject.Parse(indexFile.ReadToEnd());
|
||||
|
||||
foreach ((string file, JToken? value) in indexJson)
|
||||
{
|
||||
JArray items = (JArray)value!;
|
||||
|
||||
foreach (JToken item in items)
|
||||
{
|
||||
Compendium.AddReference(item.ToString(), file);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Возвращает коллекцию всех доступных загружаемых элементов.
|
||||
/// </summary>
|
||||
/// <returns>Коллекция всех загружаемых элементов.</returns>
|
||||
public IEnumerable<LoaderEntry> Entries()
|
||||
{
|
||||
return Entries([.. Compendium.Types.Keys]);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Возвращает коллекцию доступных загружаемых элементов.
|
||||
/// </summary>
|
||||
/// <param name="types">Тип, которые нужно включить в результат.</param>
|
||||
/// <returns>Коллекция загружаемых элементов, соответствующих предоставленному типу.</returns>
|
||||
public IEnumerable<LoaderEntry> Entries(string types)
|
||||
{
|
||||
return Entries([types]);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Возвращает коллекцию доступных загружаемых элементов.
|
||||
/// </summary>
|
||||
/// <param name="types">Типы, которые нужно включить в результат.</param>
|
||||
/// <returns>Коллекция загружаемых элементов, соответствующих предоставленным типам.</returns>
|
||||
public IEnumerable<LoaderEntry> Entries(string[] types)
|
||||
{
|
||||
foreach (string file in Compendium.Files(types))
|
||||
{
|
||||
using StreamReader fileReader = new(Path.Combine(Importer.Dir, $"{file}.json"));
|
||||
JArray array = JArray.Parse(fileReader.ReadToEnd());
|
||||
|
||||
foreach (JToken item in array)
|
||||
{
|
||||
string? id = item.SelectToken("id")?.ToString();
|
||||
|
||||
if (id is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
LoaderEntry entry = new(id, file, (JObject)item);
|
||||
|
||||
if (types.Contains(entry.Type))
|
||||
{
|
||||
yield return entry;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Класс, представляющий загружаемый элемент.
|
||||
/// </summary>
|
||||
public class LoaderEntry(string id, string file, JObject itemJson) : AonprdReference(id, file)
|
||||
{
|
||||
/// <summary>
|
||||
/// Возвращает JSON-объект, содерждающий данные о элементе.
|
||||
/// </summary>
|
||||
public JObject Json { get; } = itemJson;
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,4 @@
|
||||
using Konsole;
|
||||
using Newtonsoft.Json.Linq;
|
||||
using Newtonsoft.Json.Linq;
|
||||
using Pf2eDataScrapper.Importers.Repositories;
|
||||
using Pf2eDataScrapper.Processors;
|
||||
|
||||
@@ -8,13 +7,12 @@ namespace Pf2eDataScrapper.Importers.Implementations.Aonprd;
|
||||
/// <summary>
|
||||
/// Основной класс для работы с репозиторием данных Aonprd.
|
||||
/// </summary>
|
||||
internal class AonprdRepository(AbstractImporter importer, string searchUrl)
|
||||
: AbstractImporterRepository(importer)
|
||||
internal class AonprdRepository(AonprdImporter importer) : AbstractImporterRepository(importer)
|
||||
{
|
||||
/// <summary>
|
||||
/// URL для доступа к API Aonprd.
|
||||
/// </summary>
|
||||
public readonly string SearchUrl = searchUrl;
|
||||
public readonly string SearchUrl = "https://elasticsearch.aonprd.com";
|
||||
|
||||
/// <summary>
|
||||
/// Текущая ревизия данных. Значение обновляется при обновлении репозитория.
|
||||
@@ -96,7 +94,7 @@ internal class AonprdRepository(AbstractImporter importer, string searchUrl)
|
||||
|
||||
Revision = Convert.ToInt32(
|
||||
page.SelectToken("_index")?.ToObject<string>()?.Replace("aon", "")
|
||||
?? throw new Exception("Can't get revision")
|
||||
?? throw new("Не удаётся получить ревизию Архивов Нефиса.")
|
||||
);
|
||||
|
||||
string revisionFile = Path.Combine(RepositoryDirPath, "revision.txt");
|
||||
|
||||
@@ -1,67 +1,106 @@
|
||||
namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Compendium;
|
||||
using Newtonsoft.Json.Linq;
|
||||
|
||||
namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Compendium;
|
||||
|
||||
/// <summary>
|
||||
/// Класс хранения данных компендиума.
|
||||
/// </summary>
|
||||
internal class AonprdCompendium
|
||||
internal sealed class AonprdCompendium(AonprdRepository repository) : AbstractCompendium
|
||||
{
|
||||
/// <summary>
|
||||
/// Словарь данных компендиума.
|
||||
/// </summary>
|
||||
public readonly Dictionary<string, AonprdType> Types = [];
|
||||
public override AonprdRepository Repository { get; } = repository;
|
||||
|
||||
/// <summary>
|
||||
/// Добавить ссылку на сущность с указанием файла, в котором она хранится.
|
||||
/// Загрузка данных компендиума.
|
||||
/// </summary>
|
||||
/// <param name="id">Идентификатор сущности.</param>
|
||||
/// <param name="file">Имя файла.</param>
|
||||
public void AddReference(string id, string file)
|
||||
public void Load()
|
||||
{
|
||||
AonprdReference reference = new(id, file);
|
||||
using StreamReader indexFile = new(
|
||||
Path.Combine(Repository.RepositoryDirPath, "index.json")
|
||||
);
|
||||
JObject indexJson = JObject.Parse(indexFile.ReadToEnd());
|
||||
|
||||
if (Types.TryGetValue(reference.Type, out AonprdType? type))
|
||||
foreach ((string file, JToken? value) in indexJson)
|
||||
{
|
||||
type.Add(reference);
|
||||
}
|
||||
else
|
||||
JArray items = (JArray)value!;
|
||||
|
||||
foreach (JToken item in items)
|
||||
{
|
||||
Types.Add(reference.Type, new(reference));
|
||||
AonprdCompendiumItem compendiumItem = new(item.ToString(), file);
|
||||
|
||||
Items.Add(compendiumItem);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Коллекция всех записей компендиума.
|
||||
/// </summary>
|
||||
public List<AonprdCompendiumItem> Items { get; } = [];
|
||||
|
||||
/// <summary>
|
||||
/// Возвращает количество элементов в компендиуме.
|
||||
/// </summary>
|
||||
/// <returns>Количество элементов.</returns>
|
||||
public int Count() => Types.Values.Sum(t => t.Count);
|
||||
public override int Count() => Items.Count;
|
||||
|
||||
/// <summary>
|
||||
/// Возвращает количество элементов для указанного типа данных.
|
||||
/// </summary>
|
||||
/// <param name="type">Тип данных.</param>
|
||||
/// <returns>Количество элементов.</returns>
|
||||
public int Count(string type) => Types.Values.First(t => t.Type == type).Count;
|
||||
public override int Count(string type) => Items.Count(i => i.Type == type);
|
||||
|
||||
/// <summary>
|
||||
/// Возвращает количество элементов для указанных типов данных.
|
||||
/// Возвращает список всех файлов.
|
||||
/// </summary>
|
||||
/// <param name="types">Список типов данных.</param>
|
||||
/// <returns>Количество элементов.</returns>
|
||||
public int Count(IEnumerable<string> types) =>
|
||||
Types.Values.Where(t => types.Contains(t.Type)).Sum(t => t.Count);
|
||||
/// <returns>Список искомых файлов.</returns>
|
||||
private IEnumerable<string> Files() => Items.Select(t => t.File).Distinct();
|
||||
|
||||
/// <summary>
|
||||
/// Возвращает список файлов для выбора сущностей указанного типа данных.
|
||||
/// Возвращает список файлов, в которых есть сущности указанного типа данных.
|
||||
/// </summary>
|
||||
/// <param name="type">Тип данных.</param>
|
||||
/// <returns>Список искомых файлов.</returns>
|
||||
public IEnumerable<string> Files(string type) => Types.Values.First(t => t.Type == type).Files;
|
||||
private IEnumerable<string> Files(string type) =>
|
||||
Items.Where(t => t.Type == type).Select(t => t.File).Distinct();
|
||||
|
||||
/// <summary>
|
||||
/// Возвращает список файлов для выбора сущностей указанных типов данных.
|
||||
/// Получение итератора сущностей. Если указан тип <paramref name="type"/> - то только указанного типа.
|
||||
/// </summary>
|
||||
/// <param name="types">Список типов данных.</param>
|
||||
/// <returns>Список искомых файлов.</returns>
|
||||
public IEnumerable<string> Files(IEnumerable<string> types) =>
|
||||
Types.Values.Where(t => types.Contains(t.Type)).SelectMany(t => t.Files).Distinct();
|
||||
/// <param name="type">Тип сущностей, которые нужно получить.</param>
|
||||
/// <returns>Итератор сущностей.</returns>
|
||||
public override IEnumerable<Tuple<AbstractCompendiumItem, JObject>> Iterate(string? type = null)
|
||||
{
|
||||
foreach (string file in type is null ? Files() : Files(type))
|
||||
{
|
||||
Dictionary<string, AonprdCompendiumItem> fileReferences = Items
|
||||
.Where(i => i.File == file)
|
||||
.ToDictionary(i => i.Id);
|
||||
|
||||
using StreamReader fileReader = new(
|
||||
Path.Combine(Repository.RepositoryDirPath, $"{file}.json")
|
||||
);
|
||||
JArray array = JArray.Parse(fileReader.ReadToEnd());
|
||||
|
||||
foreach (JToken item in array)
|
||||
{
|
||||
string? id = item.SelectToken("id")?.ToString();
|
||||
|
||||
if (
|
||||
id is null
|
||||
|| !fileReferences.TryGetValue(id, out AonprdCompendiumItem? reference)
|
||||
)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (type is not null && reference.Type != type)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
yield return new(reference, (JObject)item);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+4
-4
@@ -5,22 +5,22 @@
|
||||
/// </summary>
|
||||
/// <param name="id">Идентификатор сущности.</param>
|
||||
/// <param name="file">Имя файла.</param>
|
||||
internal class AonprdReference(string id, string file)
|
||||
internal class AonprdCompendiumItem(string id, string file) : AbstractCompendiumItem
|
||||
{
|
||||
/// <summary>
|
||||
/// Идентификатор сущности.
|
||||
/// </summary>
|
||||
public readonly string Id = id;
|
||||
public override string Id { get; } = id;
|
||||
|
||||
/// <summary>
|
||||
/// Тип сущности.
|
||||
/// </summary>
|
||||
public readonly string Type = ResolveType(id);
|
||||
public override string Type { get; } = ResolveType(id);
|
||||
|
||||
/// <summary>
|
||||
/// Имя файла, в котором хранится сущность.
|
||||
/// </summary>
|
||||
public readonly string File = file;
|
||||
public string File { get; } = file;
|
||||
|
||||
/// <summary>
|
||||
/// Определение типа сущности как префикса идентификатора.
|
||||
@@ -1,41 +0,0 @@
|
||||
namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Compendium;
|
||||
|
||||
/// <summary>
|
||||
/// Класс типа элементов с Архива Нефиса.
|
||||
/// </summary>
|
||||
internal class AonprdType(string type)
|
||||
{
|
||||
/// <summary>
|
||||
/// Идентификатор типа данных.
|
||||
/// </summary>
|
||||
public readonly string Type = type;
|
||||
|
||||
/// <summary>
|
||||
/// Список ссылок на сущности, связанных с этим типом.
|
||||
/// </summary>
|
||||
public readonly List<AonprdReference> Items = [];
|
||||
|
||||
public AonprdType(AonprdReference reference)
|
||||
: this(reference.Type)
|
||||
{
|
||||
Items.Add(reference);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Добавляет новую ссылку в список.
|
||||
/// </summary>
|
||||
public void Add(AonprdReference reference)
|
||||
{
|
||||
Items.Add(reference);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Возвращает уникальные файлы, использованные в ссылках.
|
||||
/// </summary>
|
||||
public List<string> Files => Items.Select(x => x.File).Distinct().ToList();
|
||||
|
||||
/// <summary>
|
||||
/// Число элементов в списке ссылок.
|
||||
/// </summary>
|
||||
public int Count => Items.Count;
|
||||
}
|
||||
+7
-13
@@ -1,17 +1,11 @@
|
||||
using Newtonsoft.Json.Linq;
|
||||
using Pf2eDataScrapper.Importers.Implementations.Aonprd.Helpers;
|
||||
using Pf2eDataScrapper.Processors;
|
||||
using Pf2eDataScrapper.Processors;
|
||||
|
||||
namespace Pf2eDataScrapper.Importers.Implementations.Aonprd;
|
||||
namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Helpers;
|
||||
|
||||
/// <summary>
|
||||
/// Класс для работы с JSON-объектом при обработки данных с Архивов Нефиса.
|
||||
/// </summary>
|
||||
/// <param name="input">JSON-объект, который будет использоваться для доступа к данным.</param>
|
||||
internal class AonprdProcessorInput(string name, JObject input) : ProcessorInput(name, input)
|
||||
internal static class AonprdProcessorInputExtension
|
||||
{
|
||||
/// <inheritdoc/>
|
||||
public override T Enum<T>(string value)
|
||||
public static T Enum<T>(this ProcessorInput input, string value)
|
||||
where T : Enum
|
||||
{
|
||||
try
|
||||
{
|
||||
@@ -19,7 +13,7 @@ internal class AonprdProcessorInput(string name, JObject input) : ProcessorInput
|
||||
}
|
||||
catch
|
||||
{
|
||||
return base.Enum<T>(value);
|
||||
return input.Enum<T>(value);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -28,7 +22,7 @@ internal class AonprdProcessorInput(string name, JObject input) : ProcessorInput
|
||||
/// </summary>
|
||||
/// <param name="input">Входная строка с данными.</param>
|
||||
/// <returns>Структура, хранящая информацию о количетсве действий.</returns>
|
||||
public ActionStruct ResolveActionString(string input)
|
||||
public static ActionStruct ResolveActionString(this ProcessorInput _, string input)
|
||||
{
|
||||
ActionStruct result = new();
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
using Pf2eDataScrapper.Importers.Implementations.Aonprd.DataBuilder.Processors;
|
||||
using Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors;
|
||||
using Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors;
|
||||
using Pf2eModel.Model.Base;
|
||||
using Pf2eModel.Model.Base.Source;
|
||||
using Pf2eModel.Model.Common.I18n;
|
||||
|
||||
+1
-1
@@ -27,7 +27,7 @@ namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors;
|
||||
/// <param name="input">Объект обработки вводимого JSON.</param>
|
||||
/// <seealso cref="ProcessorAttribute"/>
|
||||
internal abstract class AbstractEntityProcessor(
|
||||
AonprdProcessorInput input,
|
||||
ProcessorInput input,
|
||||
Pf2eDbContext context,
|
||||
StringHelper stringHelper,
|
||||
DatabaseHelper helper,
|
||||
|
||||
+2
-1
@@ -2,6 +2,7 @@
|
||||
using Pf2eDataScrapper.Helpers;
|
||||
using Pf2eDataScrapper.Importers.Implementations.Aonprd.DataBuilder.Helpers;
|
||||
using Pf2eDataScrapper.Importers.Implementations.Aonprd.Helpers;
|
||||
using Pf2eDataScrapper.Processors;
|
||||
using Pf2eModel;
|
||||
using Pf2eModel.Model.Base;
|
||||
using Pf2eModel.Model.Common.I18n;
|
||||
@@ -16,7 +17,7 @@ namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors;
|
||||
/// для которых указан атрибут <see cref="ProcessorAttribute"/>.
|
||||
/// </summary>
|
||||
internal abstract class AbstractSubclassProcessor(
|
||||
AonprdProcessorInput input,
|
||||
ProcessorInput input,
|
||||
Pf2eDbContext context,
|
||||
StringHelper stringHelper,
|
||||
DatabaseHelper helper,
|
||||
|
||||
-170
@@ -1,170 +0,0 @@
|
||||
using Pf2eDataScrapper.App;
|
||||
using Pf2eDataScrapper.Processors;
|
||||
using Pf2eDataScrapper.SchemaBuilder;
|
||||
|
||||
namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Common;
|
||||
|
||||
[Processor(
|
||||
"aonprd:schema",
|
||||
Description = "Построение схемы данных",
|
||||
Group = ProcessorGroupEnum.AonprdPreparation
|
||||
)]
|
||||
[ProcessorDependencies(["aonprd:repository"])]
|
||||
internal class AonprdSchemaBuildProcessor(
|
||||
AonprdImporter importer,
|
||||
ProcessProgress progress,
|
||||
ApplicationSettings settings
|
||||
) : IProcessor
|
||||
{
|
||||
public AonprdImporter Importer { get; } = importer;
|
||||
public ProcessProgress Progress { get; } = progress;
|
||||
public ApplicationSettings Settings { get; } = settings;
|
||||
|
||||
/// <summary>
|
||||
/// Загузчик данных с сайта Архивов Нефеса.
|
||||
/// </summary>
|
||||
public AonprdLoader Loader => Importer.Loader;
|
||||
|
||||
/// <summary>
|
||||
/// Дополнительные схемы данных.
|
||||
/// </summary>
|
||||
public readonly Dictionary<string, SchemaNode> Schemas = new()
|
||||
{
|
||||
["_subclass"] = new() { Path = "_subclass" },
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// Построение схемы данных.
|
||||
/// </summary>
|
||||
public void Process()
|
||||
{
|
||||
Progress.Local.Reset(Loader.ItemCount);
|
||||
|
||||
foreach (AonprdLoader.LoaderEntry entry in Loader.Entries())
|
||||
{
|
||||
Progress.Local.Start(entry.Type);
|
||||
GetSchema(entry).AddValue(entry.Json);
|
||||
|
||||
if (_subclassSchemaTypes.Contains(entry.Type))
|
||||
{
|
||||
Schemas["_subclass"].AddValue(entry.Json);
|
||||
}
|
||||
|
||||
Progress.Local.End();
|
||||
}
|
||||
|
||||
Progress.Local.Complete();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Сохранение постренных схем данных.
|
||||
/// </summary>
|
||||
public void Save()
|
||||
{
|
||||
string targetDir = Settings.OutputDir;
|
||||
|
||||
targetDir = Path.Combine(targetDir, Loader.Importer.Slug);
|
||||
|
||||
foreach (KeyValuePair<string, SchemaNode> kvp in Schemas)
|
||||
{
|
||||
SaveSchema(targetDir, kvp.Key, kvp.Value);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Список типов, являющихся подклассами.
|
||||
/// </summary>
|
||||
private readonly string[] _subclassSchemaTypes =
|
||||
[
|
||||
"research-field",
|
||||
"apparition",
|
||||
"practice",
|
||||
"instinct",
|
||||
"muse",
|
||||
"cause",
|
||||
"tenet",
|
||||
"doctrine",
|
||||
"druidic-order",
|
||||
"methodology",
|
||||
"element",
|
||||
"hybrid-study",
|
||||
"mystery",
|
||||
"conscious-mind",
|
||||
"subconscious-mind",
|
||||
"hunters-edge",
|
||||
"racket",
|
||||
"bloodline",
|
||||
"style",
|
||||
"implement",
|
||||
"lesson",
|
||||
"patron",
|
||||
"arcane-school",
|
||||
"arcane-thesis",
|
||||
"way",
|
||||
"innovation",
|
||||
"ikon",
|
||||
"epithet",
|
||||
];
|
||||
|
||||
/// <summary>
|
||||
/// Выбор целевого элемента схемы данных для ввода.
|
||||
/// </summary>
|
||||
/// <param name="entry">Входные данные.</param>
|
||||
/// <returns>Целевой элемент схемы данных.</returns>
|
||||
private SchemaNode GetSchema(AonprdLoader.LoaderEntry entry)
|
||||
{
|
||||
string key = entry.Type;
|
||||
|
||||
if (Schemas.TryGetValue(key, out SchemaNode? value))
|
||||
{
|
||||
return value;
|
||||
}
|
||||
|
||||
value = new()
|
||||
{
|
||||
// Resolver = Resolver,
|
||||
Path = key,
|
||||
};
|
||||
Schemas[key] = value;
|
||||
|
||||
return value;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Сохранение отдельной схемы данных.
|
||||
/// </summary>
|
||||
/// <param name="dir">Целевой каталог.</param>
|
||||
/// <param name="name">Название схемы.</param>
|
||||
/// <param name="schema">Сохраняемая схема.</param>
|
||||
protected void SaveSchema(string dir, string name, SchemaNode schema)
|
||||
{
|
||||
char[] invalids = Path.GetInvalidFileNameChars();
|
||||
string fileName = string.Join(
|
||||
"_",
|
||||
schema.Path.Split(invalids, StringSplitOptions.RemoveEmptyEntries)
|
||||
)
|
||||
.TrimEnd('.');
|
||||
string path = Path.Combine(dir, fileName + ".yaml");
|
||||
|
||||
string? dirPath = Path.GetDirectoryName(path);
|
||||
if (dirPath is not null && !Directory.Exists(dirPath))
|
||||
{
|
||||
Directory.CreateDirectory(dirPath);
|
||||
}
|
||||
|
||||
using StreamWriter writer = new(path);
|
||||
|
||||
schema.Beautify();
|
||||
|
||||
writer.WriteLine(
|
||||
SchemaNode.FormatYaml(
|
||||
new Dictionary<string, string>()
|
||||
{
|
||||
["name"] = name,
|
||||
["values"] = schema.Values.ToString(),
|
||||
},
|
||||
schema.Types[0].GetYamlData()
|
||||
)
|
||||
);
|
||||
}
|
||||
}
|
||||
+10
-5
@@ -1,4 +1,5 @@
|
||||
using Pf2eDataScrapper.App;
|
||||
using Pf2eDataScrapper.Importers.Implementations.Aonprd.Compendium;
|
||||
using Pf2eDataScrapper.Processors;
|
||||
|
||||
namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Common;
|
||||
@@ -8,19 +9,23 @@ namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Common;
|
||||
Description = "Обновление репозитория",
|
||||
Group = ProcessorGroupEnum.AonprdPreparation
|
||||
)]
|
||||
internal class AonprdRepositoryUpdateProcessor(AonprdImporter importer, ProcessProgress progress)
|
||||
: IProcessor
|
||||
internal class RepositoryUpdateProcessor(
|
||||
AonprdRepository repository,
|
||||
AonprdCompendium compendium,
|
||||
ProcessProgress progress
|
||||
) : IProcessor
|
||||
{
|
||||
private AonprdImporter Importer { get; } = importer;
|
||||
public AonprdRepository Repository { get; } = repository;
|
||||
public AonprdCompendium Compendium { get; } = compendium;
|
||||
public ProcessProgress Progress { get; } = progress;
|
||||
|
||||
public void Process()
|
||||
{
|
||||
Importer.Repository.Update(Progress);
|
||||
Repository.Update(Progress);
|
||||
}
|
||||
|
||||
public void Save()
|
||||
{
|
||||
Importer.Loader.Prepare();
|
||||
Compendium.Load();
|
||||
}
|
||||
}
|
||||
+92
@@ -0,0 +1,92 @@
|
||||
using Pf2eDataScrapper.App;
|
||||
using Pf2eDataScrapper.Importers.Implementations.Aonprd.Compendium;
|
||||
using Pf2eDataScrapper.Processors;
|
||||
using Pf2eDataScrapper.SchemaBuilder;
|
||||
|
||||
namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Common;
|
||||
|
||||
[Processor(
|
||||
"aonprd:schema",
|
||||
Description = "Построение схемы данных",
|
||||
Group = ProcessorGroupEnum.AonprdPreparation
|
||||
)]
|
||||
[ProcessorDependencies(["aonprd:repository"])]
|
||||
internal class SchemaBuildProcessor(
|
||||
AonprdImporter importer,
|
||||
AonprdCompendium compendium,
|
||||
ProcessProgress progress,
|
||||
ApplicationSettings settings
|
||||
) : AbstractSchemaBuildProcessor
|
||||
{
|
||||
/// <inheritdoc cref="AonprdImporter"/>
|
||||
protected AonprdImporter Importer { get; } = importer;
|
||||
|
||||
/// <inheritdoc cref="AonprdCompendium"/>
|
||||
protected override AonprdCompendium Compendium { get; } = compendium;
|
||||
|
||||
/// <inheritdoc cref="ProcessProgress"/>
|
||||
protected override ProcessProgress Progress { get; } = progress;
|
||||
|
||||
/// <inheritdoc cref="ApplicationSettings"/>
|
||||
protected ApplicationSettings Settings { get; } = settings;
|
||||
|
||||
/// <inheritdoc/>
|
||||
protected override string SavePath => Path.Combine(Settings.OutputDirPath, Importer.Slug);
|
||||
|
||||
/// <inheritdoc/>
|
||||
protected override AonprdTypeResolver Resolver { get; } = new();
|
||||
|
||||
/// <summary>
|
||||
/// Список типов, являющихся подклассами.
|
||||
/// </summary>
|
||||
private readonly string[] _subclassSchemaTypes =
|
||||
[
|
||||
"research-field",
|
||||
"apparition",
|
||||
"practice",
|
||||
"instinct",
|
||||
"muse",
|
||||
"cause",
|
||||
"tenet",
|
||||
"doctrine",
|
||||
"druidic-order",
|
||||
"methodology",
|
||||
"element",
|
||||
"hybrid-study",
|
||||
"mystery",
|
||||
"conscious-mind",
|
||||
"subconscious-mind",
|
||||
"hunters-edge",
|
||||
"racket",
|
||||
"bloodline",
|
||||
"style",
|
||||
"implement",
|
||||
"lesson",
|
||||
"patron",
|
||||
"arcane-school",
|
||||
"arcane-thesis",
|
||||
"way",
|
||||
"innovation",
|
||||
"ikon",
|
||||
"epithet",
|
||||
];
|
||||
|
||||
protected override ICollection<string> GetSchemaNodes(AbstractCompendiumItem item)
|
||||
{
|
||||
List<string> nodes = [item.Type];
|
||||
|
||||
// Нода подкласса
|
||||
if (_subclassSchemaTypes.Contains(item.Type))
|
||||
{
|
||||
nodes.Add("_subclass");
|
||||
}
|
||||
|
||||
return nodes;
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
internal class AonprdTypeResolver : TypeResolver
|
||||
{
|
||||
//
|
||||
}
|
||||
+1
-1
@@ -13,7 +13,7 @@ namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Common;
|
||||
Description = "Создание цели внешних ссылок",
|
||||
Group = ProcessorGroupEnum.AonprdPreparation
|
||||
)]
|
||||
internal class SkillProcessor(AonprdProcessorInput input, Pf2eDbContext context)
|
||||
internal class SkillProcessor(ProcessorInput input, Pf2eDbContext context)
|
||||
: AbstractInputProcessor(input)
|
||||
{
|
||||
public Pf2eDbContext Context { get; } = context;
|
||||
|
||||
+1
-1
@@ -13,7 +13,7 @@ namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Common;
|
||||
Group = ProcessorGroupEnum.AonprdPreparation
|
||||
)]
|
||||
[ProcessorDependencies(["aonprd:repository"])]
|
||||
internal class SourceProcessor(AonprdProcessorInput input, Pf2eDbContext context)
|
||||
internal class SourceProcessor(ProcessorInput input, Pf2eDbContext context)
|
||||
: AbstractInputProcessor(input)
|
||||
{
|
||||
public Pf2eDbContext Context { get; } = context;
|
||||
|
||||
+2
-1
@@ -2,6 +2,7 @@
|
||||
using Pf2eDataScrapper.Helpers;
|
||||
using Pf2eDataScrapper.Importers.Implementations.Aonprd.DataBuilder.Helpers;
|
||||
using Pf2eDataScrapper.Importers.Implementations.Aonprd.Helpers;
|
||||
using Pf2eDataScrapper.Processors;
|
||||
using Pf2eModel;
|
||||
using Pf2eModel.Model.Base;
|
||||
using Pf2eModel.Model.Common;
|
||||
@@ -21,7 +22,7 @@ namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Subclasse
|
||||
["aonprd:trait", "aonprd:source", "aonprd:class", "aonprd:spell", "aonprd:skill"]
|
||||
)]
|
||||
internal class ApparitionProcessor(
|
||||
AonprdProcessorInput input,
|
||||
ProcessorInput input,
|
||||
Pf2eDbContext context,
|
||||
StringHelper stringHelper,
|
||||
DatabaseHelper helper,
|
||||
|
||||
+2
-1
@@ -2,6 +2,7 @@
|
||||
using Pf2eDataScrapper.Helpers;
|
||||
using Pf2eDataScrapper.Importers.Implementations.Aonprd.DataBuilder.Helpers;
|
||||
using Pf2eDataScrapper.Importers.Implementations.Aonprd.Helpers;
|
||||
using Pf2eDataScrapper.Processors;
|
||||
using Pf2eModel;
|
||||
using Pf2eModel.Model.Enum;
|
||||
|
||||
@@ -15,7 +16,7 @@ namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Types;
|
||||
)]
|
||||
[ProcessorDependencies(["aonprd:trait", "aonprd:source"])]
|
||||
internal class ActionProcessor(
|
||||
AonprdProcessorInput input,
|
||||
ProcessorInput input,
|
||||
Pf2eDbContext context,
|
||||
StringHelper stringHelper,
|
||||
DatabaseHelper helper,
|
||||
|
||||
+2
-1
@@ -3,6 +3,7 @@ using Pf2eDataScrapper.App;
|
||||
using Pf2eDataScrapper.Helpers;
|
||||
using Pf2eDataScrapper.Importers.Implementations.Aonprd.DataBuilder.Helpers;
|
||||
using Pf2eDataScrapper.Importers.Implementations.Aonprd.Helpers;
|
||||
using Pf2eDataScrapper.Processors;
|
||||
using Pf2eModel;
|
||||
using Pf2eModel.Model.Enum;
|
||||
using Pf2eModel.Model.Structure;
|
||||
@@ -17,7 +18,7 @@ namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Types;
|
||||
)]
|
||||
[ProcessorDependencies(["aonprd:trait", "aonprd:source", "aonprd:language"])]
|
||||
internal class AncestryProcessor(
|
||||
AonprdProcessorInput input,
|
||||
ProcessorInput input,
|
||||
Pf2eDbContext context,
|
||||
StringHelper stringHelper,
|
||||
DatabaseHelper helper,
|
||||
|
||||
+2
-1
@@ -2,6 +2,7 @@
|
||||
using Pf2eDataScrapper.Helpers;
|
||||
using Pf2eDataScrapper.Importers.Implementations.Aonprd.DataBuilder.Helpers;
|
||||
using Pf2eDataScrapper.Importers.Implementations.Aonprd.Helpers;
|
||||
using Pf2eDataScrapper.Processors;
|
||||
using Pf2eModel;
|
||||
using Pf2eModel.Model.Enum;
|
||||
using Pf2eModel.Model.Structure;
|
||||
@@ -16,7 +17,7 @@ namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Types;
|
||||
)]
|
||||
[ProcessorDependencies(["aonprd:trait", "aonprd:source"])]
|
||||
internal class ArchetypeProcessor(
|
||||
AonprdProcessorInput input,
|
||||
ProcessorInput input,
|
||||
Pf2eDbContext context,
|
||||
StringHelper stringHelper,
|
||||
DatabaseHelper helper,
|
||||
|
||||
+2
-1
@@ -3,6 +3,7 @@ using Pf2eDataScrapper.App;
|
||||
using Pf2eDataScrapper.Helpers;
|
||||
using Pf2eDataScrapper.Importers.Implementations.Aonprd.DataBuilder.Helpers;
|
||||
using Pf2eDataScrapper.Importers.Implementations.Aonprd.Helpers;
|
||||
using Pf2eDataScrapper.Processors;
|
||||
using Pf2eModel;
|
||||
using Pf2eModel.Model.Enum;
|
||||
using Pf2eModel.Model.Structure;
|
||||
@@ -18,7 +19,7 @@ namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Types;
|
||||
)]
|
||||
[ProcessorDependencies(["aonprd:trait", "aonprd:source"])]
|
||||
internal class BackgroundProcessor(
|
||||
AonprdProcessorInput input,
|
||||
ProcessorInput input,
|
||||
Pf2eDbContext context,
|
||||
StringHelper stringHelper,
|
||||
DatabaseHelper helper,
|
||||
|
||||
+2
-1
@@ -4,6 +4,7 @@ using Pf2eDataScrapper.App;
|
||||
using Pf2eDataScrapper.Helpers;
|
||||
using Pf2eDataScrapper.Importers.Implementations.Aonprd.DataBuilder.Helpers;
|
||||
using Pf2eDataScrapper.Importers.Implementations.Aonprd.Helpers;
|
||||
using Pf2eDataScrapper.Processors;
|
||||
using Pf2eModel;
|
||||
using Pf2eModel.Model.Common;
|
||||
using Pf2eModel.Model.Common.I18n;
|
||||
@@ -22,7 +23,7 @@ namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Types;
|
||||
)]
|
||||
[ProcessorDependencies(["aonprd:trait", "aonprd:source", "aonprd:skill"])]
|
||||
internal partial class ClassProcessor(
|
||||
AonprdProcessorInput input,
|
||||
ProcessorInput input,
|
||||
Pf2eDbContext context,
|
||||
StringHelper stringHelper,
|
||||
DatabaseHelper helper,
|
||||
|
||||
@@ -3,6 +3,7 @@ using Pf2eDataScrapper.App;
|
||||
using Pf2eDataScrapper.Helpers;
|
||||
using Pf2eDataScrapper.Importers.Implementations.Aonprd.DataBuilder.Helpers;
|
||||
using Pf2eDataScrapper.Importers.Implementations.Aonprd.Helpers;
|
||||
using Pf2eDataScrapper.Processors;
|
||||
using Pf2eModel;
|
||||
using Pf2eModel.Model.Enum;
|
||||
using Pf2eModel.Model.Structure;
|
||||
@@ -17,7 +18,7 @@ namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Types;
|
||||
)]
|
||||
[ProcessorDependencies(["aonprd:trait", "aonprd:source", "aonprd:archetype"])]
|
||||
internal class FeatProcessor(
|
||||
AonprdProcessorInput input,
|
||||
ProcessorInput input,
|
||||
Pf2eDbContext context,
|
||||
StringHelper stringHelper,
|
||||
DatabaseHelper helper,
|
||||
@@ -44,7 +45,7 @@ internal class FeatProcessor(
|
||||
? FeatCategory.Class
|
||||
: FeatCategory.Uncategorized;
|
||||
|
||||
AonprdProcessorInput.ActionStruct action = Input.ResolveActionString(
|
||||
AonprdProcessorInputExtension.ActionStruct action = Input.ResolveActionString(
|
||||
Input.GetOptional<string>("actions") ?? ""
|
||||
);
|
||||
Object.Actions = action.Actions;
|
||||
|
||||
+2
-1
@@ -2,6 +2,7 @@
|
||||
using Pf2eDataScrapper.Helpers;
|
||||
using Pf2eDataScrapper.Importers.Implementations.Aonprd.DataBuilder.Helpers;
|
||||
using Pf2eDataScrapper.Importers.Implementations.Aonprd.Helpers;
|
||||
using Pf2eDataScrapper.Processors;
|
||||
using Pf2eModel;
|
||||
using Pf2eModel.Model.Enum;
|
||||
using Pf2eModel.Model.Structure;
|
||||
@@ -16,7 +17,7 @@ namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Types;
|
||||
)]
|
||||
[ProcessorDependencies(["aonprd:trait", "aonprd:source"])]
|
||||
internal class LanguageProcessor(
|
||||
AonprdProcessorInput input,
|
||||
ProcessorInput input,
|
||||
Pf2eDbContext context,
|
||||
StringHelper stringHelper,
|
||||
DatabaseHelper helper,
|
||||
|
||||
+2
-1
@@ -4,6 +4,7 @@ using Pf2eDataScrapper.App;
|
||||
using Pf2eDataScrapper.Helpers;
|
||||
using Pf2eDataScrapper.Importers.Implementations.Aonprd.DataBuilder.Helpers;
|
||||
using Pf2eDataScrapper.Importers.Implementations.Aonprd.Helpers;
|
||||
using Pf2eDataScrapper.Processors;
|
||||
using Pf2eModel;
|
||||
using Pf2eModel.Model.Common.I18n;
|
||||
using Pf2eModel.Model.Enum;
|
||||
@@ -20,7 +21,7 @@ namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Types;
|
||||
)]
|
||||
[ProcessorDependencies(["aonprd:trait", "aonprd:source", "aonprd:skill"])]
|
||||
internal partial class RitualProcessor(
|
||||
AonprdProcessorInput input,
|
||||
ProcessorInput input,
|
||||
Pf2eDbContext context,
|
||||
StringHelper stringHelper,
|
||||
DatabaseHelper helper,
|
||||
|
||||
+2
-1
@@ -3,6 +3,7 @@ using Pf2eDataScrapper.App;
|
||||
using Pf2eDataScrapper.Helpers;
|
||||
using Pf2eDataScrapper.Importers.Implementations.Aonprd.DataBuilder.Helpers;
|
||||
using Pf2eDataScrapper.Importers.Implementations.Aonprd.Helpers;
|
||||
using Pf2eDataScrapper.Processors;
|
||||
using Pf2eModel;
|
||||
using Pf2eModel.Model.Base.Source;
|
||||
using Pf2eModel.Model.Base.Targets;
|
||||
@@ -20,7 +21,7 @@ namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Types.Rul
|
||||
)]
|
||||
[ProcessorDependencies(["aonprd:trait", "aonprd:source", "aonprd:rules-toc"])]
|
||||
internal class RuleProcessor(
|
||||
AonprdProcessorInput input,
|
||||
ProcessorInput input,
|
||||
Pf2eDbContext context,
|
||||
StringHelper stringHelper,
|
||||
DatabaseHelper helper,
|
||||
|
||||
+14
-7
@@ -5,17 +5,24 @@
|
||||
// так что ставить порядок ВСЕХ правил в архиве не получится, только тех, что есть в содержании.
|
||||
// Ну а пока просто расставляем родительские и дочерние элементы.
|
||||
|
||||
using System.Data;
|
||||
using Pf2eDataScrapper.App;
|
||||
using Pf2eDataScrapper.Importers.Implementations.Aonprd.Helpers;
|
||||
using Pf2eDataScrapper.Processors;
|
||||
using Pf2eModel.Model.Structure;
|
||||
using Pf2eModel;
|
||||
using Rule = Pf2eModel.Model.Structure.Rule;
|
||||
|
||||
namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.DataBuilder.Processors.Types.Rules;
|
||||
namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Types.Rules;
|
||||
|
||||
[AonprdProcessor("rules-order", null, ["rules"], false)]
|
||||
internal class RulesOrderProcessor(AonprdProcessorInput input) : AbstractProcessor(input)
|
||||
[Processor(
|
||||
"aonprd:rules-order",
|
||||
Description = "Создание цели внешних ссылок",
|
||||
Group = ProcessorGroupEnum.AonprdEntities
|
||||
)]
|
||||
[ProcessorDependencies(["aonprd:rules"])]
|
||||
internal class RulesOrderProcessor(Pf2eDbContext context) : IProcessor
|
||||
{
|
||||
public Pf2eDbContext Context { get; } = context;
|
||||
|
||||
/// <summary>
|
||||
/// Структура генерации содержания правил.
|
||||
/// </summary>
|
||||
@@ -27,7 +34,7 @@ internal class RulesOrderProcessor(AonprdProcessorInput input) : AbstractProcess
|
||||
private readonly List<OrderItem> _orderItems = [];
|
||||
|
||||
/// <inheritdoc/>
|
||||
public override void Process()
|
||||
public void Process()
|
||||
{
|
||||
Toc.SaveFile();
|
||||
|
||||
@@ -149,7 +156,7 @@ internal class RulesOrderProcessor(AonprdProcessorInput input) : AbstractProcess
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public override void Save()
|
||||
public void Save()
|
||||
{
|
||||
Context.SaveChanges();
|
||||
}
|
||||
|
||||
+14
-5
@@ -2,15 +2,24 @@
|
||||
using System.Web;
|
||||
using AngleSharp;
|
||||
using AngleSharp.Dom;
|
||||
using Pf2eDataScrapper.App;
|
||||
using Pf2eDataScrapper.Importers.Implementations.Aonprd.Helpers;
|
||||
using Pf2eDataScrapper.Processors;
|
||||
using Pf2eModel;
|
||||
using Pf2eModel.Model.Base.Source;
|
||||
|
||||
namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.DataBuilder.Processors.Types.Rules;
|
||||
namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Types.Rules;
|
||||
|
||||
[AonprdProcessor("rules-toc", null, ["source"], false)]
|
||||
internal partial class RulesTocProcessor(AonprdProcessorInput input) : AbstractProcessor(input)
|
||||
[Processor(
|
||||
"aonprd:rules-toc",
|
||||
Description = "Создание цели внешних ссылок",
|
||||
Group = ProcessorGroupEnum.AonprdEntities
|
||||
)]
|
||||
[ProcessorDependencies(["aonprd:source"])]
|
||||
internal partial class RulesTocProcessor(Pf2eDbContext context) : IProcessor
|
||||
{
|
||||
public Pf2eDbContext Context { get; } = context;
|
||||
|
||||
/// <summary>
|
||||
/// Селектор HTML-элемента, в котором находится содержание.
|
||||
/// </summary>
|
||||
@@ -27,7 +36,7 @@ internal partial class RulesTocProcessor(AonprdProcessorInput input) : AbstractP
|
||||
public readonly RuleTocCollection Toc = RuleTocCollection.Instance;
|
||||
|
||||
/// <inheritdoc/>
|
||||
public override void Process()
|
||||
public void Process()
|
||||
{
|
||||
IDocument document = DownloadPage();
|
||||
|
||||
@@ -121,7 +130,7 @@ internal partial class RulesTocProcessor(AonprdProcessorInput input) : AbstractP
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public override void Save() { }
|
||||
public void Save() { }
|
||||
|
||||
/// <summary>
|
||||
/// Получить скомпилированное регулярное выражение для получения HTML-ссылки.
|
||||
|
||||
+2
-1
@@ -3,6 +3,7 @@ using Pf2eDataScrapper.App;
|
||||
using Pf2eDataScrapper.Helpers;
|
||||
using Pf2eDataScrapper.Importers.Implementations.Aonprd.DataBuilder.Helpers;
|
||||
using Pf2eDataScrapper.Importers.Implementations.Aonprd.Helpers;
|
||||
using Pf2eDataScrapper.Processors;
|
||||
using Pf2eModel;
|
||||
using Pf2eModel.Model.Common.I18n;
|
||||
using Pf2eModel.Model.Enum;
|
||||
@@ -19,7 +20,7 @@ namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Types;
|
||||
)]
|
||||
[ProcessorDependencies(["aonprd:trait", "aonprd:source"])]
|
||||
internal class SpellProcessor(
|
||||
AonprdProcessorInput input,
|
||||
ProcessorInput input,
|
||||
Pf2eDbContext context,
|
||||
StringHelper stringHelper,
|
||||
DatabaseHelper helper,
|
||||
|
||||
+2
-1
@@ -2,6 +2,7 @@
|
||||
using Pf2eDataScrapper.Helpers;
|
||||
using Pf2eDataScrapper.Importers.Implementations.Aonprd.DataBuilder.Helpers;
|
||||
using Pf2eDataScrapper.Importers.Implementations.Aonprd.Helpers;
|
||||
using Pf2eDataScrapper.Processors;
|
||||
using Pf2eModel;
|
||||
using Pf2eModel.Model.Enum;
|
||||
using Pf2eModel.Model.Structure;
|
||||
@@ -16,7 +17,7 @@ namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Types;
|
||||
)]
|
||||
[ProcessorDependencies(["aonprd:external"])]
|
||||
internal class TraitProcessor(
|
||||
AonprdProcessorInput input,
|
||||
ProcessorInput input,
|
||||
Pf2eDbContext context,
|
||||
StringHelper stringHelper,
|
||||
DatabaseHelper helper,
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
namespace Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.Compendium;
|
||||
namespace Pf2eDataScrapper.Importers.Implementations.FoundryEn.Compendium;
|
||||
|
||||
/// <summary>
|
||||
/// Тип документов компендиума.
|
||||
+12
-13
@@ -1,6 +1,6 @@
|
||||
using Newtonsoft.Json.Linq;
|
||||
|
||||
namespace Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.Compendium;
|
||||
namespace Pf2eDataScrapper.Importers.Implementations.FoundryEn.Compendium;
|
||||
|
||||
/// <summary>
|
||||
/// Папка компендиума.
|
||||
@@ -10,32 +10,32 @@ internal class CompendiumFolder
|
||||
/// <summary>
|
||||
/// Идентификатор папки.
|
||||
/// </summary>
|
||||
public readonly string Id;
|
||||
public string Id { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Название папки.
|
||||
/// </summary>
|
||||
public readonly string Name;
|
||||
public string Name { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Путь к папке.
|
||||
/// </summary>
|
||||
public readonly string? Path;
|
||||
public string? Path { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Указатель, что папка содержится в других папках.
|
||||
/// </summary>
|
||||
public readonly bool IsContained;
|
||||
|
||||
/// <summary>
|
||||
/// Родительская папка.
|
||||
/// </summary>
|
||||
public CompendiumFolder? Parent;
|
||||
public bool IsContained { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Тип документов в папке.
|
||||
/// </summary>
|
||||
public readonly CompendiumDocumentType Type;
|
||||
public CompendiumDocumentType Type { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Родительская папка.
|
||||
/// </summary>
|
||||
public CompendiumFolder? Parent { get; set; } = null;
|
||||
|
||||
/// <summary>
|
||||
/// Конструктор папки компендиума из JSON-объекта.
|
||||
@@ -60,8 +60,7 @@ internal class CompendiumFolder
|
||||
IsContained = false;
|
||||
}
|
||||
|
||||
Type = (CompendiumDocumentType)
|
||||
Enum.Parse(typeof(CompendiumDocumentType), json["type"]!.ToString());
|
||||
Type = Enum.Parse<CompendiumDocumentType>(json["type"]!.ToString());
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
+31
@@ -0,0 +1,31 @@
|
||||
namespace Pf2eDataScrapper.Importers.Implementations.FoundryEn.Compendium;
|
||||
|
||||
/// <summary>
|
||||
/// Коллекция папок компендиума
|
||||
/// </summary>
|
||||
internal class CompendiumFolderCollection
|
||||
{
|
||||
/// <summary>
|
||||
/// Список загруженных папок.
|
||||
/// </summary>
|
||||
public List<CompendiumFolder> Items { get; } = [];
|
||||
|
||||
/// <summary>
|
||||
/// Добавление информации о папке.
|
||||
/// </summary>
|
||||
/// <param name="folder">Добавляемая папка.</param>
|
||||
public void Add(CompendiumFolder folder)
|
||||
{
|
||||
Items.Add(folder);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Получение папки по идентификатору
|
||||
/// </summary>
|
||||
/// <param name="id">Искомый идентификатор</param>
|
||||
/// <returns>Искомая папка</returns>
|
||||
public CompendiumFolder? GetById(string id)
|
||||
{
|
||||
return Items.FirstOrDefault(x => x.Id == id);
|
||||
}
|
||||
}
|
||||
+25
@@ -0,0 +1,25 @@
|
||||
namespace Pf2eDataScrapper.Importers.Implementations.FoundryEn.Compendium;
|
||||
|
||||
/// <summary>
|
||||
/// Ссылка на сущность с указанием файла, в котором она хранится.
|
||||
/// </summary>
|
||||
/// <param name="id">Идентификатор сущности.</param>
|
||||
/// <param name="type">Тип сущности.</param>
|
||||
/// <param name="file">Имя файла, в котором хранится сущность.</param>
|
||||
internal class FoundryEnCompendiumItem(string id, string type, string file) : AbstractCompendiumItem
|
||||
{
|
||||
/// <summary>
|
||||
/// Идентификатор сущности.
|
||||
/// </summary>
|
||||
public override string Id { get; } = id;
|
||||
|
||||
/// <summary>
|
||||
/// Тип сущности.
|
||||
/// </summary>
|
||||
public override string Type { get; } = type;
|
||||
|
||||
/// <summary>
|
||||
/// Имя файла, в котором хранится сущность.
|
||||
/// </summary>
|
||||
public string File { get; } = file;
|
||||
}
|
||||
+10
-8
@@ -13,15 +13,17 @@ using Pf2eModel.Model.Common.I18n;
|
||||
using Pf2eModel.Model.Enum;
|
||||
using Pf2eModel.Model.Structure;
|
||||
|
||||
namespace Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.DataBuilder;
|
||||
namespace Pf2eDataScrapper.Importers.Implementations.FoundryEn.DataBuilder;
|
||||
|
||||
internal abstract class AbstractProcessor(ProcessorInput input)
|
||||
internal abstract class FoundryEnAbstractProcessor(
|
||||
DatabaseHelper databaseHelper,
|
||||
Pf2eDbContext context,
|
||||
ProcessorInput input
|
||||
) : AbstractInputProcessor(input)
|
||||
{
|
||||
protected DatabaseHelper Helper = DatabaseHelper.Instance;
|
||||
protected DatabaseHelper Helper { get; } = databaseHelper;
|
||||
|
||||
public readonly Pf2eDbContext Context = DatabaseHelper.DbContext;
|
||||
|
||||
public readonly ProcessorInput Input = input;
|
||||
public Pf2eDbContext Context { get; } = context;
|
||||
|
||||
public Entity Entity { get; set; } = new();
|
||||
|
||||
@@ -29,7 +31,7 @@ internal abstract class AbstractProcessor(ProcessorInput input)
|
||||
|
||||
public abstract IEntity Object { get; }
|
||||
|
||||
public virtual void Process()
|
||||
public override void Process()
|
||||
{
|
||||
Entity.Id = Helper.NextEntityId;
|
||||
Entity.Type = Type;
|
||||
@@ -95,7 +97,7 @@ internal abstract class AbstractProcessor(ProcessorInput input)
|
||||
}
|
||||
}
|
||||
|
||||
public void Save()
|
||||
public override void Save()
|
||||
{
|
||||
Context.SaveChanges();
|
||||
}
|
||||
+16
-2
@@ -1,11 +1,25 @@
|
||||
using Newtonsoft.Json.Linq;
|
||||
using Pf2eDataScrapper.App;
|
||||
using Pf2eDataScrapper.Helpers;
|
||||
using Pf2eDataScrapper.Importers.Implementations.FoundryEn.DataBuilder;
|
||||
using Pf2eDataScrapper.Processors;
|
||||
using Pf2eModel;
|
||||
using Pf2eModel.Model.Enum;
|
||||
|
||||
namespace Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.DataBuilder.Processors;
|
||||
|
||||
[Pf2eTypeProcessor("Item.action")]
|
||||
internal class ActionProcessor(ProcessorInput input) : AbstractProcessor(input)
|
||||
[Processor(
|
||||
"foundry-en:action",
|
||||
InnerType = "action",
|
||||
Description = "Создание цели внешних ссылок",
|
||||
Group = ProcessorGroupEnum.FoundryEnEntities
|
||||
)]
|
||||
[ProcessorDependencies(["foundry-en:compendium"])]
|
||||
internal class ActionProcessor(
|
||||
DatabaseHelper databaseHelper,
|
||||
Pf2eDbContext context,
|
||||
ProcessorInput input
|
||||
) : FoundryEnAbstractProcessor(databaseHelper, context, input)
|
||||
{
|
||||
public override EntityType Type => EntityType.Action;
|
||||
|
||||
+16
-2
@@ -1,12 +1,26 @@
|
||||
using Newtonsoft.Json.Linq;
|
||||
using Pf2eDataScrapper.App;
|
||||
using Pf2eDataScrapper.Helpers;
|
||||
using Pf2eDataScrapper.Importers.Implementations.FoundryEn.DataBuilder;
|
||||
using Pf2eDataScrapper.Processors;
|
||||
using Pf2eModel;
|
||||
using Pf2eModel.Model.Enum;
|
||||
using Pf2eModel.Model.Structure;
|
||||
|
||||
namespace Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.DataBuilder.Processors;
|
||||
|
||||
[Pf2eTypeProcessor("Item.condition")]
|
||||
internal class ConditionProcessor(ProcessorInput input) : AbstractProcessor(input)
|
||||
[Processor(
|
||||
"foundry-en:condition",
|
||||
InnerType = "Item.condition",
|
||||
Description = "Создание цели внешних ссылок",
|
||||
Group = ProcessorGroupEnum.FoundryEnEntities
|
||||
)]
|
||||
[ProcessorDependencies(["foundry-en:compendium"])]
|
||||
internal class ConditionProcessor(
|
||||
DatabaseHelper databaseHelper,
|
||||
Pf2eDbContext context,
|
||||
ProcessorInput input
|
||||
) : FoundryEnAbstractProcessor(databaseHelper, context, input)
|
||||
{
|
||||
public override EntityType Type => EntityType.Condition;
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
using Newtonsoft.Json.Linq;
|
||||
using Pf2eDataScrapper.Importers.Implementations.FoundryEn.Compendium;
|
||||
|
||||
namespace Pf2eDataScrapper.Importers.Implementations.FoundryEn;
|
||||
|
||||
/// <summary>
|
||||
/// Загрузчик данных компендиума.
|
||||
/// </summary>
|
||||
/// <param name="repository">Репозиторий данных.</param>
|
||||
internal sealed class FoundryEnCompendium(
|
||||
FoundryEnRepository repository,
|
||||
CompendiumFolderCollection folders
|
||||
) : AbstractCompendium
|
||||
{
|
||||
/// <summary>
|
||||
/// Репозиторий данных.
|
||||
/// </summary>
|
||||
public override FoundryEnRepository Repository { get; } = repository;
|
||||
|
||||
/// <summary>
|
||||
/// Набор папок компендиума
|
||||
/// </summary>
|
||||
public CompendiumFolderCollection Folders { get; } = folders;
|
||||
|
||||
/// <summary>
|
||||
/// Коллекция всех записей компендиума.
|
||||
/// </summary>
|
||||
public List<FoundryEnCompendiumItem> Items { get; } = [];
|
||||
|
||||
/// <summary>
|
||||
/// Возвращает количество элементов в компендиуме.
|
||||
/// </summary>
|
||||
/// <returns>Количество элементов.</returns>
|
||||
public override int Count() => Items.Count;
|
||||
|
||||
/// <summary>
|
||||
/// Возвращает количество элементов для указанного типа данных.
|
||||
/// </summary>
|
||||
/// <param name="type">Тип данных.</param>
|
||||
/// <returns>Количество элементов.</returns>
|
||||
public override int Count(string type) => Items.Count(i => i.Type == type);
|
||||
|
||||
/// <summary>
|
||||
/// Получение итератора сущностей. Если указан тип <paramref name="type"/> - то только указанного типа.
|
||||
/// </summary>
|
||||
/// <param name="type">Тип сущностей, которые нужно получить.</param>
|
||||
/// <returns>Итератор сущностей.</returns>
|
||||
public override IEnumerable<Tuple<AbstractCompendiumItem, JObject>> Iterate(string? type = null)
|
||||
{
|
||||
IEnumerable<FoundryEnCompendiumItem> items = type is null
|
||||
? Items
|
||||
: Items.Where(i => i.Type == type);
|
||||
|
||||
foreach (FoundryEnCompendiumItem item in items)
|
||||
{
|
||||
using StreamReader fileReader = new(
|
||||
Path.Combine(Repository.RepositoryDirPath, item.File)
|
||||
);
|
||||
|
||||
yield return new(item, JObject.Parse(fileReader.ReadToEnd()));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Pf2eDataScrapper.Importers.Implementations.FoundryEn.Compendium;
|
||||
using Pf2eDataScrapper.Importers.Implementations.FoundryEn.DataBuilder;
|
||||
using Pf2eDataScrapper.Importers.Repositories;
|
||||
|
||||
namespace Pf2eDataScrapper.Importers.Implementations.FoundryEn;
|
||||
|
||||
/// <summary>
|
||||
/// Импортёр для модуля Pf2e FoundryVTT.
|
||||
/// </summary>
|
||||
internal class FoundryEnImporter(FoundryEnRepository repository, FoundryEnCompendium compendium)
|
||||
: AbstractImporter
|
||||
{
|
||||
/// <summary>
|
||||
/// Название внешней ссылки.
|
||||
/// </summary>
|
||||
public const string TargetName = "Pf2e FoundryVTT module";
|
||||
|
||||
/// <summary>
|
||||
/// Отображаемое имя импортёра.
|
||||
/// </summary>
|
||||
public override string Name => TargetName;
|
||||
|
||||
/// <summary>
|
||||
/// Короткое имя импортёра.
|
||||
/// </summary>
|
||||
public override string Slug => "foundry-en";
|
||||
|
||||
public override FoundryEnRepository Repository { get; } = repository;
|
||||
|
||||
/// <summary>
|
||||
/// Загрузчик данных компендиума.
|
||||
/// </summary>
|
||||
public override FoundryEnCompendium Compendium { get; } = compendium;
|
||||
|
||||
#region "Внедрение зависимостей"
|
||||
|
||||
public static void RegisterServices(IServiceCollection services)
|
||||
{
|
||||
services.AddSingleton<FoundryEnImporter>();
|
||||
services.AddSingleton<FoundryEnRepository>();
|
||||
services.AddSingleton<FoundryEnCompendium>();
|
||||
|
||||
services.AddSingleton<CompendiumFolderCollection>();
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Git-репозиторий данных FoundryVTT-модуля "pf2e".
|
||||
/// </summary>
|
||||
/// <param name="importer"><inheritdoc cref="FoundryEnImporter"/></param>
|
||||
internal class FoundryEnRepository(FoundryEnImporter importer) : GitRepository(importer, GitUrl)
|
||||
{
|
||||
public const string GitUrl = "https://github.com/foundryvtt/pf2e.git";
|
||||
}
|
||||
+123
@@ -0,0 +1,123 @@
|
||||
using Newtonsoft.Json.Linq;
|
||||
using Pf2eDataScrapper.App;
|
||||
using Pf2eDataScrapper.Importers.Implementations.FoundryEn.Compendium;
|
||||
using Pf2eDataScrapper.Processors;
|
||||
|
||||
namespace Pf2eDataScrapper.Importers.Implementations.FoundryEn.Processors.Preparation;
|
||||
|
||||
[Processor(
|
||||
"foundry-en:compendium-folders",
|
||||
Description = "Загрузка каталогов компендиума",
|
||||
Group = ProcessorGroupEnum.FoundryEnPreparation
|
||||
)]
|
||||
[ProcessorDependencies(["foundry-en:repository"])]
|
||||
internal class CompendiumFolderProcessor(
|
||||
FoundryEnRepository repository,
|
||||
CompendiumFolderCollection folders
|
||||
) : IProcessor
|
||||
{
|
||||
private FoundryEnRepository Repository { get; } = repository;
|
||||
private CompendiumFolderCollection Folders { get; } = folders;
|
||||
|
||||
private string Dir => Repository.RepositoryDirPath;
|
||||
private int NextFolderId { get; set; } = 0;
|
||||
|
||||
public void Process()
|
||||
{
|
||||
LoadPacks();
|
||||
ScanFolders();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Загрузка основных папок.
|
||||
/// </summary>
|
||||
private void LoadPacks()
|
||||
{
|
||||
using StreamReader systemFile = new(Path.Combine(Dir, "static", "system.json"));
|
||||
JObject systemData = JObject.Parse(systemFile.ReadToEnd());
|
||||
|
||||
foreach (JToken pack in systemData["packs"]!)
|
||||
{
|
||||
CompendiumFolder folder = new((JObject)pack);
|
||||
|
||||
Folders.Add(folder);
|
||||
}
|
||||
|
||||
if (systemData.TryGetValue("packFolders", out JToken? packFolders))
|
||||
{
|
||||
ProcessPackFolders((JArray)packFolders, null);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Обработка папок компендиума в поле `packFolders`.
|
||||
/// </summary>
|
||||
/// <param name="folders"></param>
|
||||
/// <param name="parent"></param>
|
||||
private void ProcessPackFolders(JArray folders, CompendiumFolder? parent)
|
||||
{
|
||||
foreach (JToken token in folders)
|
||||
{
|
||||
JObject obj = (JObject)token;
|
||||
|
||||
string id = $"folder-{NextFolderId++}";
|
||||
string name = obj["name"]!.ToString();
|
||||
|
||||
CompendiumFolder folder = new(id, name) { Parent = parent };
|
||||
Folders.Add(folder);
|
||||
|
||||
if (obj.TryGetValue("packs", out JToken? packs))
|
||||
{
|
||||
foreach (JToken jToken in packs)
|
||||
{
|
||||
string packId = jToken.ToString();
|
||||
CompendiumFolder? child = Folders.GetById(packId);
|
||||
|
||||
if (child is not null)
|
||||
{
|
||||
child.Parent = folder;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (obj.TryGetValue("folders", out JToken? innerFolders))
|
||||
{
|
||||
ProcessPackFolders((JArray)innerFolders, folder);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void ScanFolders()
|
||||
{
|
||||
// Поиск _folders.json
|
||||
foreach (CompendiumFolder parent in Folders.Items)
|
||||
{
|
||||
if (parent.Path is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
string dirPath = Path.Combine(Dir, parent.Path);
|
||||
|
||||
foreach (
|
||||
string foldersFilePath in Directory.GetFiles(
|
||||
dirPath,
|
||||
"_folders.json",
|
||||
SearchOption.AllDirectories
|
||||
)
|
||||
)
|
||||
{
|
||||
using StreamReader file = new(foldersFilePath);
|
||||
JArray data = JArray.Parse(file.ReadToEnd());
|
||||
|
||||
foreach (JToken pack in data)
|
||||
{
|
||||
CompendiumFolder folder = new((JObject)pack) { Parent = parent };
|
||||
Folders.Add(folder);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void Save() { }
|
||||
}
|
||||
+104
@@ -0,0 +1,104 @@
|
||||
using Newtonsoft.Json.Linq;
|
||||
using Pf2eDataScrapper.App;
|
||||
using Pf2eDataScrapper.Importers.Implementations.FoundryEn.Compendium;
|
||||
using Pf2eDataScrapper.Processors;
|
||||
|
||||
namespace Pf2eDataScrapper.Importers.Implementations.FoundryEn.Processors.Preparation;
|
||||
|
||||
[Processor(
|
||||
"foundry-en:compendium",
|
||||
Description = "Загрузка компендиума",
|
||||
Group = ProcessorGroupEnum.FoundryEnPreparation
|
||||
)]
|
||||
[ProcessorDependencies(["foundry-en:compendium-folders"])]
|
||||
internal class CompendiumItemsProcessor(
|
||||
FoundryEnRepository repository,
|
||||
FoundryEnCompendium compendium,
|
||||
ProcessProgress progress
|
||||
) : IProcessor
|
||||
{
|
||||
private FoundryEnRepository Repository { get; } = repository;
|
||||
private FoundryEnCompendium Compendium { get; } = compendium;
|
||||
public ProcessProgress Progress { get; } = progress;
|
||||
|
||||
private string Dir => Repository.RepositoryDirPath;
|
||||
private List<CompendiumFolder> RootFolders { get; } =
|
||||
compendium.Folders.Items.FindAll(f => f.Path != null);
|
||||
|
||||
public void Process()
|
||||
{
|
||||
Progress.Local.Reset(CountItems());
|
||||
|
||||
foreach (CompendiumFolder folder in RootFolders)
|
||||
{
|
||||
string dirPath = Path.Combine(Dir, folder.Path!);
|
||||
|
||||
foreach (
|
||||
string filePath in Directory.GetFiles(
|
||||
dirPath,
|
||||
"*.json",
|
||||
SearchOption.AllDirectories
|
||||
)
|
||||
)
|
||||
{
|
||||
string relativePath = Path.GetRelativePath(filePath, Dir);
|
||||
Progress.Local.Start(relativePath);
|
||||
|
||||
FoundryEnCompendiumItem? item = ProcessFile(relativePath, folder);
|
||||
|
||||
if (item is not null)
|
||||
{
|
||||
Compendium.Items.Add(item);
|
||||
}
|
||||
|
||||
Progress.Local.End();
|
||||
}
|
||||
}
|
||||
|
||||
Progress.Local.Complete();
|
||||
}
|
||||
|
||||
private FoundryEnCompendiumItem? ProcessFile(string relativePath, CompendiumFolder folder)
|
||||
{
|
||||
FileInfo file = new(Path.Combine(Dir, relativePath));
|
||||
|
||||
if (file.Name == "_folders.json")
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
using StreamReader reader = new(file.FullName);
|
||||
JObject itemJson = JObject.Parse(reader.ReadToEnd());
|
||||
|
||||
CompendiumFolder? itemFolder = folder;
|
||||
|
||||
JToken? itemJsonFolder = itemJson.SelectToken("folder");
|
||||
if (itemJsonFolder is not null)
|
||||
{
|
||||
itemFolder = Compendium.Folders.GetById(itemJsonFolder.ToString());
|
||||
}
|
||||
|
||||
if (itemFolder is null)
|
||||
{
|
||||
throw new(
|
||||
$"Не удаётся определить папку, к которой относится элемент {file.Name} (путь: {file.FullName})"
|
||||
);
|
||||
}
|
||||
|
||||
string? type = itemJson.SelectToken("type")?.ToString();
|
||||
|
||||
return new(file.Name, itemFolder.Type + (type is not null ? $".{type}" : ""), relativePath);
|
||||
}
|
||||
|
||||
private int CountItems()
|
||||
{
|
||||
return RootFolders.Sum(folder =>
|
||||
{
|
||||
string dirPath = Path.Combine(Dir, folder.Path!);
|
||||
return Directory.GetFiles(dirPath, "*.json", SearchOption.AllDirectories).Length
|
||||
- Directory.GetFiles(dirPath, "_folders.json", SearchOption.AllDirectories).Length;
|
||||
});
|
||||
}
|
||||
|
||||
public void Save() { }
|
||||
}
|
||||
+23
@@ -0,0 +1,23 @@
|
||||
using Pf2eDataScrapper.App;
|
||||
using Pf2eDataScrapper.Processors;
|
||||
|
||||
namespace Pf2eDataScrapper.Importers.Implementations.FoundryEn.Processors.Preparation;
|
||||
|
||||
[Processor(
|
||||
"foundry-en:repository",
|
||||
Description = "Обновление репозитория",
|
||||
Group = ProcessorGroupEnum.FoundryEnPreparation
|
||||
)]
|
||||
internal class RepositoryUpdateProcessor(FoundryEnRepository repository, ProcessProgress progress)
|
||||
: IProcessor
|
||||
{
|
||||
public FoundryEnRepository Repository { get; } = repository;
|
||||
public ProcessProgress Progress { get; } = progress;
|
||||
|
||||
public void Process()
|
||||
{
|
||||
Repository.Update(Progress);
|
||||
}
|
||||
|
||||
public void Save() { }
|
||||
}
|
||||
+99
@@ -0,0 +1,99 @@
|
||||
using Pf2eDataScrapper.App;
|
||||
using Pf2eDataScrapper.Importers.Implementations.FoundryEn.Compendium;
|
||||
using Pf2eDataScrapper.Processors;
|
||||
using Pf2eDataScrapper.SchemaBuilder;
|
||||
|
||||
namespace Pf2eDataScrapper.Importers.Implementations.FoundryEn.Processors.Preparation;
|
||||
|
||||
[Processor(
|
||||
"foundry-en:schema",
|
||||
Description = "Построение схемы данных",
|
||||
Group = ProcessorGroupEnum.AonprdPreparation
|
||||
)]
|
||||
[ProcessorDependencies(["foundry-en:compendium"])]
|
||||
internal class SchemaBuildProcessor(
|
||||
FoundryEnImporter importer,
|
||||
FoundryEnRepository repository,
|
||||
FoundryEnCompendium compendium,
|
||||
ProcessProgress progress,
|
||||
ApplicationSettings settings
|
||||
) : AbstractSchemaBuildProcessor
|
||||
{
|
||||
/// <inheritdoc cref="FoundryEnImporter"/>
|
||||
protected FoundryEnImporter Importer { get; } = importer;
|
||||
|
||||
/// <inheritdoc cref="FoundryEnRepository"/>
|
||||
protected FoundryEnRepository Repository { get; } = repository;
|
||||
|
||||
/// <inheritdoc cref="FoundryEnCompendium"/>
|
||||
protected override FoundryEnCompendium Compendium { get; } = compendium;
|
||||
|
||||
/// <inheritdoc cref="ProcessProgress"/>
|
||||
protected override ProcessProgress Progress { get; } = progress;
|
||||
|
||||
/// <inheritdoc cref="ApplicationSettings"/>
|
||||
protected ApplicationSettings Settings { get; } = settings;
|
||||
|
||||
/// <inheritdoc/>
|
||||
protected override string SavePath => Path.Combine(Settings.OutputDirPath, Importer.Slug);
|
||||
|
||||
/// <summary>
|
||||
/// Определитель типов данных.
|
||||
/// </summary>
|
||||
protected override SchemaTypeResolver Resolver { get; } = new();
|
||||
|
||||
/// <summary>
|
||||
/// Список типов, являющихся экипировкой.
|
||||
/// </summary>
|
||||
private readonly string[] _equipmentTypes =
|
||||
[
|
||||
"Item.equipment",
|
||||
"Item.armor",
|
||||
"Item.backpack",
|
||||
"Item.consumable",
|
||||
"Item.kit",
|
||||
"Item.shield",
|
||||
"Item.treasure",
|
||||
"Item.weapon",
|
||||
];
|
||||
|
||||
/// <summary>
|
||||
/// Выбор целевых элементом схемы данных для типа элемента.
|
||||
/// </summary>
|
||||
/// <param name="type">Тип входного элемента.</param>
|
||||
/// <returns>Список целевых элементов схемы данных.</returns>
|
||||
private List<SchemaNode> GetSchemaNodes(string type)
|
||||
{
|
||||
List<SchemaNode> nodes = [];
|
||||
|
||||
// Общая нода
|
||||
if (!Schemas.TryGetValue(type, out SchemaNode? typeNode))
|
||||
{
|
||||
typeNode = new() { Resolver = Resolver, Path = type };
|
||||
Schemas[type] = typeNode;
|
||||
}
|
||||
|
||||
nodes.Add(typeNode);
|
||||
|
||||
// Нода подкласса
|
||||
if (_equipmentTypes.Contains(type))
|
||||
{
|
||||
nodes.Add(Schemas["Equipment"]);
|
||||
}
|
||||
|
||||
return nodes;
|
||||
}
|
||||
|
||||
protected override ICollection<string> GetSchemaNodes(AbstractCompendiumItem item)
|
||||
{
|
||||
List<string> nodes = [item.Type];
|
||||
|
||||
// Нода подкласса
|
||||
if (_equipmentTypes.Contains(item.Type))
|
||||
{
|
||||
nodes.Add("Equipment");
|
||||
}
|
||||
|
||||
return nodes;
|
||||
}
|
||||
}
|
||||
+2
-2
@@ -2,9 +2,9 @@
|
||||
using Pf2eDataScrapper.SchemaBuilder;
|
||||
using Pf2eDataScrapper.SchemaBuilder.Types;
|
||||
|
||||
namespace Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.SchemaBuilder;
|
||||
namespace Pf2eDataScrapper.Importers.Implementations.FoundryEn.Processors.Preparation;
|
||||
|
||||
internal class CompendiumSchemaTypeResolver : TypeResolver
|
||||
internal class SchemaTypeResolver : TypeResolver
|
||||
{
|
||||
public class FoundryRulesSchemaType(SchemaNode parent) : IgnoreSchemaType(parent)
|
||||
{
|
||||
-26
@@ -1,26 +0,0 @@
|
||||
namespace Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.Compendium;
|
||||
|
||||
/// <summary>
|
||||
/// Коллекция папок компендиума
|
||||
/// </summary>
|
||||
internal class CompendiumFolderCollection : List<CompendiumFolder>
|
||||
{
|
||||
/// <summary>
|
||||
/// Получение папки по идентификатору
|
||||
/// </summary>
|
||||
/// <param name="id">Искомый идентификатор</param>
|
||||
/// <returns>Искомая папка</returns>
|
||||
public CompendiumFolder? GetById(string id)
|
||||
{
|
||||
return this.FirstOrDefault(x => x.Id == id);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Фильтрация требуемых папок компендиума.
|
||||
/// </summary>
|
||||
/// <returns></returns>
|
||||
public CompendiumFolderCollection Filter()
|
||||
{
|
||||
return [.. this.FindAll(x => x.Type == CompendiumDocumentType.Item)];
|
||||
}
|
||||
}
|
||||
@@ -1,178 +0,0 @@
|
||||
using Newtonsoft.Json.Linq;
|
||||
using Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.Compendium;
|
||||
|
||||
namespace Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry;
|
||||
|
||||
/// <summary>
|
||||
/// Загрузчик данных компендиума
|
||||
/// </summary>
|
||||
/// <param name="importer">Импортёр</param>
|
||||
internal class CompendiumLoader(Pf2EFoundryImporter importer)
|
||||
{
|
||||
/// <summary>
|
||||
/// Импортёр
|
||||
/// </summary>
|
||||
public readonly Pf2EFoundryImporter Importer = importer;
|
||||
|
||||
/// <summary>
|
||||
/// Набор папок компендиума
|
||||
/// </summary>
|
||||
public CompendiumFolderCollection Collection = [];
|
||||
|
||||
/// <summary>
|
||||
/// Счётчик папок
|
||||
/// </summary>
|
||||
private int _folderCount;
|
||||
|
||||
/// <summary>
|
||||
/// Общее количество элементов в компендиуме
|
||||
/// </summary>
|
||||
public int ItemCount { get; protected set; }
|
||||
|
||||
/// <summary>
|
||||
/// Обработка папок компендиума в поле `packFolders`.
|
||||
/// </summary>
|
||||
/// <param name="folders"></param>
|
||||
/// <param name="parent"></param>
|
||||
protected void ProcessPackFolders(JArray folders, CompendiumFolder? parent)
|
||||
{
|
||||
foreach (JToken token in folders)
|
||||
{
|
||||
JObject obj = (JObject)token;
|
||||
|
||||
string id = $"folder-{_folderCount++}";
|
||||
string name = obj["name"]!.ToString();
|
||||
|
||||
CompendiumFolder folder = new(id, name) { Parent = parent };
|
||||
Collection.Add(folder);
|
||||
|
||||
if (obj.TryGetValue("packs", out JToken? packs))
|
||||
{
|
||||
foreach (JToken jToken in packs)
|
||||
{
|
||||
string packId = jToken.ToString();
|
||||
CompendiumFolder? child = Collection.GetById(packId);
|
||||
|
||||
if (child is not null)
|
||||
{
|
||||
child.Parent = folder;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (obj.TryGetValue("folders", out JToken? innerFolders))
|
||||
{
|
||||
ProcessPackFolders((JArray)innerFolders, folder);
|
||||
}
|
||||
}
|
||||
|
||||
Collection = Collection.Filter();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Подготовка набора папок компендиума
|
||||
/// </summary>
|
||||
public void Prepare()
|
||||
{
|
||||
using StreamReader systemFile = new(Path.Combine(Importer.Dir, "static", "system.json"));
|
||||
JObject systemData = JObject.Parse(systemFile.ReadToEnd());
|
||||
|
||||
foreach (JToken pack in systemData["packs"]!)
|
||||
{
|
||||
CompendiumFolder folder = new((JObject)pack);
|
||||
|
||||
Collection.Add(folder);
|
||||
}
|
||||
|
||||
if (systemData.TryGetValue("packFolders", out JToken? packFolders))
|
||||
{
|
||||
ProcessPackFolders((JArray)packFolders, null);
|
||||
}
|
||||
|
||||
// Поиск _folders.json
|
||||
foreach (CompendiumFolder parent in Collection.ToList())
|
||||
{
|
||||
if (parent.Path is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
string dirPath = Path.Combine(Importer.Dir, parent.Path);
|
||||
|
||||
ItemCount += Directory.GetFiles(dirPath, "*.json", SearchOption.AllDirectories).Length;
|
||||
|
||||
foreach (
|
||||
string foldersFilePath in Directory.GetFiles(
|
||||
dirPath,
|
||||
"_folders.json",
|
||||
SearchOption.AllDirectories
|
||||
)
|
||||
)
|
||||
{
|
||||
ItemCount--;
|
||||
using StreamReader file = new(foldersFilePath);
|
||||
JArray data = JArray.Parse(file.ReadToEnd());
|
||||
|
||||
foreach (JToken pack in data)
|
||||
{
|
||||
CompendiumFolder folder = new((JObject)pack) { Parent = parent };
|
||||
Collection.Add(folder);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// foreach (var folder in Collection)
|
||||
// {
|
||||
// Console.WriteLine(folder);
|
||||
// }
|
||||
|
||||
Collection = Collection.Filter();
|
||||
|
||||
Console.WriteLine($"Total items: {ItemCount}");
|
||||
}
|
||||
|
||||
public IEnumerable<LoaderEnrty> Enrties()
|
||||
{
|
||||
foreach (CompendiumFolder folder in Collection.FindAll(f => f.Path is not null))
|
||||
{
|
||||
string dirPath = Path.Combine(Importer.Dir, folder.Path!);
|
||||
|
||||
foreach (
|
||||
string filePath in Directory.GetFiles(
|
||||
dirPath,
|
||||
"*.json",
|
||||
SearchOption.AllDirectories
|
||||
)
|
||||
)
|
||||
{
|
||||
FileInfo file = new(filePath);
|
||||
|
||||
if (file.Name == "_folders.json")
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
using StreamReader reader = new(file.FullName);
|
||||
JObject itemJson = JObject.Parse(reader.ReadToEnd());
|
||||
|
||||
CompendiumFolder? itemFolder = folder;
|
||||
if (itemJson.TryGetValue("folder", out JToken? itemJsonFolder))
|
||||
{
|
||||
itemFolder = Collection.GetById(itemJsonFolder.ToString());
|
||||
}
|
||||
|
||||
yield return new LoaderEnrty(file, itemJson, itemFolder ?? folder);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public class LoaderEnrty(FileInfo file, JObject itemJson, CompendiumFolder folder)
|
||||
{
|
||||
public FileInfo File { get; } = file;
|
||||
public JObject Json { get; } = itemJson;
|
||||
public CompendiumFolder Folder { get; } = folder;
|
||||
public string Type { get; } =
|
||||
itemJson["type"]?.ToString()
|
||||
?? throw new Exception($"Expected to have `type` field in file {file.FullName}");
|
||||
}
|
||||
}
|
||||
-88
@@ -1,88 +0,0 @@
|
||||
using System.Reflection;
|
||||
using Konsole;
|
||||
using Newtonsoft.Json.Linq;
|
||||
using Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.Compendium;
|
||||
using Pf2eDataScrapper.Processors;
|
||||
using Pf2eModel;
|
||||
|
||||
namespace Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.DataBuilder;
|
||||
|
||||
internal class PF2eFoundryDataBuilder
|
||||
{
|
||||
/// <summary>
|
||||
/// Загрузчик данных.
|
||||
/// </summary>
|
||||
public readonly CompendiumLoader Loader;
|
||||
|
||||
/// <summary>
|
||||
/// Коллекция папок компендиума.
|
||||
/// </summary>
|
||||
public CompendiumFolderCollection Collection => Loader.Collection;
|
||||
|
||||
public Dictionary<string, ConstructorInfo> Processors = [];
|
||||
|
||||
public PF2eFoundryDataBuilder(CompendiumLoader loader)
|
||||
{
|
||||
Loader = loader;
|
||||
PrepareProcessors();
|
||||
}
|
||||
|
||||
public void PrepareProcessors()
|
||||
{
|
||||
foreach (Type type in Assembly.GetExecutingAssembly().GetTypes())
|
||||
{
|
||||
if (type.GetCustomAttributes(typeof(Pf2eTypeProcessor), true).Length <= 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
Pf2eTypeProcessor? attribute = type.GetCustomAttribute<Pf2eTypeProcessor>();
|
||||
Processors.Add(attribute!.Type, type.GetConstructor([typeof(ProcessorInput)])!);
|
||||
}
|
||||
}
|
||||
|
||||
public AbstractProcessor? GetProcessor(string type, JObject input)
|
||||
{
|
||||
return Processors.TryGetValue(type, out ConstructorInfo? constructor)
|
||||
? (AbstractProcessor)constructor.Invoke([new ProcessorInput(input)])
|
||||
: null;
|
||||
}
|
||||
|
||||
public void TruncateTables()
|
||||
{
|
||||
Console.WriteLine("Truncating all old data...");
|
||||
|
||||
using Pf2eDbContext context = new();
|
||||
string cmd = context.Entities.Truncate() + "\n" + context.TruncateAll();
|
||||
|
||||
Console.WriteLine(cmd);
|
||||
|
||||
Console.WriteLine("Truncation completed!");
|
||||
}
|
||||
|
||||
public void Build()
|
||||
{
|
||||
Console.WriteLine("Building data...");
|
||||
|
||||
ProgressBar bar = new(Loader.ItemCount);
|
||||
int i = 0;
|
||||
|
||||
foreach (CompendiumLoader.LoaderEnrty entry in Loader.Enrties())
|
||||
{
|
||||
bar.Refresh(i++, $"{entry.Folder.Name}/{entry.File.Name}");
|
||||
|
||||
AbstractProcessor? processor = GetProcessor(
|
||||
$"{entry.Folder.Type}.{entry.Type}",
|
||||
entry.Json
|
||||
);
|
||||
if (processor is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
processor.Process();
|
||||
processor.Save();
|
||||
}
|
||||
|
||||
bar.Refresh(Loader.ItemCount, "Done!");
|
||||
}
|
||||
}
|
||||
-9
@@ -1,9 +0,0 @@
|
||||
namespace Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.DataBuilder;
|
||||
|
||||
[AttributeUsage(AttributeTargets.Class)]
|
||||
internal class Pf2eTypeProcessor(string type, int priority = 0) : Attribute
|
||||
{
|
||||
public readonly string Type = type;
|
||||
|
||||
public readonly int Priority = priority;
|
||||
}
|
||||
@@ -1,67 +0,0 @@
|
||||
using Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.DataBuilder;
|
||||
using Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.SchemaBuilder;
|
||||
using Pf2eDataScrapper.Importers.Repositories;
|
||||
|
||||
namespace Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry;
|
||||
|
||||
/// <summary>
|
||||
/// Импортёр для модуля Pf2e FoundryVTT.
|
||||
/// </summary>
|
||||
internal class Pf2EFoundryImporter : AbstractImporter
|
||||
{
|
||||
/// <summary>
|
||||
/// Отображаемое имя импортёра.
|
||||
/// </summary>
|
||||
public override string Name => "Pf2e FoundryVTT module";
|
||||
|
||||
/// <summary>
|
||||
/// Короткое имя импортёра.
|
||||
/// </summary>
|
||||
public override string Slug => "pf2e";
|
||||
|
||||
public override AbstractImporterRepository Repository =>
|
||||
new GitRepository(this, "https://github.com/foundryvtt/pf2e.git");
|
||||
|
||||
/// <summary>
|
||||
/// Загрузчик данных компендиума.
|
||||
/// </summary>
|
||||
public CompendiumLoader Loader { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Конструктор импортёра.
|
||||
/// </summary>
|
||||
public Pf2EFoundryImporter()
|
||||
{
|
||||
Loader = new CompendiumLoader(this);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Метод подготовки данных для импорта.
|
||||
/// </summary>
|
||||
public override void Prepare()
|
||||
{
|
||||
Loader.Prepare();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Импорт данных.
|
||||
/// </summary>
|
||||
public override void Import()
|
||||
{
|
||||
PF2eFoundryDataBuilder builder = new(Loader);
|
||||
|
||||
builder.TruncateTables();
|
||||
builder.Build();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Построение схем данных
|
||||
/// </summary>
|
||||
public override void BuildSchemas()
|
||||
{
|
||||
CompendiumSchemaBuilder builder = new(Loader);
|
||||
|
||||
builder.BuildSchemas();
|
||||
builder.Save(EnvSettings.OutputDir);
|
||||
}
|
||||
}
|
||||
-197
@@ -1,197 +0,0 @@
|
||||
using Konsole;
|
||||
using Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.Compendium;
|
||||
using Pf2eDataScrapper.SchemaBuilder;
|
||||
|
||||
namespace Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.SchemaBuilder;
|
||||
|
||||
/// <summary>
|
||||
/// Класс построения схемы компендиумов.
|
||||
/// </summary>
|
||||
/// <param name="loader">Загрузчик данных.</param>
|
||||
internal class CompendiumSchemaBuilder(CompendiumLoader loader)
|
||||
{
|
||||
/// <summary>
|
||||
/// Загрузчик данных.
|
||||
/// </summary>
|
||||
public CompendiumLoader Loader = loader;
|
||||
|
||||
/// <summary>
|
||||
/// Коллекция папок компендиума.
|
||||
/// </summary>
|
||||
public CompendiumFolderCollection Collection = loader.Collection;
|
||||
|
||||
/// <summary>
|
||||
/// Объект для определения типов полей в компендиумах PF2E.
|
||||
/// </summary>
|
||||
public CompendiumSchemaTypeResolver Resolver = new();
|
||||
|
||||
/// <summary>
|
||||
/// Схемы данных по папкам.
|
||||
/// </summary>
|
||||
public Dictionary<CompendiumFolder, SchemaNode> PackSchemas = [];
|
||||
|
||||
/// <summary>
|
||||
/// Схемы данных по типам.
|
||||
/// </summary>
|
||||
public Dictionary<string, SchemaNode> TypeSchemas = [];
|
||||
|
||||
private readonly string[] _equipmentTypes =
|
||||
[
|
||||
"equipment",
|
||||
"armor",
|
||||
"backpack",
|
||||
"consumable",
|
||||
"kit",
|
||||
"shield",
|
||||
"treasure",
|
||||
"weapon",
|
||||
];
|
||||
|
||||
private SchemaNode GetPackSchema(CompendiumFolder folder)
|
||||
{
|
||||
if (!PackSchemas.TryGetValue(folder, out SchemaNode? value))
|
||||
{
|
||||
value = new SchemaNode() { Resolver = Resolver, Path = folder.Name };
|
||||
PackSchemas[folder] = value;
|
||||
}
|
||||
|
||||
return value;
|
||||
}
|
||||
|
||||
private SchemaNode GetTypeSchema(string key)
|
||||
{
|
||||
if (!TypeSchemas.TryGetValue(key, out SchemaNode? value))
|
||||
{
|
||||
value = new SchemaNode() { Resolver = Resolver, Path = key };
|
||||
TypeSchemas[key] = value;
|
||||
}
|
||||
|
||||
return value;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Получение набора схем, к которым относится проверяемый объект.
|
||||
/// </summary>
|
||||
/// <param name="folder">Папка компендиума, в которой находится объект.</param>
|
||||
/// <param name="itemType">Указанный тип объекта.</param>
|
||||
/// <returns></returns>
|
||||
protected List<SchemaNode> GetItemSchemas(CompendiumFolder folder, string? itemType)
|
||||
{
|
||||
string key = folder.Type.ToString() + (itemType is not null ? $".{itemType}" : "");
|
||||
|
||||
return _equipmentTypes.Contains(itemType)
|
||||
? [GetPackSchema(folder), GetTypeSchema(key), GetTypeSchema("Equipment")]
|
||||
: [GetPackSchema(folder), GetTypeSchema(key)];
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Построение схем данных.
|
||||
/// </summary>
|
||||
public void BuildSchemas()
|
||||
{
|
||||
Console.WriteLine("Building schemas...");
|
||||
|
||||
ProgressBar bar = new(Loader.ItemCount);
|
||||
int i = 0;
|
||||
|
||||
foreach (CompendiumLoader.LoaderEnrty entry in Loader.Enrties())
|
||||
{
|
||||
bar.Refresh(i++, entry.Folder.Name);
|
||||
|
||||
List<SchemaNode> schemas = GetItemSchemas(entry.Folder, entry.Type);
|
||||
|
||||
foreach (SchemaNode schema in schemas)
|
||||
{
|
||||
schema.AddValue(entry.Json);
|
||||
}
|
||||
}
|
||||
|
||||
bar.Refresh(Loader.ItemCount, "Done!");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Сохранение данных.
|
||||
/// </summary>
|
||||
/// <param name="targetDir">Папка для сохранения.</param>
|
||||
public void Save(string targetDir)
|
||||
{
|
||||
Console.WriteLine("Dumping schemas...");
|
||||
|
||||
ProgressBar bar = new(PackSchemas.Count + TypeSchemas.Count);
|
||||
int i = 0;
|
||||
|
||||
targetDir = Path.Combine(targetDir, Loader.Importer.Slug);
|
||||
|
||||
if (!Directory.Exists(targetDir))
|
||||
{
|
||||
Directory.CreateDirectory(targetDir);
|
||||
}
|
||||
Array.ForEach(Directory.GetFiles(@targetDir), File.Delete);
|
||||
|
||||
// Packs
|
||||
string packsDir = Path.Combine(targetDir, "packs");
|
||||
Directory.CreateDirectory(packsDir);
|
||||
|
||||
foreach (KeyValuePair<CompendiumFolder, SchemaNode> pair in PackSchemas)
|
||||
{
|
||||
bar.Refresh(i++, pair.Key.Name);
|
||||
SaveSchema(packsDir, pair.Key.Name, pair.Value);
|
||||
}
|
||||
|
||||
// Types
|
||||
string typesDir = Path.Combine(targetDir, "types");
|
||||
Directory.CreateDirectory(typesDir);
|
||||
foreach (KeyValuePair<string, SchemaNode> pair in TypeSchemas)
|
||||
{
|
||||
bar.Refresh(i++, pair.Key);
|
||||
SaveSchema(typesDir, pair.Key, pair.Value);
|
||||
}
|
||||
|
||||
Console.WriteLine("Done!");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Сохранение конкретной схемы.
|
||||
/// </summary>
|
||||
/// <param name="dir">Корневая папка для сохранения этой категории схем.</param>
|
||||
/// <param name="name">Название сохраняемой схемы.</param>
|
||||
/// <param name="schema">Сохраняемая схема.</param>
|
||||
protected void SaveSchema(string dir, string name, SchemaNode schema)
|
||||
{
|
||||
char[] invalids = Path.GetInvalidFileNameChars();
|
||||
string fileName = string.Join(
|
||||
"_",
|
||||
schema.Path.Split(invalids, StringSplitOptions.RemoveEmptyEntries)
|
||||
)
|
||||
.TrimEnd('.');
|
||||
|
||||
string path = Path.Combine(dir, fileName + ".yaml");
|
||||
|
||||
string? dirPath = Path.GetDirectoryName(path);
|
||||
|
||||
if (dirPath is not null && !Directory.Exists(dirPath))
|
||||
{
|
||||
Directory.CreateDirectory(dirPath);
|
||||
}
|
||||
|
||||
using StreamWriter writer = new(path);
|
||||
|
||||
schema.Beautify();
|
||||
|
||||
writer.WriteLine(
|
||||
SchemaNode.FormatYaml(
|
||||
new Dictionary<string, string>()
|
||||
{
|
||||
["name"] = name,
|
||||
["values"] = schema.Values.ToString(),
|
||||
},
|
||||
schema.Types[0].GetYamlData()
|
||||
)
|
||||
);
|
||||
|
||||
//writer.WriteLine(name);
|
||||
//writer.WriteLine($"Different values: {schema.Values}");
|
||||
//writer.WriteLine("=====================================");
|
||||
//writer.Write(schema.ToString());
|
||||
}
|
||||
}
|
||||
+1
-1
@@ -1,7 +1,7 @@
|
||||
using Pf2eDataScrapper.App;
|
||||
using Pf2eModel;
|
||||
|
||||
namespace Pf2eDataScrapper.Processors;
|
||||
namespace Pf2eDataScrapper.Processors.General;
|
||||
|
||||
[Processor("truncate", Description = "Отчистка таблиц БД", Group = ProcessorGroupEnum.General)]
|
||||
internal class DatabaseTruncateProcessor : IProcessor
|
||||
@@ -0,0 +1,106 @@
|
||||
using System;
|
||||
using System.ComponentModel;
|
||||
using System.Diagnostics;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Newtonsoft.Json.Linq;
|
||||
using Pf2eDataScrapper.App;
|
||||
using Pf2eDataScrapper.Importers;
|
||||
using Pf2eDataScrapper.Importers.Implementations.Aonprd;
|
||||
using Pf2eDataScrapper.Importers.Implementations.FoundryEn;
|
||||
|
||||
namespace Pf2eDataScrapper.Processors;
|
||||
|
||||
internal sealed class ProcessorExecutor(
|
||||
IServiceProvider serviceProvider,
|
||||
ProcessorCollection collection,
|
||||
ProcessProgress progress
|
||||
)
|
||||
{
|
||||
public IServiceProvider ServiceProvider { get; } = serviceProvider;
|
||||
public ProcessorCollection Collection { get; } = collection;
|
||||
public ProcessProgress Progress { get; } = progress;
|
||||
|
||||
public void Start()
|
||||
{
|
||||
Collection.SortItems();
|
||||
|
||||
Progress.Processors.Reset(Collection.Items.Count);
|
||||
|
||||
foreach (ProcessorCollectionItem item in Collection.Items)
|
||||
{
|
||||
Progress.Processors.Start(item.Description);
|
||||
|
||||
Process(item);
|
||||
|
||||
Progress.Processors.End();
|
||||
}
|
||||
|
||||
Progress.Processors.Complete();
|
||||
}
|
||||
|
||||
private void Process(ProcessorCollectionItem item)
|
||||
{
|
||||
Thread thread = new(() =>
|
||||
{
|
||||
foreach (IProcessor processor in IterateProcessors(item))
|
||||
{
|
||||
processor.Process();
|
||||
processor.Save();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private IEnumerable<IProcessor> IterateProcessors(ProcessorCollectionItem item)
|
||||
{
|
||||
if (!item.IsCollectionProcessor)
|
||||
{
|
||||
Progress.Local.Reset(1);
|
||||
Progress.Local.Start(item.Description);
|
||||
yield return (IProcessor)ActivatorUtilities.CreateInstance(ServiceProvider, item.Type);
|
||||
Progress.Local.Complete();
|
||||
yield break;
|
||||
}
|
||||
|
||||
if (item.Group == ProcessorGroupEnum.General)
|
||||
{
|
||||
throw new(
|
||||
$"Обработчик {item.Description} помечен как общий (поле Group == General) и коллекционный (поле InnerType), что запрещено."
|
||||
);
|
||||
}
|
||||
|
||||
Type importerType = item.Group switch
|
||||
{
|
||||
ProcessorGroupEnum.AonprdPreparation
|
||||
or ProcessorGroupEnum.AonprdEntities
|
||||
or ProcessorGroupEnum.AonprdSubclasses => typeof(AonprdImporter),
|
||||
ProcessorGroupEnum.FoundryEnPreparation or ProcessorGroupEnum.FoundryEnEntities =>
|
||||
typeof(FoundryEnImporter),
|
||||
_ => throw new($"Не могу выбрать импортёр для группы {item.Group}"),
|
||||
};
|
||||
|
||||
AbstractImporter importer = (AbstractImporter)
|
||||
ActivatorUtilities.CreateInstance(ServiceProvider, importerType);
|
||||
|
||||
Progress.Local.Reset(importer.Compendium.Count(item.InnerType!));
|
||||
|
||||
foreach (
|
||||
(AbstractCompendiumItem compendiumItem, JObject json) in importer.Compendium.Iterate(
|
||||
item.InnerType!
|
||||
)
|
||||
)
|
||||
{
|
||||
Progress.Local.Start(compendiumItem.Id);
|
||||
|
||||
yield return (IProcessor)
|
||||
ActivatorUtilities.CreateInstance(
|
||||
ServiceProvider,
|
||||
item.Type,
|
||||
new ProcessorInput(compendiumItem.Id, json)
|
||||
);
|
||||
|
||||
Progress.Local.End();
|
||||
}
|
||||
|
||||
Progress.Local.Complete();
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,8 @@
|
||||
namespace Pf2eDataScrapper;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Pf2eDataScrapper.Importers.Implementations.Aonprd;
|
||||
using Pf2eDataScrapper.Importers.Implementations.FoundryEn;
|
||||
|
||||
namespace Pf2eDataScrapper;
|
||||
|
||||
internal static class Program
|
||||
{
|
||||
@@ -6,7 +10,7 @@ internal static class Program
|
||||
/// The main entry point for the application.
|
||||
/// </summary>
|
||||
[STAThread]
|
||||
static void Main()
|
||||
public static void Main()
|
||||
{
|
||||
// To customize application configuration such as set high DPI settings or default font,
|
||||
// see https://aka.ms/applicationconfiguration.
|
||||
@@ -18,6 +22,22 @@ internal static class Program
|
||||
Application.SetColorMode(SystemColorMode.System);
|
||||
#pragma warning restore WFO5001
|
||||
|
||||
Application.Run(new ImportApplication());
|
||||
IServiceCollection services = RegisterServices();
|
||||
|
||||
IServiceProvider serviceProvider = services.BuildServiceProvider();
|
||||
|
||||
Application.Run(ActivatorUtilities.CreateInstance<ImportApplication>(serviceProvider));
|
||||
}
|
||||
|
||||
private static IServiceCollection RegisterServices()
|
||||
{
|
||||
IServiceCollection services = new ServiceCollection();
|
||||
|
||||
ImportApplication.RegisterServices(services);
|
||||
|
||||
AonprdImporter.RegisterServices(services);
|
||||
FoundryEnImporter.RegisterServices(services);
|
||||
|
||||
return services;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,117 @@
|
||||
using Newtonsoft.Json.Linq;
|
||||
using Pf2eDataScrapper.Importers;
|
||||
using Pf2eDataScrapper.Processors;
|
||||
|
||||
namespace Pf2eDataScrapper.SchemaBuilder;
|
||||
|
||||
internal abstract class AbstractSchemaBuildProcessor : IProcessor
|
||||
{
|
||||
/// <summary>
|
||||
/// Связаный компендиум
|
||||
/// </summary>
|
||||
protected abstract AbstractCompendium Compendium { get; }
|
||||
|
||||
/// <inheritdoc cref="ProcessProgress"/>
|
||||
protected abstract ProcessProgress Progress { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Определитель типов данных.
|
||||
/// </summary>
|
||||
protected abstract TypeResolver Resolver { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Путь к каталогу для сохранения вывода.
|
||||
/// </summary>
|
||||
protected abstract string SavePath { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Набор схем данных.
|
||||
/// </summary>
|
||||
protected Dictionary<string, SchemaNode> Schemas { get; } = [];
|
||||
|
||||
/// <summary>
|
||||
/// Получить список названий элементов структуры данных на основе вводимого элемента.
|
||||
/// </summary>
|
||||
/// <param name="item">Вводимый элемент</param>
|
||||
/// <returns>Коллекция названий элементов структуры данных/</returns>
|
||||
protected abstract ICollection<string> GetSchemaNodes(AbstractCompendiumItem item);
|
||||
|
||||
/// <summary>
|
||||
/// Построение схемы данных.
|
||||
/// </summary>
|
||||
public void Process()
|
||||
{
|
||||
Progress.Local.Reset(Compendium.Count());
|
||||
|
||||
foreach ((AbstractCompendiumItem reference, JObject json) in Compendium.Iterate())
|
||||
{
|
||||
Progress.Local.Start(reference.ToString()!);
|
||||
|
||||
foreach (string nodeName in GetSchemaNodes(reference))
|
||||
{
|
||||
if (!Schemas.TryGetValue(nodeName, out SchemaNode? node))
|
||||
{
|
||||
node = new() { Path = nodeName, Resolver = Resolver };
|
||||
Schemas.Add(nodeName, node);
|
||||
}
|
||||
|
||||
node.AddValue(json);
|
||||
}
|
||||
|
||||
Progress.Local.End();
|
||||
}
|
||||
|
||||
Progress.Local.Complete();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Сохранение постренных схем данных.
|
||||
/// </summary>
|
||||
public void Save()
|
||||
{
|
||||
string targetDir = SavePath;
|
||||
|
||||
foreach (KeyValuePair<string, SchemaNode> kvp in Schemas)
|
||||
{
|
||||
SaveSchema(targetDir, kvp.Key, kvp.Value);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Сохранение отдельной схемы данных.
|
||||
/// </summary>
|
||||
/// <param name="dir">Целевой каталог.</param>
|
||||
/// <param name="name">Название схемы.</param>
|
||||
/// <param name="schema">Сохраняемая схема.</param>
|
||||
protected void SaveSchema(string dir, string name, SchemaNode schema)
|
||||
{
|
||||
char[] invalids = Path.GetInvalidFileNameChars();
|
||||
string fileName = string.Join(
|
||||
"_",
|
||||
schema.Path.Split(invalids, StringSplitOptions.RemoveEmptyEntries)
|
||||
)
|
||||
.TrimEnd('.');
|
||||
string path = Path.Combine(dir, fileName + ".yaml");
|
||||
|
||||
string? dirPath = Path.GetDirectoryName(path);
|
||||
if (dirPath is not null && !Directory.Exists(dirPath))
|
||||
{
|
||||
Directory.CreateDirectory(dirPath);
|
||||
}
|
||||
|
||||
using StreamWriter writer = new(path);
|
||||
|
||||
schema.Beautify();
|
||||
|
||||
writer.WriteLine(
|
||||
SchemaNode.FormatYaml(
|
||||
new Dictionary<string, string>()
|
||||
{
|
||||
["name"] = name,
|
||||
["values"] = schema.Values.ToString(),
|
||||
},
|
||||
schema.Types[0].GetYamlData()
|
||||
)
|
||||
);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user