Переосмысление аттрибутов переберающих процессоров и устаревшых элементов

This commit is contained in:
2025-05-14 15:44:32 +04:00
parent e6f53fc074
commit 995029ce8d
36 changed files with 1036 additions and 68 deletions
+34 -5
View File
@@ -22,15 +22,44 @@ public class ProcessorAttribute(string name) : Attribute
/// </summary>
public ProcessorGroupEnum Group { get; init; } = ProcessorGroupEnum.General;
/// <summary>
/// Коллекция названий обработчиков, которые должны быть выполнены до этого оброботчика.
/// </summary>
public string? InnerType { get; init; } = null;
/// <summary>
/// Процессор включён по умолчанию.
/// </summary>
public bool Active { get; init; } = true;
/// <summary>
/// Разрешить выполнять процессов в параллельном режиме
/// </summary>
public bool Parallel { get; init; } = true;
/// <summary>
/// Обработчик набора элементов
/// </summary>
public virtual bool Collection { get; } = false;
}
/// <summary>
/// Аттрибут обработчика набор элементов.
/// </summary>
/// <param name="name">Название обработчика.</param>
[AttributeUsage(AttributeTargets.Class)]
public class ProcessorCollectionAttribute(string name) : ProcessorAttribute(name)
{
/// <summary>
/// Название типов, который обрабатывает этот оброботчик.
/// <c>null</c> - обработать все типы.
/// </summary>
public required string? InnerType { get; init; }
/// <summary>
/// Тип используемого компендиума.
/// </summary>
public required ProcessorCompendiumEnum CompendiumType { get; init; }
/// <summary>
/// Обработчик набора элементов.
/// </summary>
public override bool Collection { get; } = true;
}
/// <summary>
+24 -5
View File
@@ -24,7 +24,7 @@ internal class ProcessorCollection
/// </summary>
/// <param name="name"></param>
/// <returns></returns>
public ProcessorCollectionItem this[string name] => Items.First(item => item.Name == name);
public ProcessorCollectionItem this[string name] => AllItems.First(item => item.Name == name);
/// <summary>
/// Конструктор класса <see cref="ProcessorCollection"/>.
@@ -65,7 +65,19 @@ internal class ProcessorCollection
ProcessorDependenciesAttribute? dependenciesAttribute =
type.GetCustomAttribute<ProcessorDependenciesAttribute>();
bool isCollection = processorAttribute.InnerType is not null;
bool isCollection = processorAttribute.Collection;
string? innerType = null;
ProcessorCompendiumEnum? compendium = null;
if (processorAttribute.GetType() == typeof(ProcessorCollectionAttribute))
{
ProcessorCollectionAttribute processorCollectionAttribute =
(ProcessorCollectionAttribute)processorAttribute;
innerType = processorCollectionAttribute.InnerType;
compendium = processorCollectionAttribute.CompendiumType;
}
return new()
{
@@ -74,8 +86,9 @@ internal class ProcessorCollection
Description = processorAttribute.Description,
Group = processorAttribute.Group,
Dependencies = dependenciesAttribute?.Dependencies ?? [],
IsCollectionProcessor = isCollection,
InnerType = processorAttribute.InnerType,
IsCollection = isCollection,
InnerType = innerType,
Compendium = compendium,
Active = processorAttribute.Active,
};
}
@@ -216,10 +229,16 @@ internal class ProcessorCollectionItem
/// Флаг, указывающий, обрабатывает ли обработчик коллекцию элементов.
/// Внутренний тип элементов указывается в <see cref="InnerType"/>.
/// </summary>
public required bool IsCollectionProcessor { get; init; }
public required bool IsCollection { get; init; }
/// <summary>
/// Флаг, указывающий внутренний тип коллекции обрабатываемых элементов.
/// </summary>
public string? InnerType { get; init; }
/// <summary>
/// Название типов, который обрабатывает этот оброботчик.
/// <c>null</c> - обработать все типы.
/// </summary>
public ProcessorCompendiumEnum? Compendium { get; init; }
}
@@ -24,6 +24,12 @@ public enum ProcessorGroupEnum
FoundryEnEntities,
}
public enum ProcessorCompendiumEnum
{
Aonprd,
FoundryEn,
}
public static class ProcessorGroupEnumExtension
{
public static string GetDescription(this ProcessorGroupEnum group)
@@ -13,6 +13,7 @@ using Pf2eDataScrapper.Processors;
using Pf2eModel;
using Pf2eModel.Model;
using Pf2eModel.Model.Base;
using Pf2eModel.Model.Base.Legacy;
using Pf2eModel.Model.Base.Source;
using Pf2eModel.Model.Base.Targets;
using Pf2eModel.Model.Common.I18n;
@@ -160,8 +161,9 @@ internal abstract class AbstractEntityProcessor(
{
Entity = Entity,
Target = Globals.AonprdTarget,
ExternalId = Input.GetRequired<string>("id"),
Suffix = Input.GetRequired<string>("url"),
Md5 = Md5(Input.Input.ToString()),
Md5 = Input.Md5(),
};
Context.EntityExternals.Add(external);
@@ -223,11 +225,6 @@ internal abstract class AbstractEntityProcessor(
return;
}
if (legacyIdArray.Count > 1)
{
throw new("Несколько отметок ремастер-сущности на устаревшие!");
}
string[] legacyIds = legacyIdArray.ToArray().Select(t => t.ToString()).ToArray();
if (legacyIds.Length == 0)
@@ -345,19 +342,5 @@ internal abstract class AbstractEntityProcessor(
return JsonConvert.SerializeObject(data, new JsonSerializerSettings());
}
/// <summary>
/// Создание контрольной суммы объекта.
/// </summary>
/// <param name="input">Строковое представление вычисляемого объекта.</param>
/// <param name="isLowercase">Использовать шестнадцатеричные цифры в нижнем регистре.</param>
/// <returns>Контрольная сумма объекта.</returns>
public static string Md5(string input, bool isLowercase = false)
{
using MD5 md5 = MD5.Create();
byte[] byteHash = md5.ComputeHash(Encoding.UTF8.GetBytes(input));
string hash = BitConverter.ToString(byteHash).Replace("-", "");
return isLowercase ? hash.ToLower() : hash;
}
protected static readonly ConcurrentBag<LegacyEntityHandler> LegacyHandlers = [];
}
@@ -0,0 +1,125 @@
using Newtonsoft.Json.Linq;
using Pf2eDataScrapper.App;
using Pf2eDataScrapper.Importers.Implementations.Aonprd.DataBuilder.Helpers;
using Pf2eDataScrapper.Importers.Implementations.Aonprd.Helpers;
using Pf2eDataScrapper.Processors;
using Pf2eModel;
using Pf2eModel.Model.Base.Legacy;
using Pf2eModel.Model.Base.Source;
using Pf2eModel.Model.Common.I18n;
namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Common;
[ProcessorCollection(
"aonprd:legacy",
InnerType = null,
CompendiumType = ProcessorCompendiumEnum.Aonprd,
Description = "Создание ссылок на устаревшие сущности",
Group = ProcessorGroupEnum.AonprdPreparation
)]
[ProcessorDependencies(["aonprd:source", "aonprd:external"])]
internal class LegacyProcessor(
ProcessorInput input,
Pf2eDbContext context,
AonprdBuilderGlobals globals,
StringHelper stringHelper
) : AbstractInputProcessor(input)
{
public Pf2eDbContext Context { get; } = context;
public AonprdBuilderGlobals Globals { get; } = globals;
public StringHelper StringHelper { get; } = stringHelper;
private LegacyEntity LegacyEntity { get; set; } = null!;
public override void Process()
{
JArray? remasterIdArray = Input.GetOptional<JArray>("remaster_id");
if (Input.GetOptional<JArray>("remaster_id") is null)
{
throw new SilentSkipItemException();
}
LegacyEntity = new()
{
Name = { [LocaleType.English] = Input.GetOptional<string>("name", "sidebar-29") },
};
AddSource();
AddExternal();
}
/// <summary>
/// Генерация источников сущности.
/// </summary>
protected virtual void AddSource()
{
foreach (LegacyEntitySource entitySource in GetSources())
{
LegacyEntity.Sources.Add(entitySource);
}
}
/// <summary>
/// Получение списка источников сущности.
/// </summary>
/// <returns></returns>
public List<LegacyEntitySource> GetSources()
{
string[] sourceNames = Input
.GetRequired<JArray>("source")
.Select(token => token.ToString())
.ToArray();
if (sourceNames.Length == 0)
{
throw new(
$"Source for entity with id \"{LegacyEntity.Name[LocaleType.English]}\" "
+ $"(Aonprd id \"{Input.GetRequired<string>("id")}\") is not found"
);
}
List<LegacyEntitySource> entitySources = [];
foreach (string sourceName in sourceNames)
{
Source source =
Context.Sources.FirstOrDefault(s => s.Name == sourceName)
?? throw new($"Source \"{sourceName}\" did not found");
LegacyEntitySource entitySource = new()
{
LegacyEntity = LegacyEntity,
Source = source,
Suffix = StringHelper.ResolveSource(Input.GetRequired<string>("source_markdown")),
IsRemaster = Input.GetOptional<JArray>("legacy_id") is not null,
};
entitySources.Add(entitySource);
}
return entitySources;
}
/// <summary>
/// Генерация внешних ссылок сущности.
/// </summary>
protected virtual void AddExternal()
{
LegacyEntityExternal external = new()
{
LegacyEntity = LegacyEntity,
Target = Globals.AonprdTarget,
ExternalId = Input.GetRequired<string>("id"),
Suffix = Input.GetRequired<string>("url"),
Md5 = Input.Md5(),
};
Context.LegacyEntityExternals.Add(external);
}
public override void Save()
{
Context.LegacyEntities.Add(LegacyEntity);
}
}
@@ -7,8 +7,9 @@ using Pf2eModel.Model.Enum;
namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Common;
[Processor(
[ProcessorCollection(
"aonprd:skill",
CompendiumType = ProcessorCompendiumEnum.Aonprd,
InnerType = "skill",
Description = "Навыки",
Group = ProcessorGroupEnum.AonprdPreparation
@@ -6,9 +6,10 @@ using Pf2eModel.Model.Enum;
namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Common;
[Processor(
[ProcessorCollection(
"aonprd:source",
InnerType = "source",
CompendiumType = ProcessorCompendiumEnum.Aonprd,
Description = "Источники",
Group = ProcessorGroupEnum.AonprdPreparation
)]
@@ -12,9 +12,10 @@ using Pf2eModel.Model.Structure;
namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Subclasses;
[Processor(
[ProcessorCollection(
"aonprd:apparition",
InnerType = "apparition",
CompendiumType = ProcessorCompendiumEnum.Aonprd,
Description = "Apparitions",
Group = ProcessorGroupEnum.AonprdSubclasses
)]
@@ -8,9 +8,10 @@ using Pf2eModel.Model.Enum;
namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Types;
[Processor(
[ProcessorCollection(
"aonprd:action",
InnerType = "action",
CompendiumType = ProcessorCompendiumEnum.Aonprd,
Description = "Действия",
Group = ProcessorGroupEnum.AonprdEntities
)]
@@ -10,9 +10,10 @@ using Pf2eModel.Model.Structure;
namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Types;
[Processor(
[ProcessorCollection(
"aonprd:ancestry",
InnerType = "ancestry",
CompendiumType = ProcessorCompendiumEnum.Aonprd,
Description = "Родословные",
Group = ProcessorGroupEnum.AonprdEntities
)]
@@ -9,9 +9,10 @@ using Pf2eModel.Model.Structure;
namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Types;
[Processor(
[ProcessorCollection(
"aonprd:archetype",
InnerType = "archetype",
CompendiumType = ProcessorCompendiumEnum.Aonprd,
Description = "Архетипы",
Group = ProcessorGroupEnum.AonprdEntities
)]
@@ -11,9 +11,10 @@ using Pf2eModel.Model.Structure.BackgroundRelation;
namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Types;
[Processor(
[ProcessorCollection(
"aonprd:background",
InnerType = "background",
CompendiumType = ProcessorCompendiumEnum.Aonprd,
Description = "Предыстории",
Group = ProcessorGroupEnum.AonprdEntities
)]
@@ -15,9 +15,10 @@ using static System.Text.RegularExpressions.Regex;
namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Types;
[Processor(
[ProcessorCollection(
"aonprd:class",
InnerType = "class",
CompendiumType = ProcessorCompendiumEnum.Aonprd,
Description = "Классы",
Group = ProcessorGroupEnum.AonprdEntities
)]
@@ -10,9 +10,10 @@ using Pf2eModel.Model.Structure;
namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Types;
[Processor(
[ProcessorCollection(
"aonprd:feat",
InnerType = "feat",
CompendiumType = ProcessorCompendiumEnum.Aonprd,
Description = "Черты",
Group = ProcessorGroupEnum.AonprdEntities
)]
@@ -9,9 +9,10 @@ using Pf2eModel.Model.Structure;
namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Types;
[Processor(
[ProcessorCollection(
"aonprd:language",
InnerType = "language",
CompendiumType = ProcessorCompendiumEnum.Aonprd,
Description = "Языки",
Group = ProcessorGroupEnum.AonprdEntities
)]
@@ -13,9 +13,10 @@ using Pf2eModel.Model.Structure.RitualRelation;
namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Types;
[Processor(
[ProcessorCollection(
"aonprd:ritual",
InnerType = "ritual",
CompendiumType = ProcessorCompendiumEnum.Aonprd,
Description = "Ритуалы",
Group = ProcessorGroupEnum.AonprdEntities
)]
@@ -5,6 +5,7 @@ using Pf2eDataScrapper.Importers.Implementations.Aonprd.DataBuilder.Helpers;
using Pf2eDataScrapper.Importers.Implementations.Aonprd.Helpers;
using Pf2eDataScrapper.Processors;
using Pf2eModel;
using Pf2eModel.Model.Base.Legacy;
using Pf2eModel.Model.Base.Source;
using Pf2eModel.Model.Base.Targets;
using Pf2eModel.Model.Common.I18n;
@@ -13,10 +14,11 @@ using Rule = Pf2eModel.Model.Structure.Rule;
namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Types.Rules;
[Processor(
[ProcessorCollection(
"aonprd:rules",
InnerType = "rules",
Description = "Правила",
CompendiumType = ProcessorCompendiumEnum.Aonprd,
Description = "Создание цели внешних ссылок",
Group = ProcessorGroupEnum.AonprdEntities
)]
[ProcessorDependencies(["aonprd:trait", "aonprd:source", "aonprd:rules-toc"])]
@@ -12,9 +12,10 @@ using Pf2eModel.Model.Structure.SpellRelation;
namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Types;
[Processor(
[ProcessorCollection(
"aonprd:spell",
InnerType = "spell",
CompendiumType = ProcessorCompendiumEnum.Aonprd,
Description = "Заклинания",
Group = ProcessorGroupEnum.AonprdEntities
)]
@@ -9,13 +9,14 @@ using Pf2eModel.Model.Structure;
namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Types;
[Processor(
[ProcessorCollection(
"aonprd:trait",
InnerType = "trait",
CompendiumType = ProcessorCompendiumEnum.Aonprd,
Description = "Признаки",
Group = ProcessorGroupEnum.AonprdEntities
)]
[ProcessorDependencies(["aonprd:external", "aonprd:source"])]
[ProcessorDependencies(["aonprd:legacy"])]
internal class TraitProcessor(
ProcessorInput input,
Pf2eDbContext context,
@@ -7,9 +7,10 @@ using Pf2eModel.Model.Enum;
namespace Pf2eDataScrapper.Importers.Implementations.FoundryEn.Processors.Entities;
[Processor(
[ProcessorCollection(
"foundry-en:action",
InnerType = "action",
CompendiumType = ProcessorCompendiumEnum.FoundryEn,
Description = "Действия",
Group = ProcessorGroupEnum.FoundryEnEntities,
Active = false
@@ -8,9 +8,10 @@ using Pf2eModel.Model.Structure;
namespace Pf2eDataScrapper.Importers.Implementations.FoundryEn.Processors.Entities;
[Processor(
[ProcessorCollection(
"foundry-en:condition",
InnerType = "Item.condition",
CompendiumType = ProcessorCompendiumEnum.FoundryEn,
Description = "Состояния",
Group = ProcessorGroupEnum.FoundryEnEntities,
Active = false
@@ -7,6 +7,7 @@ using Pf2eDataScrapper.Processors;
using Pf2eModel;
using Pf2eModel.Model;
using Pf2eModel.Model.Base;
using Pf2eModel.Model.Base.Legacy;
using Pf2eModel.Model.Base.Source;
using Pf2eModel.Model.Base.Targets;
using Pf2eModel.Model.Common.I18n;
@@ -59,6 +59,10 @@ internal sealed class ProcessorExecutor(
{
Progress.Information(exception.Message);
}
catch (SilentSkipItemException)
{
//
}
}
Context.SaveChanges();
};
@@ -85,7 +89,7 @@ internal sealed class ProcessorExecutor(
private IEnumerable<IProcessor> IterateProcessors(ProcessorCollectionItem item)
{
if (!item.IsCollectionProcessor)
if (!item.IsCollection)
{
Progress.Local.Reset(1);
Progress.Local.Start(item.Description);
@@ -101,21 +105,20 @@ internal sealed class ProcessorExecutor(
);
}
AbstractCompendium compendium = item.Group switch
AbstractCompendium compendium = item.Compendium switch
{
ProcessorGroupEnum.AonprdPreparation
or ProcessorGroupEnum.AonprdEntities
or ProcessorGroupEnum.AonprdSubclasses => AonprdCompendium,
ProcessorGroupEnum.FoundryEnPreparation or ProcessorGroupEnum.FoundryEnEntities =>
FoundryEnCompendium,
ProcessorCompendiumEnum.Aonprd => AonprdCompendium,
ProcessorCompendiumEnum.FoundryEn => FoundryEnCompendium,
_ => throw new($"Не могу выбрать импортёр для группы {item.Group}"),
};
Progress.Local.Reset(compendium.Count(item.InnerType!));
Progress.Local.Reset(
item.InnerType is null ? compendium.Count() : compendium.Count(item.InnerType)
);
foreach (
(AbstractCompendiumItem compendiumItem, JObject json) in compendium.Iterate(
item.InnerType!
item.InnerType
)
)
{
@@ -150,3 +153,8 @@ public class LegacyItemException(string name) : SkipItemException(name, "Уст
/// <param name="reason">Причина пропуска.</param>
public class SkipItemException(string name, string reason)
: Exception($"Пропускаю элемент '{name}', указана причина: {reason}.") { }
/// <summary>
/// Исключение, показывающее, что обработка сущности пропускается без вывода сообщения.
/// </summary>
public class SilentSkipItemException : Exception { }
+15 -1
View File
@@ -1,4 +1,6 @@
using Newtonsoft.Json.Linq;
using System.Security.Cryptography;
using System.Text;
using Newtonsoft.Json.Linq;
using NpgsqlTypes;
using Pf2eModel.Model.Common.I18n;
@@ -181,6 +183,18 @@ internal class ProcessorInput(string name, JObject input, IEnumerable<IEnumSolve
throw new InvalidEnumStringException(typeof(T), value);
}
/// <summary>
/// Создание контрольной суммы объекта.
/// </summary>
/// <param name="isLowercase">Использовать шестнадцатеричные цифры в нижнем регистре.</param>
/// <returns>Контрольная сумма объекта.</returns>
public string Md5(bool isLowercase = false)
{
byte[] byteHash = MD5.HashData(Encoding.UTF8.GetBytes(Input.ToString()));
string hash = Convert.ToHexString(byteHash).Replace("-", "");
return isLowercase ? hash.ToLower() : hash;
}
/// <summary>
/// Исключение при неверном значении для перечисленния.
/// </summary>