From 81695587f1fa473d052386bc1d54c92e5d59f4d8 Mon Sep 17 00:00:00 2001 From: kozhilya Date: Wed, 21 May 2025 18:36:32 +0400 Subject: [PATCH] =?UTF-8?q?=D0=9D=D0=B0=D1=87=D0=B0=D0=BB=D0=BE=20=D0=B1?= =?UTF-8?q?=D0=BE=D1=80=D1=8C=D0=B1=D1=8B=20=D1=81=20=D0=B4=D1=83=D0=B1?= =?UTF-8?q?=D0=BB=D0=B8=D1=80=D1=83=D1=8E=D1=89=D0=B8=D0=BC=D0=B8=D1=81?= =?UTF-8?q?=D1=8F=20slug-=D0=B0=D0=BC=D0=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Pf2eDataScrapper/Helpers/EntityView.cs | 8 ++ .../Aonprd/Helpers/AonprdBuilderGlobals.cs | 106 ++++++++++++++++++ .../Aonprd/Helpers/StringHelper.cs | 5 +- .../Processors/AbstractEntityProcessor.cs | 4 +- .../Finalization/LegacyRemasterProcessors.cs | 16 ++- .../Subclasses/ApparitionProcessor.cs | 5 +- .../Processors/Types/ActionProcessor.cs | 13 ++- .../Processors/Types/Rules/RuleProcessor.cs | 6 +- .../Types/Rules/RulesTocProcessor.cs | 18 ++- .../EntityViewer/Component/ViewControl.xaml | 14 ++- .../WPF/EntityViewer/EntityForm.xaml | 2 +- .../WPF/ProgressForm/ProgressForm.xaml.cs | 2 +- 12 files changed, 176 insertions(+), 23 deletions(-) diff --git a/Pf2eDataScrapper/Helpers/EntityView.cs b/Pf2eDataScrapper/Helpers/EntityView.cs index fb818db..59c2b2b 100644 --- a/Pf2eDataScrapper/Helpers/EntityView.cs +++ b/Pf2eDataScrapper/Helpers/EntityView.cs @@ -110,6 +110,14 @@ public class EntityView .ThenInclude(c => c.Training) .AsQueryable() ), + new Getter( + EntityType.Subclass, + context => + context + .Subclasses.Include(s => s.Trainings) + .ThenInclude(t => t.Training) + .Include(s => s.Entities) + ), ]; /// diff --git a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Helpers/AonprdBuilderGlobals.cs b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Helpers/AonprdBuilderGlobals.cs index 97cbc53..0446482 100644 --- a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Helpers/AonprdBuilderGlobals.cs +++ b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Helpers/AonprdBuilderGlobals.cs @@ -27,4 +27,110 @@ internal class AonprdBuilderGlobals /// Конструктор класса . /// public AonprdBuilderGlobals() { } + + /// + /// Получить суффикс для ссылки. + /// + /// Идентификатор Aounprd. + /// Суффикс для ссылки или пустая строка. + public string GetSlugSuffix(string id) + { + return SlugSuffixes.TryGetValue(id, out var slug) ? $"-{slug}" : string.Empty; + } + + /// + /// Словарей суффиксов для сущностей + /// + public Dictionary SlugSuffixes { get; } = + new() + { + ["feat-5997"] = "legacy", + ["feat-5730"] = "dragonblood", + ["feat-6469"] = "general", + ["feat-4344"] = "archetype", + ["feat-3549"] = "vampire", + ["feat-7380"] = "mythic", + ["feat-5737"] = "dragonblood", + ["feat-6284"] = "bounty-hunter", + ["feat-2664"] = "archetype", + ["feat-6883"] = "kitsune", + ["feat-6209"] = "investigator", + ["feat-6477"] = "undead-slayer", + ["feat-7637"] = "dandy", + ["feat-7004"] = "yaoguai", + ["feat-4380"] = "stonebound", + }; + + /// + /// Словарь допольнительных проверок связей. + /// Ключ - устаревшая сущность, значение - обновлённая. + /// + public Dictionary LegacyRemasterExtra { get; } = + new() + { + ["archetype-83"] = "archetype-129", + ["archetype-86"] = "archetype-198", + ["feat-5997"] = "feat-7108", + ["feat-4088"] = "feat-5491", + ["feat-1177"] = "feat-4553", + ["feat-2986"] = "feat-4340", + ["feat-1753"] = "feat-6031", + ["feat-2729"] = "feat-4077", + ["feat-480"] = "feat-4850", + ["feat-2989"] = "feat-4343", + ["feat-2987"] = "feat-4341", + ["feat-2990"] = "feat-4346", + ["feat-936"] = "feat-2269", + ["feat-2992"] = "feat-4350", + ["feat-2991"] = "feat-4349", + ["feat-2742"] = "feat-4078", + ["feat-1169"] = "feat-5055", + ["feat-2743"] = "feat-4079", + ["feat-963"] = "feat-4519", + ["feat-1171"] = "feat-4858", + ["feat-1167"] = "feat-5880", + ["feat-455"] = "feat-6007", + ["feat-2993"] = "feat-4352", + ["language-91"] = "language-180", + ["ritual-31"] = "ritual-191", + ["spell-571"] = "spell-1975", + ["spell-1052"] = "spell-1309", + ["spell-1053"] = "spell-1306", + ["spell-843"] = "spell-2376", + ["spell-578"] = "spell-1566", + ["spell-551"] = "spell-2003", + ["spell-553"] = "spell-2012", + ["spell-646"] = "spell-1394", + ["spell-1054"] = "spell-1305", + ["spell-559"] = "spell-2417", + ["spell-1055"] = "spell-1311", + ["spell-1056"] = "spell-1310", + ["spell-841"] = "spell-1107", + ["spell-1057"] = "spell-1307", + ["spell-560"] = "spell-2418", + ["spell-1058"] = "spell-1304", + ["spell-641"] = "spell-1413", + ["spell-1059"] = "spell-1308", + }; + + /// + /// Словарь сущностей, которые нужно слить с другой/другими. + /// Ключ - пропускаемая сущность. Значение - целевая сущность. + /// + public Dictionary MergeSources { get; } = new() { ["spell-654"] = "spell-819" }; } + +/* + Проверка уникальности: + +SELECT + E.type, + E.slug, + COUNT(*), + string_agg(E.id::text, ', ' ORDER BY E.id), + string_agg(EE.external_id::text, ', ' ORDER BY E.id) +FROM public.entities AS E +INNER JOIN public.entity_externals AS EE ON EE.entity_id = E.id +GROUP BY E.type, E.slug +HAVING COUNT(*) > 1 AND E.type NOT IN ('rule', 'action') + */ diff --git a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Helpers/StringHelper.cs b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Helpers/StringHelper.cs index 93b1f87..8ee6e66 100644 --- a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Helpers/StringHelper.cs +++ b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Helpers/StringHelper.cs @@ -40,8 +40,9 @@ internal partial class StringHelper { MarkdownLinkSplit result = new(); - Match match = LinkRegex().Match(input); - if (match.Captures.Count > 0) + MatchCollection matches = LinkRegex().Matches(input); + + foreach (Match match in matches) { result.Links.Add( new() diff --git a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/AbstractEntityProcessor.cs b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/AbstractEntityProcessor.cs index f0d7780..2c4e175 100644 --- a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/AbstractEntityProcessor.cs +++ b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/AbstractEntityProcessor.cs @@ -139,7 +139,9 @@ internal abstract class AbstractEntityProcessor( Entity.Img = ""; Entity.Description = GetDescription(); Entity.Rarity = Input.EnumOptional("rarity", EntityRarity.Common); - Entity.Slug = Helper.GenerateSlug(Entity.Name[LocaleType.English]); + Entity.Slug = + Helper.GenerateSlug(Entity.Name[LocaleType.English]) + + Globals.GetSlugSuffix(Input.GetRequired("id")); Entity.IsTranslated = false; Entity.IsChecked = false; diff --git a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Finalization/LegacyRemasterProcessors.cs b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Finalization/LegacyRemasterProcessors.cs index 5963fc1..2961ffc 100644 --- a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Finalization/LegacyRemasterProcessors.cs +++ b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Finalization/LegacyRemasterProcessors.cs @@ -1,10 +1,12 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; using System.Linq; using Microsoft.EntityFrameworkCore; using Newtonsoft.Json.Linq; using Pf2eDataScrapper.App; using Pf2eDataScrapper.Processors; using Pf2eModel; +using Pf2eModel.Model.Base; using Pf2eModel.Model.Base.Targets; namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Finalization; @@ -28,11 +30,23 @@ internal class LegacyRemasterProcessors(ProcessorInput input) : AbstractInputPro Context.EntityExternals.AsNoTracking().FirstOrDefault(x => x.ExternalId == aonprdId) ?? throw new SilentSkipItemException(); int entityId = entity.EntityId; + Entity legacyEntity = + Context.Entities.FirstOrDefault(e => e.Id == entityId) + ?? throw new Exception($"Неизвестная сущность с ID {entityId}"); foreach (int remasterId in GetEntityIds("remaster_id")) { Context.LegacyRemasters.Add(new() { LegacyId = entityId, RemasterId = remasterId }); + Entity remasterEntity = + Context.Entities.AsNoTracking().FirstOrDefault(e => e.Id == remasterId) + ?? throw new Exception($"Неизвестная сущность с ID {remasterId}"); + + if (legacyEntity.Slug == remasterEntity.Slug) + { + legacyEntity.Slug += "-legacy"; + } } + /* foreach (int legacyId in GetEntityIds("legacy_id")) { diff --git a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Subclasses/ApparitionProcessor.cs b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Subclasses/ApparitionProcessor.cs index b0451aa..7dd7b8e 100644 --- a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Subclasses/ApparitionProcessor.cs +++ b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Subclasses/ApparitionProcessor.cs @@ -19,7 +19,8 @@ namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Subclasse InnerType = "apparition", CompendiumType = ProcessorCompendiumEnum.Aonprd, Description = "Apparitions", - Group = ProcessorGroupEnum.AonprdSubclasses + Group = ProcessorGroupEnum.AonprdSubclasses, + Active = false )] [ProcessorDependencies( ["aonprd:trait", "aonprd:source", "aonprd:class", "aonprd:spell", "aonprd:skill"] @@ -44,7 +45,7 @@ internal class ApparitionProcessor( public override void ProcessObject() { // Spells - string spellMd = Input.GetRequired("spells_markdown"); + string spellMd = Input.GetRequired("spell_markdown"); List spellUrls = StringHelper .ParseMarkdownLinks(spellMd) .Links.Select(l => l.Url) diff --git a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/ActionProcessor.cs b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/ActionProcessor.cs index 50f0559..e142199 100644 --- a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/ActionProcessor.cs +++ b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/ActionProcessor.cs @@ -34,10 +34,7 @@ internal class ActionProcessor( /// public override void Process() { - if (Input.GetRequired("markdown").StartsWith("**Activate**")) - { - throw new SkipItemException(Input.GetRequired("name"), "activating action"); - } + Check(); base.Process(); @@ -45,6 +42,14 @@ internal class ActionProcessor( Object.Extra = GenerateJsonString("school"); } + public void Check() + { + if (Input.GetRequired("markdown").StartsWith("**Activate")) + { + throw new SkipItemException(Input.GetRequired("name"), "activating action"); + } + } + /// protected override string[] AddedDescriptionPaths() => [ diff --git a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/Rules/RuleProcessor.cs b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/Rules/RuleProcessor.cs index 729f807..eb6deb2 100644 --- a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/Rules/RuleProcessor.cs +++ b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/Rules/RuleProcessor.cs @@ -51,7 +51,7 @@ internal class RuleProcessor( AddExternal(); AddTraits(); - RuleTocBook book = Toc[GetSource().Id]; + RuleTocBook book = Toc[GetSourceId()]; string[] breadcrumbs = (Input.GetOptional("breadcrumbs") ?? []) .Select(token => token.ToString()) @@ -119,11 +119,11 @@ internal class RuleProcessor( /// Получение источника, к которому относится правило. /// /// Источник, к которому относится правило. - private Source GetSource() + private int GetSourceId() { foreach (EntitySource entitySource in Entity.Sources) { - return entitySource.Source; + return entitySource.SourceId; } throw new("Source not found"); diff --git a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/Rules/RulesTocProcessor.cs b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/Rules/RulesTocProcessor.cs index 6ba76db..af2f841 100644 --- a/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/Rules/RulesTocProcessor.cs +++ b/Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/Rules/RulesTocProcessor.cs @@ -1,4 +1,5 @@ -using System.Linq; +using System; +using System.Linq; using System.Text.RegularExpressions; using System.Web; using AngleSharp; @@ -116,7 +117,20 @@ internal partial class RulesTocProcessor(Pf2eDbContext context) : IProcessor { IConfiguration config = Configuration.Default.WithDefaultLoader(); IBrowsingContext context = BrowsingContext.New(config); - IDocument document = context.OpenAsync(Url).Result; + IDocument document = null!; + bool completed = false; + + while (!completed){ + try + { + document = context.OpenAsync(Url).Result; + completed = true; + } + catch (AggregateException) + { + // + } + } IElement root = document.QuerySelector(Selector) ?? throw new("Broken page download"); diff --git a/Pf2eDataScrapper/WPF/EntityViewer/Component/ViewControl.xaml b/Pf2eDataScrapper/WPF/EntityViewer/Component/ViewControl.xaml index 32ff84b..be95e75 100644 --- a/Pf2eDataScrapper/WPF/EntityViewer/Component/ViewControl.xaml +++ b/Pf2eDataScrapper/WPF/EntityViewer/Component/ViewControl.xaml @@ -140,17 +140,19 @@ - - + Background="Transparent" + BorderThickness="0" + IsReadOnly="True" + Text="{Binding Value, Mode=OneWay}"> + - - + + diff --git a/Pf2eDataScrapper/WPF/EntityViewer/EntityForm.xaml b/Pf2eDataScrapper/WPF/EntityViewer/EntityForm.xaml index e1055bb..6132c5f 100644 --- a/Pf2eDataScrapper/WPF/EntityViewer/EntityForm.xaml +++ b/Pf2eDataScrapper/WPF/EntityViewer/EntityForm.xaml @@ -50,7 +50,7 @@ ResizeDirection="Rows" />