Начало борьбы с дублирующимися slug-ами

2025-05-21 18:36:32 +04:00
parent 96b4d6da31
commit 81695587f1
12 changed files with 176 additions and 23 deletions
@@ -110,6 +110,14 @@ public class EntityView
                    .ThenInclude(c => c.Training)
                    .AsQueryable()
        ),
+        new Getter<Subclass>(
+            EntityType.Subclass,
+            context =>
+                context
+                    .Subclasses.Include(s => s.Trainings)
+                    .ThenInclude(t => t.Training)
+                    .Include(s => s.Entities)
+        ),
    ];

    /// <summary>
@@ -27,4 +27,110 @@ internal class AonprdBuilderGlobals
    /// Конструктор класса <see cref="AonprdBuilderGlobals"/>.
    /// </summary>
    public AonprdBuilderGlobals() { }
+
+    /// <summary>
+    /// Получить суффикс для ссылки.
+    /// </summary>
+    /// <param name="id">Идентификатор Aounprd.</param>
+    /// <returns>Суффикс для ссылки или пустая строка.</returns>
+    public string GetSlugSuffix(string id)
+    {
+        return SlugSuffixes.TryGetValue(id, out var slug) ? $"-{slug}" : string.Empty;
+    }
+
+    /// <summary>
+    /// Словарей суффиксов для сущностей
+    /// </summary>
+    public Dictionary<string, string> SlugSuffixes { get; } =
+        new()
+        {
+            ["feat-5997"] = "legacy",
+            ["feat-5730"] = "dragonblood",
+            ["feat-6469"] = "general",
+            ["feat-4344"] = "archetype",
+            ["feat-3549"] = "vampire",
+            ["feat-7380"] = "mythic",
+            ["feat-5737"] = "dragonblood",
+            ["feat-6284"] = "bounty-hunter",
+            ["feat-2664"] = "archetype",
+            ["feat-6883"] = "kitsune",
+            ["feat-6209"] = "investigator",
+            ["feat-6477"] = "undead-slayer",
+            ["feat-7637"] = "dandy",
+            ["feat-7004"] = "yaoguai",
+            ["feat-4380"] = "stonebound",
+        };
+
+    /// <summary>
+    /// Словарь допольнительных проверок связей.
+    /// Ключ - устаревшая сущность, значение - обновлённая.
+    /// </summary>
+    public Dictionary<string, string> LegacyRemasterExtra { get; } =
+        new()
+        {
+            ["archetype-83"] = "archetype-129",
+            ["archetype-86"] = "archetype-198",
+            ["feat-5997"] = "feat-7108",
+            ["feat-4088"] = "feat-5491",
+            ["feat-1177"] = "feat-4553",
+            ["feat-2986"] = "feat-4340",
+            ["feat-1753"] = "feat-6031",
+            ["feat-2729"] = "feat-4077",
+            ["feat-480"] = "feat-4850",
+            ["feat-2989"] = "feat-4343",
+            ["feat-2987"] = "feat-4341",
+            ["feat-2990"] = "feat-4346",
+            ["feat-936"] = "feat-2269",
+            ["feat-2992"] = "feat-4350",
+            ["feat-2991"] = "feat-4349",
+            ["feat-2742"] = "feat-4078",
+            ["feat-1169"] = "feat-5055",
+            ["feat-2743"] = "feat-4079",
+            ["feat-963"] = "feat-4519",
+            ["feat-1171"] = "feat-4858",
+            ["feat-1167"] = "feat-5880",
+            ["feat-455"] = "feat-6007",
+            ["feat-2993"] = "feat-4352",
+            ["language-91"] = "language-180",
+            ["ritual-31"] = "ritual-191",
+            ["spell-571"] = "spell-1975",
+            ["spell-1052"] = "spell-1309",
+            ["spell-1053"] = "spell-1306",
+            ["spell-843"] = "spell-2376",
+            ["spell-578"] = "spell-1566",
+            ["spell-551"] = "spell-2003",
+            ["spell-553"] = "spell-2012",
+            ["spell-646"] = "spell-1394",
+            ["spell-1054"] = "spell-1305",
+            ["spell-559"] = "spell-2417",
+            ["spell-1055"] = "spell-1311",
+            ["spell-1056"] = "spell-1310",
+            ["spell-841"] = "spell-1107",
+            ["spell-1057"] = "spell-1307",
+            ["spell-560"] = "spell-2418",
+            ["spell-1058"] = "spell-1304",
+            ["spell-641"] = "spell-1413",
+            ["spell-1059"] = "spell-1308",
+        };
+
+    /// <summary>
+    /// Словарь сущностей, которые нужно слить с другой/другими.
+    /// Ключ - пропускаемая сущность. Значение - целевая сущность.
+    /// </summary>
+    public Dictionary<string, string> MergeSources { get; } = new() { ["spell-654"] = "spell-819" };
 }
+
+/*
+ Проверка уникальности:
+
+SELECT
+    E.type,
+    E.slug,
+    COUNT(*),
+    string_agg(E.id::text, ', ' ORDER BY E.id),
+    string_agg(EE.external_id::text, ', ' ORDER BY E.id)
+FROM public.entities AS E
+INNER JOIN public.entity_externals AS EE ON EE.entity_id = E.id
+GROUP BY E.type, E.slug
+HAVING COUNT(*) > 1 AND E.type NOT IN ('rule', 'action')
+ */
@@ -40,8 +40,9 @@ internal partial class StringHelper
    {
        MarkdownLinkSplit result = new();

-        Match match = LinkRegex().Match(input);
-        if (match.Captures.Count > 0)
+        MatchCollection matches = LinkRegex().Matches(input);
+
+        foreach (Match match in matches)
        {
            result.Links.Add(
                new()
@@ -139,7 +139,9 @@ internal abstract class AbstractEntityProcessor(
        Entity.Img = "";
        Entity.Description = GetDescription();
        Entity.Rarity = Input.EnumOptional("rarity", EntityRarity.Common);
-        Entity.Slug = Helper.GenerateSlug(Entity.Name[LocaleType.English]);
+        Entity.Slug =
+            Helper.GenerateSlug(Entity.Name[LocaleType.English])
+            + Globals.GetSlugSuffix(Input.GetRequired<string>("id"));
        Entity.IsTranslated = false;
        Entity.IsChecked = false;

@@ -1,10 +1,12 @@
-using System.Collections.Generic;
+using System;
+using System.Collections.Generic;
 using System.Linq;
 using Microsoft.EntityFrameworkCore;
 using Newtonsoft.Json.Linq;
 using Pf2eDataScrapper.App;
 using Pf2eDataScrapper.Processors;
 using Pf2eModel;
+using Pf2eModel.Model.Base;
 using Pf2eModel.Model.Base.Targets;

 namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Finalization;
@@ -28,11 +30,23 @@ internal class LegacyRemasterProcessors(ProcessorInput input) : AbstractInputPro
            Context.EntityExternals.AsNoTracking().FirstOrDefault(x => x.ExternalId == aonprdId)
            ?? throw new SilentSkipItemException();
        int entityId = entity.EntityId;
+        Entity legacyEntity =
+            Context.Entities.FirstOrDefault(e => e.Id == entityId)
+            ?? throw new Exception($"Неизвестная сущность с ID {entityId}");

        foreach (int remasterId in GetEntityIds("remaster_id"))
        {
            Context.LegacyRemasters.Add(new() { LegacyId = entityId, RemasterId = remasterId });
+            Entity remasterEntity =
+                Context.Entities.AsNoTracking().FirstOrDefault(e => e.Id == remasterId)
+                ?? throw new Exception($"Неизвестная сущность с ID {remasterId}");
+
+            if (legacyEntity.Slug == remasterEntity.Slug)
+            {
+                legacyEntity.Slug += "-legacy";
+            }
        }
+
        /*
        foreach (int legacyId in GetEntityIds("legacy_id"))
        {
@@ -19,7 +19,8 @@ namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Subclasse
    InnerType = "apparition",
    CompendiumType = ProcessorCompendiumEnum.Aonprd,
    Description = "Apparitions",
-    Group = ProcessorGroupEnum.AonprdSubclasses
+    Group = ProcessorGroupEnum.AonprdSubclasses,
+    Active = false
 )]
 [ProcessorDependencies(
    ["aonprd:trait", "aonprd:source", "aonprd:class", "aonprd:spell", "aonprd:skill"]
@@ -44,7 +45,7 @@ internal class ApparitionProcessor(
    public override void ProcessObject()
    {
        // Spells
-        string spellMd = Input.GetRequired<string>("spells_markdown");
+        string spellMd = Input.GetRequired<string>("spell_markdown");
        List<string> spellUrls = StringHelper
            .ParseMarkdownLinks(spellMd)
            .Links.Select(l => l.Url)
@@ -34,10 +34,7 @@ internal class ActionProcessor(
    /// <inheritdoc/>
    public override void Process()
    {
-        if (Input.GetRequired<string>("markdown").StartsWith("**Activate**"))
-        {
-            throw new SkipItemException(Input.GetRequired<string>("name"), "activating action");
-        }
+        Check();

        base.Process();

@@ -45,6 +42,14 @@ internal class ActionProcessor(
        Object.Extra = GenerateJsonString("school");
    }

+    public void Check()
+    {
+        if (Input.GetRequired<string>("markdown").StartsWith("**Activate"))
+        {
+            throw new SkipItemException(Input.GetRequired<string>("name"), "activating action");
+        }
+    }
+
    /// <inheritdoc/>
    protected override string[] AddedDescriptionPaths() =>
        [
@@ -51,7 +51,7 @@ internal class RuleProcessor(
        AddExternal();
        AddTraits();

-        RuleTocBook book = Toc[GetSource().Id];
+        RuleTocBook book = Toc[GetSourceId()];

        string[] breadcrumbs = (Input.GetOptional<JArray>("breadcrumbs") ?? [])
            .Select(token => token.ToString())
@@ -119,11 +119,11 @@ internal class RuleProcessor(
    /// Получение источника, к которому относится правило.
    /// </summary>
    /// <returns>Источник, к которому относится правило.</returns>
-    private Source GetSource()
+    private int GetSourceId()
    {
        foreach (EntitySource entitySource in Entity.Sources)
        {
-            return entitySource.Source;
+            return entitySource.SourceId;
        }

        throw new("Source not found");
@@ -1,4 +1,5 @@
-using System.Linq;
+using System;
+using System.Linq;
 using System.Text.RegularExpressions;
 using System.Web;
 using AngleSharp;
@@ -116,7 +117,20 @@ internal partial class RulesTocProcessor(Pf2eDbContext context) : IProcessor
    {
        IConfiguration config = Configuration.Default.WithDefaultLoader();
        IBrowsingContext context = BrowsingContext.New(config);
-        IDocument document = context.OpenAsync(Url).Result;
+        IDocument document = null!;
+        bool completed = false;
+
+        while (!completed){
+            try
+            {
+                document = context.OpenAsync(Url).Result;
+                completed = true;
+            }
+            catch (AggregateException)
+            {
+                //
+            }
+        }

        IElement root = document.QuerySelector(Selector) ?? throw new("Broken page download");

@@ -140,17 +140,19 @@
                                    </MultiBinding>
                                </TextBlock.Foreground>
                            </TextBlock>
-                            <TextBlock
+                            <TextBox
                                HorizontalAlignment="Center"
                                VerticalAlignment="Center"
-                                Text="{Binding Value}"
-                                TextTrimming="CharacterEllipsis">
-                                <TextBlock.Foreground>
+                                Background="Transparent"
+                                BorderThickness="0"
+                                IsReadOnly="True"
+                                Text="{Binding Value, Mode=OneWay}">
+                                <TextBox.Foreground>
                                    <MultiBinding Converter="{StaticResource ColorAdapterConverter}">
                                        <Binding Path="AccentColor" />
                                    </MultiBinding>
-                                </TextBlock.Foreground>
-                            </TextBlock>
+                                </TextBox.Foreground>
+                            </TextBox>
                        </StackPanel>
                    </Grid>

@@ -50,7 +50,7 @@
            ResizeDirection="Rows" />

        <Button
-            Margin="3,4,0,0"
+            Margin="3,4,3,0"
            Click="Button_Click"
            Content="&lt;&lt;&lt; Вернуться в начало" />

@@ -60,7 +60,7 @@ internal partial class ProgressForm : Window
    }
    private void Progress_ProgressComplete(ProcessProgress sender)
    {
-        //(Application.Current as ImportApplication)!.CurrentStage = ApplicationStage.Viewer;
+        (Application.Current as ImportApplication)!.CurrentStage = ApplicationStage.Viewer;
    }

    public event ImportApplication.ApplicationStateChangeDelegate? ApplicationStateChange;