Обновление README.md

Pf2eDataScrapper/Importers/Implementations/Aonprd/DataBuilder/AonprdDataBuilder.cs

@@ -80,18 +80,20 @@ internal class AonprdDataBuilder(AonprdLoader loader)
     {
         ConfigureWindow();
 
+        ProcessProgressBar.Refresh(-1, "");
+
         foreach (AonprdProcessorCollection.ProcessorItem item in Processors.Iterate())
         {
             ProcessProgressBar.Next(item.Attribute.Name);
 
+            SubProcessProgressBar.Refresh(-1, "Starting...");
+
             if (item.Attribute.Type is null)
             {
                 RunProcessor(item, []);
                 continue;
             }
 
-            SubProcessProgressBar.Refresh(0, "");
-
             SubProcessProgressBar.Max = Compendium.Count([item.Attribute.Type]);
 
             foreach (AonprdLoader.LoaderEntry entry in Loader.Entries(item.Attribute.Type))

Pf2eDataScrapper/Importers/Implementations/Aonprd/DataBuilder/AonprdProcessorAttribute.cs

@@ -1,20 +1,30 @@
 namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.DataBuilder;
 
 [AttributeUsage(AttributeTargets.Class)]
-public class AonprdProcessorAttribute(string name, string? type, string[] dependencies) : Attribute
+public class AonprdProcessorAttribute(
+    string name,
+    string? type,
+    string[] dependencies,
+    bool includeDefaults = true
+) : Attribute
 {
+    private static readonly string[] DefaultDependencies = ["trait", "source"];
     public readonly string Name = name;
 
     public readonly string? Type = type;
 
-    public readonly IEnumerable<string> Dependencies = dependencies;
+    public readonly IEnumerable<string> Dependencies =
+    [
+        .. includeDefaults ? DefaultDependencies : [],
+        .. dependencies,
+    ];
 
     public AonprdProcessorAttribute(string type)
-        : this(type, type, ["trait", "source"]) { }
+        : this(type, type, DefaultDependencies) { }
 
-    public AonprdProcessorAttribute(string type, string[] dependencies)
+    public AonprdProcessorAttribute(string type, string[] dependencies, bool includeDefaults = true)
-        : this(type, type, dependencies) { }
+        : this(type, type, dependencies, includeDefaults) { }
 
     public AonprdProcessorAttribute(string name, string type)
-        : this(name, type, ["trait", "source"]) { }
+        : this(name, type, DefaultDependencies) { }
 }

Pf2eDataScrapper/Importers/Implementations/Aonprd/DataBuilder/AonprdProcessorCollection.cs

@@ -116,6 +116,12 @@ internal class AonprdProcessorCollection
 
     public void UseOnlyNames(string[] processorNames)
     {
+        if (processorNames.Length == 0)
+        {
+            Items = _allItems;
+            return;
+        }
+
         Dictionary<string, bool> nameFlags = processorNames.ToDictionary(name => name, _ => false);
 
         while (nameFlags.Values.Any(v => !v))

Pf2eDataScrapper/Importers/Implementations/Aonprd/DataBuilder/Helpers/AonprdBuilderGlobals.cs

@@ -8,5 +8,5 @@ internal class AonprdBuilderGlobals
 
     private AonprdBuilderGlobals() { }
 
-    public static AonprdBuilderGlobals Instance = new();
+    public static readonly AonprdBuilderGlobals Instance = new();
 }

Pf2eDataScrapper/Importers/Implementations/Aonprd/DataBuilder/Processors/Common/ExternalProcessor.cs

@@ -2,7 +2,7 @@
 
 namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.DataBuilder.Processors.Common;
 
-[AonprdProcessor("external", null, [])]
+[AonprdProcessor("external", null, [], false)]
 internal class ExternalProcessor(AonprdProcessorInput input) : AbstractProcessor(input)
 {
     public ExternalTarget AonprdTarget = null!;

Pf2eDataScrapper/Importers/Implementations/Aonprd/DataBuilder/Processors/Common/SkillProcessor.cs

@@ -4,7 +4,7 @@ using Pf2eModel.Model.Enum;
 
 namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.DataBuilder.Processors.Common;
 
-[AonprdProcessor("skill", [])]
+[AonprdProcessor("skill", [], false)]
 internal class SkillProcessor(AonprdProcessorInput input) : AbstractProcessor(input)
 {
     public Training Training = null!;

Pf2eDataScrapper/Importers/Implementations/Aonprd/DataBuilder/Processors/Common/SourceProcessor.cs

@@ -3,7 +3,7 @@ using Pf2eModel.Model.Enum;
 
 namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.DataBuilder.Processors.Common;
 
-[AonprdProcessor("source", [])]
+[AonprdProcessor("source", [], false)]
 internal class SourceProcessor(AonprdProcessorInput input) : AbstractProcessor(input)
 {
     protected Source Source = null!;

Pf2eDataScrapper/Importers/Implementations/Aonprd/DataBuilder/Processors/Types/RuleProcessor.cs

@@ -1,115 +0,0 @@
-using System.Data;
-using Microsoft.EntityFrameworkCore.Migrations.Operations;
-using Newtonsoft.Json.Linq;
-using Pf2eModel.Model.Common.I18n;
-using Pf2eModel.Model.Enum;
-using Pf2eModel.Model.Structure;
-using Rule = Pf2eModel.Model.Structure.Rule;
-
-namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.DataBuilder.Processors.Types;
-
-[AonprdProcessor("rules")]
-internal class RuleProcessor(AonprdProcessorInput input) : AbstractEntityProcessor(input)
-{
-    public override EntityType Type => EntityType.Rule;
-
-    public override Rule Object { get; } = new();
-
-    public override void Add() => Context.Rules.Add(Object);
-
-    public override void Process()
-    {
-        base.Process();
-
-        string[] breadcrumbs = (Input.GetOptional<JArray>("breadcrumbs") ?? [])
-            .Select(token => token.ToString())
-            .ToArray();
-
-        RuleStructureItem item = GetStructureItem(
-            [.. breadcrumbs, Entity.Name[LocaleType.English]]
-        );
-
-        item.IsHandled = true;
-        item.RuleId = Object.Id;
-        item.Id = Input.GetRequired<string>("id");
-
-        if (item.Parent?.IsHandled ?? false)
-        {
-            Object.ParentId = item.Parent.RuleId!;
-            // Object.Order = item.Parent.ChildrenOrder?.IndexOf(item.Id) ?? 0;
-            Object.Order = 0;
-        }
-
-        item.ChildrenOrder = [];
-
-        List<int> childrenIds = item
-            .Children.Select(i => i.RuleId)
-            .Where(id => id != null)
-            .OfType<int>()
-            .ToList();
-
-        List<Rule> rules = Context.Rules.Where(i => childrenIds.Contains(i.Id)).ToList();
-
-        foreach (Rule rule in rules)
-        {
-            rule.Order = 0;
-        }
-        /*
-         * TODO
-         * Обработка порядка из полей
-         *
-          "next_link": {
-             "label": "Chapter 3: Classes",
-             "url": "/Rules.aspx?ID=134"
-           },
-           "previous_link": {
-             "label": "Chapter 1: Introduction",
-             "url": "/Rules.aspx?ID=1"
-           },
-         */
-    }
-
-    private static readonly List<RuleStructureItem> Roots = [];
-
-    private RuleStructureItem GetStructureItem(string[] breadcrumbs, RuleStructureItem? from = null)
-    {
-        while (true)
-        {
-            List<RuleStructureItem> children = from?.Children ?? Roots;
-
-            if (breadcrumbs.Length == 0)
-            {
-                return from!;
-            }
-
-            string root = breadcrumbs[0];
-            RuleStructureItem? item = children.FirstOrDefault(i => i.Name == root);
-
-            if (item is null)
-            {
-                item = new(root, from);
-                children.Add(item);
-            }
-
-            breadcrumbs = breadcrumbs[1..];
-            from = item;
-        }
-    }
-}
-
-internal class RuleStructureItem(string name, RuleStructureItem? parent)
-{
-    public string? Id { get; set; } = null;
-
-    public string Name { get; set; } = name;
-
-    public int? RuleId { get; set; } = null;
-
-    public bool IsHandled { get; set; } = false;
-
-    public RuleStructureItem? Parent { get; set; } = parent;
-
-    public List<string>? ChildrenOrder { get; set; } = null;
-
-    public List<RuleStructureItem> Children { get; } = [];
-}

Pf2eDataScrapper/Importers/Implementations/Aonprd/DataBuilder/Processors/Types/Rules/RuleProcessor.cs

@@ -0,0 +1,115 @@
+using System.Data;
+using Microsoft.EntityFrameworkCore.Migrations.Operations;
+using Newtonsoft.Json.Linq;
+using Pf2eModel.Model.Base.Source;
+using Pf2eModel.Model.Base.Targets;
+using Pf2eModel.Model.Common.I18n;
+using Pf2eModel.Model.Enum;
+using Pf2eModel.Model.Structure;
+using Rule = Pf2eModel.Model.Structure.Rule;
+
+namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.DataBuilder.Processors.Types.Rules;
+
+[AonprdProcessor("rules", ["rules-toc"])]
+internal class RuleProcessor(AonprdProcessorInput input) : AbstractEntityProcessor(input)
+{
+    public override EntityType Type => EntityType.Rule;
+
+    public override Rule Object { get; } = new();
+
+    public override void Add() => Context.Rules.Add(Object);
+
+    public readonly RuleTocCollection Toc = RuleTocCollection.Instance;
+
+    public override void Process()
+    {
+        // base.Process();
+        AddEntity();
+        AddSource();
+        AddExternal();
+        AddTraits();
+
+        RuleTocBook book = Toc[GetSource().Id];
+
+        string[] breadcrumbs = (Input.GetOptional<JArray>("breadcrumbs") ?? [])
+            .Select(token => token.ToString())
+            .ToArray();
+
+        string id = GetExternal().Suffix.Replace("/Rules.aspx?ID=", "rules-");
+        RuleTocItem? item;
+
+        if (breadcrumbs.Length > 0)
+        {
+            RuleTocItem parent = book.Get(breadcrumbs);
+            item = parent.Children.FirstOrDefault(child => child.Id == id);
+
+            if (item is null)
+            {
+                item = new(Entity.Name[LocaleType.English], parent);
+                parent.Children.Add(item);
+            }
+        }
+        else
+        {
+            item = book.Children.First(child => child.Id == id);
+        }
+
+        item.Id = Input.GetRequired<string>("id");
+        item.RuleId = Object.Id;
+        item.IsHandled = true;
+
+        if (item.Parent?.IsHandled ?? false)
+        {
+            Object.ParentId = item.Parent.RuleId!;
+            Object.Order = item.Parent.ChildrenOrder?.IndexOf(item.Id) ?? -1;
+        }
+
+        // ЗАМЕТКА
+        //
+        // Индекс Архивов не включает поля "next_link" и "previous_link"!
+
+        /*
+        if (item.ChildrenOrder is not null)
+        {
+            return;
+        }
+
+        item.PreviousId = Input
+            .GetOptional<string>("previous_link")
+            ?.Replace("/Rules.aspx?ID=", "rules-");
+        item.NextId = Input.GetOptional<string>("next_link")?.Replace("/Rules.aspx?ID=", "rules-");
+
+        if (item.Parent is not null)
+        {
+            if (item.PreviousId is null)
+            {
+                item.Parent.FirstChildId = item.Id;
+            }
+            if (item.NextId is null)
+            {
+                item.Parent.LastChildId = item.Id;
+            }
+        }
+        */
+    }
+
+    private Source GetSource()
+    {
+        foreach (EntitySource entitySource in Entity.Sources)
+        {
+            return entitySource.Source;
+        }
+
+        throw new("Source not found");
+    }
+
+    private EntityExternal GetExternal()
+    {
+        foreach (EntityExternal entityExternal in Entity.Externals)
+        {
+            return entityExternal;
+        }
+
+        throw new("External not found");
+    }
+}

Pf2eDataScrapper/Importers/Implementations/Aonprd/DataBuilder/Processors/Types/Rules/RuleToc.cs

@@ -0,0 +1,219 @@
+using System.Linq;
+using System.Runtime.CompilerServices;
+using LinqToDB.DataProvider.ClickHouse;
+using Pf2eModel.Model.Base.Source;
+
+namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.DataBuilder.Processors.Types.Rules;
+
+internal class RuleTocCollection
+{
+    public List<RuleTocBook> Books { get; } = [];
+
+    public RuleTocBook this[int id] =>
+        Books.FirstOrDefault(i => i.SourceId == id) ?? throw new($"Book with id {id} not found");
+
+    public RuleTocBook this[Source source]
+    {
+        get
+        {
+            RuleTocBook? result = Books.FirstOrDefault(i => i.SourceId == source.Id);
+
+            if (result is not null)
+            {
+                return result;
+            }
+
+            result = new(source);
+            Books.Add(result);
+            return result;
+        }
+    }
+
+    public RuleTocBook? FindName(string name)
+    {
+        foreach (RuleTocBook book in Books)
+        {
+            if (book.IsNameExists(name))
+            {
+                return book;
+            }
+        }
+
+        return null;
+    }
+
+    public RuleTocItem? FindEntityId(int id)
+    {
+        foreach (RuleTocBook book in Books)
+        {
+            RuleTocItem? result = book.GetByEntityId(id);
+
+            if (result is not null)
+            {
+                return result;
+            }
+        }
+
+        return null;
+    }
+
+    public static RuleTocCollection Instance { get; } = new();
+
+    public override string ToString()
+    {
+        return string.Join("\n", Books.Select(book => book.ToString()).ToList());
+    }
+
+    public void SaveFile()
+    {
+        string fileName =
+            $"rules-toc.{new DateTimeOffset(DateTime.UtcNow).ToUnixTimeSeconds()}.debug.yaml";
+
+        using FileStream fileStream = File.Create(fileName);
+        using StreamWriter writer = new(fileStream);
+
+        writer.Write(ToString());
+    }
+}
+
+internal class RuleTocBook(Source source)
+{
+    public int SourceId { get; } = source.Id;
+    public string SourceName { get; } = source.Name;
+
+    public List<RuleTocItem> Children { get; } = [];
+
+    public RuleTocItem Get(string[] breadcrumbs)
+    {
+        RuleTocItem? node = null;
+
+        while (true)
+        {
+            List<RuleTocItem> children = node?.Children ?? Children;
+
+            if (breadcrumbs.Length == 0)
+            {
+                return node!;
+            }
+
+            string root = breadcrumbs[0];
+            RuleTocItem? item = children.FirstOrDefault(i => i.Name == root);
+
+            if (item is null)
+            {
+                item = new(root, node);
+                children.Add(item);
+            }
+
+            breadcrumbs = breadcrumbs[1..];
+            node = item;
+        }
+    }
+
+    public bool IsNameExists(string name, RuleTocItem? node = null)
+    {
+        foreach (RuleTocItem item in node?.Children ?? Children)
+        {
+            if (item.Name == name || IsNameExists(name, item))
+            {
+                return true;
+            }
+        }
+
+        return false;
+    }
+
+    public RuleTocItem? GetByEntityId(int id, RuleTocItem? node = null)
+    {
+        if (node?.RuleId == id)
+        {
+            return node;
+        }
+
+        foreach (RuleTocItem item in node?.Children ?? Children)
+        {
+            RuleTocItem? result = GetByEntityId(id, item);
+
+            if (result is not null)
+            {
+                return result;
+            }
+        }
+
+        return null;
+    }
+
+    public override string ToString()
+    {
+        return $"\"{SourceName}\":\n"
+            + string.Join(
+                "\n",
+                Children.Select(child => RuleTocItem.Indent(child.ToString())).ToList()
+            );
+    }
+}
+
+internal class RuleTocItem(string name, RuleTocItem? parent)
+{
+    public string? Id { get; set; } = null;
+
+    public string Name { get; } = name;
+
+    public int? RuleId { get; set; } = null;
+
+    public bool IsHandled { get; set; } = false;
+
+    public RuleTocItem? Parent { get; } = parent;
+
+    public List<string>? ChildrenOrder { get; set; } = null;
+
+    public List<RuleTocItem> Children { get; } = [];
+
+    public string? PreviousId { get; set; }
+    public string? NextId { get; set; }
+
+    public string? FirstChildId { get; set; }
+    public string? LastChildId { get; set; }
+
+    public override string ToString()
+    {
+        List<string> rows =
+        [
+            $"{Name}:",
+            $"{Tab}aonprd: {Id ?? "null"}",
+            $"{Tab}entity: {RuleId?.ToString() ?? "null"}",
+        ];
+
+        if (Children.Count == 0)
+        {
+            // rows.Add($"{TAB}children: None");
+            return string.Join("\n", rows);
+        }
+
+        rows.Add($"{Tab}children:");
+
+        foreach (RuleTocItem child in Children)
+        {
+            rows.Add(child.Indent(2));
+        }
+
+        return string.Join("\n", rows);
+    }
+
+    public string Indent(int amount = 1)
+    {
+        string result = ToString();
+        for (int i = 0; i < amount; i++)
+        {
+            result = Indent(result);
+        }
+        return result;
+    }
+
+    public static string Indent(string str)
+    {
+        return Tab + str.Replace("\n", $"\n{Tab}");
+    }
+
+    public const string Tab = "    ";
+}

Pf2eDataScrapper/Importers/Implementations/Aonprd/DataBuilder/Processors/Types/Rules/RulesOrderProcessor.cs

@@ -0,0 +1,135 @@
+// ЗАМЕТКА
+//
+// В теории, этот метод должен был дополнительно устанавливать порядок разделов.
+// Но индекс Архивов не включает поля "next_link" и "previous_link",
+// так что ставить порядок ВСЕХ правил в архиве не получится, только тех, что есть в содержании.
+// Ну а пока просто расставляем родительские и дочерние элементы.
+
+using System.Data;
+using Pf2eModel.Model.Structure;
+using Rule = Pf2eModel.Model.Structure.Rule;
+
+namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.DataBuilder.Processors.Types.Rules;
+
+[AonprdProcessor("rules-order", null, ["rules"], false)]
+internal class RulesOrderProcessor(AonprdProcessorInput input) : AbstractProcessor(input)
+{
+    public readonly RuleTocCollection Toc = RuleTocCollection.Instance;
+
+    private readonly List<OrderItem> OrderItems = [];
+
+    public override void Process()
+    {
+        Toc.SaveFile();
+
+        /*
+        foreach (RuleTocBook book in Toc.Books)
+        {
+            BuildIdOrderDictionary(book.Children);
+        }
+        */
+
+        int count = Context.Rules.Count();
+        const int perPage = 50;
+
+        for (int skip = 0; skip < count; skip += perPage)
+        {
+            Context.Rules.Skip(skip).Take(perPage).ToList().ForEach(Process);
+        }
+    }
+
+    private void Process(Rule rule)
+    {
+        RuleTocItem? item = Toc.FindEntityId(rule.Id);
+
+        if (item is null)
+        {
+            throw new($"Rule with id {rule.Id} not found in TOC");
+        }
+
+        rule.ParentId = item.Parent?.RuleId;
+
+        // Дети будут раставлены автоматически по связи
+    }
+
+    private void BuildIdOrderDictionary(RuleTocItem item)
+    {
+        List<RuleTocItem> items = [];
+
+        if (item.ChildrenOrder is null)
+        {
+            RuleTocItem? child = item.Children.FirstOrDefault(i => i.Id == item.FirstChildId);
+
+            while (child is not null)
+            {
+                items.Add(child);
+                child = item.Children.FirstOrDefault(i => i.Id == child.NextId);
+            }
+        }
+        else
+        {
+            foreach (string id in item.ChildrenOrder)
+            {
+                RuleTocItem? child = item.Children.FirstOrDefault(i => i.Id == id);
+                if (child is null)
+                {
+                    throw new($"Child with id {id} not found in {item.Name}");
+                }
+
+                items.Add(child);
+            }
+        }
+
+        if (items.Count != item.Children.Count)
+        {
+            throw new($"Children count mismatch for {item.Name}");
+        }
+
+        BuildIdOrderDictionary(items);
+    }
+
+    private void BuildIdOrderDictionary(List<RuleTocItem> items)
+    {
+        for (int i = 0; i < items.Count; i++)
+        {
+            RuleTocItem item = items[i];
+
+            if (item.RuleId is null)
+            {
+                throw new($"RuleId is null for {item.Name} (id {item.Id}");
+            }
+
+            AddOrderItem((int)item.RuleId!, i, null);
+            BuildIdOrderDictionary(item);
+        }
+    }
+
+    private void AddOrderItem(int id, int order, int? parentId)
+    {
+        if (OrderItems.Any(item => item.Id == id))
+        {
+            throw new($"Order item with id {id} already exists");
+        }
+
+        OrderItems.Add(
+            new()
+            {
+                Id = id,
+                Order = order,
+                ParentId = parentId,
+            }
+        );
+    }
+
+    public override void Save()
+    {
+        Context.SaveChanges();
+    }
+
+    private record OrderItem
+    {
+        public int Id { get; init; }
+        public int Order { get; init; }
+        public int? ParentId { get; init; }
+    }
+}

Pf2eDataScrapper/Importers/Implementations/Aonprd/DataBuilder/Processors/Types/Rules/RulesTocProcessor.cs

@@ -0,0 +1,104 @@
+using System.Text.RegularExpressions;
+using System.Web;
+using AngleSharp;
+using AngleSharp.Dom;
+using Pf2eModel.Model.Base.Source;
+
+namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.DataBuilder.Processors.Types.Rules;
+
+[AonprdProcessor("rules-toc", null, ["source"], false)]
+internal partial class RulesTocProcessor(AonprdProcessorInput input) : AbstractProcessor(input)
+{
+    private const string Selector = "#ctl00_RadDrawer1_Content_MainContent_DetailedOutput";
+    private const string Url = "https://2e.aonprd.com/Rules.aspx";
+
+    public readonly RuleTocCollection Toc = RuleTocCollection.Instance;
+
+    public override void Process()
+    {
+        IDocument document = DownloadPage();
+
+        foreach (IElement block in document.QuerySelector(Selector)!.Children)
+        {
+            IElement summaryElement =
+                block.QuerySelector("summary") ?? throw new("Can't determine <summary> element");
+
+            string sourceName = summaryElement.TextContent;
+            Source source =
+                Context.Sources.FirstOrDefault(s => s.Name == sourceName)
+                ?? throw new($"Source {sourceName} not found");
+
+            RuleTocBook book = Toc[source];
+
+            for (int i = 1; i < block.Children.Length; i++)
+            {
+                IElement element = block.Children[i];
+
+                if (element.TagName != "B")
+                {
+                    throw new($"Expected <B> tag, but got <{element.TagName}>");
+                }
+
+                RuleTocItem item = ParseLink(element.InnerHtml);
+                item.ChildrenOrder = [];
+
+                book.Children.Add(item);
+
+                if (element.NextElementSibling?.TagName != "UL")
+                {
+                    continue;
+                }
+
+                IElement ul = block.Children[++i];
+
+                foreach (IElement li in ul.Children)
+                {
+                    RuleTocItem child = ParseLink(li.InnerHtml);
+
+                    item.Children.Add(child);
+                    item.ChildrenOrder.Add(child.Id!);
+                }
+            }
+        }
+
+        Toc.SaveFile();
+    }
+
+    private RuleTocItem ParseLink(string value, RuleTocItem? parent = null)
+    {
+        Match linkMatch = LinkRegex().Match(value);
+
+        if (!linkMatch.Success)
+        {
+            throw new($"Can't parse link {value}");
+        }
+
+        return new(HttpUtility.HtmlDecode(linkMatch.Groups[2].Value).Trim(), parent)
+        {
+            Id = $"rules-{linkMatch.Groups[1].Value}",
+        };
+    }
+
+    private IDocument DownloadPage()
+    {
+        IConfiguration config = Configuration.Default.WithDefaultLoader();
+        IBrowsingContext context = BrowsingContext.New(config);
+        IDocument document = context.OpenAsync(Url).Result;
+
+        IElement root = document.QuerySelector(Selector) ?? throw new("Broken page download");
+
+        string html = root.OuterHtml;
+
+        foreach (string s in new string[] { "br", "\r", "\n", "<u>", "</u>", "&amp;NoRedirect=1" })
+        {
+            html = html.Replace(s, "");
+        }
+
+        return context.OpenAsync(req => req.Content(html)).Result;
+    }
+
+    public override void Save() { }
+
+    [GeneratedRegex(@"<a href=""Rules\.aspx\?ID=(\d+)"">(.*?)</a>")]
+    public static partial Regex LinkRegex();
+}

Pf2eDataScrapper/Importers/Implementations/Aonprd/DataBuilder/Processors/Types/TraitProcessor.cs

@@ -3,7 +3,7 @@ using Pf2eModel.Model.Structure;
 
 namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.DataBuilder.Processors.Types;
 
-[AonprdProcessor("trait", ["external"])]
+[AonprdProcessor("trait", ["external"], false)]
 internal class TraitProcessor(AonprdProcessorInput input) : AbstractEntityProcessor(input)
 {
     public override EntityType Type => EntityType.Trait;

Pf2eDataScrapper/Pf2eDataScrapper.csproj

@@ -7,6 +7,7 @@
     <Nullable>enable</Nullable>
   </PropertyGroup>
   <ItemGroup>
+    <PackageReference Include="AngleSharp" Version="1.3.0" />
     <PackageReference Include="dotenv.net" Version="3.2.1" />
     <PackageReference Include="EFCore.NamingConventions" Version="9.0.0" />
     <PackageReference Include="Goblinfactory.Konsole" Version="6.2.2" />

Pf2eDataScrapper/Program.cs

@@ -11,7 +11,7 @@ Pf2EFoundryImporter importer = new();
 
 AonprdImporter importer = new();
 
-importer.ProcessorNames = ["rules"];
+// importer.ProcessorNames = ["rules", "rules-order"];
 
 importer.Prepare();
 importer.Import();

Pf2eModel/Migrations/20250508101742_1.Designer.cs

@@ -13,7 +13,7 @@ using Pf2eModel.Model.Enum;
 namespace Pf2eModel.Migrations
 {
     [DbContext(typeof(Pf2eDbContext))]
-    [Migration("20250507112626_1")]
+    [Migration("20250508101742_1")]
     partial class _1
     {
         /// <inheritdoc />
@@ -236,22 +236,30 @@ namespace Pf2eModel.Migrations
 
             modelBuilder.Entity("Pf2eModel.Model.Base.Source.EntitySource", b =>
                 {
+                    b.Property<int>("Id")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("integer")
+                        .HasColumnName("id")
+                        .HasColumnOrder(0);
+
+                    NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property<int>("Id"));
+
                     b.Property<int>("EntityId")
                         .HasColumnType("integer")
                         .HasColumnName("entity_id")
                         .HasColumnOrder(1);
 
-                    b.Property<int>("SourceId")
-                        .HasColumnType("integer")
-                        .HasColumnName("source_id")
-                        .HasColumnOrder(2);
-
                     b.Property<bool>("IsRemaster")
                         .HasColumnType("boolean")
                         .HasColumnName("is_remaster")
                         .HasColumnOrder(4)
                         .HasComment("Флаг, указывающий, что объект является частью ремастера");
 
+                    b.Property<int>("SourceId")
+                        .HasColumnType("integer")
+                        .HasColumnName("source_id")
+                        .HasColumnOrder(2);
+
                     b.Property<string>("Suffix")
                         .IsRequired()
                         .HasMaxLength(255)
@@ -260,9 +268,12 @@ namespace Pf2eModel.Migrations
                         .HasColumnOrder(3)
                         .HasComment("Номер страницы, эррата и т.д.");
 
-                    b.HasKey("EntityId", "SourceId")
+                    b.HasKey("Id")
                         .HasName("pk_entity_sources");
 
+                    b.HasIndex("EntityId")
+                        .HasDatabaseName("ix_entity_sources_entity_id");
+
                     b.HasIndex("SourceId")
                         .HasDatabaseName("ix_entity_sources_source_id");
 

Pf2eModel/Migrations/20250508101742_1.cs

@@ -469,6 +469,8 @@ namespace Pf2eModel.Migrations
                 name: "entity_sources",
                 columns: table => new
                 {
+                    id = table.Column<int>(type: "integer", nullable: false)
+                        .Annotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn),
                     entity_id = table.Column<int>(type: "integer", nullable: false),
                     source_id = table.Column<int>(type: "integer", nullable: false),
                     suffix = table.Column<string>(type: "character varying(255)", maxLength: 255, nullable: false, comment: "Номер страницы, эррата и т.д."),
@@ -476,7 +478,7 @@ namespace Pf2eModel.Migrations
                 },
                 constraints: table =>
                 {
-                    table.PrimaryKey("pk_entity_sources", x => new { x.entity_id, x.source_id });
+                    table.PrimaryKey("pk_entity_sources", x => x.id);
                     table.ForeignKey(
                         name: "fk_entity_sources_entities_entity_id",
                         column: x => x.entity_id,
@@ -1260,6 +1262,11 @@ namespace Pf2eModel.Migrations
                 table: "entity_keyword",
                 column: "keywords_id");
 
+            migrationBuilder.CreateIndex(
+                name: "ix_entity_sources_entity_id",
+                table: "entity_sources",
+                column: "entity_id");
+
             migrationBuilder.CreateIndex(
                 name: "ix_entity_sources_source_id",
                 table: "entity_sources",

Pf2eModel/Migrations/Pf2eDbContextModelSnapshot.cs

@@ -233,22 +233,30 @@ namespace Pf2eModel.Migrations
 
             modelBuilder.Entity("Pf2eModel.Model.Base.Source.EntitySource", b =>
                 {
+                    b.Property<int>("Id")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("integer")
+                        .HasColumnName("id")
+                        .HasColumnOrder(0);
+
+                    NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property<int>("Id"));
+
                     b.Property<int>("EntityId")
                         .HasColumnType("integer")
                         .HasColumnName("entity_id")
                         .HasColumnOrder(1);
 
-                    b.Property<int>("SourceId")
-                        .HasColumnType("integer")
-                        .HasColumnName("source_id")
-                        .HasColumnOrder(2);
-
                     b.Property<bool>("IsRemaster")
                         .HasColumnType("boolean")
                         .HasColumnName("is_remaster")
                         .HasColumnOrder(4)
                         .HasComment("Флаг, указывающий, что объект является частью ремастера");
 
+                    b.Property<int>("SourceId")
+                        .HasColumnType("integer")
+                        .HasColumnName("source_id")
+                        .HasColumnOrder(2);
+
                     b.Property<string>("Suffix")
                         .IsRequired()
                         .HasMaxLength(255)
@@ -257,9 +265,12 @@ namespace Pf2eModel.Migrations
                         .HasColumnOrder(3)
                         .HasComment("Номер страницы, эррата и т.д.");
 
-                    b.HasKey("EntityId", "SourceId")
+                    b.HasKey("Id")
                         .HasName("pk_entity_sources");
 
+                    b.HasIndex("EntityId")
+                        .HasDatabaseName("ix_entity_sources_entity_id");
+
                     b.HasIndex("SourceId")
                         .HasDatabaseName("ix_entity_sources_source_id");
 

Pf2eModel/Model/Base/Source/EntitySource.cs

@@ -4,9 +4,13 @@ using Microsoft.EntityFrameworkCore;
 
 namespace Pf2eModel.Model.Base.Source;
 
-[PrimaryKey(nameof(EntityId), nameof(SourceId))]
 public class EntitySource
 {
+    [Key]
+    [DatabaseGenerated(DatabaseGeneratedOption.Identity)]
+    [Column(Order = 0)]
+    public int Id { get; set; }
+
     [Column(Order = 1)]
     public int EntityId { get; set; }
 

Pf2eModel/Pf2eDbContext.cs

@@ -125,6 +125,7 @@ public partial class Pf2eDbContext : DbContext
 
         // optionsBuilder.LogTo(Console.WriteLine);
         optionsBuilder.UseSnakeCaseNamingConvention();
+        optionsBuilder.EnableSensitiveDataLogging();
         optionsBuilder.UseNpgsql(
             databaseConnection,
             builder =>

README.md

@@ -10,6 +10,8 @@
     - `OUTPUT_DIR` - путь к месту сохранения схем;
     - `DATABASE_CONNECTION` - строка подключения к БД (только PostreSQL).
 - Перейдите а папку `Pf2eModel` (`cd ../Pf2eModel`) и запустите `dotnet ef database update` для создания структуры БД в командной строке.
+- Запустите проект `Pf2eDataScrapper` для парсинга информации.
+- Запустите проект `Pf2eViewer` для просмотра полученной инфомрации.
 
 ## Роли проектов
 
@@ -65,7 +67,7 @@ schema.AddValue(data); // Добавление данных
 
 ## Aonprd (Архивы Нефиса)
 
-Готово: **5** из 70
+Готово: **15** из 70
 
 ### Общее
 - [x] action.yaml
@@ -80,19 +82,19 @@ schema.AddValue(data); // Добавление данных
 - [ ] curse.yaml
 - [ ] deviant-ability-classification.yaml
 - [ ] disease.yaml
-- [ ] heritage.yaml
+- [x] heritage.yaml
 - [ ] kingdom-event.yaml
 - [ ] kingdom-structure.yaml
 - [x] language.yaml
 - [ ] mythic-calling.yaml
 - [ ] plane.yaml
 - [ ] relic.yaml
-- [ ] ritual.yaml
+- [x] ritual.yaml
 - [ ] sidebar.yaml
 - [ ] skill-general-action.yaml
 - [x] skill.yaml
 - [x] source.yaml
-- [ ] spell.yaml
+- [x] spell.yaml
 - [ ] tradition.yaml
 - [x] trait.yaml
 - [ ] warfare-army.yaml
@@ -103,7 +105,7 @@ schema.AddValue(data); // Добавление данных
 **Тексты**
 - [ ] article.yaml
 - [ ] category-page.yaml
-- [ ] rules.yaml
+- [x] rules.yaml
 
 **Боги**
 - [ ] deity.yaml
@@ -116,7 +118,7 @@ schema.AddValue(data); // Добавление данных
 
 **Подклассы**
 - [ ] research-field.yaml (Alchemist)
-- [ ] apparition.yaml (Animist)
+- [x] apparition.yaml (Animist)
 - [ ] practice.yaml (Animist)
 - [ ] instinct.yaml (Barbarian)
 - [ ] muse.yaml (Bard)