Первая версия обработчиков

2025-04-14 18:44:38 +04:00
parent 492323a4c0
commit 8277294319
6 changed files with 261 additions and 0 deletions
@@ -0,0 +1,24 @@
+using Newtonsoft.Json.Linq;
+using Slugify;
+
+namespace Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.DataBuilder
+{
+    internal abstract class AbstractEntityProcessor(JObject input) : AbstractProcessor(input)
+    {
+        public abstract string DbType { get; }
+
+        public override void Process()
+        {
+            Add("name", "name");
+            Add("img", "img");
+            Add("description", "system.description.value");
+            Add("description_gm", "system.description.gm");
+            Add("type", DbType);
+            Add("rarity", "system.traits.rarity");
+
+            SlugHelper sh = new();
+            Data.Add("slug", sh.GenerateSlug(Data["name"]?.ToString() ?? throw new NullReferenceException("'name' is null")));
+        }
+
+    }
+}
@@ -0,0 +1,30 @@
+using Newtonsoft.Json.Linq;
+using Npgsql;
+
+namespace Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.DataBuilder
+{
+    internal abstract class AbstractProcessor(JObject input)
+    {
+        public readonly JObject Input = input;
+
+        public Pf2eDatabase Database = Pf2eDatabase.Instance;
+
+        public DataBuilderHelper Helper = DataBuilderHelper.Instance;
+
+        public abstract string Table { get; }
+
+        public Dictionary<string, object?> Data = [];
+
+        protected void Add(string key, string path)
+        {
+            Data.Add(key, DataBuilderHelper.JsonGet(Input, path));
+        }
+
+        public abstract void Process();
+
+        public int Save()
+        {
+            return Helper.Insert(Table, Data);
+        }
+    }
+}
@@ -0,0 +1,82 @@
+using Newtonsoft.Json.Linq;
+using Npgsql;
+
+namespace Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.DataBuilder
+{
+    internal class DataBuilderHelper
+    {
+        public Pf2eDatabase Database = Pf2eDatabase.Instance;
+
+        private DataBuilderHelper()
+        {
+        }
+
+        public NpgsqlCommand PrepareInsert(string table, Dictionary<string, object?> data, bool returnId)
+        {
+            List<string> columns = [];
+            List<string> patterns = [];
+
+            foreach (var key in data.Keys)
+            {
+                columns.Add(key);
+                patterns.Add($":{key.ToUpper()}");
+            }
+
+            string dml = $"INSERT INTO {table} (\"{string.Join("\", \"", columns)}\") VALUES ({string.Join(", ", patterns)})" + (returnId ? " RETURNING id" : "");
+
+            NpgsqlCommand cmd = Database.GetCommand(dml);
+            foreach (var kvp in data)
+            {
+                cmd.Parameters.AddWithValue(":" + kvp.Key.ToUpper(), kvp.Value ?? DBNull.Value);
+            }
+
+            Console.WriteLine(cmd.CommandText);
+            return cmd;
+        }
+
+        public int Insert(string table, Dictionary<string, object?> data)
+        {
+            var cmd = PrepareInsert(table, data, true);
+            return (int)(cmd.ExecuteScalar() ?? throw new Exception("Not a number"));
+        }
+
+        public void InsertBlind(string table, Dictionary<string, object?> data)
+        {
+            var cmd = PrepareInsert(table, data, false);
+            cmd.ExecuteNonQuery();
+        }
+
+        public static object? JsonGet(JObject root, string path)
+        {
+            var token = root.SelectToken(path);
+
+            if (token == null)
+            {
+                return null;
+            }
+
+            if (token.Type == JTokenType.String)
+            {
+                return token.ToString();
+            }
+            else if (token.Type == JTokenType.Integer)
+            {
+                return token.ToObject<int>();
+            }
+            else if (token.Type == JTokenType.Float)
+            {
+                return token.ToObject<float>();
+            }
+            else if (token.Type == JTokenType.Boolean)
+            {
+                return token.ToObject<bool>();
+            }
+            else
+            {
+                return token.ToString();
+            }
+        }
+
+        public static DataBuilderHelper Instance = new();
+    }
+}
@@ -0,0 +1,85 @@
+using Konsole;
+using Newtonsoft.Json.Linq;
+using Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.Compendium;
+using System.Reflection;
+
+namespace Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.DataBuilder
+{
+    internal class PF2eFoundryDataBuilder
+    {
+        public Pf2eDatabase Database = Pf2eDatabase.Instance;
+
+        /// <summary>
+        /// Загрузчик данных.
+        /// </summary>
+        public readonly CompendiumLoader Loader;
+
+        /// <summary>
+        /// Коллекция папок компендиума.
+        /// </summary>
+        public CompendiumFolderCollection Collection => Loader.Collection;
+
+        public Dictionary<string, ConstructorInfo> Processors = [];
+
+        public PF2eFoundryDataBuilder(CompendiumLoader loader)
+        {
+            Loader = loader;
+            PrepareProcessors();
+        }
+
+        public void PrepareProcessors()
+        {
+            foreach (Type type in Assembly.GetExecutingAssembly().GetTypes())
+            {
+                if (type.GetCustomAttributes(typeof(Pf2eTypeProcessor), true).Length > 0)
+                {
+                    var attribute = type.GetCustomAttribute<Pf2eTypeProcessor>();
+                    Processors.Add(attribute!.Type, type.GetConstructor(new[] { typeof(JObject) })!);
+                }
+            }
+        }
+
+        public AbstractProcessor? GetProcessor(string type, JObject input)
+        {
+            if (Processors.TryGetValue(type, out var constructor))
+            {
+                return (AbstractProcessor)constructor.Invoke([input]);
+            }
+
+            return null;
+        }
+
+        public void TruncateTables()
+        {
+            Console.WriteLine("Truncating all old data...");
+
+            Database.ExecuteCommand($"SELECT f_truncate_tables();");
+
+            Console.WriteLine("Truncation completed!");
+        }
+
+        public void Build()
+        {
+            Console.WriteLine("Building data...");
+
+            var bar = new ProgressBar(Loader.ItemCount);
+            var i = 0;
+
+            foreach (var entry in Loader.Enrties())
+            {
+                bar.Refresh(i++, $"{entry.Folder.Name}/{entry.File.Name}");
+
+                var processor = GetProcessor($"{entry.Folder.Type}.{entry.Type}", entry.Json);
+                if (processor is null)
+                {
+                    continue;
+                }
+
+                processor.Process();
+                processor.Save();
+            }
+
+            bar.Refresh(Loader.ItemCount, "Done!");
+        }
+    }
+}
@@ -0,0 +1,11 @@
+using System.Reflection;
+
+namespace Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.DataBuilder
+{
+    [AttributeUsage(AttributeTargets.Class)]
+    internal class Pf2eTypeProcessor(string type) : Attribute
+    {
+        public readonly string Type = type;
+        
+    }
+}
@@ -0,0 +1,29 @@
+using Newtonsoft.Json.Linq;
+
+namespace Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.DataBuilder.Processors
+{
+    [Pf2eTypeProcessor("Item.condition")]
+    internal class ConditionProcessor(JObject input) : AbstractEntityProcessor(input)
+    {
+        public override string Table => "condition";
+
+        public override string DbType => "condition";
+
+        public override void Process()
+        {
+            base.Process();
+
+            Add("group", "system.group");
+
+            JObject extra = new()
+                {
+                    { "duration", Input.SelectToken("duration") },
+                    { "overrides", Input.SelectToken("overrides") },
+                    { "references", Input.SelectToken("references") },
+                    { "value", Input.SelectToken("value") }
+                };
+
+            Data.Add("extra", extra.ToString(Newtonsoft.Json.Formatting.Indented));
+        }
+    }
+}