Отказ от прямой работы с БД в обработчиках и переход на EF

Pf2eDataScrapper/.env.example

@@ -1 +1,4 @@
 DATA_DIR=D:\path\to\directory\with\repositories
+OUTPUT_DIR=D:\path\to\output\directory
+
+DATABASE_CONNECTION=Host=localhost;Port=5433;Database=usersdb2;Username=postgres;Password=password

Pf2eDataScrapper/EnvSettings.cs

@@ -36,5 +36,6 @@ namespace Pf2eDataScrapper
         /// </summary>
         public static string OutputDir => new(OutputDirPath);
 
+        public static string DatabaseConnection => EnvReader.GetStringValue("DATABASE_CONNECTION");
     }
 }

Pf2eDataScrapper/Importers/AbstractImporter.cs

@@ -35,5 +35,9 @@
         /// </summary>
         public abstract void Import();
 
+        /// <summary>
+        /// Построение схем данных
+        /// </summary>
+        public abstract void BuildSchemas();
     }
 }

Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/Compendium/CompendiumFolderCollection.cs

@@ -21,7 +21,8 @@
         /// <returns></returns>
         public CompendiumFolderCollection Filter()
         {
-            return [.. this.FindAll(x => x.Type != CompendiumDocumentType.Actor)];
+            // return [.. this.FindAll(x => x.Type != CompendiumDocumentType.Actor)];
+            return [.. this.FindAll(x => x.Type != CompendiumDocumentType.Actor && x.Name == "Conditions")];
         }
     }
 }

Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/CompendiumLoader.cs

@@ -1,4 +1,5 @@
-using Newtonsoft.Json.Linq;
+using Microsoft.Extensions.Logging;
+using Newtonsoft.Json.Linq;
 using Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.Compendium;
 
 namespace Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry
@@ -119,12 +120,52 @@ namespace Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry
                 }
             }
 
-            foreach (var folder in Collection)
-            {
-                Console.WriteLine(folder);
-            }
+            // foreach (var folder in Collection)
+            // {
+            //     Console.WriteLine(folder);
+            // }
+
+            Collection = Collection.Filter();
 
             Console.WriteLine($"Total items: {ItemCount}");
         }
+
+        public IEnumerable<LoaderEnrty> Enrties()
+        {
+            foreach (var folder in Collection.FindAll(f => f.Path is not null))
+            {
+                var dirPath = Path.Combine(Importer.Dir, folder.Path!);
+
+                foreach (var filePath in Directory.GetFiles(dirPath, "*.json", SearchOption.AllDirectories))
+                {
+                    var file = new FileInfo(filePath);
+
+                    if (file.Name == "_folders.json")
+                    {
+                        continue;
+                    }
+
+                    using var reader = new StreamReader(file.FullName);
+                    var itemJson = JObject.Parse(reader.ReadToEnd());
+
+                    var itemFolder = folder;
+                    if (itemJson.TryGetValue("folder", out var itemJsonFolder))
+                    {
+                        itemFolder = Collection.GetById(itemJsonFolder.ToString());
+                    }
+
+                    yield return new LoaderEnrty(file, itemJson, itemFolder ?? folder);
+                }
+            }
+        }
+
+        public class LoaderEnrty(FileInfo file, JObject itemJson, CompendiumFolder folder)
+        {
+            public FileInfo File { get; } = file;
+            public JObject Json { get; } = itemJson;
+            public CompendiumFolder Folder { get; } = folder;
+            public string Type { get; } = itemJson["type"]?.ToString() 
+                ?? throw new Exception($"Expected to have `type` field in file {file.FullName}");
+        }
     }
 }

Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/DataBuilder/AbstractProcessor.cs

@@ -0,0 +1,43 @@
+using LinqToDB.DataProvider.SapHana;
+using Newtonsoft.Json.Linq;
+using Npgsql;
+
+namespace Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.DataBuilder
+{
+    internal abstract class AbstractProcessor(JObject input)
+    {
+        public readonly Pf2eTestContext Context = new();
+
+        public readonly JObject Input = input;
+
+        public abstract void Process();
+
+        public void Save()
+        {
+            Context.SaveChanges();
+        }
+
+
+
+        protected T GetRequired<T>(string path)
+        {
+            var token = Input.SelectToken(path) ?? throw new NullReferenceException($"'{path}' is null");
+
+            return token.ToObject<T>() ?? throw new NullReferenceException($"'{path}' is null");
+        }
+
+        protected T? GetOptional<T>(string path)
+        {
+            var token = Input.SelectToken(path) ?? null;
+
+            if (token is null)
+            {
+                return default!;
+            }
+
+            return token.ToObject<T>() ?? default;
+        }
+
+
+    }
+}

Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/DataBuilder/DataBuilderHelper.cs

@@ -0,0 +1,82 @@
+using Newtonsoft.Json.Linq;
+using Npgsql;
+
+namespace Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.DataBuilder
+{
+    internal class DataBuilderHelper
+    {
+        public Pf2eDatabase Database = Pf2eDatabase.Instance;
+
+        private DataBuilderHelper()
+        {
+        }
+
+        public NpgsqlCommand PrepareInsert(string table, Dictionary<string, object?> data, bool returnId)
+        {
+            List<string> columns = [];
+            List<string> patterns = [];
+
+            foreach (var key in data.Keys)
+            {
+                columns.Add(key);
+                patterns.Add($":{key.ToUpper()}");
+            }
+
+            string dml = $"INSERT INTO {table} (\"{string.Join("\", \"", columns)}\") VALUES ({string.Join(", ", patterns)})" + (returnId ? " RETURNING id" : "");
+
+            NpgsqlCommand cmd = Database.GetCommand(dml);
+            foreach (var kvp in data)
+            {
+                cmd.Parameters.AddWithValue(":" + kvp.Key.ToUpper(), kvp.Value ?? DBNull.Value);
+            }
+
+            Console.WriteLine(cmd.CommandText);
+            return cmd;
+        }
+
+        public int Insert(string table, Dictionary<string, object?> data)
+        {
+            var cmd = PrepareInsert(table, data, true);
+            return (int)(cmd.ExecuteScalar() ?? throw new Exception("Not a number"));
+        }
+
+        public void InsertBlind(string table, Dictionary<string, object?> data)
+        {
+            var cmd = PrepareInsert(table, data, false);
+            cmd.ExecuteNonQuery();
+        }
+
+        public static object? JsonGet(JObject root, string path)
+        {
+            var token = root.SelectToken(path);
+
+            if (token == null)
+            {
+                return null;
+            }
+
+            if (token.Type == JTokenType.String)
+            {
+                return token.ToString();
+            }
+            else if (token.Type == JTokenType.Integer)
+            {
+                return token.ToObject<int>();
+            }
+            else if (token.Type == JTokenType.Float)
+            {
+                return token.ToObject<float>();
+            }
+            else if (token.Type == JTokenType.Boolean)
+            {
+                return token.ToObject<bool>();
+            }
+            else
+            {
+                return token.ToString();
+            }
+        }
+
+        public static DataBuilderHelper Instance = new();
+    }
+}

Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/DataBuilder/PF2eFoundryDataBuilder.cs

@@ -0,0 +1,85 @@
+using Konsole;
+using Newtonsoft.Json.Linq;
+using Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.Compendium;
+using System.Reflection;
+
+namespace Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.DataBuilder
+{
+    internal class PF2eFoundryDataBuilder
+    {
+        public Pf2eDatabase Database = Pf2eDatabase.Instance;
+
+        /// <summary>
+        /// Загрузчик данных.
+        /// </summary>
+        public readonly CompendiumLoader Loader;
+
+        /// <summary>
+        /// Коллекция папок компендиума.
+        /// </summary>
+        public CompendiumFolderCollection Collection => Loader.Collection;
+
+        public Dictionary<string, ConstructorInfo> Processors = [];
+
+        public PF2eFoundryDataBuilder(CompendiumLoader loader)
+        {
+            Loader = loader;
+            PrepareProcessors();
+        }
+
+        public void PrepareProcessors()
+        {
+            foreach (Type type in Assembly.GetExecutingAssembly().GetTypes())
+            {
+                if (type.GetCustomAttributes(typeof(Pf2eTypeProcessor), true).Length > 0)
+                {
+                    var attribute = type.GetCustomAttribute<Pf2eTypeProcessor>();
+                    Processors.Add(attribute!.Type, type.GetConstructor(new[] { typeof(JObject) })!);
+                }
+            }
+        }
+
+        public AbstractProcessor? GetProcessor(string type, JObject input)
+        {
+            if (Processors.TryGetValue(type, out var constructor))
+            {
+                return (AbstractProcessor)constructor.Invoke([input]);
+            }
+
+            return null;
+        }
+
+        public void TruncateTables()
+        {
+            Console.WriteLine("Truncating all old data...");
+
+            Database.ExecuteCommand($"SELECT f_truncate_tables();");
+
+            Console.WriteLine("Truncation completed!");
+        }
+
+        public void Build()
+        {
+            Console.WriteLine("Building data...");
+
+            var bar = new ProgressBar(Loader.ItemCount);
+            var i = 0;
+
+            foreach (var entry in Loader.Enrties())
+            {
+                bar.Refresh(i++, $"{entry.Folder.Name}/{entry.File.Name}");
+
+                var processor = GetProcessor($"{entry.Folder.Type}.{entry.Type}", entry.Json);
+                if (processor is null)
+                {
+                    continue;
+                }
+
+                processor.Process();
+                processor.Save();
+            }
+
+            bar.Refresh(Loader.ItemCount, "Done!");
+        }
+    }
+}

Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/DataBuilder/Pf2eTypeProcessor.cs

@@ -0,0 +1,11 @@
+using System.Reflection;
+
+namespace Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.DataBuilder
+{
+    [AttributeUsage(AttributeTargets.Class)]
+    internal class Pf2eTypeProcessor(string type) : Attribute
+    {
+        public readonly string Type = type;
+        
+    }
+}

Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/DataBuilder/Processors/ConditionProcessor.cs

@@ -0,0 +1,39 @@
+using Newtonsoft.Json.Linq;
+using Pf2eDataScrapper.Model;
+using Slugify;
+using static LinqToDB.Common.Configuration;
+
+namespace Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.DataBuilder.Processors
+{
+    [Pf2eTypeProcessor("Item.condition")]
+    internal class ConditionProcessor(JObject input) : AbstractProcessor(input)
+    {
+        public override void Process()
+        {
+            JObject extra = new()
+                {
+                    { "duration", Input.SelectToken("duration") },
+                    { "overrides", Input.SelectToken("overrides") },
+                    { "references", Input.SelectToken("references") },
+                    { "value", Input.SelectToken("value") }
+                };
+
+            Condition condition = new()
+            {
+                Name = GetRequired<string>("name"),
+                Img = GetRequired<string>("img"),
+                Description = GetRequired<string>("system.description.value"),
+                DescriptionGm = GetOptional<string>("system.description.gm"),
+                // Type = 
+                IsTranslated = false,
+                IsChecked = false,
+
+                // Group = GetOptional<string>("system.group");
+                Extra = extra.ToString(Newtonsoft.Json.Formatting.Indented)
+            };
+            condition.Slug = new SlugHelper().GenerateSlug(condition.Name);
+
+            Context.Conditions.Add(condition);
+        }
+    }
+}

Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/Pf2eFoundryImporter.cs

@@ -1,4 +1,7 @@
-namespace Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry
+using Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.DataBuilder;
+using Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.SchemaBuilder;
+
+namespace Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry
 {
     /// <summary>
     /// Импортёр для модуля Pf2e FoundryVTT.
@@ -46,7 +49,21 @@
         /// </summary>
         public override void Import()
         {
-            Loader.Prepare();
+            var builder = new PF2eFoundryDataBuilder(Loader);
+
+            builder.TruncateTables();
+            builder.Build();
+        }
+
+        /// <summary>
+        /// Построение схем данных
+        /// </summary>
+        public override void BuildSchemas()
+        {
+            var builder = new CompendiumSchemaBuilder(Loader);
+
+            builder.BuildSchemas();
+            builder.Save(EnvSettings.OutputDir);
         }
     }
 }

Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/SchemaBuilder/CompendiumSchemaBuilder.cs

@@ -1,8 +1,9 @@
 using Konsole;
 using Newtonsoft.Json.Linq;
+using Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.Compendium;
 using Pf2eDataScrapper.SchemaBuilder;
 
-namespace Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.Compendium
+namespace Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.SchemaBuilder
 {
     /// <summary>
     /// Класс построения схемы компендиумов.
@@ -97,36 +98,15 @@ namespace Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.Compendium
             var bar = new ProgressBar(Loader.ItemCount);
             var i = 0;
 
-            foreach (var folder in Collection.FindAll(f => f.Path is not null))
+            foreach (var entry in Loader.Enrties())
             {
-                var dirPath = Path.Combine(Loader.Importer.Dir, folder.Path!);
+                bar.Refresh(i++, entry.Folder.Name);
 
-                foreach (var filePath in Directory.GetFiles(dirPath, "*.json", SearchOption.AllDirectories))
+                var schemas = GetItemSchemas(entry.Folder, entry.Type);
+
+                foreach (var schema in schemas)
                 {
-                    bar.Refresh(i++, folder.Name);
-
-                    var file = new FileInfo(filePath);
-                
-                    if (file.Name == "_folders.json")
-                    {
-                        continue;
-                    }
-                
-                    using var reader = new StreamReader(file.FullName);
-                    var itemJson = JObject.Parse(reader.ReadToEnd());
-
-                    var itemFolder = folder;
-                    if (itemJson.TryGetValue("folder", out var itemJsonFolder))
-                    {
-                        itemFolder = Collection.GetById(itemJsonFolder.ToString());
-                    }
-
-                    var schemas = GetItemSchemas(itemFolder ?? folder, itemJson["type"]?.ToString());
-
-                    foreach (var schema in schemas)
-                    {
-                        schema.AddValue(itemJson);
-                    }
+                    schema.AddValue(entry.Json);
                 }
             }
 

Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/SchemaBuilder/CompendiumSchemaTypeResolver.cs

@@ -7,7 +7,7 @@ using Newtonsoft.Json.Linq;
 using Pf2eDataScrapper.SchemaBuilder;
 using Pf2eDataScrapper.SchemaBuilder.Types;
 
-namespace Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.Compendium
+namespace Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.SchemaBuilder
 {
     internal class CompendiumSchemaTypeResolver : TypeResolver
     {

Pf2eDataScrapper/Model/Action.cs

@@ -0,0 +1,29 @@
+using System;
+using System.Collections.Generic;
+
+namespace Pf2eDataScrapper.Model;
+
+public partial class Action
+{
+    public int Id { get; set; }
+
+    public string Name { get; set; } = null!;
+
+    public string Img { get; set; } = null!;
+
+    public string Description { get; set; } = null!;
+
+    public string? DescriptionGm { get; set; }
+
+    public string Slug { get; set; } = null!;
+
+    public bool? IsTranslated { get; set; }
+
+    public bool? IsChecked { get; set; }
+
+    public int? Actions { get; set; }
+
+    public bool? DeathNote { get; set; }
+
+    public string? Extra { get; set; }
+}

Pf2eDataScrapper/Model/Ancestry.cs

@@ -0,0 +1,37 @@
+using System;
+using System.Collections.Generic;
+
+namespace Pf2eDataScrapper.Model;
+
+public partial class Ancestry
+{
+    public int Id { get; set; }
+
+    public string Name { get; set; } = null!;
+
+    public string Img { get; set; } = null!;
+
+    public string Description { get; set; } = null!;
+
+    public string? DescriptionGm { get; set; }
+
+    public string Slug { get; set; } = null!;
+
+    public bool? IsTranslated { get; set; }
+
+    public bool? IsChecked { get; set; }
+
+    public int Hp { get; set; }
+
+    public int Reach { get; set; }
+
+    public int Speed { get; set; }
+
+    public virtual ICollection<AncestryBoost> AncestryBoosts { get; set; } = new List<AncestryBoost>();
+
+    public virtual ICollection<AncestryItem> AncestryItems { get; set; } = new List<AncestryItem>();
+
+    public virtual ICollection<AncestryLanguage> AncestryLanguages { get; set; } = new List<AncestryLanguage>();
+
+    public virtual ICollection<Heritage> Heritages { get; set; } = new List<Heritage>();
+}

Pf2eDataScrapper/Model/AncestryBoost.cs

@@ -0,0 +1,17 @@
+using System;
+using System.Collections.Generic;
+
+namespace Pf2eDataScrapper.Model;
+
+public partial class AncestryBoost
+{
+    public int Id { get; set; }
+
+    public int AncestryId { get; set; }
+
+    public bool Boost { get; set; }
+
+    public string? Value { get; set; }
+
+    public virtual Ancestry Ancestry { get; set; } = null!;
+}

Pf2eDataScrapper/Model/AncestryItem.cs

@@ -0,0 +1,17 @@
+using System;
+using System.Collections.Generic;
+
+namespace Pf2eDataScrapper.Model;
+
+public partial class AncestryItem
+{
+    public int AncestryId { get; set; }
+
+    public int ItemId { get; set; }
+
+    public int Level { get; set; }
+
+    public virtual Ancestry Ancestry { get; set; } = null!;
+
+    public virtual Entity Item { get; set; } = null!;
+}

Pf2eDataScrapper/Model/AncestryLanguage.cs

@@ -0,0 +1,17 @@
+using System;
+using System.Collections.Generic;
+
+namespace Pf2eDataScrapper.Model;
+
+public partial class AncestryLanguage
+{
+    public int AncestryId { get; set; }
+
+    public int LanguageId { get; set; }
+
+    public bool IsAdditional { get; set; }
+
+    public virtual Ancestry Ancestry { get; set; } = null!;
+
+    public virtual Language Language { get; set; } = null!;
+}

Pf2eDataScrapper/Model/Background.cs

@@ -0,0 +1,29 @@
+using System;
+using System.Collections.Generic;
+
+namespace Pf2eDataScrapper.Model;
+
+public partial class Background
+{
+    public int Id { get; set; }
+
+    public string Name { get; set; } = null!;
+
+    public string Img { get; set; } = null!;
+
+    public string Description { get; set; } = null!;
+
+    public string? DescriptionGm { get; set; }
+
+    public string Slug { get; set; } = null!;
+
+    public bool? IsTranslated { get; set; }
+
+    public bool? IsChecked { get; set; }
+
+    public virtual ICollection<BackgroundBoost> BackgroundBoosts { get; set; } = new List<BackgroundBoost>();
+
+    public virtual ICollection<BackgroundItem> BackgroundItems { get; set; } = new List<BackgroundItem>();
+
+    public virtual ICollection<BackgroundSkill> BackgroundSkills { get; set; } = new List<BackgroundSkill>();
+}

Pf2eDataScrapper/Model/BackgroundBoost.cs

@@ -0,0 +1,15 @@
+using System;
+using System.Collections.Generic;
+
+namespace Pf2eDataScrapper.Model;
+
+public partial class BackgroundBoost
+{
+    public int Id { get; set; }
+
+    public int BackgroundId { get; set; }
+
+    public string? Value { get; set; }
+
+    public virtual Background Background { get; set; } = null!;
+}

Pf2eDataScrapper/Model/BackgroundItem.cs

@@ -0,0 +1,17 @@
+using System;
+using System.Collections.Generic;
+
+namespace Pf2eDataScrapper.Model;
+
+public partial class BackgroundItem
+{
+    public int BackgroundId { get; set; }
+
+    public int EntityId { get; set; }
+
+    public int? Level { get; set; }
+
+    public virtual Background Background { get; set; } = null!;
+
+    public virtual Entity Entity { get; set; } = null!;
+}

Pf2eDataScrapper/Model/BackgroundSkill.cs

@@ -0,0 +1,17 @@
+using System;
+using System.Collections.Generic;
+
+namespace Pf2eDataScrapper.Model;
+
+public partial class BackgroundSkill
+{
+    public int Id { get; set; }
+
+    public int BackgroundId { get; set; }
+
+    public string Name { get; set; } = null!;
+
+    public bool IsSkill { get; set; }
+
+    public virtual Background Background { get; set; } = null!;
+}

Pf2eDataScrapper/Model/CampaignFeature.cs

@@ -0,0 +1,31 @@
+using System;
+using System.Collections.Generic;
+
+namespace Pf2eDataScrapper.Model;
+
+public partial class CampaignFeature
+{
+    public int Id { get; set; }
+
+    public string Name { get; set; } = null!;
+
+    public string Img { get; set; } = null!;
+
+    public string Description { get; set; } = null!;
+
+    public string? DescriptionGm { get; set; }
+
+    public string Slug { get; set; } = null!;
+
+    public bool? IsTranslated { get; set; }
+
+    public bool? IsChecked { get; set; }
+
+    public int? Level { get; set; }
+
+    public int Actions { get; set; }
+
+    public string? Prerequisites { get; set; }
+
+    public string? Extra { get; set; }
+}

Pf2eDataScrapper/Model/Class.cs

@@ -0,0 +1,37 @@
+using System;
+using System.Collections.Generic;
+
+namespace Pf2eDataScrapper.Model;
+
+public partial class Class
+{
+    public int Id { get; set; }
+
+    public string Name { get; set; } = null!;
+
+    public string Img { get; set; } = null!;
+
+    public string Description { get; set; } = null!;
+
+    public string? DescriptionGm { get; set; }
+
+    public string Slug { get; set; } = null!;
+
+    public bool? IsTranslated { get; set; }
+
+    public bool? IsChecked { get; set; }
+
+    public int Hp { get; set; }
+
+    public int Perception { get; set; }
+
+    public bool Spellcasting { get; set; }
+
+    public int AdditionalTrainedSkills { get; set; }
+
+    public virtual ICollection<ClassKeyAbility> ClassKeyAbilities { get; set; } = new List<ClassKeyAbility>();
+
+    public virtual ICollection<ClassProgression> ClassProgressions { get; set; } = new List<ClassProgression>();
+
+    public virtual ICollection<ClassTraining> ClassTrainings { get; set; } = new List<ClassTraining>();
+}

Pf2eDataScrapper/Model/ClassKeyAbility.cs

@@ -0,0 +1,15 @@
+using System;
+using System.Collections.Generic;
+
+namespace Pf2eDataScrapper.Model;
+
+public partial class ClassKeyAbility
+{
+    public int Id { get; set; }
+
+    public int ClassId { get; set; }
+
+    public string Value { get; set; } = null!;
+
+    public virtual Class Class { get; set; } = null!;
+}

Pf2eDataScrapper/Model/ClassProgression.cs

@@ -0,0 +1,19 @@
+using System;
+using System.Collections.Generic;
+
+namespace Pf2eDataScrapper.Model;
+
+public partial class ClassProgression
+{
+    public int Id { get; set; }
+
+    public int ClassId { get; set; }
+
+    public int Level { get; set; }
+
+    public int? EntityId { get; set; }
+
+    public virtual Class Class { get; set; } = null!;
+
+    public virtual Entity? Entity { get; set; }
+}

Pf2eDataScrapper/Model/ClassTraining.cs

@@ -0,0 +1,21 @@
+using System;
+using System.Collections.Generic;
+
+namespace Pf2eDataScrapper.Model;
+
+public partial class ClassTraining
+{
+    public int Id { get; set; }
+
+    public int ClassId { get; set; }
+
+    public int? Item { get; set; }
+
+    public string? Name { get; set; }
+
+    public int Stage { get; set; }
+
+    public virtual Class Class { get; set; } = null!;
+
+    public virtual Training? ItemNavigation { get; set; }
+}

Pf2eDataScrapper/Model/Condition.cs

@@ -0,0 +1,25 @@
+using System;
+using System.Collections.Generic;
+
+namespace Pf2eDataScrapper.Model;
+
+public partial class Condition
+{
+    public int Id { get; set; }
+
+    public string Name { get; set; } = null!;
+
+    public string Img { get; set; } = null!;
+
+    public string Description { get; set; } = null!;
+
+    public string? DescriptionGm { get; set; }
+
+    public string Slug { get; set; } = null!;
+
+    public bool? IsTranslated { get; set; }
+
+    public bool? IsChecked { get; set; }
+
+    public string? Extra { get; set; }
+}

Pf2eDataScrapper/Model/Deity.cs

@@ -0,0 +1,35 @@
+using System;
+using System.Collections.Generic;
+
+namespace Pf2eDataScrapper.Model;
+
+public partial class Deity
+{
+    public int Id { get; set; }
+
+    public string Name { get; set; } = null!;
+
+    public string Img { get; set; } = null!;
+
+    public string Description { get; set; } = null!;
+
+    public string? DescriptionGm { get; set; }
+
+    public string Slug { get; set; } = null!;
+
+    public bool? IsTranslated { get; set; }
+
+    public bool? IsChecked { get; set; }
+
+    public bool SanctificationMust { get; set; }
+
+    public virtual ICollection<DeityAttribute> DeityAttributes { get; set; } = new List<DeityAttribute>();
+
+    public virtual ICollection<DeityDomain> DeityDomains { get; set; } = new List<DeityDomain>();
+
+    public virtual ICollection<DeitySpell> DeitySpells { get; set; } = new List<DeitySpell>();
+
+    public virtual ICollection<DeityWeapon> DeityWeapons { get; set; } = new List<DeityWeapon>();
+
+    public virtual ICollection<Training> Skills { get; set; } = new List<Training>();
+}

Pf2eDataScrapper/Model/DeityAttribute.cs

@@ -0,0 +1,13 @@
+using System;
+using System.Collections.Generic;
+
+namespace Pf2eDataScrapper.Model;
+
+public partial class DeityAttribute
+{
+    public int DeityId { get; set; }
+
+    public string Attribute { get; set; } = null!;
+
+    public virtual Deity Deity { get; set; } = null!;
+}

Pf2eDataScrapper/Model/DeityDomain.cs

@@ -0,0 +1,17 @@
+using System;
+using System.Collections.Generic;
+
+namespace Pf2eDataScrapper.Model;
+
+public partial class DeityDomain
+{
+    public int Id { get; set; }
+
+    public int DeityId { get; set; }
+
+    public bool IsPrimary { get; set; }
+
+    public string Value { get; set; } = null!;
+
+    public virtual Deity Deity { get; set; } = null!;
+}

Pf2eDataScrapper/Model/DeitySpell.cs

@@ -0,0 +1,19 @@
+using System;
+using System.Collections.Generic;
+
+namespace Pf2eDataScrapper.Model;
+
+public partial class DeitySpell
+{
+    public int Id { get; set; }
+
+    public int DeityId { get; set; }
+
+    public int Level { get; set; }
+
+    public int SpellId { get; set; }
+
+    public virtual Deity Deity { get; set; } = null!;
+
+    public virtual Spell Spell { get; set; } = null!;
+}

Pf2eDataScrapper/Model/DeityWeapon.cs

@@ -0,0 +1,15 @@
+using System;
+using System.Collections.Generic;
+
+namespace Pf2eDataScrapper.Model;
+
+public partial class DeityWeapon
+{
+    public int Id { get; set; }
+
+    public int DeityId { get; set; }
+
+    public string Weapon { get; set; } = null!;
+
+    public virtual Deity Deity { get; set; } = null!;
+}

Pf2eDataScrapper/Model/Entity.cs

@@ -0,0 +1,37 @@
+using System;
+using System.Collections.Generic;
+
+namespace Pf2eDataScrapper.Model;
+
+public partial class Entity
+{
+    public int Id { get; set; }
+
+    public string Name { get; set; } = null!;
+
+    public string Img { get; set; } = null!;
+
+    public string Description { get; set; } = null!;
+
+    public string? DescriptionGm { get; set; }
+
+    public string Slug { get; set; } = null!;
+
+    public bool? IsTranslated { get; set; }
+
+    public bool? IsChecked { get; set; }
+
+    public virtual ICollection<AncestryItem> AncestryItems { get; set; } = new List<AncestryItem>();
+
+    public virtual ICollection<BackgroundItem> BackgroundItems { get; set; } = new List<BackgroundItem>();
+
+    public virtual ICollection<ClassProgression> ClassProgressions { get; set; } = new List<ClassProgression>();
+
+    public virtual ICollection<EntityAlias> EntityAliases { get; set; } = new List<EntityAlias>();
+
+    public virtual ICollection<EntityExternal> EntityExternals { get; set; } = new List<EntityExternal>();
+
+    public virtual Frequency? Frequency { get; set; }
+
+    public virtual ICollection<Trait> Traits { get; set; } = new List<Trait>();
+}

Pf2eDataScrapper/Model/EntityAlias.cs

@@ -0,0 +1,15 @@
+using System;
+using System.Collections.Generic;
+
+namespace Pf2eDataScrapper.Model;
+
+public partial class EntityAlias
+{
+    public int Id { get; set; }
+
+    public int EntityId { get; set; }
+
+    public string Alias { get; set; } = null!;
+
+    public virtual Entity Entity { get; set; } = null!;
+}

Pf2eDataScrapper/Model/EntityExternal.cs

@@ -0,0 +1,21 @@
+using System;
+using System.Collections.Generic;
+
+namespace Pf2eDataScrapper.Model;
+
+public partial class EntityExternal
+{
+    public int EntityId { get; set; }
+
+    public int TargetId { get; set; }
+
+    public string Path { get; set; } = null!;
+
+    public string Suffix { get; set; } = null!;
+
+    public string Md5 { get; set; } = null!;
+
+    public virtual Entity Entity { get; set; } = null!;
+
+    public virtual ExternalTarget Target { get; set; } = null!;
+}

Pf2eDataScrapper/Model/EntityKeyword.cs

@@ -0,0 +1,15 @@
+using System;
+using System.Collections.Generic;
+
+namespace Pf2eDataScrapper.Model;
+
+public partial class EntityKeyword
+{
+    public int EntityId { get; set; }
+
+    public int KeywordId { get; set; }
+
+    public virtual Entity Entity { get; set; } = null!;
+
+    public virtual Keyword Keyword { get; set; } = null!;
+}

Pf2eDataScrapper/Model/EntitySource.cs

@@ -0,0 +1,19 @@
+using System;
+using System.Collections.Generic;
+
+namespace Pf2eDataScrapper.Model;
+
+public partial class EntitySource
+{
+    public int EntityId { get; set; }
+
+    public int SourceId { get; set; }
+
+    public string Suffix { get; set; } = null!;
+
+    public bool IsRemaster { get; set; }
+
+    public virtual Entity Entity { get; set; } = null!;
+
+    public virtual Source Source { get; set; } = null!;
+}

Pf2eDataScrapper/Model/ExternalTarget.cs

@@ -0,0 +1,16 @@
+using Pf2eDataScrapper.Model;
+using System;
+using System.Collections.Generic;
+
+namespace Pf2eDataScrapper;
+
+public partial class ExternalTarget
+{
+    public int Id { get; set; }
+
+    public string Name { get; set; } = null!;
+
+    public string Icon { get; set; } = null!;
+
+    public virtual ICollection<EntityExternal> EntityExternals { get; set; } = new List<EntityExternal>();
+}

Pf2eDataScrapper/Model/Feat.cs

@@ -0,0 +1,39 @@
+using System;
+using System.Collections.Generic;
+
+namespace Pf2eDataScrapper;
+
+public partial class Feat
+{
+    public int Id { get; set; }
+
+    public string Name { get; set; } = null!;
+
+    public string Img { get; set; } = null!;
+
+    public string Description { get; set; } = null!;
+
+    public string? DescriptionGm { get; set; }
+
+    public string Slug { get; set; } = null!;
+
+    public bool? IsTranslated { get; set; }
+
+    public bool? IsChecked { get; set; }
+
+    public int? Actions { get; set; }
+
+    public int Level { get; set; }
+
+    public int? LevelTaken { get; set; }
+
+    public int? MaxTakable { get; set; }
+
+    public string? Location { get; set; }
+
+    public bool? OnlyLevel1 { get; set; }
+
+    public string? Extra { get; set; }
+
+    public virtual ICollection<FeatPrerequisite> FeatPrerequisites { get; set; } = new List<FeatPrerequisite>();
+}

Pf2eDataScrapper/Model/FeatPrerequisite.cs

@@ -0,0 +1,15 @@
+using System;
+using System.Collections.Generic;
+
+namespace Pf2eDataScrapper;
+
+public partial class FeatPrerequisite
+{
+    public int Id { get; set; }
+
+    public int FeatId { get; set; }
+
+    public string Value { get; set; } = null!;
+
+    public virtual Feat Feat { get; set; } = null!;
+}

Pf2eDataScrapper/Model/Frequency.cs

@@ -0,0 +1,18 @@
+using Pf2eDataScrapper.Model;
+using System;
+using System.Collections.Generic;
+
+namespace Pf2eDataScrapper;
+
+public partial class Frequency
+{
+    public int Id { get; set; }
+
+    public int Max { get; set; }
+
+    public int Per { get; set; }
+
+    public int? Value { get; set; }
+
+    public virtual Entity IdNavigation { get; set; } = null!;
+}

Pf2eDataScrapper/Model/Heritage.cs

@@ -0,0 +1,28 @@
+using Pf2eDataScrapper.Model;
+using System;
+using System.Collections.Generic;
+
+namespace Pf2eDataScrapper;
+
+public partial class Heritage
+{
+    public int Id { get; set; }
+
+    public string Name { get; set; } = null!;
+
+    public string Img { get; set; } = null!;
+
+    public string Description { get; set; } = null!;
+
+    public string? DescriptionGm { get; set; }
+
+    public string Slug { get; set; } = null!;
+
+    public bool? IsTranslated { get; set; }
+
+    public bool? IsChecked { get; set; }
+
+    public int? AncestryId { get; set; }
+
+    public virtual Ancestry? Ancestry { get; set; }
+}

Pf2eDataScrapper/Model/Keyword.cs

@@ -0,0 +1,11 @@
+using System;
+using System.Collections.Generic;
+
+namespace Pf2eDataScrapper;
+
+public partial class Keyword
+{
+    public int Id { get; set; }
+
+    public string Slug { get; set; } = null!;
+}

Pf2eDataScrapper/Model/KeywordAlias.cs

@@ -0,0 +1,13 @@
+using System;
+using System.Collections.Generic;
+
+namespace Pf2eDataScrapper;
+
+public partial class KeywordAlias
+{
+    public int KeywordId { get; set; }
+
+    public string Value { get; set; } = null!;
+
+    public virtual Keyword Keyword { get; set; } = null!;
+}

Pf2eDataScrapper/Model/Language.cs

@@ -0,0 +1,14 @@
+using Pf2eDataScrapper.Model;
+using System;
+using System.Collections.Generic;
+
+namespace Pf2eDataScrapper;
+
+public partial class Language
+{
+    public int Id { get; set; }
+
+    public string Name { get; set; } = null!;
+
+    public virtual ICollection<AncestryLanguage> AncestryLanguages { get; set; } = new List<AncestryLanguage>();
+}

Pf2eDataScrapper/Model/Source.cs

@@ -0,0 +1,13 @@
+using System;
+using System.Collections.Generic;
+
+namespace Pf2eDataScrapper.Model;
+
+public partial class Source
+{
+    public int Id { get; set; }
+
+    public string Name { get; set; } = null!;
+
+    public string Url { get; set; } = null!;
+}

Pf2eDataScrapper/Model/Spell.cs

@@ -0,0 +1,53 @@
+using System;
+using System.Collections.Generic;
+
+namespace Pf2eDataScrapper.Model;
+
+public partial class Spell
+{
+    public int Id { get; set; }
+
+    public string Name { get; set; } = null!;
+
+    public string Img { get; set; } = null!;
+
+    public string Description { get; set; } = null!;
+
+    public string? DescriptionGm { get; set; }
+
+    public string Slug { get; set; } = null!;
+
+    public bool? IsTranslated { get; set; }
+
+    public bool? IsChecked { get; set; }
+
+    public int Level { get; set; }
+
+    public int Cost { get; set; }
+
+    public bool Counteraction { get; set; }
+
+    public bool Sustained { get; set; }
+
+    public string Duration { get; set; } = null!;
+
+    public string Range { get; set; } = null!;
+
+    public string Requirements { get; set; } = null!;
+
+    public string Target { get; set; } = null!;
+
+    public string Time { get; set; } = null!;
+
+    public bool CanHeightening { get; set; }
+
+    public virtual ICollection<DeitySpell> DeitySpells { get; set; } = new List<DeitySpell>();
+
+    public virtual SpellArea? SpellArea { get; set; }
+
+    public virtual ICollection<SpellDamage> SpellDamages { get; set; } = new List<SpellDamage>();
+
+    public virtual ICollection<SpellDefense> SpellDefenses { get; set; } = new List<SpellDefense>();
+
+    public virtual SpellRitual? SpellRitual { get; set; }
+}

Pf2eDataScrapper/Model/SpellArea.cs

@@ -0,0 +1,15 @@
+using System;
+using System.Collections.Generic;
+
+namespace Pf2eDataScrapper.Model;
+
+public partial class SpellArea
+{
+    public int SpellId { get; set; }
+
+    public string? Details { get; set; }
+
+    public int Value { get; set; }
+
+    public virtual Spell Spell { get; set; } = null!;
+}

Pf2eDataScrapper/Model/SpellDamage.cs

@@ -0,0 +1,23 @@
+using System;
+using System.Collections.Generic;
+
+namespace Pf2eDataScrapper.Model;
+
+public partial class SpellDamage
+{
+    public int Id { get; set; }
+
+    public int SpellId { get; set; }
+
+    public string Key { get; set; } = null!;
+
+    public bool? ApplyMod { get; set; }
+
+    public string Formula { get; set; } = null!;
+
+    public string? Materials { get; set; }
+
+    public string Type { get; set; } = null!;
+
+    public virtual Spell Spell { get; set; } = null!;
+}

Pf2eDataScrapper/Model/SpellDefense.cs

@@ -0,0 +1,19 @@
+using System;
+using System.Collections.Generic;
+
+namespace Pf2eDataScrapper.Model;
+
+public partial class SpellDefense
+{
+    public int Id { get; set; }
+
+    public int SpellId { get; set; }
+
+    public bool IsPassive { get; set; }
+
+    public bool? IsBasic { get; set; }
+
+    public string Statistic { get; set; } = null!;
+
+    public virtual Spell Spell { get; set; } = null!;
+}

Pf2eDataScrapper/Model/SpellRitual.cs

@@ -0,0 +1,17 @@
+using System;
+using System.Collections.Generic;
+
+namespace Pf2eDataScrapper.Model;
+
+public partial class SpellRitual
+{
+    public int SpellId { get; set; }
+
+    public string PrimaryCheck { get; set; } = null!;
+
+    public int SecondaryCasters { get; set; }
+
+    public string SecondaryChecks { get; set; } = null!;
+
+    public virtual Spell Spell { get; set; } = null!;
+}

Pf2eDataScrapper/Model/Training.cs

@@ -0,0 +1,15 @@
+using System;
+using System.Collections.Generic;
+
+namespace Pf2eDataScrapper.Model;
+
+public partial class Training
+{
+    public int Id { get; set; }
+
+    public string Key { get; set; } = null!;
+
+    public virtual ICollection<ClassTraining> ClassTrainings { get; set; } = new List<ClassTraining>();
+
+    public virtual ICollection<Deity> Deities { get; set; } = new List<Deity>();
+}

Pf2eDataScrapper/Model/Trait.cs

@@ -0,0 +1,15 @@
+using System;
+using System.Collections.Generic;
+
+namespace Pf2eDataScrapper.Model;
+
+public partial class Trait
+{
+    public int Id { get; set; }
+
+    public string Name { get; set; } = null!;
+
+    public string Description { get; set; } = null!;
+
+    public virtual ICollection<Entity> Entities { get; set; } = new List<Entity>();
+}

Pf2eDataScrapper/Pf2eDataScrapper.csproj

@@ -12,14 +12,22 @@
     <PackageReference Include="dotenv.net" Version="3.2.1" />
     <PackageReference Include="Goblinfactory.ProgressBar" Version="1.0.0" />
     <PackageReference Include="linq2db" Version="5.4.1" />
+    <PackageReference Include="Microsoft.EntityFrameworkCore" Version="9.0.4" />
+    <PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="9.0.4">
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+      <PrivateAssets>all</PrivateAssets>
+    </PackageReference>
     <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
+    <PackageReference Include="Npgsql" Version="9.0.3" />
+    <PackageReference Include="Npgsql.EntityFrameworkCore.PostgreSQL" Version="9.0.4" />
+    <PackageReference Include="Slugify.Core" Version="5.1.1" />
   </ItemGroup>
 
   <ItemGroup>
     <None Update=".env">
       <CopyToOutputDirectory>Always</CopyToOutputDirectory>
     </None>
-    <None Update="test.json">
+    <None Update="schema.pgsql">
       <CopyToOutputDirectory>Always</CopyToOutputDirectory>
     </None>
   </ItemGroup>

Pf2eDataScrapper/Pf2eDatabase.cs

@@ -0,0 +1,121 @@
+using Npgsql;
+using System.Text.RegularExpressions;
+
+namespace Pf2eDataScrapper
+{
+    public partial class Pf2eDatabase
+    {
+        public NpgsqlConnection Connection;
+
+        public readonly string Username;
+        public readonly string DatabaseName;
+
+
+        private Pf2eDatabase()
+        {
+            Username = UsernamePattern().Match(EnvSettings.DatabaseConnection).Groups[1].Value;
+            DatabaseName = DatabaseNamePattern().Match(EnvSettings.DatabaseConnection).Groups[1].Value;
+
+            Connection = new NpgsqlConnection(EnvSettings.DatabaseConnection);
+            Connection.Open();
+        }
+
+        ~Pf2eDatabase()
+        {
+            Connection.Close();
+        }
+
+        public static Pf2eDatabase Instance { get; } = new();
+
+        public NpgsqlTransaction? Transaction { get; private set; }
+
+        public void StartTransaction()
+        {
+            if (Transaction != null)
+            {
+                throw new InvalidOperationException("Transaction already started.");
+            }
+
+            Transaction = Connection.BeginTransaction();
+        }
+
+        public void Commit()
+        {
+            if (Transaction == null)
+            {
+                throw new InvalidOperationException("No transaction started.");
+            }
+
+            Transaction.Commit();
+        }
+
+        public void ExecuteCommand(string sql)
+        {
+            var command = GetCommand(sql);
+
+            command.ExecuteNonQuery();
+        }
+
+        public NpgsqlCommand GetCommand(string sql)
+        {
+            return new NpgsqlCommand(sql, Connection);
+        }
+
+
+        /*
+        public void RebuildDatabase()
+        {
+            // Закрываем текущее соединение
+            Connection.Close();
+
+            // Получение имён
+            var databaseNamePattern = DatabaseNamePattern();
+            var userConnection = databaseNamePattern.Replace(EnvSettings.DatabaseConnection, ";");
+            var databaseName = databaseNamePattern.Match(EnvSettings.DatabaseConnection).Groups[1].Value;
+            var username = UserNamePattern().Match(EnvSettings.DatabaseConnection).Groups[1].Value;
+
+            // Временное подключение к базе данных
+            NpgsqlConnection tmpConnection = new(userConnection);
+            tmpConnection.Open();
+
+            // Удаляем базу данных
+            NpgsqlCommand dropCommand = new($"DROP DATABASE \"{databaseName}\";", tmpConnection);
+            dropCommand.ExecuteNonQuery();
+
+            // Создаём новую базу данных
+            NpgsqlCommand createCommand = new($@"
+                CREATE DATABASE {databaseName}
+                    WITH OWNER = {username}
+                    ENCODING = 'UTF8'
+                    CONNECTION LIMIT = -1;
+                ", tmpConnection);
+            createCommand.ExecuteNonQuery();
+
+            string[] permissionCommands = [
+                $"GRANT TEMPORARY, CONNECT ON DATABASE {databaseName} TO PUBLIC;",
+                $"GRANT ALL ON DATABASE {databaseName} TO {username} WITH GRANT OPTION;", 
+                $"GRANT ALL ON DATABASE {databaseName} TO postgres;"
+            ];
+
+            foreach (var command in permissionCommands)
+            {
+                NpgsqlCommand permissionCommand = new(command, tmpConnection);
+                permissionCommand.ExecuteNonQuery();
+            }
+
+            // Закрываем временное соединение
+            tmpConnection.Close();
+
+            // Заново создаём соединение
+            Connection = new NpgsqlConnection(EnvSettings.DatabaseConnection);
+        }
+
+        */
+
+        [GeneratedRegex(@"(?:^|;)Database=(.*?)(?:$|;)")]
+        private static partial Regex DatabaseNamePattern();
+
+        [GeneratedRegex(@"(?:^|;)User=(.*?)(?:$|;)")]
+        private static partial Regex UsernamePattern();
+    }
+}

Pf2eDataScrapper/Program.cs

@@ -1,46 +1,24 @@
-using Newtonsoft.Json.Linq;
-using Pf2eDataScrapper;
-using Pf2eDataScrapper.Importers;
+using Pf2eDataScrapper;
 using Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry;
-using Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.Compendium;
-using Pf2eDataScrapper.SchemaBuilder;
 
 EnvSettings.Load();
 
-Console.WriteLine("Hello, World!");
-
-//var loader = new GitLoader(EnvSettings.RepositoryDirPath);
-
 
 var importer = new Pf2EFoundryImporter();
 
-//loader.UpdateImporter(importer);
-
 importer.Prepare();
 
-var builder = new CompendiumSchemaBuilder(importer.Loader);
-
-builder.BuildSchemas();
-builder.Save(EnvSettings.OutputDir);
-
-// var q = importer.Loader.BuildSchemas();
-// SchemaDumper.Dump(q);
-
-
-
-
-
-
-return;
-
-JObject data;
-
-using (var reader = new StreamReader("test.json"))
+try
 {
-    data = JObject.Parse(reader.ReadToEnd());
+    importer.Import();
 }
+catch (Exception ex)
+{
+    Console.WriteLine(ex);
+    //Console.WriteLine(ex.Message);
+    //Console.WriteLine(ex.InnerException);
+    //Console.WriteLine(ex.StackTrace);
+}
+// importer.BuildSchemas();
 
-SchemaNode schema = new();
-schema.AddValue(data);
 
-Console.WriteLine(schema);

Pf2eDataScrapper/schema.pgsql

@@ -1,4 +1,4 @@
--- Common Enums
+-- Common Enums
 CREATE TYPE "enum_size" AS ENUM (
   'med',
   'lanuages',
@@ -65,7 +65,7 @@ ALTER TABLE "entity_aliases" ADD FOREIGN KEY ("entity_id") REFERENCES "entity" (
 -- Keywords
 
 CREATE TABLE "keyword" (
-  "id" INT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY
+  "id" INT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
   "slug" varchar(50) NOT NULL
 );
 
@@ -199,7 +199,7 @@ CREATE TYPE "spell_damage_kinds" AS ENUM (
 );
 
 CREATE TABLE "spell" (
-  "id" int PRIMARY KEY,
+  -- "id" int PRIMARY KEY,
   "level" int NOT NULL,
   "cost" int NOT NULL,
   "counteraction" bool NOT NULL,
@@ -209,9 +209,10 @@ CREATE TABLE "spell" (
   "requirements" varchar NOT NULL,
   "target" varchar NOT NULL,
   "time" varchar NOT NULL,
-  "can_heightening" bool NOT NULL
+  "can_heightening" bool NOT NULL,
+  PRIMARY KEY ("id")
 ) INHERITS ("entity");
-ALTER TABLE "spell" ADD FOREIGN KEY ("id") REFERENCES "entity" ("id");
+-- ALTER TABLE "spell" ADD FOREIGN KEY ("id") REFERENCES "entity" ("id");
 
 CREATE TABLE "spell_area" (
   "spell_id" int PRIMARY KEY,
@@ -273,27 +274,29 @@ CREATE TYPE "ancestry_vision" AS ENUM (
 );
 
 CREATE TABLE "action" (
-  "id" int PRIMARY KEY,
-  "type" action_type NOT NULL,
+  -- "id" int PRIMARY KEY,
+  "action_type" action_type NOT NULL,
   "actions" int,
   "category" action_category,
   "death_note" bool,
-  "extra" text
+  "extra" text,
+  PRIMARY KEY ("id")
 ) INHERITS ("entity");
-ALTER TABLE "action" ADD FOREIGN KEY ("id") REFERENCES "entity" ("id");
+-- ALTER TABLE "action" ADD FOREIGN KEY ("id") REFERENCES "entity" ("id");
 
 
 -- Ancestry
 
 CREATE TABLE "ancestry" (
-  "id" int PRIMARY KEY,
+  -- "id" int PRIMARY KEY,
   "hp" int NOT NULL,
   "reach" int NOT NULL,
   "size" enum_size NOT NULL,
   "speed" int NOT NULL,
-  "vision" ancestry_vision NOT NULL
+  "vision" ancestry_vision NOT NULL,
+  PRIMARY KEY ("id")
 ) INHERITS ("entity");
-ALTER TABLE "ancestry" ADD FOREIGN KEY ("id") REFERENCES "entity" ("id");
+-- ALTER TABLE "ancestry" ADD FOREIGN KEY ("id") REFERENCES "entity" ("id");
 
 CREATE TABLE "ancestry_languages" (
   "ancestry_id" int,
@@ -325,9 +328,10 @@ ALTER TABLE "ancestry_items" ADD FOREIGN KEY ("item_id") REFERENCES "entity" ("i
 -- Background
 
 CREATE TABLE "background" (
-  "id" int PRIMARY KEY
+  -- "id" int PRIMARY KEY,
+  PRIMARY KEY ("id")
 ) INHERITS ("entity");
-ALTER TABLE "background" ADD FOREIGN KEY ("id") REFERENCES "entity" ("id");
+-- ALTER TABLE "background" ADD FOREIGN KEY ("id") REFERENCES "entity" ("id");
 
 CREATE TABLE "background_items" (
   "background_id" int,
@@ -365,15 +369,16 @@ CREATE TYPE "campaign_feature_category" AS ENUM (
 );
 
 CREATE TABLE "campaign_feature" (
-  "id" int PRIMARY KEY,
+  -- "id" int PRIMARY KEY,
   "level" int,
-  "type" action_type NOT NULL,
+  "action_type" action_type NOT NULL,
   "actions" int NOT NULL,
   "category" campaign_feature_category NOT NULL,
   "prerequisites" varchar,
-  "extra" text
+  "extra" text,
+  PRIMARY KEY ("id")
 ) INHERITS ("entity");
-ALTER TABLE "campaign_feature" ADD FOREIGN KEY ("id") REFERENCES "entity" ("id");
+-- ALTER TABLE "campaign_feature" ADD FOREIGN KEY ("id") REFERENCES "entity" ("id");
 
 
 -- Class
@@ -388,13 +393,14 @@ CREATE TYPE "class_progression_type" AS ENUM (
 );
 
 CREATE TABLE "class" (
-  "id" int PRIMARY KEY,
+  -- "id" int PRIMARY KEY,
   "hp" int NOT NULL,
   "perception" int NOT NULL,
   "spellcasting" bool NOT NULL,
-  "additionalTrainedSkills" int NOT NULL
+  "additionalTrainedSkills" int NOT NULL,
+  PRIMARY KEY ("id")
 ) INHERITS ("entity");
-ALTER TABLE "class" ADD FOREIGN KEY ("id") REFERENCES "entity" ("id");
+-- ALTER TABLE "class" ADD FOREIGN KEY ("id") REFERENCES "entity" ("id");
 
 CREATE TABLE "class_key_abilities" (
   "id" INT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
@@ -435,11 +441,12 @@ CREATE TYPE "condition_group" AS ENUM (
 );
 
 CREATE TABLE "condition" (
-  "id" int PRIMARY KEY,
+  -- "id" int PRIMARY KEY,
   "group" condition_group,
-  "extra" text
+  "extra" text,
+  PRIMARY KEY ("id")
 ) INHERITS ("entity");
-ALTER TABLE "condition" ADD FOREIGN KEY ("id") REFERENCES "entity" ("id");
+-- ALTER TABLE "condition" ADD FOREIGN KEY ("id") REFERENCES "entity" ("id");
 
 
 -- Deity
@@ -463,13 +470,14 @@ CREATE TYPE "deity_sanctification" AS ENUM (
 );
 
 CREATE TABLE "deity" (
-  "id" int PRIMARY KEY,
+  -- "id" int PRIMARY KEY,
   "category" deity_category NOT NULL,
   "font" deity_fonts NOT NULL,
   "sanctification_must" bool NOT NULL,
-  "sanctification_type" deity_sanctification NOT NULL
+  "sanctification_type" deity_sanctification NOT NULL,
+  PRIMARY KEY ("id")
 ) INHERITS ("entity");
-ALTER TABLE "deity" ADD FOREIGN KEY ("id") REFERENCES "entity" ("id");
+-- ALTER TABLE "deity" ADD FOREIGN KEY ("id") REFERENCES "entity" ("id");
 
 CREATE TABLE "deity_attributes" (
   "deity_id" int,
@@ -513,12 +521,6 @@ ALTER TABLE "deity_domains" ADD FOREIGN KEY ("deity_id") REFERENCES "deity" ("id
 
 -- Feat
 
-CREATE TYPE "feat_action_type" AS ENUM (
-  'passive',
-  'action',
-  'reaction',
-  'free'
-);
 CREATE TYPE "feat_category" AS ENUM (
   'ancestryfeature',
   'classfeature',
@@ -534,8 +536,8 @@ CREATE TYPE "feat_category" AS ENUM (
 );
 
 CREATE TABLE "feat" (
-  "id" int PRIMARY KEY,
-  "action_type" feat_action_type NOT NULL,
+  -- "id" int PRIMARY KEY,
+  "action_type" action_type NOT NULL,
   "actions" int,
   "category" feat_category NOT NULL,
   "level" int NOT NULL,
@@ -543,9 +545,10 @@ CREATE TABLE "feat" (
   "max_takable" int,
   "location" varchar(12),
   "only_level_1" bool,
-  "extra" text
+  "extra" text,
+  PRIMARY KEY ("id")
 ) INHERITS ("entity");
-ALTER TABLE "feat" ADD FOREIGN KEY ("id") REFERENCES "entity" ("id");
+-- ALTER TABLE "feat" ADD FOREIGN KEY ("id") REFERENCES "entity" ("id");
 
 CREATE TABLE "feat_prerequisites" (
   "id" INT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
@@ -558,8 +561,9 @@ ALTER TABLE "feat_prerequisites" ADD FOREIGN KEY ("feat_id") REFERENCES "feat" (
 -- Heritage
 
 CREATE TABLE "heritage" (
-  "id" int PRIMARY KEY,
-  "ancestry_id" int
+  -- "id" int PRIMARY KEY,
+  "ancestry_id" int,
+  PRIMARY KEY ("id")
 ) INHERITS ("entity");
-ALTER TABLE "heritage" ADD FOREIGN KEY ("id") REFERENCES "entity" ("id");
+-- ALTER TABLE "heritage" ADD FOREIGN KEY ("id") REFERENCES "entity" ("id");
 ALTER TABLE "heritage" ADD FOREIGN KEY ("ancestry_id") REFERENCES "ancestry" ("id");

Pf2eDataScrapper/test.json

@@ -1,97 +0,0 @@
-{
-    "_id": "fekYhQe1qrvwarBL",
-    "folder": "bGvA4p5QGO30eC0r",
-    "img": "systems/pf2e/icons/default-icons/hazard.svg",
-    "items": [
-        {
-            "_id": "vsxjIPIKMzsQXXfM",
-            "img": "systems/pf2e/icons/actions/Reaction.webp",
-            "name": "Death Throes",
-            "sort": 0,
-            "system": {
-                "actionType": {
-                    "value": "reaction"
-                },
-                "actions": {
-                    "value": null
-                },
-                "category": null,
-                "description": {
-                    "value": "<p><strong>Trigger</strong> A creature touches the vampire's corpse</p>\n<hr />\n<p><strong>Effect</strong> A surge of void energy fills the room as the corpse lurches as though staked. The haunt deals @Damage[10d8[void]] damage to creatures within 30 feet of the corpse; this damage heals undead creatures for the same amount. Living creatures must attempt a @Check[will|dc:30|traits:damaging-effect] save.</p>\n<hr />\n<p><strong>Critical Success</strong> The creature is unaffected.</p>\n<p><strong>Success</strong> The creature takes half damage and becomes @UUID[Compendium.pf2e.conditionitems.Item.Drained]{Drained 1}. This drained condition is reduced by 1 after 10 minutes.</p>\n<p><strong>Failure</strong> The creature takes full damage and becomes drained 1.</p>\n<p><strong>Critical Failure</strong> The creature takes double damage and is @UUID[Compendium.pf2e.conditionitems.Item.Drained]{Drained 2}.</p>"
-                },
-                "publication": {
-                    "license": "OGL",
-                    "remaster": false,
-                    "title": ""
-                },
-                "rules": [],
-                "slug": null,
-                "traits": {
-                    "rarity": "common",
-                    "value": []
-                }
-            },
-            "type": "action"
-        }
-    ],
-    "name": "Benefactor's End",
-    "system": {
-        "attributes": {
-            "ac": {
-                "value": 0
-            },
-            "hardness": 0,
-            "hp": {
-                "details": "",
-                "max": 0,
-                "temp": 0,
-                "value": 0
-            },
-            "stealth": {
-                "details": "<p>(expert)</p>",
-                "value": 20
-            }
-        },
-        "creatureType": "",
-        "details": {
-            "description": "<p>A spectral assailant drives a stake into a dying vampire's heart; the vampire collapses and releases a surge of void energy.</p>",
-            "disable": "<p>@Check[religion|dc:28] (expert) to drain away the haunt's void energy</p>",
-            "isComplex": false,
-            "level": {
-                "value": 9
-            },
-            "publication": {
-                "license": "OGL",
-                "remaster": false,
-                "title": "Pathfinder #183: Field of Maidens"
-            },
-            "reset": "<p>Unless disabled, the haunt materializes again on the next moonrise.</p>",
-            "routine": ""
-        },
-        "saves": {
-            "fortitude": {
-                "saveDetail": "",
-                "value": 0
-            },
-            "reflex": {
-                "saveDetail": "",
-                "value": 0
-            },
-            "will": {
-                "saveDetail": "",
-                "value": 0
-            }
-        },
-        "statusEffects": [],
-        "traits": {
-            "rarity": "common",
-            "size": {
-                "value": "med"
-            },
-            "value": [
-                "haunt"
-            ]
-        }
-    },
-    "type": "hazard"
-}