Compare commits

...
2 Commits
14 changed files with 5383 additions and 75 deletions
@@ -37,7 +37,7 @@ internal class AonprdDataBuilder(AonprdLoader loader)
IConsole bars = window.OpenBox("Progress", Console.WindowWidth, 4, new BoxStyle { Body = Colors.WhiteOnBlack }); IConsole bars = window.OpenBox("Progress", Console.WindowWidth, 4, new BoxStyle { Body = Colors.WhiteOnBlack });
Window output = new Window(0, 4, Console.WindowWidth, Console.WindowHeight - 4, window); Window output = new(0, 4, Console.WindowWidth, Console.WindowHeight - 4, window);
ProgressBar bar1 = new(bars, Loader.ItemCount, 15); ProgressBar bar1 = new(bars, Loader.ItemCount, 15);
ProgressBar bar2 = new(bars, 1, 15); ProgressBar bar2 = new(bars, 1, 15);
@@ -54,7 +54,7 @@ internal class AonprdDataBuilder(AonprdLoader loader)
{ {
bar2.Refresh(i++, $"{entry.Id}"); bar2.Refresh(i++, $"{entry.Id}");
ProcessorInput input = new ProcessorInput(entry.Json); AonprdProcessorInput input = new(entry.Json);
AbstractProcessor processor = (AbstractProcessor) item.Constructor.Invoke([input]); AbstractProcessor processor = (AbstractProcessor) item.Constructor.Invoke([input]);
try try
@@ -35,7 +35,7 @@ public static class AonprdEnumResolver
}), }),
]; ];
public static T ResolveEventArgs<T>(string value) where T : struct, Enum public static T Resolve<T>(string value) where T : Enum
{ {
Type type = typeof(T); Type type = typeof(T);
@@ -56,7 +56,7 @@ public static class AonprdEnumResolver
public abstract Type Type { get; } public abstract Type Type { get; }
} }
private class Solver<T>(Dictionary<string, T> data) : Solver where T : struct, Enum private class Solver<T>(Dictionary<string, T> data) : Solver where T : Enum
{ {
public override Type Type { get; } = typeof(T); public override Type Type { get; } = typeof(T);
public Dictionary<string, T> Data { get; } = data; public Dictionary<string, T> Data { get; } = data;
@@ -0,0 +1,18 @@
using Newtonsoft.Json.Linq;
namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.DataBuilder;
internal class AonprdProcessorInput(JObject input) : ProcessorInput(input)
{
public override T Enum<T>(string value)
{
try
{
return AonprdEnumResolver.Resolve<T>(value);
}
catch
{
return base.Enum<T>(value);
}
}
}
@@ -1,5 +1,4 @@
using Newtonsoft.Json.Linq; using Newtonsoft.Json.Linq;
using Pf2eDataScrapper.Importers.Implementations.Aonprd.DataBuilder;
using Pf2eModel.Model; using Pf2eModel.Model;
using Pf2eModel.Model.Base; using Pf2eModel.Model.Base;
using Pf2eModel.Model.Base.Source; using Pf2eModel.Model.Base.Source;
@@ -11,13 +10,13 @@ using System.Text;
namespace Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.DataBuilder; namespace Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.DataBuilder;
internal abstract class AbstractProcessor(JObject input) internal abstract class AbstractProcessor(ProcessorInput input)
{ {
protected DatabaseHelper Helper = DatabaseHelper.Instance; protected DatabaseHelper Helper = DatabaseHelper.Instance;
public readonly Pf2eDbContext Context = DatabaseHelper.DbContext; public readonly Pf2eDbContext Context = DatabaseHelper.DbContext;
public readonly JObject Input = input; public readonly ProcessorInput Input = input;
public Entity Entity { get; set; } = new(); public Entity Entity { get; set; } = new();
@@ -30,15 +29,15 @@ internal abstract class AbstractProcessor(JObject input)
Entity.Id = Helper.NextEntityId; Entity.Id = Helper.NextEntityId;
Entity.Type = Type; Entity.Type = Type;
Entity.Name = GetRequired<string>("name"); Entity.Name = Input.GetRequired<string>("name");
Entity.Img = GetRequired<string>("img"); Entity.Img = Input.GetRequired<string>("img");
Entity.Description = GetRequired<string>("system.description.value"); Entity.Description = Input.GetRequired<string>("system.description.value");
Entity.IsTranslated = false; Entity.IsTranslated = false;
Entity.IsChecked = false; Entity.IsChecked = false;
Entity.Slug = Helper.GenerateSlug(Entity.Name); Entity.Slug = Helper.GenerateSlug(Entity.Name);
Entity.Rarity = EnumOptional<EntityRarity>("system.traits.rarity", EntityRarity.Common); Entity.Rarity = Input.EnumOptional<EntityRarity>("system.traits.rarity", EntityRarity.Common);
Object.Id = Entity.Id; Object.Id = Entity.Id;
@@ -59,10 +58,10 @@ internal abstract class AbstractProcessor(JObject input)
{ {
Entity = Entity, Entity = Entity,
Source = GetSource(GetRequired<string>("system.publication.title")), Source = GetSource(Input.GetRequired<string>("system.publication.title")),
Suffix = "", Suffix = "",
IsRemaster = GetRequired<bool>("system.publication.remaster") IsRemaster = Input.GetRequired<bool>("system.publication.remaster")
}; };
Context.EntitySources.Add(source); Context.EntitySources.Add(source);
@@ -71,13 +70,13 @@ internal abstract class AbstractProcessor(JObject input)
{ {
Entity = Entity, Entity = Entity,
Target = GetTarget(), Target = GetTarget(),
Suffix = GetRequired<string>("_id"), Suffix = Input.GetRequired<string>("_id"),
Md5 = Md5(Input.ToString()) Md5 = Md5(Input.Input.ToString())
}; };
Context.EntityExternals.Add(external); Context.EntityExternals.Add(external);
// Traits // Traits
JToken? traitsJson = Input.SelectToken("system.traits.value"); JToken? traitsJson = Input.Input.SelectToken("system.traits.value");
if (traitsJson?.Type != JTokenType.Array) if (traitsJson?.Type != JTokenType.Array)
{ {
@@ -95,33 +94,6 @@ internal abstract class AbstractProcessor(JObject input)
Context.SaveChanges(); Context.SaveChanges();
} }
public T GetRequired<T>(string path)
{
JToken token = Input.SelectToken(path) ?? throw new NullReferenceException($"'{path}' is null");
return token.ToObject<T>() ?? throw new NullReferenceException($"'{path}' is null");
}
public T? GetOptional<T>(string path)
{
JToken? token = Input.SelectToken(path) ?? null;
return (token is null || token.Type == JTokenType.Null) ? default! : token.ToObject<T>() ?? default;
}
public T EnumRequired<T>(string path) where T : Enum
{
string value = GetOptional<string>(path) ?? throw new NullReferenceException($"Value in path {path} null or not exists");
return ProcessorInput.Enum<T>(value);
}
public T EnumOptional<T>(string path, T defaultValue) where T : Enum
{
string? value = GetOptional<string>(path);
return value is null ? defaultValue : ProcessorInput.Enum<T>(value);
}
protected Source GetSource(string name) protected Source GetSource(string name)
{ {
@@ -36,13 +36,13 @@ internal class PF2eFoundryDataBuilder
continue; continue;
} }
Pf2eTypeProcessor? attribute = type.GetCustomAttribute<Pf2eTypeProcessor>(); Pf2eTypeProcessor? attribute = type.GetCustomAttribute<Pf2eTypeProcessor>();
Processors.Add(attribute!.Type, type.GetConstructor([typeof(JObject)])!); Processors.Add(attribute!.Type, type.GetConstructor([typeof(ProcessorInput)])!);
} }
} }
public AbstractProcessor? GetProcessor(string type, JObject input) public AbstractProcessor? GetProcessor(string type, JObject input)
{ {
return Processors.TryGetValue(type, out ConstructorInfo? constructor) ? (AbstractProcessor) constructor.Invoke([input]) : null; return Processors.TryGetValue(type, out ConstructorInfo? constructor) ? (AbstractProcessor) constructor.Invoke([new ProcessorInput(input)]) : null;
} }
public void TruncateTables() public void TruncateTables()
@@ -4,7 +4,7 @@ using Pf2eModel.Model.Enum;
namespace Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.DataBuilder.Processors; namespace Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.DataBuilder.Processors;
[Pf2eTypeProcessor("Item.action")] [Pf2eTypeProcessor("Item.action")]
internal class ActionProcessor(JObject input) : AbstractProcessor(input) internal class ActionProcessor(ProcessorInput input) : AbstractProcessor(input)
{ {
public override EntityType Type => EntityType.Action; public override EntityType Type => EntityType.Action;
@@ -21,7 +21,7 @@ internal class ActionProcessor(JObject input) : AbstractProcessor(input)
*/ */
JObject extra = new() JObject extra = new()
{ {
{ "selfEffect", Input.SelectToken("system.selfEffect") } { "selfEffect", Input.Input.SelectToken("system.selfEffect") }
}; };
Object.Extra = extra.ToString(Newtonsoft.Json.Formatting.None); Object.Extra = extra.ToString(Newtonsoft.Json.Formatting.None);
@@ -5,7 +5,7 @@ using Pf2eModel.Model.Structure;
namespace Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.DataBuilder.Processors; namespace Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.DataBuilder.Processors;
[Pf2eTypeProcessor("Item.condition")] [Pf2eTypeProcessor("Item.condition")]
internal class ConditionProcessor(JObject input) : AbstractProcessor(input) internal class ConditionProcessor(ProcessorInput input) : AbstractProcessor(input)
{ {
public override EntityType Type => EntityType.Condition; public override EntityType Type => EntityType.Condition;
@@ -17,14 +17,14 @@ internal class ConditionProcessor(JObject input) : AbstractProcessor(input)
JObject extra = new() JObject extra = new()
{ {
{ "duration", Input.SelectToken("system.duration") }, { "duration", Input.Input.SelectToken("system.duration") },
{ "overrides", Input.SelectToken("system.overrides") }, { "overrides", Input.Input.SelectToken("system.overrides") },
{ "references", Input.SelectToken("system.references") }, { "references", Input.Input.SelectToken("system.references") },
{ "value", Input.SelectToken("system.value") } { "value", Input.Input.SelectToken("system.value") }
}; };
Object.Extra = extra.ToString(Newtonsoft.Json.Formatting.None); Object.Extra = extra.ToString(Newtonsoft.Json.Formatting.None);
Object.Group = EnumOptional<ConditionGroup>("system.group", ConditionGroup.Unknown); Object.Group = Input.EnumOptional<ConditionGroup>("system.group", ConditionGroup.Unknown);
Context.Conditions.Add(Object); Context.Conditions.Add(Object);
} }
@@ -1,41 +1,53 @@
using Newtonsoft.Json.Linq; using Newtonsoft.Json.Linq;
using NpgsqlTypes; using NpgsqlTypes;
namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.DataBuilder; namespace Pf2eDataScrapper.Importers;
internal class ProcessorInput(JObject input) internal class ProcessorInput(JObject input)
{ {
public readonly JObject Input = input; public readonly JObject Input = input;
public T GetRequired<T>(string path) public virtual T GetRequired<T>(string path)
{ {
JToken token = Input.SelectToken(path) ?? throw new NullReferenceException($"'{path}' is null"); JToken token = Input.SelectToken(path) ?? throw new NullReferenceException($"'{path}' is null");
return token.ToObject<T>() ?? throw new NullReferenceException($"'{path}' is null"); return token.ToObject<T>() ?? throw new NullReferenceException($"'{path}' is null");
} }
public T? GetOptional<T>(string path) public virtual T? GetOptional<T>(string path)
{ {
JToken? token = Input.SelectToken(path) ?? null; JToken? token = Input.SelectToken(path) ?? null;
return (token is null || token.Type == JTokenType.Null) ? default! : token.ToObject<T>() ?? default; return (token is null || token.Type == JTokenType.Null) ? default! : token.ToObject<T>() ?? default;
} }
public T EnumRequired<T>(string path) where T : Enum public virtual T EnumRequired<T>(string path) where T : Enum
{ {
string value = GetOptional<string>(path) ?? throw new NullReferenceException($"Value in path {path} null or not exists"); string value = GetOptional<string>(path) ?? throw new NullReferenceException($"Value in path {path} null or not exists");
return Enum<T>(value); return Enum<T>(value);
} }
public T EnumOptional<T>(string path, T defaultValue) where T : Enum public virtual T EnumOptional<T>(string path, T defaultValue) where T : Enum
{ {
string? value = GetOptional<string>(path); string? value = GetOptional<string>(path);
return value is null ? defaultValue : Enum<T>(value); if (value is null)
{
return defaultValue;
}
try
{
return Enum<T>(value);
}
catch (InvalidEnumStringException)
{
return defaultValue;
}
} }
public static T Enum<T>(string value) where T : Enum public virtual T Enum<T>(string value) where T : Enum
{ {
foreach (System.Reflection.FieldInfo field in typeof(T).GetFields()) foreach (System.Reflection.FieldInfo field in typeof(T).GetFields())
{ {
@@ -45,10 +57,6 @@ internal class ProcessorInput(JObject input)
{ {
variants.Add(pgAttribute.PgName); variants.Add(pgAttribute.PgName);
} }
if (Attribute.GetCustomAttribute(field, typeof(AonprdEnumAttribute)) is AonprdEnumAttribute aonprdAttribute)
{
variants.AddRange(aonprdAttribute.Values);
}
if (variants.Contains(value)) if (variants.Contains(value))
{ {
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
@@ -112,11 +112,12 @@ public partial class Pf2eDbContext : DbContext
protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder)
{ {
DotEnv.Load(); DotEnv.Load(options: new DotEnvOptions(probeForEnv: true, probeLevelsToSearch: 2));
string databaseCnnection = EnvReader.GetStringValue("DATABASE_CONNECTION");
// optionsBuilder.LogTo(Console.WriteLine); // optionsBuilder.LogTo(Console.WriteLine);
optionsBuilder.UseSnakeCaseNamingConvention(); optionsBuilder.UseSnakeCaseNamingConvention();
optionsBuilder.UseNpgsql(EnvReader.GetStringValue("DATABASE_CONNECTION"), optionsBuilder.UseNpgsql(databaseCnnection,
builder => builder =>
{ {
builder builder
+14 -9
View File
@@ -4,21 +4,26 @@
<TargetFramework>net8.0</TargetFramework> <TargetFramework>net8.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings> <ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable> <Nullable>enable</Nullable>
<GenerateRuntimeConfigurationFiles>True</GenerateRuntimeConfigurationFiles>
</PropertyGroup> </PropertyGroup>
<ItemGroup> <ItemGroup>
<None Include="..\Pf2eDataScrapper\.env" Link=".env"> <None Include="..\Pf2eDataScrapper\.env" Link=".env">
<CopyToOutputDirectory>Always</CopyToOutputDirectory> <CopyToOutputDirectory>Always</CopyToOutputDirectory>
</None> </None>
</ItemGroup> </ItemGroup>
<ItemGroup> <ItemGroup>
<PackageReference Include="dotenv.net" Version="3.2.1" /> <PackageReference Include="dotenv.net" Version="3.2.1" />
<PackageReference Include="EFCore.NamingConventions" Version="9.0.0" /> <PackageReference Include="EFCore.NamingConventions" Version="9.0.0" />
<PackageReference Include="linq2db" Version="5.4.1" /> <PackageReference Include="linq2db" Version="5.4.1" />
<PackageReference Include="linq2db.EntityFrameworkCore" Version="9.0.0" /> <PackageReference Include="linq2db.EntityFrameworkCore" Version="9.0.0" />
<PackageReference Include="Npgsql" Version="9.0.3" /> <PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="9.0.4">
<PackageReference Include="Npgsql.EntityFrameworkCore.PostgreSQL" Version="9.0.4" /> <!--<PrivateAssets>all</PrivateAssets>-->
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Include="Npgsql" Version="9.0.3" />
<PackageReference Include="Npgsql.EntityFrameworkCore.PostgreSQL" Version="9.0.4" />
</ItemGroup> </ItemGroup>
</Project> </Project>