Compare commits
2
Commits
a0eb739363
...
10e3e274d5
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
10e3e274d5 | ||
|
|
efaa6c71d7 |
@@ -10,7 +10,7 @@ public class DatabaseHelper
|
||||
|
||||
|
||||
private int _id = 0;
|
||||
|
||||
|
||||
public int NextEntityId { get => ++_id; set => _id = value; }
|
||||
|
||||
private DatabaseHelper()
|
||||
|
||||
+12
-12
@@ -77,13 +77,13 @@ internal abstract partial class AbstractProcessor(ProcessorInput input)
|
||||
|
||||
Context.EntityAliases.Add(alias);
|
||||
|
||||
var legacyIdArray = Input.GetOptional<JArray>("legacy_id");
|
||||
JArray? legacyIdArray = Input.GetOptional<JArray>("legacy_id");
|
||||
|
||||
if (legacyIdArray is not null)
|
||||
{
|
||||
string legacyId = legacyIdArray[0].ToString();
|
||||
|
||||
if (LegacyAliases.TryGetValue(legacyId, out var legacyName))
|
||||
if (LegacyAliases.TryGetValue(legacyId, out string? legacyName))
|
||||
{
|
||||
if (legacyName != Entity.Name)
|
||||
{
|
||||
@@ -126,7 +126,7 @@ internal abstract partial class AbstractProcessor(ProcessorInput input)
|
||||
}
|
||||
string sourceMarkdown = Input.GetRequired<string>("source_markdown");
|
||||
Match match = SourceSuffixRegex().Match(sourceMarkdown);
|
||||
|
||||
|
||||
EntitySource entitySource = new()
|
||||
{
|
||||
Entity = Entity,
|
||||
@@ -145,11 +145,11 @@ internal abstract partial class AbstractProcessor(ProcessorInput input)
|
||||
AonprdTarget = Context.ExternalTargets.FirstOrDefault(t => t.Name == AonprdImporter.SourceName);
|
||||
|
||||
AonprdTarget ??= new()
|
||||
{
|
||||
Name = AonprdImporter.SourceName,
|
||||
Icon = "http://www.google.com/s2/favicons?domain=2e.aonprd.com",
|
||||
Prefix = "https://2e.aonprd.com",
|
||||
};
|
||||
{
|
||||
Name = AonprdImporter.SourceName,
|
||||
Icon = "http://www.google.com/s2/favicons?domain=2e.aonprd.com",
|
||||
Prefix = "https://2e.aonprd.com",
|
||||
};
|
||||
}
|
||||
|
||||
EntityExternal external = new()
|
||||
@@ -171,14 +171,14 @@ internal abstract partial class AbstractProcessor(ProcessorInput input)
|
||||
throw new Exception("'trait' is expected to be an array");
|
||||
}
|
||||
|
||||
var names = traitsJson.ToArray().Select(jToken => jToken.ToString());
|
||||
var traitEntities = Context.Traits
|
||||
IEnumerable<string> names = traitsJson.ToArray().Select(jToken => jToken.ToString());
|
||||
List<Entity> traitEntities = Context.Traits
|
||||
.SelectMany(t => t.Entity.Aliases)
|
||||
.Where(a => names.Contains(a.Alias))
|
||||
.Select(a => a.Entity)
|
||||
.ToList();
|
||||
|
||||
var traits = Context.Traits
|
||||
List<Trait> traits = Context.Traits
|
||||
.Where(t => traitEntities.Contains(t.Entity))
|
||||
.ToList();
|
||||
|
||||
@@ -249,7 +249,7 @@ internal abstract partial class AbstractProcessor(ProcessorInput input)
|
||||
using MD5 md5 = MD5.Create();
|
||||
byte[] byteHash = md5.ComputeHash(Encoding.UTF8.GetBytes(input));
|
||||
string hash = BitConverter.ToString(byteHash).Replace("-", "");
|
||||
return (isLowercase) ? hash.ToLower() : hash;
|
||||
return isLowercase ? hash.ToLower() : hash;
|
||||
}
|
||||
|
||||
protected static ExternalTarget? AonprdTarget = null;
|
||||
|
||||
@@ -1,11 +1,5 @@
|
||||
using Konsole;
|
||||
using Newtonsoft.Json.Linq;
|
||||
using Pf2eDataScrapper.Importers.Implementations.Aonprd.Compendium;
|
||||
using Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.Compendium;
|
||||
using Pf2eDataScrapper.Model.Base;
|
||||
using System;
|
||||
using System.ComponentModel;
|
||||
using System.Reflection;
|
||||
using static Pf2eDataScrapper.Importers.Implementations.Aonprd.DataBuilder.AbstractProcessor;
|
||||
|
||||
namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.DataBuilder;
|
||||
@@ -39,29 +33,29 @@ internal class AonprdDataBuilder(AonprdLoader loader)
|
||||
{
|
||||
Console.WriteLine("Building data...");
|
||||
|
||||
var window = new Window(0, 0, Console.WindowWidth, Console.WindowHeight).Concurrent();
|
||||
ConcurrentWriter window = new Window(0, 0, Console.WindowWidth, Console.WindowHeight).Concurrent();
|
||||
|
||||
var bars = window.OpenBox("Progress", Console.WindowWidth, 4, new BoxStyle { Body = Colors.WhiteOnBlack });
|
||||
IConsole bars = window.OpenBox("Progress", Console.WindowWidth, 4, new BoxStyle { Body = Colors.WhiteOnBlack });
|
||||
|
||||
var output = new Window(0, 4, Console.WindowWidth, Console.WindowHeight - 4, window);
|
||||
Window output = new Window(0, 4, Console.WindowWidth, Console.WindowHeight - 4, window);
|
||||
|
||||
ProgressBar bar1 = new(bars, Loader.ItemCount, 15);
|
||||
ProgressBar bar2 = new(bars, 1, 15);
|
||||
int i = 0;
|
||||
|
||||
foreach (var item in Processors.Items)
|
||||
foreach (AonprdProcessorCollection.ProcessorItem item in Processors.Items)
|
||||
{
|
||||
bar1.Refresh(i++, $"{item.Attribute.Name}");
|
||||
|
||||
bar2.Max = Compendium.Count([item.Attribute.Type]);
|
||||
int j = 0;
|
||||
|
||||
foreach(var entry in Loader.Entries(item.Attribute.Type))
|
||||
foreach (AonprdLoader.LoaderEntry entry in Loader.Entries(item.Attribute.Type))
|
||||
{
|
||||
bar2.Refresh(i++, $"{entry.Id}");
|
||||
|
||||
var input = new ProcessorInput(entry.Json);
|
||||
var processor = (AbstractProcessor) item.Constructor.Invoke([input]);
|
||||
ProcessorInput input = new ProcessorInput(entry.Json);
|
||||
AbstractProcessor processor = (AbstractProcessor) item.Constructor.Invoke([input]);
|
||||
|
||||
try
|
||||
{
|
||||
|
||||
+1
-7
@@ -1,10 +1,4 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.DataBuilder;
|
||||
namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.DataBuilder;
|
||||
|
||||
[AttributeUsage(AttributeTargets.Field)]
|
||||
internal class AonprdEnumAttribute(string value) : Attribute
|
||||
|
||||
+2
-2
@@ -9,13 +9,13 @@ public class AonprdProcessorAttribute(string name, string type, string[] depende
|
||||
|
||||
public readonly IEnumerable<string> Dependencies = dependencies;
|
||||
|
||||
public AonprdProcessorAttribute(string type) : this(type, type, [])
|
||||
public AonprdProcessorAttribute(string type) : this(type, type, ["trait"])
|
||||
{
|
||||
}
|
||||
public AonprdProcessorAttribute(string type, string[] dependencies) : this(type, type, dependencies)
|
||||
{
|
||||
}
|
||||
public AonprdProcessorAttribute(string name, string type) : this(name, type, [])
|
||||
public AonprdProcessorAttribute(string name, string type) : this(name, type, ["trait"])
|
||||
{
|
||||
}
|
||||
}
|
||||
|
||||
+5
-5
@@ -33,7 +33,7 @@ internal class AonprdProcessorCollection
|
||||
|
||||
ConstructorInfo constructor = type.GetConstructor([typeof(ProcessorInput)])!;
|
||||
|
||||
var collision = Items.FirstOrDefault(i => i.Attribute.Name == attribute.Name);
|
||||
ProcessorItem? collision = Items.FirstOrDefault(i => i.Attribute.Name == attribute.Name);
|
||||
if (collision is not null)
|
||||
{
|
||||
throw new Exception($"Multiply attributes 'AonprdProcessor' have same name '{attribute.Name}'." +
|
||||
@@ -50,14 +50,14 @@ internal class AonprdProcessorCollection
|
||||
|
||||
while (priorities.Count < Items.Count)
|
||||
{
|
||||
foreach (var item in Items)
|
||||
foreach (ProcessorItem item in Items)
|
||||
{
|
||||
if (item.Priority > 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var dependencyPriorities = item.Attribute.Dependencies.Select(d => priorities.TryGetValue(d, out int value) ? value : -1);
|
||||
IEnumerable<int> dependencyPriorities = item.Attribute.Dependencies.Select(d => priorities.TryGetValue(d, out int value) ? value : -1);
|
||||
|
||||
if (dependencyPriorities.Any(p => p == -1))
|
||||
{
|
||||
@@ -69,14 +69,14 @@ internal class AonprdProcessorCollection
|
||||
}
|
||||
}
|
||||
|
||||
Items.Sort((a, b) => a.Priority - b.Priority);
|
||||
Items.Sort((a, b) => a.Priority - b.Priority);
|
||||
}
|
||||
|
||||
public IEnumerable<ProcessorItem> Iterate()
|
||||
{
|
||||
SortItems();
|
||||
|
||||
foreach (var item in Items)
|
||||
foreach (ProcessorItem item in Items)
|
||||
{
|
||||
yield return item;
|
||||
}
|
||||
|
||||
+25
@@ -0,0 +1,25 @@
|
||||
using Pf2eDataScrapper.Model.Enum;
|
||||
using Pf2eDataScrapper.Model.Structure;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.DataBuilder.Processors;
|
||||
|
||||
[AonprdProcessor("action")]
|
||||
internal class ActionProcessor(ProcessorInput input) : AbstractProcessor(input)
|
||||
{
|
||||
public override EntityType Type => EntityType.Trait;
|
||||
|
||||
public override Model.Structure.Action Object { get; } = new();
|
||||
|
||||
public override void Process()
|
||||
{
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
+1
-1
@@ -3,7 +3,7 @@ using Pf2eDataScrapper.Model.Structure;
|
||||
|
||||
namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.DataBuilder.Processors;
|
||||
|
||||
[AonprdProcessor("trait")]
|
||||
[AonprdProcessor("trait", [])]
|
||||
internal class TraitProcessor(ProcessorInput input) : AbstractProcessor(input)
|
||||
{
|
||||
public override EntityType Type => EntityType.Trait;
|
||||
|
||||
-38
@@ -1,38 +0,0 @@
|
||||
using Newtonsoft.Json.Linq;
|
||||
using Pf2eDataScrapper.Model.Enum;
|
||||
using Pf2eDataScrapper.Model.Structure;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.DataBuilder.Processors;
|
||||
|
||||
[AonprdProcessor("trait_traits", "trait", ["trait"])]
|
||||
internal class TraitTraitsProcessor(ProcessorInput input) : AbstractProcessor(input)
|
||||
{
|
||||
public override EntityType Type => EntityType.Trait;
|
||||
|
||||
public override Trait Object { get; } = new();
|
||||
|
||||
public override void Process()
|
||||
{
|
||||
string id = Input.GetRequired<string>("id");
|
||||
string name = Input.GetRequired<string>("name");
|
||||
|
||||
if (Input.GetOptional<JArray>("remaster_id") is not null)
|
||||
{
|
||||
throw new LegacyItemException(name);
|
||||
}
|
||||
|
||||
// base.Process();
|
||||
string url = Input.GetRequired<string>("url");
|
||||
|
||||
var external = Context.EntityExternals.FirstOrDefault(x => x.Suffix == url) ?? throw new Exception("Entity not found?");
|
||||
Entity = external.Entity;
|
||||
|
||||
AddTraits();
|
||||
}
|
||||
|
||||
}
|
||||
+3
-7
@@ -1,17 +1,13 @@
|
||||
using LinqToDB.Common;
|
||||
using Newtonsoft.Json.Linq;
|
||||
using NpgsqlTypes;
|
||||
using Newtonsoft.Json.Linq;
|
||||
using Pf2eDataScrapper.Importers.Implementations.Aonprd.DataBuilder;
|
||||
using Pf2eDataScrapper.Model;
|
||||
using Pf2eDataScrapper.Model.Base;
|
||||
using Pf2eDataScrapper.Model.Base.Source;
|
||||
using Pf2eDataScrapper.Model.Base.Targets;
|
||||
using Pf2eDataScrapper.Model.Common;
|
||||
using Pf2eDataScrapper.Model.Enum;
|
||||
using Pf2eDataScrapper.Model.Structure;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using static LinqToDB.Reflection.Methods.LinqToDB.Insert;
|
||||
|
||||
namespace Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.DataBuilder;
|
||||
|
||||
@@ -88,7 +84,7 @@ internal abstract class AbstractProcessor(JObject input)
|
||||
throw new Exception("system.traits.value is expected to be an array");
|
||||
}
|
||||
|
||||
foreach(Trait trait in GetTraits(traitsJson.ToArray().Select(jToken => jToken.ToString())))
|
||||
foreach (Trait trait in GetTraits(traitsJson.ToArray().Select(jToken => jToken.ToString())))
|
||||
{
|
||||
Entity.Traits.Add(trait);
|
||||
}
|
||||
@@ -198,6 +194,6 @@ internal abstract class AbstractProcessor(JObject input)
|
||||
using MD5 md5 = MD5.Create();
|
||||
byte[] byteHash = md5.ComputeHash(Encoding.UTF8.GetBytes(input));
|
||||
string hash = BitConverter.ToString(byteHash).Replace("-", "");
|
||||
return (isLowercase) ? hash.ToLower() : hash;
|
||||
return isLowercase ? hash.ToLower() : hash;
|
||||
}
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,37 +1,35 @@
|
||||
using System;
|
||||
using Microsoft.EntityFrameworkCore.Migrations;
|
||||
using Microsoft.EntityFrameworkCore.Migrations;
|
||||
|
||||
#nullable disable
|
||||
|
||||
namespace Pf2eDataScrapper.Migrations
|
||||
namespace Pf2eDataScrapper.Migrations;
|
||||
|
||||
/// <inheritdoc />
|
||||
public partial class DateOnly : Migration
|
||||
{
|
||||
/// <inheritdoc />
|
||||
public partial class DateOnly : Migration
|
||||
protected override void Up(MigrationBuilder migrationBuilder)
|
||||
{
|
||||
/// <inheritdoc />
|
||||
protected override void Up(MigrationBuilder migrationBuilder)
|
||||
{
|
||||
migrationBuilder.AlterColumn<DateOnly>(
|
||||
name: "release_date",
|
||||
table: "sources",
|
||||
type: "date",
|
||||
nullable: true,
|
||||
oldClrType: typeof(DateTime),
|
||||
oldType: "timestamp with time zone",
|
||||
oldNullable: true);
|
||||
}
|
||||
migrationBuilder.AlterColumn<DateOnly>(
|
||||
name: "release_date",
|
||||
table: "sources",
|
||||
type: "date",
|
||||
nullable: true,
|
||||
oldClrType: typeof(DateTime),
|
||||
oldType: "timestamp with time zone",
|
||||
oldNullable: true);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override void Down(MigrationBuilder migrationBuilder)
|
||||
{
|
||||
migrationBuilder.AlterColumn<DateTime>(
|
||||
name: "release_date",
|
||||
table: "sources",
|
||||
type: "timestamp with time zone",
|
||||
nullable: true,
|
||||
oldClrType: typeof(DateOnly),
|
||||
oldType: "date",
|
||||
oldNullable: true);
|
||||
}
|
||||
/// <inheritdoc />
|
||||
protected override void Down(MigrationBuilder migrationBuilder)
|
||||
{
|
||||
migrationBuilder.AlterColumn<DateTime>(
|
||||
name: "release_date",
|
||||
table: "sources",
|
||||
type: "timestamp with time zone",
|
||||
nullable: true,
|
||||
oldClrType: typeof(DateOnly),
|
||||
oldType: "date",
|
||||
oldNullable: true);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,7 +2,6 @@
|
||||
using Pf2eDataScrapper.Model.Base.Keywords;
|
||||
using Pf2eDataScrapper.Model.Base.Source;
|
||||
using Pf2eDataScrapper.Model.Base.Targets;
|
||||
using Pf2eDataScrapper.Model.Common;
|
||||
using Pf2eDataScrapper.Model.Enum;
|
||||
using Pf2eDataScrapper.Model.Structure;
|
||||
using System.ComponentModel.DataAnnotations;
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
using NpgsqlTypes;
|
||||
using Pf2eDataScrapper.Importers.Implementations.Aonprd.DataBuilder;
|
||||
|
||||
namespace Pf2eDataScrapper.Model.Enum;
|
||||
|
||||
|
||||
@@ -5,14 +5,16 @@ namespace Pf2eDataScrapper.Model.Enum;
|
||||
public enum ActionType
|
||||
{
|
||||
[PgName("action")]
|
||||
action,
|
||||
Action,
|
||||
|
||||
[PgName("free")]
|
||||
free,
|
||||
Free,
|
||||
|
||||
[PgName("reaction")]
|
||||
reaction,
|
||||
Reaction,
|
||||
|
||||
[PgName("passive")]
|
||||
passive,
|
||||
Passive,
|
||||
|
||||
|
||||
}
|
||||
@@ -319,7 +319,7 @@ public partial class Pf2eDbContext
|
||||
modelBuilder.Entity<SpellRitual>(entity => { });
|
||||
|
||||
// Trait
|
||||
modelBuilder.Entity<Trait>(entity =>
|
||||
modelBuilder.Entity<Trait>(entity =>
|
||||
{
|
||||
BindEntity(entity);
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user