kozhilya/Pf2eDataScrapper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Изменение метода вывода данных

Browse Source
This commit is contained in:
kozhilya
2025-03-29 15:29:50 +04:00
parent 7afc44a599
commit d78b466ff0
14 changed files with 345 additions and 190 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Pf2eDataScrapper/Importers/AbstractImporter.cs
+1
View File
@@ -28,6 +28,7 @@ namespace Pf2eDataScrapper.Importers
/// </summary>
public string Dir => Path.Combine(EnvSettings.DataDirPath, Slug);
public abstract void Prepare();
public abstract void Import();
}
Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/Compendium/CompendiumFolder.cs
+45 -5
View File
@@ -3,17 +3,57 @@ using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Newtonsoft.Json.Linq;
namespace Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.Compendium
{
internal class CompendiumFolder(string name, string path, CompendiumDocumentType type)
internal class CompendiumFolder
{
public string Name = name;
public readonly string Id;
public string Path = path;
public readonly string Name;
public CompendiumDocumentType Type = type;
public readonly string? Path;
public DirectoryInfo Directory = new(path);
public readonly bool IsContained;
public CompendiumFolder? Parent;
public readonly CompendiumDocumentType Type;
public CompendiumFolder(JObject json)
{
if (json.ContainsKey("_id"))
{
// Вложенная папка через _folders.json
Id = json["_id"]!.ToString();
Name = json["name"]!.ToString();
Path = null;
IsContained = true;
}
else
{
// Корневой пак в system.json
Id = json["name"]!.ToString();
Name = json["label"]!.ToString();
Path = json["path"]!.ToString();
IsContained = false;
}
Type = (CompendiumDocumentType)Enum.Parse(typeof(CompendiumDocumentType), json["type"]!.ToString());
}
public CompendiumFolder(string id, string name)
{
Id = id;
Name = name;
Path = null;
IsContained = false;
}
public override string ToString()
{
return $"CompendiumFolder([Id={Id}][Type={Type}][Name={Name}][Path={Path}][Parent={Parent?.Name ?? "null"}])";
}
}
}
Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/Compendium/CompendiumFolderCollection.cs
+19
View File
@@ -0,0 +1,19 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.Compendium
{
internal class CompendiumFolderCollection : List<CompendiumFolder>
{
public CompendiumFolder? GetById(string id)
{
return this.FirstOrDefault(x => x.Id == id);
}
}
}
Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/Compendium/CompendiumPack.cs
-31
View File
@@ -1,31 +0,0 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.Compendium
{
/// <summary>
/// Implementation of `CompendiumPack` class from pf2e voundry module
/// </summary>
internal class CompendiumPack
{
public string PackId;
public string PackDir;
public CompendiumDocumentType DocumentType;
public string SystemId;
public CompendiumPack(string PackDir, object[] parsedData, object[] parsedFolders)
{
}
//public static CompendiumPack LoadJson(string path)
//{
// ret
//}
}
}
Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/Compendium/CompendiumSchemaBuilder.cs
+153
View File
@@ -0,0 +1,153 @@
using Konsole;
using Newtonsoft.Json.Linq;
using Pf2eDataScrapper.SchemaBuilder;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.Compendium
{
internal class CompendiumSchemaBuilder(CompendiumLoader loader)
{
public CompendiumLoader Loader = loader;
public CompendiumFolderCollection Collection = loader.Collection;
public CompendiumSchemaTypeResolver Resolver = new();
public Dictionary<CompendiumFolder, SchemaNode> PackSchemas = [];
public Dictionary<string, SchemaNode> TypeSchemas = [];
protected List<SchemaNode> GetItemSchemas(CompendiumFolder folder, string? itemType)
{
if (!PackSchemas.ContainsKey(folder))
{
PackSchemas[folder] = new SchemaNode()
{
Resolver = Resolver,
Path = folder.Name
};
}
var key = folder.Type.ToString() + (itemType is not null ? $".{itemType}" : "");
if (!TypeSchemas.ContainsKey(key))
{
TypeSchemas[key] = new SchemaNode()
{
Resolver = Resolver,
Path = key
};
}
return [PackSchemas[folder], TypeSchemas[key]];
}
public void BuildSchemas()
{
Console.WriteLine("Building schemas...");
var bar = new ProgressBar(Loader.ItemCount);
var i = 0;
foreach (var folder in Collection.FindAll(f => f.Path is not null))
{
var dirPath = Path.Combine(Loader.Importer.Dir, folder.Path!);
foreach (var filePath in Directory.GetFiles(dirPath, "*.json", SearchOption.AllDirectories))
{
bar.Refresh(i++, folder.Name);
var file = new FileInfo(filePath);
if (file.Name == "_folders.json")
{
continue;
}
using var reader = new StreamReader(file.FullName);
var itemJson = JObject.Parse(reader.ReadToEnd());
var itemFolder = folder;
if (itemJson.TryGetValue("folder", out var itemJsonFolder))
{
itemFolder = Collection.GetById(itemJsonFolder.ToString());
}
var schemas = GetItemSchemas(itemFolder ?? folder, itemJson["type"]?.ToString());
foreach (var schema in schemas)
{
schema.AddValue(itemJson);
}
}
}
bar.Refresh(Loader.ItemCount, "Done!");
}
public void Save(string targetDir)
{
Console.WriteLine("Dumping schemas...");
var bar = new ProgressBar(PackSchemas.Count + TypeSchemas.Count);
var i = 0;
targetDir = Path.Combine(targetDir, "pf2e");
if (!Directory.Exists(targetDir))
{
Directory.CreateDirectory(targetDir);
}
Array.ForEach(Directory.GetFiles(@targetDir), File.Delete);
// Packs
var packsDir = Path.Combine(targetDir, "packs");
Directory.CreateDirectory(packsDir);
foreach (var pair in PackSchemas)
{
bar.Refresh(i++, pair.Key.Name);
SaveSchema(packsDir, pair.Key.Name, pair.Value);
}
// Types
var typesDir = Path.Combine(targetDir, "types");
Directory.CreateDirectory(typesDir);
foreach (var pair in TypeSchemas)
{
bar.Refresh(i++, pair.Key);
SaveSchema(typesDir, pair.Key, pair.Value);
}
Console.WriteLine("Done!");
}
protected void SaveSchema(string dir, string name, SchemaNode schema)
{
var invalids = System.IO.Path.GetInvalidFileNameChars();
var fileName = string.Join("_", schema.Path.Split(invalids, StringSplitOptions.RemoveEmptyEntries)).TrimEnd('.');
var path = Path.Combine(dir, fileName + ".txt");
var dirPath = Path.GetDirectoryName(path);
if (!Directory.Exists(dirPath))
{
Directory.CreateDirectory(dirPath);
}
using var writer = new StreamWriter(path);
writer.WriteLine(name);
writer.WriteLine($"Different values: {schema.Values}");
writer.WriteLine("=====================================");
writer.Write(schema.ToString());
}
}
}
Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/Compendium/CompendiumSchemaTypeResolver.cs
+12 -5
View File
@@ -22,9 +22,16 @@ namespace Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.Compendium
}
}
List<string> dictionaryPaths = [
"abomination-vaults-bestiary.items.[].system.overlays",
"abomination-vaults-bestiary.items.[].system.damageRolls",
protected readonly List<string> DictionaryPaths = [
"Actor.character.items.[].system.items",
"Item.ancestry.system.boosts",
"Item.background.system.boosts",
"Item.background.system.items",
"Item.class.system.items",
"Item.feat.system.subfeatures.proficiencies",
"Item.kit.system.items",
"Item.spell.system.damage",
"Item.spell.system.heightening.damage"
];
protected bool ShouldBeDictionary(SchemaNode parent, JToken? value)
@@ -36,8 +43,8 @@ namespace Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.Compendium
//var p = (JProperty)value.Parent;
//return dictionaryPaths.Any(path => path == $"{parent.Path}.{p.Name}");
return dictionaryPaths.Any(path => path == parent.Path);
//return DictionaryPaths.Any(path => path == $"{parent.Path}.{p.Name}");
return DictionaryPaths.Any(path => path == parent.Path);
}
public override AbstractSchemaType Resolve(SchemaNode parent, JToken? value)
Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/CompendiumLoader.cs
+71 -85
View File
@@ -7,121 +7,107 @@ using Pf2eDataScrapper.SchemaBuilder;
namespace Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry
{
internal partial class CompendiumLoader(Pf2eFoundryImporter importer)
internal class CompendiumLoader(Pf2eFoundryImporter importer)
{
public readonly Pf2eFoundryImporter Importer = importer;
public List<CompendiumFolder> Folders = [];
public CompendiumFolderCollection Collection = [];
private static string SlugifyName(string s)
private int _folderCount = 0;
public int ItemCount { get; protected set; } = 0;
protected void ProcessPackFolders(JArray folders, CompendiumFolder? parent)
{
s = SkippableCharactersRegex().Replace(s, "");
s = NonAlphanumericRegex().Replace(s.ToLower(), "-");
foreach (var token in folders)
{
JObject obj = (JObject)token;
return s;
var id = $"folder-{_folderCount++}";
var name = obj["name"]!.ToString();
var folder = new CompendiumFolder(id, name)
{
Parent = parent
};
Collection.Add(folder);
if (obj.TryGetValue("packs", out var packs))
{
foreach (var jToken in packs!)
{
var packId = jToken.ToString();
var child = Collection.GetById(packId);
if (child is not null)
{
child.Parent = folder;
}
}
}
if (obj.TryGetValue("folders", out var innerFolders))
{
ProcessPackFolders((JArray)innerFolders!, folder);
}
}
}
private static CompendiumDocumentType GetType(string s)
{
return (CompendiumDocumentType)Enum.Parse(typeof(CompendiumDocumentType), s);
}
void ResolveFolders()
public void Prepare()
{
using var systemFile = new StreamReader(Path.Combine(Importer.Dir, "static", "system.json"));
var systemData = JObject.Parse(systemFile.ReadToEnd());
foreach (var pack in systemData["packs"])
foreach (var pack in systemData["packs"]!)
{
var packPath = Path.Combine(importer.Dir, pack["path"].ToString());
var packType = GetType(pack["type"].ToString());
var packName = pack["path"].ToString().Replace("packs/", "");
var folder = new CompendiumFolder((JObject)pack);
if (!Directory.Exists(packPath))
Collection.Add(folder);
}
if (systemData.TryGetValue("packFolders", out var packFolders))
{
ProcessPackFolders((JArray)packFolders, null);
}
// Поиск _folders.json
foreach (var parent in Collection.ToList())
{
if (parent.Path is null)
{
throw new Exception("Pack not found: " + packPath);
continue;
}
var folder = new CompendiumFolder(packName, packPath, packType);
Folders.Add(folder);
var dirPath = Path.Combine(Importer.Dir, parent.Path);
//var innerFolderPath = Path.Combine(folder.Directory.FullName, "_folders.json");
//if (File.Exists(innerFolderPath))
//{
// using var innerFolderFile = new StreamReader(innerFolderPath);
// var innerFolderData = JArray.Parse(innerFolderFile.ReadToEnd());
ItemCount += Directory.GetFiles(dirPath, "*.json", SearchOption.AllDirectories)
.Length;
// foreach (var item in innerFolderData)
// {
// var innerPackPath = Path.Combine(packPath, SlugifyName(item["name"].ToString()));
// var innerPackType = GetType(item["type"].ToString());
// var innerPackName = packName + "/" + SlugifyName(item["name"].ToString());
// if (!Directory.Exists(packPath))
// {
// throw new Exception("Pack not found: " + innerPackPath);
// }
// var innerFolder = new CompendiumFolder(innerPackName, innerPackPath, innerPackType);
// Folders.Add(innerFolder);
// }
//}
}
foreach (var item in Folders)
{
Console.WriteLine($"{item.Directory.FullName} => {item.Type}");
}
}
public Dictionary<string, SchemaNode> BuildSchemas()
{
Dictionary<string, SchemaNode> Result = [];
var bar = new ProgressBar(Folders.Count);
var i = 0;
var resolver = new CompendiumSchemaTypeResolver();
foreach (var folder in Folders)
{
bar.Refresh(i++, folder.Name);
var schema = new SchemaNode() {
Resolver = resolver,
Path = folder.Name
};
foreach (var filePath in Directory.GetFiles(folder.Directory.FullName, "*.json", SearchOption.AllDirectories))
foreach (var foldersFilePath in Directory.GetFiles(dirPath, "_folders.json", SearchOption.AllDirectories))
{
var file = new FileInfo(filePath);
ItemCount--;
using var file = new StreamReader(foldersFilePath);
var data = JArray.Parse(file.ReadToEnd());
if (file.Name == "_folders.json")
foreach (var pack in data)
{
continue;
var folder = new CompendiumFolder((JObject)pack) { Parent = parent };
Collection.Add(folder);
}
using var reader = new StreamReader(file.FullName); // Открытие файла
var data = JObject.Parse(reader.ReadToEnd()); // Загрузка данных
schema.AddValue(data); // Добавление данных
}
Result.Add(folder.Name, schema);
}
bar.Refresh(Folders.Count, "Done!");
return Result;
foreach (var folder in Collection)
{
Console.WriteLine(folder);
}
Console.WriteLine($"Total items: {ItemCount}");
}
public void Run()
{
this.ResolveFolders();
this.Prepare();
}
[GeneratedRegex("[^a-zA-Z0-9]+")]
private static partial Regex NonAlphanumericRegex();
[GeneratedRegex("[\'?!]")]
private static partial Regex SkippableCharactersRegex();
}
}
Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/Pf2eFoundryImporter.cs
+5
View File
@@ -30,6 +30,11 @@ namespace Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry
Loader = new CompendiumLoader(this);
}
public override void Prepare()
{
Loader.Prepare();
}
public override void Import()
{
Loader.Run();
Pf2eDataScrapper/Program.cs
+9 -3
View File
@@ -2,6 +2,7 @@
using Pf2eDataScrapper;
using Pf2eDataScrapper.Importers;
using Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry;
using Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.Compendium;
using Pf2eDataScrapper.SchemaBuilder;
EnvSettings.Load();
@@ -15,10 +16,15 @@ var importer = new Pf2eFoundryImporter();
//loader.UpdateImporter(importer);
importer.Import();
importer.Prepare();
var q = importer.Loader.BuildSchemas();
SchemaDumper.Dump(q);
var builder = new CompendiumSchemaBuilder(importer.Loader);
builder.BuildSchemas();
builder.Save(EnvSettings.OutputDir);
// var q = importer.Loader.BuildSchemas();
// SchemaDumper.Dump(q);
Pf2eDataScrapper/SchemaBuilder/ArraySchemaType.cs
+2 -3
View File
@@ -53,10 +53,9 @@ namespace Pf2eDataScrapper.SchemaBuilder
public override string ToString()
{
return string.Join("\n" + Tab, [
GetTypeName(),
GetTypeName() + $" {{{Parent?.Path ?? ""}}}",
$"Length: min={MinCount} max={MaxCount}",
"Value type:",
Tab + Indent(ElementSchema.ToString())
"Value type: " + Indent(ElementSchema.ToString())
]);
}
}
Pf2eDataScrapper/SchemaBuilder/DictionarySchemaType.cs
+1 -1
View File
@@ -42,7 +42,7 @@ namespace Pf2eDataScrapper.SchemaBuilder
public override string ToString()
{
return string.Join("\n" + Tab, [
GetTypeName(),
GetTypeName() + $" {{{Parent?.Path ?? ""}}}",
"Key:" + Indent(KeyNode.ToString()),
"Value:" + Indent(ItemNode.ToString())
]);
Pf2eDataScrapper/SchemaBuilder/ObjectSchemaType.cs
+2 -2
View File
@@ -30,7 +30,7 @@ namespace Pf2eDataScrapper.SchemaBuilder
foreach (var pair in obj)
{
var field = Fields.Where(f => f.Name == pair.Key).FirstOrDefault();
var field = Fields.FirstOrDefault(f => f.Name == pair.Key);
if (field is null)
{
@@ -67,7 +67,7 @@ namespace Pf2eDataScrapper.SchemaBuilder
public override string ToString()
{
List<string> rows = ["object"];
List<string> rows = [$"object {{{Parent?.Path ?? ""}}}"];
foreach (var field in Fields)
{
Pf2eDataScrapper/SchemaBuilder/StaticSchemaType.cs
+25 -6
View File
@@ -7,7 +7,7 @@ namespace Pf2eDataScrapper.SchemaBuilder
protected JTokenType Type = type;
protected Dictionary<JToken, int> Values = [];
public override bool IsCompatibleValue(JToken value)
{
return value.Type == Type;
@@ -22,17 +22,31 @@ namespace Pf2eDataScrapper.SchemaBuilder
public override void Beautify()
{
// Do nothing
if (Type == JTokenType.String)
{
var newValues = new Dictionary<JToken, int>();
foreach (var pair in Values)
{
var newValue = "'" + pair.Key.ToString().Trim().Replace("'", "\\'") + "'";
newValues.TryAdd(newValue, 0);
newValues[newValue] += pair.Value;
}
Values = newValues;
}
Values = Values.OrderByDescending(pair => pair.Value).ToDictionary(pair => pair.Key, pair => pair.Value);
}
public override string GetTypeName()
{
return Type.ToString();
return Type.ToString().ToLower();
}
public override string ToString()
{
var result = Type.ToString();
var result = GetTypeName() + $" {{{Parent?.Path ?? ""}}}";
var count = Values.Count;
if (count == Root.Values)
@@ -41,7 +55,13 @@ namespace Pf2eDataScrapper.SchemaBuilder
return result;
}
result += $" (different values: {count})";
if (count == 1)
{
result += $" (same value: {Values.Keys.ToList()[0]})";
return result;
}
result += $" ({count} different values)";
if (count is > ElementLimit or <= 0)
{
@@ -50,7 +70,6 @@ namespace Pf2eDataScrapper.SchemaBuilder
result += "\n";
result = Values.Aggregate(result, (current, value) => current + $"{Tab}{value.Key} (x{value.Value})\n");
return result.Trim();
Pf2eDataScrapper/SchemaDumper.cs
-49
View File
@@ -1,49 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Pf2eDataScrapper.SchemaBuilder;
namespace Pf2eDataScrapper
{
internal class SchemaDumper(Dictionary<string, SchemaNode> data)
{
public readonly Dictionary<string, SchemaNode> Data = data;
public readonly string OutputDir = EnvSettings.OutputDir;
public void Run()
{
Console.WriteLine("Dumping schema...");
foreach (var pair in Data)
{
var path = Path.Combine(OutputDir, pair.Key + ".txt");
var dir = Path.GetDirectoryName(path);
if (!Directory.Exists(dir))
{
Directory.CreateDirectory(dir);
}
var schema = pair.Value;
using var writer = new StreamWriter(path);
writer.WriteLine(pair.Key);
writer.Write(schema.ToString());
}
Console.WriteLine("Done!");
}
public static void Dump(Dictionary<string, SchemaNode> data)
{
(new SchemaDumper(data)).Run();
}
}
}