From 042b6e23c676ce994b4eb33fb11b1ca98e558042 Mon Sep 17 00:00:00 2001 From: Benjamin Date: Sat, 29 Nov 2025 16:44:09 -0600 Subject: [PATCH] Rebuilt CPK tools - Rebuilt how CPKs are read and written to, speeding up CPK repacking and massively reducing allocations --- .gitignore | 1 + CpkTools/ColumnFlags.cs | 24 + CpkTools/Comparers/FileEntryIdComparer.cs | 15 + CpkTools/CpkTools.csproj | 13 + CpkTools/Endian/EndianReader.cs | 176 +++ CpkTools/Endian/EndianWriter.cs | 185 +++ CpkTools/Model/Column.cs | 6 + CpkTools/Model/Cpk.cs | 645 ++++++++++ CpkTools/Model/FileEntry.cs | 30 + CpkTools/Model/Row.cs | 40 + CpkTools/Model/Utf.cs | 149 +++ CpkTools/Tools.cs | 60 + ShinRyuModManager-CE.slnx | 1 + .../ModLoadOrder/CpkPatcher.cs | 78 +- .../ModLoadOrder/Dependency/CPKGen/CPK.cs | 1046 ----------------- .../ModLoadOrder/Dependency/CPKGen/Endian.cs | 155 --- .../ModLoadOrder/Dependency/CPKGen/Program.cs | 106 -- .../ModLoadOrder/Dependency/CPKGen/Tools.cs | 72 -- ShinRyuModManager-CE/ModLoadOrder/Mods/Mod.cs | 6 +- .../ShinRyuModManager-CE.csproj | 1 + .../UserInterface/Views/MainWindow.axaml.cs | 8 +- 21 files changed, 1426 insertions(+), 1391 deletions(-) create mode 100644 CpkTools/ColumnFlags.cs create mode 100644 CpkTools/Comparers/FileEntryIdComparer.cs create mode 100644 CpkTools/CpkTools.csproj create mode 100644 CpkTools/Endian/EndianReader.cs create mode 100644 CpkTools/Endian/EndianWriter.cs create mode 100644 CpkTools/Model/Column.cs create mode 100644 CpkTools/Model/Cpk.cs create mode 100644 CpkTools/Model/FileEntry.cs create mode 100644 CpkTools/Model/Row.cs create mode 100644 CpkTools/Model/Utf.cs create mode 100644 CpkTools/Tools.cs delete mode 100644 ShinRyuModManager-CE/ModLoadOrder/Dependency/CPKGen/CPK.cs delete mode 100644 ShinRyuModManager-CE/ModLoadOrder/Dependency/CPKGen/Endian.cs delete mode 100644 ShinRyuModManager-CE/ModLoadOrder/Dependency/CPKGen/Program.cs delete mode 100644 ShinRyuModManager-CE/ModLoadOrder/Dependency/CPKGen/Tools.cs diff --git a/.gitignore b/.gitignore index 0a4f853..41cfca5 100644 --- a/.gitignore +++ b/.gitignore @@ -495,3 +495,4 @@ fabric.properties ### Custom ### launchSettings.json .idea/.idea.ShinRyuModManager-CE/.idea/sonarlint.xml +dist/ diff --git a/CpkTools/ColumnFlags.cs b/CpkTools/ColumnFlags.cs new file mode 100644 index 0000000..fd6f18e --- /dev/null +++ b/CpkTools/ColumnFlags.cs @@ -0,0 +1,24 @@ +namespace CpkTools; + +public enum StorageFlags { + StorageMask = 0xF0, + StorageNone = 0x00, + StorageZero = 0x10, + StorageConstant = 0x30, + StoragePerrow = 0x50, +} + +public enum TypeFlags { + TypeMask = 0x0F, + TypeData = 0x0B, + TypeString = 0x0A, + TypeFloat = 0x08, + Type8Byte2 = 0x07, + Type8Byte = 0x06, + Type4Byte2 = 0x05, + Type4Byte = 0x04, + Type2Byte2 = 0x03, + Type2Byte = 0x02, + Type1Byte2 = 0x01, + Type1Byte = 0x00, +} diff --git a/CpkTools/Comparers/FileEntryIdComparer.cs b/CpkTools/Comparers/FileEntryIdComparer.cs new file mode 100644 index 0000000..d203a62 --- /dev/null +++ b/CpkTools/Comparers/FileEntryIdComparer.cs @@ -0,0 +1,15 @@ +using CpkTools.Model; + +namespace CpkTools.Comparers; + +public sealed class FileEntryIdComparer : IComparer { + public static readonly FileEntryIdComparer Instance = new(); + + public int Compare(FileEntry? x, FileEntry? y) { + if (ReferenceEquals(x, y)) return 0; + if (x is null) return -1; + if (y is null) return 1; + + return x.Id.CompareTo(y.Id); + } +} diff --git a/CpkTools/CpkTools.csproj b/CpkTools/CpkTools.csproj new file mode 100644 index 0000000..0af915f --- /dev/null +++ b/CpkTools/CpkTools.csproj @@ -0,0 +1,13 @@ + + + + net10.0 + enable + enable + + + + + + + diff --git a/CpkTools/Endian/EndianReader.cs b/CpkTools/Endian/EndianReader.cs new file mode 100644 index 0000000..6ae4f1a --- /dev/null +++ b/CpkTools/Endian/EndianReader.cs @@ -0,0 +1,176 @@ +using System.Buffers.Binary; +using CommunityToolkit.HighPerformance; + +namespace CpkTools.Endian; + +public sealed class EndianReader : IDisposable { + public bool IsLittleEndian { get; set; } + public long Position { get => BaseStream.Position; } + public Stream BaseStream { get; } + + public EndianReader(Memory data, bool isLittleEndian = false) { + BaseStream = data.AsStream(); + IsLittleEndian = isLittleEndian; + } + + public EndianReader(byte[] data, bool isLittleEndian = false) { + BaseStream = new MemoryStream(data); + IsLittleEndian = isLittleEndian; + } + + public EndianReader(Stream stream, bool isLittleEndian = false) { + BaseStream = stream; + IsLittleEndian = isLittleEndian; + } + + public Half ReadHalf() { + Span buffer = stackalloc byte[2]; + + BaseStream.ReadExactly(buffer); + + return IsLittleEndian + ? BinaryPrimitives.ReadHalfLittleEndian(buffer) + : BinaryPrimitives.ReadHalfBigEndian(buffer); + } + + public float ReadSingle() { + Span buffer = stackalloc byte[4]; + + BaseStream.ReadExactly(buffer); + + return IsLittleEndian + ? BinaryPrimitives.ReadSingleLittleEndian(buffer) + : BinaryPrimitives.ReadSingleBigEndian(buffer); + } + + public double ReadDouble() { + Span buffer = stackalloc byte[8]; + + BaseStream.ReadExactly(buffer); + + return IsLittleEndian + ? BinaryPrimitives.ReadDoubleLittleEndian(buffer) + : BinaryPrimitives.ReadDoubleBigEndian(buffer); + } + + public short ReadInt16() { + Span buffer = stackalloc byte[2]; + + BaseStream.ReadExactly(buffer); + + return IsLittleEndian + ? BinaryPrimitives.ReadInt16LittleEndian(buffer) + : BinaryPrimitives.ReadInt16BigEndian(buffer); + } + + public int ReadInt32() { + Span buffer = stackalloc byte[4]; + + BaseStream.ReadExactly(buffer); + + return IsLittleEndian + ? BinaryPrimitives.ReadInt32LittleEndian(buffer) + : BinaryPrimitives.ReadInt32BigEndian(buffer); + } + + public long ReadInt64() { + Span buffer = stackalloc byte[8]; + + BaseStream.ReadExactly(buffer); + + return IsLittleEndian + ? BinaryPrimitives.ReadInt64LittleEndian(buffer) + : BinaryPrimitives.ReadInt64BigEndian(buffer); + } + + public ushort ReadUInt16() { + Span buffer = stackalloc byte[2]; + + BaseStream.ReadExactly(buffer); + + return IsLittleEndian + ? BinaryPrimitives.ReadUInt16LittleEndian(buffer) + : BinaryPrimitives.ReadUInt16BigEndian(buffer); + } + + public uint ReadUInt32() { + Span buffer = stackalloc byte[4]; + + BaseStream.ReadExactly(buffer); + + return IsLittleEndian + ? BinaryPrimitives.ReadUInt32LittleEndian(buffer) + : BinaryPrimitives.ReadUInt32BigEndian(buffer); + } + + public ulong ReadUInt64() { + Span buffer = stackalloc byte[8]; + + BaseStream.ReadExactly(buffer); + + return IsLittleEndian + ? BinaryPrimitives.ReadUInt64LittleEndian(buffer) + : BinaryPrimitives.ReadUInt64BigEndian(buffer); + } + + public byte ReadByte() { + var value = BaseStream.ReadByte(); + + if (value == -1) + throw new EndOfStreamException(); + + return (byte)value; + } + + public byte[] ReadBytes(int count) { + ArgumentOutOfRangeException.ThrowIfNegative(count); + + if (count == 0) { + return []; + } + + var result = new byte[count]; + var numRead = BaseStream.ReadAtLeast(result, result.Length, throwOnEndOfStream: false); + + if (numRead != result.Length) { + // Trim array. This should happen on EOF & possibly net streams. + result = result[..numRead]; + } + + return result; + } + + public int ReadStreamInto(Stream dest, int length) { + ArgumentOutOfRangeException.ThrowIfNegative(length); + + if (length == 0) { + return 0; + } + + var buffer = new byte[80 * 1024]; + var remaining = length; + var totalRead = 0; + + while (remaining > 0) { + var toRead = Math.Min(buffer.Length, remaining); + var read = BaseStream.Read(buffer, 0, toRead); + + if (read == 0) // EOF + break; + + dest.Write(buffer, 0, read); + totalRead += read; + remaining -= read; + } + + return totalRead; + } + + public void Seek(long offset, SeekOrigin origin) { + BaseStream.Seek(offset, origin); + } + + public void Dispose() { + BaseStream.Dispose(); + } +} diff --git a/CpkTools/Endian/EndianWriter.cs b/CpkTools/Endian/EndianWriter.cs new file mode 100644 index 0000000..8e8f2ba --- /dev/null +++ b/CpkTools/Endian/EndianWriter.cs @@ -0,0 +1,185 @@ +using System.Buffers.Binary; +using CommunityToolkit.HighPerformance; +using CpkTools.Model; + +namespace CpkTools.Endian; + +public sealed class EndianWriter : IDisposable { + public bool IsLittleEndian { get; set; } + public long Position { get => BaseStream.Position; } + public Stream BaseStream { get; } + + public EndianWriter(Memory data, bool isLittleEndian = false) { + BaseStream = data.AsStream(); + IsLittleEndian = isLittleEndian; + } + + public EndianWriter(Stream stream, bool isLittleEndian = false) { + BaseStream = stream; + IsLittleEndian = isLittleEndian; + } + + public void Write(bool value) { + BaseStream.WriteByte(Convert.ToByte(value)); + } + + public void Write(char value) { + Write((ushort)value); + } + + public void Write(Half value) { + Span buffer = stackalloc byte[2]; + + if (IsLittleEndian) { + BinaryPrimitives.WriteHalfLittleEndian(buffer, value); + } else { + BinaryPrimitives.WriteHalfBigEndian(buffer, value); + } + + BaseStream.Write(buffer); + } + + public void Write(float value) { + Span buffer = stackalloc byte[4]; + + if (IsLittleEndian) { + BinaryPrimitives.WriteSingleLittleEndian(buffer, value); + } else { + BinaryPrimitives.WriteSingleBigEndian(buffer, value); + } + + BaseStream.Write(buffer); + } + + public void Write(double value) { + Span buffer = stackalloc byte[8]; + + if (IsLittleEndian) { + BinaryPrimitives.WriteDoubleLittleEndian(buffer, value); + } else { + BinaryPrimitives.WriteDoubleBigEndian(buffer, value); + } + + BaseStream.Write(buffer); + } + + public void Write(short value) { + Span buffer = stackalloc byte[2]; + + if (IsLittleEndian) { + BinaryPrimitives.WriteInt16LittleEndian(buffer, value); + } else { + BinaryPrimitives.WriteInt16BigEndian(buffer, value); + } + + BaseStream.Write(buffer); + } + + public void Write(int value) { + Span buffer = stackalloc byte[4]; + + if (IsLittleEndian) { + BinaryPrimitives.WriteInt32LittleEndian(buffer, value); + } else { + BinaryPrimitives.WriteInt32BigEndian(buffer, value); + } + + BaseStream.Write(buffer); + } + + public void Write(long value) { + Span buffer = stackalloc byte[8]; + + if (IsLittleEndian) { + BinaryPrimitives.WriteInt64LittleEndian(buffer, value); + } else { + BinaryPrimitives.WriteInt64BigEndian(buffer, value); + } + + BaseStream.Write(buffer); + } + + public void Write(ushort value) { + Span buffer = stackalloc byte[2]; + + if (IsLittleEndian) { + BinaryPrimitives.WriteUInt16LittleEndian(buffer, value); + } else { + BinaryPrimitives.WriteUInt16BigEndian(buffer, value); + } + + BaseStream.Write(buffer); + } + + public void Write(uint value) { + Span buffer = stackalloc byte[4]; + + if (IsLittleEndian) { + BinaryPrimitives.WriteUInt32LittleEndian(buffer, value); + } else { + BinaryPrimitives.WriteUInt32BigEndian(buffer, value); + } + + BaseStream.Write(buffer); + } + + public void Write(ulong value) { + Span buffer = stackalloc byte[8]; + + if (IsLittleEndian) { + BinaryPrimitives.WriteUInt64LittleEndian(buffer, value); + } else { + BinaryPrimitives.WriteUInt64BigEndian(buffer, value); + } + + BaseStream.Write(buffer); + } + + public void Write(byte value) { + BaseStream.WriteByte(value); + } + + public void Write(sbyte value) { + BaseStream.WriteByte((byte)value); + } + + public void Write(byte[] value) { + BaseStream.Write(value); + } + + public void Write(Span value) { + BaseStream.Write(value); + } + + public void Write(ReadOnlySpan value) { + BaseStream.Write(value); + } + + public void Write(Stream stream) { + stream.CopyTo(BaseStream); + } + + public void Write(FileEntry entry) { + if (entry.ExtractSizeType == typeof(byte)) { + Write((byte)entry.ExtractSize); + } else if (entry.ExtractSizeType == typeof(ushort)) { + Write((ushort)entry.ExtractSize); + } else if (entry.ExtractSizeType == typeof(uint)) { + Write((uint)entry.ExtractSize); + } else if (entry.ExtractSizeType == typeof(ulong)) { + Write(entry.ExtractSize); + } else if (entry.ExtractSizeType == typeof(float)) { + Write((float)entry.ExtractSize); + } else { + throw new Exception("Not supported type!"); + } + } + + public void Seek(long offset, SeekOrigin origin) { + BaseStream.Seek(offset, origin); + } + + public void Dispose() { + BaseStream.Dispose(); + } +} diff --git a/CpkTools/Model/Column.cs b/CpkTools/Model/Column.cs new file mode 100644 index 0000000..7c212c4 --- /dev/null +++ b/CpkTools/Model/Column.cs @@ -0,0 +1,6 @@ +namespace CpkTools.Model; + +public record struct Column { + public string Name { get; set; } + public byte Flags { get; set; } +} diff --git a/CpkTools/Model/Cpk.cs b/CpkTools/Model/Cpk.cs new file mode 100644 index 0000000..6dff58c --- /dev/null +++ b/CpkTools/Model/Cpk.cs @@ -0,0 +1,645 @@ +using System.Buffers.Binary; +using System.Text; +using CpkTools.Comparers; +using CpkTools.Endian; + +namespace CpkTools.Model; + +public sealed class Cpk { + public List FileTable { get; } = []; + public ulong ContentOffset { get; private set; } + + private bool _isUtfEncrypted; + //private int _unk1; + private long _utfSize; + private Memory _utfPacket; + private Memory _cpkPacket; + private Memory _tocPacket; + private Memory _itocPacket; + private Memory _etocPacket; + private Memory _gtocPacket; + private ulong _tocOffset = ulong.MaxValue; + private ulong _etocOffset = ulong.MaxValue; + private ulong _itocOffset = ulong.MaxValue; + private ulong _gtocOffset = ulong.MaxValue; + private Utf? _utf; + private Utf? _files; + + private static ReadOnlySpan UtfHeader { get => "@UTF"u8; } + + public bool ReadCpk(string path) { + if (!File.Exists(path)) + return false; + + using var reader = new EndianReader(File.OpenRead(path), true); + + if (Tools.ReadCString(reader, 4) != "CPK ") { + return false; + } + + ReadUtfData(reader); + + _cpkPacket = _utfPacket; + + // Dump CPK + var cpkEntry = new FileEntry { + FileName = "CPK_HDR", + FileOffsetPos = reader.Position + 0x10, + FileSize = (ulong)_cpkPacket.Length, + Encrypted = _isUtfEncrypted, + FileType = "CPK" + }; + + FileTable.Add(cpkEntry); + + _utf = new Utf(); + + if (!_utf.ReadUtf(_utfPacket)) { + return false; + } + + /*try { + for (var i = 0; i < _utf.Columns.Count; i++) { + _cpkData.Add(_utf.Columns[i].Name, _utf.Rows[0][i].GetValue()!); + } + } catch (Exception ex) { + Console.WriteLine(ex); + }*/ + + ContentOffset = TryGetColumnData(_utf, 0, "ContentOffset", out var content) ? content.Value : ulong.MaxValue; + + var newEntry = CreateFileEntry("CONTENT_OFFSET", ContentOffset, content.Position, "CPK", "CONTENT", false); + + FileTable.Add(newEntry); + + var align = TryGetColumnData(_utf, 0, "Align", out var alignData) ? alignData.Value : ushort.MaxValue; + + if (TryGetColumnData(_utf, 0, "TocOffset", out var toc)) { + var entry = CreateFileEntry("TOC_HDR", toc.Value, toc.Position, "CPK", "HDR", false); + + FileTable.Add(entry); + + _tocOffset = toc.Value; + + if (!ReadToc(reader, toc.Value, ContentOffset)) + return false; + } + + if (TryGetColumnData(_utf, 0, "EtocOffset", out var etoc)) { + var entry = CreateFileEntry("ETOC_HDR", etoc.Value, etoc.Position, "CPK", "HDR", false); + + FileTable.Add(entry); + + _etocOffset = etoc.Value; + + if (!ReadEtoc(reader, etoc.Value)) + return false; + } + + if (TryGetColumnData(_utf, 0, "ItocOffset", out var itoc)) { + var entry = CreateFileEntry("ITOC_HDR", itoc.Value, itoc.Position, "CPK", "HDR", false); + + FileTable.Add(entry); + + _itocOffset = itoc.Value; + + if (!ReadItoc(reader, itoc.Value, ContentOffset, align)) + return false; + } + + if (TryGetColumnData(_utf, 0, "GtocOffset", out var gtoc)) { + var entry = CreateFileEntry("GTOC_HDR", gtoc.Value, gtoc.Position, "CPK", "HDR", false); + + FileTable.Add(entry); + + _gtocOffset = gtoc.Value; + + if (!ReadGtoc(reader, gtoc.Value)) + return false; + } + + _files = null; + + return true; + } + + public static byte[] DecompressCrilayla(byte[] input, int size) { + using var reader = new EndianReader(input); + + reader.Seek(8, SeekOrigin.Begin); // Skip CRILAYLA + + var uncompressedSize = reader.ReadInt32(); + var uncompressedHeaderOffset = reader.ReadInt32(); + var result = new byte[uncompressedSize + 0x100]; + + // Copy uncompressed 0x100 header to start of file + Array.Copy(input, uncompressedHeaderOffset + 0x10, result, 0, 0x100); + + var inputEnd = input.Length - 0x100 - 1; + var inputOffset = inputEnd; + var outputEnd = 0x100 + uncompressedSize - 1; + byte bitPool = 0; + var bitsLeft = 0; + var bytesOutput = 0; + int[] vleLens = [2, 3, 5, 8]; + + while (bytesOutput < uncompressedSize) { + if (GenNextBits(input, ref inputOffset, ref bitPool, ref bitsLeft, 1) > 0) { + var backreferenceOffset = outputEnd - bytesOutput + + GenNextBits(input, ref inputOffset, ref bitPool, ref bitsLeft, 13) + 3; + + var backreferenceLength = 3; + int vleLevel; + + for (vleLevel = 0; vleLevel < vleLens.Length; vleLevel++) { + int thisLevel = GenNextBits(input, ref inputOffset, ref bitPool, ref bitsLeft, vleLens[vleLevel]); + + backreferenceLength += thisLevel; + + if (thisLevel != ((1 << vleLens[vleLevel]) - 1)) + break; + } + + if (vleLevel == vleLens.Length) { + int thisLevel; + + do { + thisLevel = GenNextBits(input, ref inputOffset, ref bitPool, ref bitsLeft, 8); + + backreferenceLength += thisLevel; + } while (thisLevel == 255); + } + + for (var i = 0; i < backreferenceLength; i++) { + result[outputEnd - bytesOutput] = result[backreferenceOffset--]; + bytesOutput++; + } + } else { + // Verbatim byte + result[outputEnd - bytesOutput] = (byte)GenNextBits(input, ref inputOffset, ref bitPool, ref bitsLeft, 8); + + bytesOutput++; + } + } + + return result; + } + + private static FileEntry CreateFileEntry(string fileName, ulong fileOffset, long fileOffsetPos, string tocName, string fileType, bool encrypted) { + return new FileEntry { + FileName = fileName, + FileOffset = fileOffset, + FileOffsetPos = fileOffsetPos, + FileOffsetType = typeof(ulong), + TocName = tocName, + FileType = fileType, + Encrypted = encrypted + }; + } + + public void UpdateFileEntry(FileEntry fileEntry) { + if (fileEntry.FileType is not ("FILE" or "HDR")) + return; + + var updateMe = fileEntry.TocName switch { + "CPK" => _cpkPacket, + "TOC" => _tocPacket, + "ITOC" => _itocPacket, + "ETOC" => _etocPacket, + _ => throw new Exception("I need to implement this TOC!") + }; + + // Update ExtractSize + if (fileEntry.ExtractSizePos > 0) + UpdateValue(updateMe, fileEntry.ExtractSizePos, fileEntry.ExtractSize, fileEntry.ExtractSizeType!); + + // Update FileSize + if (fileEntry.FileSizePos > 0) + UpdateValue(updateMe, fileEntry.FileSizePos, fileEntry.FileSize, fileEntry.FileSizeType!); + + // Update FileOffset + if (fileEntry.FileOffsetPos > 0) { + var finalOffset = fileEntry.FileOffset - (ulong)(fileEntry.TocName == "TOC" ? 0x800 : 0); + + UpdateValue(updateMe, fileEntry.FileOffsetPos, finalOffset, fileEntry.FileOffsetType!); + } + + switch (fileEntry.TocName) { + case "CPK": + updateMe = _cpkPacket; + + break; + case "TOC": + _tocPacket = updateMe; + + break; + case "ITOC": + _itocPacket = updateMe; + + break; + case "ETOC": + updateMe = _etocPacket; + + break; + default: + throw new Exception("I need to implement this TOC!"); + } + } + + public void WriteCpk(EndianWriter writer) { + WritePacket(writer, "CPK ", 0, _cpkPacket); + + writer.Seek(0x800 - 6, SeekOrigin.Begin); + writer.Write("(c)CRI"u8); + } + + public void WriteToc(EndianWriter writer) { + WritePacket(writer, "TOC ", _tocOffset, _tocPacket); + } + + public void WriteItoc(EndianWriter writer) { + WritePacket(writer, "ITOC", _itocOffset, _itocPacket); + } + + public void WriteEtoc(EndianWriter writer) { + WritePacket(writer, "ETOC", _etocOffset, _etocPacket); + } + + public void WriteGtoc(EndianWriter writer) { + WritePacket(writer, "GTOC", _gtocOffset, _gtocPacket); + } + + private static void WritePacket(EndianWriter writer, string id, ulong position, Memory packet) { + if (position == 0xffffffffffffffff) + return; + + writer.Seek((long)position, SeekOrigin.Begin); + + DecryptUtf(packet); + + writer.Write(Encoding.ASCII.GetBytes(id)); + writer.Write(0); + writer.Write((ulong)packet.Length); + writer.Write(packet.Span); + } + + private static void UpdateValue(Memory packet, long pos, ulong value, Type type) { + ArgumentOutOfRangeException.ThrowIfLessThan(pos, 0); + + using var writer = new EndianWriter(packet); + + writer.Seek((int)pos, SeekOrigin.Begin); + + if (type == typeof(byte)) { + writer.Write((byte)value); + } else if (type == typeof(ushort)) { + writer.Write((ushort)value); + } else if (type == typeof(uint)) { + writer.Write((uint)value); + } else if (type == typeof(ulong)) { + writer.Write(value); + } else if (type == typeof(float)) { + writer.Write((float)value); + } + } + + private void ReadUtfData(EndianReader reader) { + _isUtfEncrypted = false; + reader.IsLittleEndian = true; + + //_unk1 = reader.ReadInt32(); + _ = reader.ReadInt32(); + _utfSize = reader.ReadInt64(); + _utfPacket = reader.ReadBytes((int)_utfSize); + + if (!_utfPacket.Span[..4].SequenceEqual(UtfHeader)) { + DecryptUtf(_utfPacket); + _isUtfEncrypted = true; + } + + reader.IsLittleEndian = false; + } + + private bool ReadToc(EndianReader reader, ulong tocOffset, ulong contentOffset) { + var addOffset = Math.Min(tocOffset, contentOffset); + + reader.Seek((long) tocOffset, SeekOrigin.Begin); + + if (Tools.ReadCString(reader, 4) != "TOC ") { + return false; + } + + ReadUtfData(reader); + + // Store Encrypted TOC + _tocPacket = _utfPacket; + + // Dump TOC + var tocEntry = FileTable.Single(x => x.FileName == "TOC_HDR"); + + tocEntry.Encrypted = _isUtfEncrypted; + tocEntry.FileSize = (ulong)_tocPacket.Length; + + _files = new Utf(); + + if (!_files.ReadUtf(_utfPacket)) { + return false; + } + + for (var i = 0; i < _files.Rows!.Length; i++) { + var fileSize = GetColumnData(_files, i, "FileSize"); + var extractSize = GetColumnData(_files, i, "ExtractSize"); + var fileOffset = GetColumnData(_files, i, "FileOffset"); + + FileTable.Add(new FileEntry { + TocName = "TOC", + DirName = GetColumnData(_files, i, "DirName").Value!, + FileName = GetColumnData(_files, i, "FileName").Value!, + FileSize = fileSize.Value, + FileSizePos = fileSize.Position, + FileSizeType = fileSize.Type!, + ExtractSize = extractSize.Value, + ExtractSizePos = extractSize.Position, + ExtractSizeType = extractSize.Type!, + FileOffset = fileOffset.Value + addOffset, + FileOffsetPos = fileOffset.Position, + FileOffsetType = fileOffset.Type!, + FileType = "FILE", + Id = GetColumnData(_files, i, "ID").Value, + UserString = GetColumnData(_files, i, "UserString").Value! + }); + } + + _files = null; + + return true; + } + + private bool ReadEtoc(EndianReader reader, ulong offset) { + reader.Seek((long)offset, SeekOrigin.Begin); + + if (Tools.ReadCString(reader, 4) != "ETOC") { + return false; + } + + ReadUtfData(reader); + + _etocPacket = _utfPacket; + + // Dump ETOC + var etocEntry = FileTable.Single(x => x.FileName == "ETOC_HDR"); + + etocEntry.Encrypted = _isUtfEncrypted; + etocEntry.FileSize = (ulong)_etocPacket.Length; + + _files = new Utf(); + + if (!_files.ReadUtf(_utfPacket)) { + return false; + } + + var fileEntries = FileTable.Where(x => x.FileType == "FILE").ToList(); + + for (var i = 0; i < fileEntries.Count; i++) { + FileTable[i].LocalDir = GetColumnData(_files, i, "LocalDir").Value!; + } + + return true; + } + + private bool ReadItoc(EndianReader reader, ulong itocOffset, ulong contentOffset, ushort align) { + reader.Seek((long)itocOffset, SeekOrigin.Begin); + + if (Tools.ReadCString(reader, 4) != "ITOC") { + return false; + } + + ReadUtfData(reader); + + _itocPacket = _utfPacket; + + // Dump ITOC + var itocEntry = FileTable.Single(x => x.FileName == "ITOC_HDR"); + + itocEntry.Encrypted = _isUtfEncrypted; + itocEntry.FileSize = (ulong)_itocPacket.Length; + + _files = new Utf(); + + if (!_files.ReadUtf(_utfPacket)) { + return false; + } + + var dataL = GetColumnData(_files, 0, "DataL"); + var dataH = GetColumnData(_files, 0, "DataH"); + + var fileSizeL = GetFileSizeInfo(dataL); + var fileSizeH = GetFileSizeInfo(dataH); + + var tempFiles = new List(fileSizeL.Ids.Count + fileSizeH.Ids.Count); + + AddDataInfoToFileTable(fileSizeL, tempFiles); + AddDataInfoToFileTable(fileSizeH, tempFiles); + + tempFiles.Sort(FileEntryIdComparer.Instance); + + RecalculateFileOffset(tempFiles, contentOffset, align); + + FileTable.AddRange(tempFiles); + + _files = null; + + return true; + } + + private static bool ReadGtoc(EndianReader reader, ulong offset) { + reader.Seek((long)offset, SeekOrigin.Begin); + + if (Tools.ReadCString(reader, 4) != "GTOC") { + return false; + } + + reader.Seek(0xC, SeekOrigin.Begin); // Skip header area + + return true; + } + + private static void DecryptUtf(Memory input) { + var span = input.Span; + + var m = 0x0000655F; // State + const int t = 0x00004115; // Multiplier + + for (var i = 0; i < input.Length; i++) { + var d = span[i]; // Current Byte + d = (byte)(d ^ (byte)(m & 0xFF)); + span[i] = d; + m *= t; + } + } + + private static ushort GenNextBits(byte[] input, ref int offsetP, ref byte bitPoolP, ref int bitsLeftP, int bitCount) { + ushort outBits = 0; + var numBitsProduced = 0; + + while (numBitsProduced < bitCount) { + if (bitsLeftP == 0) { + bitPoolP = input[offsetP]; + bitsLeftP = 8; + offsetP--; + } + + int bitsThisRound; + + if (bitsLeftP > (bitCount - numBitsProduced)) { + bitsThisRound = bitCount - numBitsProduced; + } else { + bitsThisRound = bitsLeftP; + } + + outBits <<= bitsThisRound; + + outBits |= (ushort)((ushort)(bitPoolP >> (bitsLeftP - bitsThisRound)) & (1 << bitsThisRound) - 1); + + bitsLeftP -= bitsThisRound; + numBitsProduced += bitsThisRound; + } + + return outBits; + } + + private static void AddDataInfoToFileTable(DataInfo dataInfo, List tempList) { + foreach (var id in dataInfo.Ids) { + var temp = new FileEntry(); + dataInfo.SizeTable.TryGetValue(id, out var sizeData); + + ulong fileSize = sizeData.Value switch { + ushort us => us, + uint ui => ui, + _ => 0 + }; + + temp.TocName = "ITOC"; + temp.FileName = id.ToString("D4"); + temp.FileSize = fileSize; + temp.FileSizePos = sizeData.Position; + temp.FileSizeType = sizeData.Type!; + + if (dataInfo.CSizeTable.TryGetValue(id, out var cSizeData)) { + ulong extractSize = cSizeData.Value switch { + ushort us => us, + uint ui => ui, + _ => 0 + }; + + temp.ExtractSize = extractSize; + temp.ExtractSizePos = cSizeData.Position; + temp.ExtractSizeType = cSizeData.Type!; + } + + temp.FileOffsetType = typeof(ulong); + temp.FileType = "FILE"; + temp.Id = id; + + tempList.Add(temp); + } + } + + private static void RecalculateFileOffset(IEnumerable entries, ulong contentOffset, ushort align) { + foreach (var entry in entries) { + var fileSize = Convert.ToInt32(entry.FileSize); + + entry.FileOffset = contentOffset; + + if (fileSize % align > 0) + contentOffset += (ulong)(fileSize + (align - (fileSize % align))); + else + contentOffset += (ulong)fileSize; + } + } + + private static bool TryGetColumnData(Utf utf, int rowIndex, string columnName, out ColumnData columnData) { + var data = GetColumnData(utf, rowIndex, columnName); + + if (data.Position == -1) { + columnData = default; + return false; + } + + columnData = data; + + return true; + } + + private static ColumnData GetColumnData(Utf utf, int rowIndex, string columnName) { + try { + var columnIndex = utf.Columns!.FindIndex(c => c.Name == columnName); + + if (columnIndex == -1) { + return new ColumnData(default!, -1, null); + } + + var row = utf.Rows![rowIndex][columnIndex]; + + var cell = row.GetValue(); + var position = row.Position; + var type = row.GetType(); + + if (cell is T value) + return new ColumnData(value, position, type); + + if (type == null) { + value = default!; + position = -1; + } else { + value = (T)Convert.ChangeType(cell, typeof(T))!; + } + + return new ColumnData(value, position, type); + } catch (Exception ex) { + Console.WriteLine(ex); + } + + return new ColumnData(default!, -1, null); + } + + private static DataInfo GetFileSizeInfo(ColumnData inputData) { + var result = new DataInfo(); + + if (inputData.Value == null) + return result; + + var utfData = new Utf(); + utfData.ReadUtf(inputData.Value); + + for (var i = 0; i < utfData.Rows!.Length; i++) { + var id = GetColumnData(utfData, i, "ID").Value; + + var fileSize = GetColumnData(utfData, i, "FileSize"); + + fileSize.Position += inputData.Position; + + result.SizeTable.Add(id, fileSize); + + var extractSize = GetColumnData(utfData, i, "ExtractSize"); + + if (extractSize.Value != null) { + extractSize.Position += inputData.Position; + + result.CSizeTable.Add(id, extractSize); + } + + result.Ids.Add(id); + } + + return result; + } +} + +internal record struct ColumnData(T? Value, long Position, Type? Type); + +internal record struct DataInfo() { + public List Ids { get; } = []; + public Dictionary> SizeTable { get; } = []; + public Dictionary> CSizeTable { get; } = []; +} diff --git a/CpkTools/Model/FileEntry.cs b/CpkTools/Model/FileEntry.cs new file mode 100644 index 0000000..cce5177 --- /dev/null +++ b/CpkTools/Model/FileEntry.cs @@ -0,0 +1,30 @@ +namespace CpkTools.Model; + +public sealed record FileEntry { + + public string DirName { get; set; } = string.Empty; + public string FileName { get; set; } = string.Empty; + + public ulong FileSize { get; set; } + public long FileSizePos { get; set; } + public Type? FileSizeType { get; set; } + + public ulong ExtractSize { get; set; } // int + public long ExtractSizePos { get; set; } + public Type? ExtractSizeType { get; set; } + + public ulong FileOffset { get; set; } + public long FileOffsetPos { get; set; } + public Type? FileOffsetType { get; set; } + + public ulong Offset { get; set; } + public int Id { get; set; } + public string UserString { get; set; } = string.Empty; + public ulong UpdateDateTime { get; set; } = 0; + public string LocalDir { get; set; } = string.Empty; + public string TocName { get; set; } = string.Empty; + + public bool Encrypted { get; set; } + + public string FileType { get; set; } = string.Empty; +} diff --git a/CpkTools/Model/Row.cs b/CpkTools/Model/Row.cs new file mode 100644 index 0000000..294ed48 --- /dev/null +++ b/CpkTools/Model/Row.cs @@ -0,0 +1,40 @@ +namespace CpkTools.Model; + +public record struct Row() { + public int Type { get; set; } = -1; + //column based datatypes + public byte UInt8 { get; set; } + public ushort UInt16 { get; set; } + public uint UInt32 { get; set; } + public ulong UInt64 { get; set; } + public float UFloat { get; set; } + public string Str { get; set; } = string.Empty; + public byte[] Data { get; set; } = []; + public long Position { get; set; } + + public object? GetValue() { + return Type switch { + 0 or 1 => UInt8, + 2 or 3 => UInt16, + 4 or 5 => UInt32, + 6 or 7 => UInt64, + 8 => UFloat, + 0xA => Str, + 0xB => Data, + _ => null + }; + } + + public new Type? GetType() { + return Type switch { + 0 or 1 => UInt8.GetType(), + 2 or 3 => UInt16.GetType(), + 4 or 5 => UInt32.GetType(), + 6 or 7 => UInt64.GetType(), + 8 => UFloat.GetType(), + 0xA => Str.GetType(), + 0xB => Data.GetType(), + _ => null + }; + } +} diff --git a/CpkTools/Model/Utf.cs b/CpkTools/Model/Utf.cs new file mode 100644 index 0000000..08a7b22 --- /dev/null +++ b/CpkTools/Model/Utf.cs @@ -0,0 +1,149 @@ +using CpkTools.Endian; + +namespace CpkTools.Model; + +public class Utf { + public List? Columns { get; private set; } + public Row[][]? Rows { get; private set; } + + private int _tableSize; + private long _rowsOffset; + private long _stringsOffset; + private long _dataOffset; + private int _tableName; + private short _numColumns; + private short _rowLength; + private int _numRows; + + public bool ReadUtf(Memory data) { + var reader = new EndianReader(data); + + return ReadUtf(reader); + } + + public bool ReadUtf(byte[] data) { + var reader = new EndianReader(data); + + return ReadUtf(reader); + } + + public bool ReadUtf(Stream stream) { + var reader = new EndianReader(stream); + + return ReadUtf(reader); + } + + private bool ReadUtf(EndianReader reader, bool leaveOpen = false) { + var offset = reader.Position; + + if (Tools.ReadCString(reader, 4) != "@UTF") + return false; + + _tableSize = reader.ReadInt32(); + _rowsOffset = reader.ReadInt32(); + _stringsOffset = reader.ReadInt32(); + _dataOffset = reader.ReadInt32(); + + // CPK Header & UTF Header are ignored, so add 8 to each offset + _rowsOffset += offset + 8; + _stringsOffset += offset + 8; + _dataOffset += offset + 8; + + _tableName = reader.ReadInt32(); + _numColumns = reader.ReadInt16(); + _rowLength = reader.ReadInt16(); + _numRows = reader.ReadInt32(); + + Columns = new List(_numColumns); + Rows = new Row[_numRows][]; + + // Read Columns + for (var i = 0; i < _numColumns; i ++) { + var column = new Column { + Flags = reader.ReadByte() + }; + + if (column.Flags == 0) { + reader.Seek(3, SeekOrigin.Current); + column.Flags = reader.ReadByte(); + } + + column.Name = Tools.ReadCString(reader, -1, reader.ReadInt32() + _stringsOffset); + Columns.Add(column); + } + + const int storageMask = (int)StorageFlags.StorageMask; + const int typeMask = (int)TypeFlags.TypeMask; + + // Read Rows + for (var y = 0; y < _numRows; y++) { + reader.Seek(_rowsOffset + (y * _rowLength), SeekOrigin.Begin); + + var currentEntry = new Row[_numColumns]; + + for (var x = 0; x < _numColumns; x++) { + var currentRow = new Row(); + var storageFlag = Columns[x].Flags & storageMask; + + switch (storageFlag) { + case (int)StorageFlags.StorageNone: + case (int)StorageFlags.StorageZero: + case (int)StorageFlags.StorageConstant: + currentEntry[x] = currentRow; + + continue; + } + + // 0x50 + currentRow.Type = Columns[x].Flags & typeMask; + currentRow.Position = reader.Position; + + switch (currentRow.Type) { + case 0 or 1: + currentRow.UInt8 = reader.ReadByte(); + + break; + case 2 or 3: + currentRow.UInt16 = reader.ReadUInt16(); + + break; + case 4 or 5: + currentRow.UInt32 = reader.ReadUInt32(); + + break; + case 6 or 7: + currentRow.UInt64 = reader.ReadUInt64(); + + break; + case 8: + currentRow.UFloat = reader.ReadSingle(); + + break; + case 0xA: + currentRow.Str = Tools.ReadCString(reader, -1, reader.ReadInt32() + _stringsOffset); + + break; + case 0xB: + var position = reader.ReadInt32() + _dataOffset; + + currentRow.Position = position; + currentRow.Data = Tools.GetData(reader, position, reader.ReadInt32()); + + break; + default: + throw new NotImplementedException(); + } + + currentEntry[x] = currentRow; + } + + Rows[y] = currentEntry; + } + + if (!leaveOpen) { + reader.Dispose(); + } + + return true; + } +} diff --git a/CpkTools/Tools.cs b/CpkTools/Tools.cs new file mode 100644 index 0000000..aadfc8a --- /dev/null +++ b/CpkTools/Tools.cs @@ -0,0 +1,60 @@ +using System.Text; +using CpkTools.Endian; + +namespace CpkTools; + +public static class Tools { + public static string ReadCString(EndianReader reader, int maxLength = -1, long lOffset = -1, Encoding? enc = null) { + enc ??= Encoding.GetEncoding(932); + + var max = maxLength == -1 ? 255 : maxLength; + var fTemp = reader.Position; + + if (lOffset >= 0) { + reader.Seek(lOffset, SeekOrigin.Begin); + } + + var length = 0; + + while (length < max && reader.ReadByte() != 0) { + length++; + } + + if (maxLength == -1) + max = length + 1; + else + max = maxLength; + + var initSeek = lOffset >= 0 ? lOffset : fTemp; + var returnSeek = lOffset >= 0 ? fTemp : fTemp + max; + + reader.Seek(initSeek, SeekOrigin.Begin); + + var bytes = reader.ReadBytes(length); + + reader.Seek(returnSeek, SeekOrigin.Begin); + + return enc.GetString(bytes); + } + + public static void DeleteFileIfExists(string sPath) { + if (File.Exists(sPath)) + File.Delete(sPath); + } + + public static string GetPath(string input) { + return Path.GetDirectoryName(input) + Path.DirectorySeparatorChar + Path.GetFileNameWithoutExtension(input); + } + + public static byte[] GetData(EndianReader reader, long offset, int size) { + var backup = reader.Position; + + reader.Seek(offset, SeekOrigin.Begin); + + var result = reader.ReadBytes(size); + + reader.Seek(backup, SeekOrigin.Begin); + + return result; + } +} diff --git a/ShinRyuModManager-CE.slnx b/ShinRyuModManager-CE.slnx index 7bf0218..f6ddc31 100644 --- a/ShinRyuModManager-CE.slnx +++ b/ShinRyuModManager-CE.slnx @@ -2,6 +2,7 @@ + diff --git a/ShinRyuModManager-CE/ModLoadOrder/CpkPatcher.cs b/ShinRyuModManager-CE/ModLoadOrder/CpkPatcher.cs index b744526..0d17f0a 100644 --- a/ShinRyuModManager-CE/ModLoadOrder/CpkPatcher.cs +++ b/ShinRyuModManager-CE/ModLoadOrder/CpkPatcher.cs @@ -1,3 +1,5 @@ +using CpkTools.Endian; +using CpkTools.Model; using Serilog; using Utils; @@ -18,31 +20,93 @@ public static async Task RepackDictionary(Dictionary> cpkDi Directory.CreateDirectory(cpkPath); foreach (var kvp in cpkDict) { - var cpkDir = cpkPath + kvp.Key; + var key = kvp.Key.Trim(Path.DirectorySeparatorChar); + + var cpkDir = Path.Combine(cpkPath, key); string origCpk; - if (!kvp.Key.Contains(".cpk")) { - origCpk = GamePath.DataPath + kvp.Key + ".cpk"; + if (!key.Contains(".cpk")) { + origCpk = Path.Combine(GamePath.DataPath, $"{key}.cpk"); } else { - origCpk = GamePath.DataPath + kvp.Key; + origCpk = Path.Combine(GamePath.DataPath, key); } if (!Directory.Exists(cpkDir)) Directory.CreateDirectory(cpkDir); foreach (var mod in kvp.Value) { - var modCpkDir = Path.Combine(GamePath.ModsPath, mod).Replace(".cpk", ""); - var cpkFiles = Directory.GetFiles(modCpkDir, "*."); + //var modCpkDir = Path.Combine(GamePath.ModsPath, mod).Replace(".cpk", ""); + var searchDir = Path.Combine(GamePath.ModsPath, mod, key); + var cpkFiles = Directory.EnumerateFiles(searchDir, "*.", SearchOption.AllDirectories); foreach (var file in cpkFiles) { File.Copy(file, Path.Combine(cpkDir, Path.GetFileName(file)), true); } } - CriPakTools.Program.Modify(origCpk, cpkDir, new DirectoryInfo(cpkDir).FullName + ".cpk"); + Modify(origCpk, cpkDir, new DirectoryInfo(cpkDir).FullName + ".cpk"); } await Task.CompletedTask; } + + private static void Modify(string inputCpk, string replaceDir, string outputCpk) { + var cpk = new Cpk(); + + cpk.ReadCpk(inputCpk); + + using var oldFile = new EndianReader(File.OpenRead(inputCpk), true); + + var files = Directory.EnumerateFiles(replaceDir, "*."); + var fileNames = new HashSet(); + + foreach (var str in files.Select(Path.GetFileNameWithoutExtension)) { + fileNames.Add(str); + } + + using var newCpk = new EndianWriter(File.OpenWrite(outputCpk), true); + + var entries = cpk.FileTable.OrderBy(x => x.FileOffset); + + foreach (var entry in entries) { + if (entry.FileType == "FILE" && (ulong)newCpk.Position < cpk.ContentOffset) { + var padLength = cpk.ContentOffset - (ulong)newCpk.Position; + + newCpk.Write(new byte[padLength]); + } + + if (!fileNames.Contains(entry.FileName)) { + oldFile.Seek((long)entry.FileOffset, SeekOrigin.Begin); + + entry.FileOffset = (ulong)newCpk.Position; + cpk.UpdateFileEntry(entry); + + _ = oldFile.ReadStreamInto(newCpk.BaseStream, (int)entry.FileSize); + } else { + using var newbie = File.OpenRead(Path.Combine(replaceDir, entry.FileName)); + + entry.FileOffset = (ulong)newCpk.Position; + entry.FileSize = (ulong)newbie.Length; + entry.ExtractSize = (ulong)newbie.Length; + cpk.UpdateFileEntry(entry); + + newCpk.Write(newbie); + } + + if ((newCpk.Position % 0x800) > 0) { + var padding = (int)(0x800 - (newCpk.Position % 0x800)); + + newCpk.Write(new byte[padding]); + } + } + + cpk.WriteCpk(newCpk); + cpk.WriteItoc(newCpk); + cpk.WriteToc(newCpk); + cpk.WriteEtoc(newCpk); + cpk.WriteGtoc(newCpk); + + Log.Information("Writing {FileName}", Path.GetFileName(inputCpk)); + } } diff --git a/ShinRyuModManager-CE/ModLoadOrder/Dependency/CPKGen/CPK.cs b/ShinRyuModManager-CE/ModLoadOrder/Dependency/CPKGen/CPK.cs deleted file mode 100644 index c51af5b..0000000 --- a/ShinRyuModManager-CE/ModLoadOrder/Dependency/CPKGen/CPK.cs +++ /dev/null @@ -1,1046 +0,0 @@ -using System.Text; -using Serilog; - -namespace CriPakTools { - public class CPK(Tools tool) { - public readonly List FileTable = new(); - public Dictionary Cpkdata; - public UTF Utf; - - private UTF _files; - - public bool ReadCPK(string sPath) { - if (!File.Exists(sPath)) - return false; - - uint Files; - ushort Align; - - var br = new EndianReader(File.OpenRead(sPath), true); - - if (Tools.ReadCString(br, 4) != "CPK ") { - br.Close(); - - return false; - } - - ReadUTFData(br); - - CpkPacket = UtfPacket; - //Dump CPK - //File.WriteAllBytes("U_CPK", CPK_packet); - - var cpakEntry = new FileEntry { - FileName = "CPK_HDR", - FileOffsetPos = br.BaseStream.Position + 0x10, - FileSize = CpkPacket.Length, - Encrypted = IsUtfEncrypted, - FileType = "CPK" - }; - - FileTable.Add(cpakEntry); - - var ms = new MemoryStream(UtfPacket); - var utfr = new EndianReader(ms, false); - - Utf = new UTF(tool); - - if (!Utf.ReadUTF(utfr)) { - br.Close(); - - return false; - } - - utfr.Close(); - ms.Close(); - - Cpkdata = new Dictionary(); - - try { - for (var i = 0; i < Utf.Columns.Count; i++) { - Cpkdata.Add(Utf.Columns[i].Name, Utf.Rows[0].Rows[i].GetValue()); - } - } catch (Exception ex) { - //MessageBox.Show(ex.ToString()); - Log.Error(ex, ""); - } - - TocOffset = (ulong)GetColumnsData2(Utf, 0, "TocOffset", 3); - var TocOffsetPos = GetColumnPosition(Utf, 0, "TocOffset"); - - EtocOffset = (ulong)GetColumnsData2(Utf, 0, "EtocOffset", 3); - var ETocOffsetPos = GetColumnPosition(Utf, 0, "EtocOffset"); - - ItocOffset = (ulong)GetColumnsData2(Utf, 0, "ItocOffset", 3); - var ITocOffsetPos = GetColumnPosition(Utf, 0, "ItocOffset"); - - GtocOffset = (ulong)GetColumnsData2(Utf, 0, "GtocOffset", 3); - var GTocOffsetPos = GetColumnPosition(Utf, 0, "GtocOffset"); - - ContentOffset = (ulong)GetColumnsData2(Utf, 0, "ContentOffset", 3); - var ContentOffsetPos = GetColumnPosition(Utf, 0, "ContentOffset"); - - FileTable.Add(CreateFileEntry("CONTENT_OFFSET", ContentOffset, typeof(ulong), ContentOffsetPos, "CPK", - "CONTENT", false)); - - Files = (uint)GetColumnsData2(Utf, 0, "Files", 2); - Align = (ushort)GetColumnsData2(Utf, 0, "Align", 1); - - if (TocOffset != 0xFFFFFFFFFFFFFFFF) { - var entry = CreateFileEntry("TOC_HDR", TocOffset, typeof(ulong), TocOffsetPos, "CPK", "HDR", - false); - - FileTable.Add(entry); - - if (!ReadTOC(br, TocOffset, ContentOffset)) - return false; - } - - if (EtocOffset != 0xFFFFFFFFFFFFFFFF) { - var entry = CreateFileEntry("ETOC_HDR", EtocOffset, typeof(ulong), ETocOffsetPos, "CPK", - "HDR", false); - - FileTable.Add(entry); - - if (!ReadETOC(br, EtocOffset)) - return false; - } - - if (ItocOffset != 0xFFFFFFFFFFFFFFFF) { - //FileEntry ITOC_entry = new FileEntry { - // FileName = "ITOC_HDR", - // FileOffset = ItocOffset, FileOffsetType = typeof(ulong), FileOffsetPos = ITocOffsetPos, - // TOCName = "CPK", - // FileType = "FILE", Encrypted = true, - //}; - - var entry = CreateFileEntry("ITOC_HDR", ItocOffset, typeof(ulong), ITocOffsetPos, "CPK", - "HDR", false); - - FileTable.Add(entry); - - if (!ReadITOC(br, ItocOffset, ContentOffset, Align)) - return false; - } - - if (GtocOffset != 0xFFFFFFFFFFFFFFFF) { - var entry = CreateFileEntry("GTOC_HDR", GtocOffset, typeof(ulong), GTocOffsetPos, "CPK", - "HDR", false); - - FileTable.Add(entry); - - if (!ReadGTOC(br, GtocOffset)) - return false; - } - - br.Close(); - - // at this point, we should have all needed file info - - //utf = null; - _files = null; - - return true; - } - - private static FileEntry CreateFileEntry(string fileName, ulong fileOffset, Type fileOffsetType, - long fileOffsetPos, string tocName, string fileType, bool encrypted) { - var entry = new FileEntry { - FileName = fileName, - FileOffset = fileOffset, - FileOffsetType = fileOffsetType, - FileOffsetPos = fileOffsetPos, - TOCName = tocName, - FileType = fileType, - Encrypted = encrypted, - }; - - return entry; - } - - public bool ReadTOC(EndianReader br, ulong tocOffset, ulong contentOffset) { - var addOffset = contentOffset < tocOffset ? contentOffset : tocOffset; - - br.BaseStream.Seek((long)tocOffset, SeekOrigin.Begin); - - if (Tools.ReadCString(br, 4) != "TOC ") { - br.Close(); - - return false; - } - - ReadUTFData(br); - - // Store unencrypted TOC - TocPacket = UtfPacket; - //Dump TOC - //File.WriteAllBytes("U_TOC", TOC_packet); - - var tocEntry = FileTable.Single(x => x.FileName.ToString() == "TOC_HDR"); - tocEntry.Encrypted = IsUtfEncrypted; - tocEntry.FileSize = TocPacket.Length; - - var ms = new MemoryStream(UtfPacket); - var utfr = new EndianReader(ms, false); - - _files = new UTF(tool); - - if (!_files.ReadUTF(utfr)) { - br.Close(); - - return false; - } - - utfr.Close(); - ms.Close(); - - for (var i = 0; i < _files.NumRows; i++) { - var temp = new FileEntry { - TOCName = "TOC", - DirName = GetColumnData(_files, i, "DirName"), - FileName = GetColumnData(_files, i, "FileName"), - FileSize = GetColumnData(_files, i, "FileSize"), - FileSizePos = GetColumnPosition(_files, i, "FileSize"), - FileSizeType = GetColumnType(_files, i, "FileSize"), - ExtractSize = GetColumnData(_files, i, "ExtractSize"), - ExtractSizePos = GetColumnPosition(_files, i, "ExtractSize"), - ExtractSizeType = GetColumnType(_files, i, "ExtractSize"), - FileOffset = ((ulong)GetColumnData(_files, i, "FileOffset") + addOffset), - FileOffsetPos = GetColumnPosition(_files, i, "FileOffset"), - FileOffsetType = GetColumnType(_files, i, "FileOffset"), - FileType = "FILE", - Offset = addOffset, - ID = GetColumnData(_files, i, "ID"), - UserString = GetColumnData(_files, i, "UserString") - }; - - FileTable.Add(temp); - } - - _files = null; - - return true; - } - - public void WriteCPK(BinaryWriter cpk) { - WritePacket(cpk, "CPK ", 0, CpkPacket); - - cpk.BaseStream.Seek(0x800 - 6, SeekOrigin.Begin); - cpk.Write("(c)CRI"u8.ToArray()); - } - - public void WriteTOC(BinaryWriter cpk) { - WritePacket(cpk, "TOC ", TocOffset, TocPacket); - } - - public void WriteITOC(BinaryWriter cpk) { - WritePacket(cpk, "ITOC", ItocOffset, ItocPacket); - } - - public void WriteETOC(BinaryWriter cpk) { - WritePacket(cpk, "ETOC", EtocOffset, EtocPacket); - } - - public void WriteGTOC(BinaryWriter cpk) { - WritePacket(cpk, "GTOC", GtocOffset, GtocPacket); - } - - public void WritePacket(BinaryWriter cpk, string id, ulong position, byte[] packet) { - if (position == 0xffffffffffffffff) - return; - - cpk.BaseStream.Seek((long)position, SeekOrigin.Begin); - var encrypted = DecryptUTF(packet); // Yes it says decrypt... - cpk.Write(Encoding.ASCII.GetBytes(id)); - cpk.Write(0); - cpk.Write((ulong)encrypted.Length); - cpk.Write(encrypted); - } - - public bool ReadITOC(EndianReader br, ulong startOffset, ulong contentOffset, ushort align) { - br.BaseStream.Seek((long)startOffset, SeekOrigin.Begin); - - if (Tools.ReadCString(br, 4) != "ITOC") { - br.Close(); - - return false; - } - - ReadUTFData(br); - - ItocPacket = UtfPacket; - //Dump ITOC - //File.WriteAllBytes("U_ITOC", ITOC_packet); - - var itocEntry = FileTable.Single(x => x.FileName.ToString() == "ITOC_HDR"); - itocEntry.Encrypted = IsUtfEncrypted; - itocEntry.FileSize = ItocPacket.Length; - - var ms = new MemoryStream(UtfPacket); - var utfr = new EndianReader(ms, false); - - _files = new UTF(tool); - - if (!_files.ReadUTF(utfr)) { - br.Close(); - - return false; - } - - utfr.Close(); - ms.Close(); - - //uint FilesL = (uint)GetColumnData(files, 0, "FilesL"); - //uint FilesH = (uint)GetColumnData(files, 0, "FilesH"); - var dataL = (byte[])GetColumnData(_files, 0, "DataL"); - var dataLPos = GetColumnPosition(_files, 0, "DataL"); - - var dataH = (byte[])GetColumnData(_files, 0, "DataH"); - var dataHPos = GetColumnPosition(_files, 0, "DataH"); - - //MemoryStream ms; - //EndianReader ir; - UTF utfDataL, utfDataH; - - var IDs = new List(); - - var SizeTable = new Dictionary(); - var SizePosTable = new Dictionary(); - var SizeTypeTable = new Dictionary(); - - var CSizeTable = new Dictionary(); - var CSizePosTable = new Dictionary(); - var CSizeTypeTable = new Dictionary(); - - ushort ID; - long pos; - Type type; - - if (dataL != null) { - ms = new MemoryStream(dataL); - utfr = new EndianReader(ms, false); - utfDataL = new UTF(tool); - utfDataL.ReadUTF(utfr); - - for (var i = 0; i < utfDataL.NumRows; i++) { - ID = (ushort)GetColumnData(utfDataL, i, "ID"); - var size1 = (ushort)GetColumnData(utfDataL, i, "FileSize"); - SizeTable.Add(ID, size1); - - pos = GetColumnPosition(utfDataL, i, "FileSize"); - SizePosTable.Add(ID, pos + dataLPos); - - type = GetColumnType(utfDataL, i, "FileSize"); - SizeTypeTable.Add(ID, type); - - if ((GetColumnData(utfDataL, i, "ExtractSize")) != null) { - size1 = (ushort)GetColumnData(utfDataL, i, "ExtractSize"); - CSizeTable.Add(ID, size1); - - pos = GetColumnPosition(utfDataL, i, "ExtractSize"); - CSizePosTable.Add(ID, pos + dataLPos); - - type = GetColumnType(utfDataL, i, "ExtractSize"); - CSizeTypeTable.Add(ID, type); - } - - IDs.Add(ID); - } - } - - if (dataH != null) { - ms = new MemoryStream(dataH); - utfr = new EndianReader(ms, false); - utfDataH = new UTF(tool); - utfDataH.ReadUTF(utfr); - - for (var i = 0; i < utfDataH.NumRows; i++) { - ID = (ushort)GetColumnData(utfDataH, i, "ID"); - var size2 = (uint)GetColumnData(utfDataH, i, "FileSize"); - SizeTable.Add(ID, size2); - - pos = GetColumnPosition(utfDataH, i, "FileSize"); - SizePosTable.Add(ID, pos + dataHPos); - - type = GetColumnType(utfDataH, i, "FileSize"); - SizeTypeTable.Add(ID, type); - - if ((GetColumnData(utfDataH, i, "ExtractSize")) != null) { - size2 = (uint)GetColumnData(utfDataH, i, "ExtractSize"); - CSizeTable.Add(ID, size2); - - pos = GetColumnPosition(utfDataH, i, "ExtractSize"); - CSizePosTable.Add(ID, pos + dataHPos); - - type = GetColumnType(utfDataH, i, "ExtractSize"); - CSizeTypeTable.Add(ID, type); - } - - IDs.Add(ID); - } - } - - //int id = 0; - var baseoffset = contentOffset; - - // Seems ITOC can mix up the IDs..... but they'll alwaysy be in order... - IDs = IDs.OrderBy(x => x).ToList(); - - foreach (var id in IDs) { - var temp = new FileEntry(); - SizeTable.TryGetValue(id, out var value); - CSizeTable.TryGetValue(id, out var value2); - - temp.TOCName = "ITOC"; - - temp.DirName = null; - temp.FileName = id.ToString("D4"); - - temp.FileSize = value; - temp.FileSizePos = SizePosTable[id]; - temp.FileSizeType = SizeTypeTable[id]; - - if (CSizeTable.Count > 0 && CSizeTable.ContainsKey(id)) { - temp.ExtractSize = value2; - temp.ExtractSizePos = CSizePosTable[id]; - temp.ExtractSizeType = CSizeTypeTable[id]; - } - - temp.FileType = "FILE"; - - temp.FileOffset = baseoffset; - temp.ID = id; - temp.UserString = null; - - FileTable.Add(temp); - - if ((value % align) > 0) - baseoffset += value + (align - (value % align)); - else - baseoffset += value; - } - - _files = null; - utfDataL = null; - utfDataH = null; - - ms.Close(); - utfr.Close(); - - return true; - } - - private void ReadUTFData(EndianReader br) { - IsUtfEncrypted = false; - br.IsLittleEndian = true; - - Unk1 = br.ReadInt32(); - UtfSize = br.ReadInt64(); - UtfPacket = br.ReadBytes((int)UtfSize); - - if (UtfPacket[0] != 0x40 && UtfPacket[1] != 0x55 && UtfPacket[2] != 0x54 && UtfPacket[3] != 0x46) //@UTF - { - UtfPacket = DecryptUTF(UtfPacket); - IsUtfEncrypted = true; - } - - br.IsLittleEndian = false; - } - - public bool ReadGTOC(EndianReader br, ulong startoffset) { - br.BaseStream.Seek((long)startoffset, SeekOrigin.Begin); - - if (Tools.ReadCString(br, 4) != "GTOC") { - br.Close(); - - return false; - } - - br.BaseStream.Seek(0xC, SeekOrigin.Current); //skip header data - - return true; - } - - public bool ReadETOC(EndianReader br, ulong startoffset) { - br.BaseStream.Seek((long)startoffset, SeekOrigin.Begin); - - if (Tools.ReadCString(br, 4) != "ETOC") { - br.Close(); - - return false; - } - - //br.BaseStream.Seek(0xC, SeekOrigin.Current); //skip header data - - ReadUTFData(br); - - EtocPacket = UtfPacket; - //Dump ETOC - //File.WriteAllBytes("U_ETOC", ETOC_packet); - - var etocEntry = FileTable.Single(x => x.FileName.ToString() == "ETOC_HDR"); - etocEntry.Encrypted = IsUtfEncrypted; - etocEntry.FileSize = EtocPacket.Length; - - var ms = new MemoryStream(UtfPacket); - var utfr = new EndianReader(ms, false); - - _files = new UTF(tool); - - if (!_files.ReadUTF(utfr)) { - br.Close(); - - return false; - } - - utfr.Close(); - ms.Close(); - - var fileEntries = FileTable.Where(x => x.FileType == "FILE").ToList(); - - for (var i = 0; i < fileEntries.Count; i++) { - FileTable[i].LocalDir = GetColumnData(_files, i, "LocalDir"); - } - - return true; - } - - public byte[] DecryptUTF(byte[] input) { - var result = new byte[input.Length]; - - var m = 0x0000655f; - const int t = 0x00004115; - - for (var i = 0; i < input.Length; i++) { - var d = input[i]; - d = (byte)(d ^ (byte)(m & 0xff)); - result[i] = d; - m *= t; - } - - return result; - } - - public byte[] DecompressCRILAYLA(byte[] input, int uSize) { - var ms = new MemoryStream(input); - var br = new EndianReader(ms, true); - - br.BaseStream.Seek(8, SeekOrigin.Begin); // Skip CRILAYLA - var uncompressedSize = br.ReadInt32(); - var uncompressedHeaderOffset = br.ReadInt32(); - var result = new byte[uncompressedSize + 0x100]; // = new byte[USize]; - // do some error checks here......... - // copy uncompressed 0x100 header to start of file - Array.Copy(input, uncompressedHeaderOffset + 0x10, result, 0, 0x100); - - var inputEnd = input.Length - 0x100 - 1; - var inputOffset = inputEnd; - var outputEnd = 0x100 + uncompressedSize - 1; - byte bitPool = 0; - int bitsLeft = 0, bytesOutput = 0; - var vleLens = new int[4] { 2, 3, 5, 8 }; - - while (bytesOutput < uncompressedSize) { - if (GenNextBits(input, ref inputOffset, ref bitPool, ref bitsLeft, 1) > 0) { - var backreferenceOffset = outputEnd - bytesOutput + - GenNextBits(input, ref inputOffset, ref bitPool, ref bitsLeft, 13) + 3; - - var backreferenceLength = 3; - int vleLevel; - - for (vleLevel = 0; vleLevel < vleLens.Length; vleLevel++) { - int thisLevel = GenNextBits(input, ref inputOffset, ref bitPool, ref bitsLeft, - vleLens[vleLevel]); - - backreferenceLength += thisLevel; - - if (thisLevel != ((1 << vleLens[vleLevel]) - 1)) - break; - } - - if (vleLevel == vleLens.Length) { - int thisLevel; - - do { - thisLevel = GenNextBits(input, ref inputOffset, ref bitPool, ref bitsLeft, 8); - backreferenceLength += thisLevel; - } while (thisLevel == 255); - } - - for (var i = 0; i < backreferenceLength; i++) { - result[outputEnd - bytesOutput] = result[backreferenceOffset--]; - bytesOutput++; - } - } else { - // verbatim byte - result[outputEnd - bytesOutput] = - (byte)GenNextBits(input, ref inputOffset, ref bitPool, ref bitsLeft, 8); - - bytesOutput++; - } - } - - br.Close(); - ms.Close(); - - return result; - } - - private static ushort GenNextBits(byte[] input, ref int offsetP, ref byte bitPoolP, ref int bitsLeftP, - int bitCount) { - ushort outBits = 0; - var numBitsProduced = 0; - - while (numBitsProduced < bitCount) { - if (bitsLeftP == 0) { - bitPoolP = input[offsetP]; - bitsLeftP = 8; - offsetP--; - } - - int bitsThisRound; - - if (bitsLeftP > (bitCount - numBitsProduced)) - bitsThisRound = bitCount - numBitsProduced; - else - bitsThisRound = bitsLeftP; - - outBits <<= bitsThisRound; - - outBits |= (ushort)((ushort)(bitPoolP >> (bitsLeftP - bitsThisRound)) & - ((1 << bitsThisRound) - 1)); - - bitsLeftP -= bitsThisRound; - numBitsProduced += bitsThisRound; - } - - return outBits; - } - - public object GetColumnsData2(UTF utf, int row, string name, int type) { - var temp = GetColumnData(utf, row, name); - - if (temp == null) { - switch (type) { - case 0: // byte - return (byte)0xFF; - case 1: // short - return (ushort)0xFFFF; - case 2: // int - return 0xFFFFFFFF; - case 3: // long - return 0xFFFFFFFFFFFFFFFF; - } - } - - if (temp is ulong tempUlong) { - return tempUlong; - } - - if (temp is uint tempUint) { - return tempUint; - } - - if (temp is ushort tempUshort) { - return tempUshort; - } - - return 0; - } - - public static object GetColumnData(UTF utf, int row, string name) { - object result = null; - - try { - for (var i = 0; i < utf.NumColumns; i++) { - if (utf.Columns[i].Name != name) - continue; - - result = utf.Rows[row].Rows[i].GetValue(); - - break; - } - } catch (Exception ex) { - Log.Error(ex, ""); - - return null; - } - - return result; - } - - public static long GetColumnPosition(UTF utf, int row, string name) { - long result = -1; - - try { - for (var i = 0; i < utf.NumColumns; i++) { - if (utf.Columns[i].Name != name) - continue; - - result = utf.Rows[row].Rows[i].Position; - - break; - } - } catch (Exception ex) { - Log.Error(ex, ""); - - return -1; - } - - return result; - } - - public static Type GetColumnType(UTF utf, int row, string name) { - Type result = null; - - try { - for (var i = 0; i < utf.NumColumns; i++) { - if (utf.Columns[i].Name != name) - continue; - - result = utf.Rows[row].Rows[i].GetType(); - - break; - } - } catch (Exception ex) { - Log.Error(ex, ""); - - return null; - } - - return result; - } - - public void UpdateFileEntry(FileEntry fileEntry) { - if (fileEntry.FileType is not ("FILE" or "HDR")) - return; - - var updateMe = fileEntry.TOCName switch { - "CPK" => CpkPacket, - "TOC" => TocPacket, - "ITOC" => ItocPacket, - "ETOC" => EtocPacket, - _ => throw new Exception("I need to implement this TOC!") - }; - - //Update ExtractSize - if (fileEntry.ExtractSizePos > 0) - UpdateValue(ref updateMe, fileEntry.ExtractSize, fileEntry.ExtractSizePos, - fileEntry.ExtractSizeType); - - //Update FileSize - if (fileEntry.FileSizePos > 0) - UpdateValue(ref updateMe, fileEntry.FileSize, fileEntry.FileSizePos, fileEntry.FileSizeType); - - //Update FileOffset - if (fileEntry.FileOffsetPos > 0) - UpdateValue(ref updateMe, fileEntry.FileOffset - (ulong)((fileEntry.TOCName == "TOC") ? 0x800 : 0), - fileEntry.FileOffsetPos, fileEntry.FileOffsetType); - - switch (fileEntry.TOCName) { - case "CPK": - updateMe = CpkPacket; - - break; - case "TOC": - TocPacket = updateMe; - - break; - case "ITOC": - ItocPacket = updateMe; - - break; - case "ETOC": - updateMe = EtocPacket; - - break; - default: - throw new Exception("I need to implement this TOC!"); - } - } - - public static void UpdateValue(ref byte[] packet, object value, long pos, Type type) { - var temp = new MemoryStream(); - temp.Write(packet, 0, packet.Length); - - var toc = new EndianWriter(temp, false); - toc.Seek((int)pos, SeekOrigin.Begin); - - value = Convert.ChangeType(value, type); - - if (type == typeof(byte)) { - toc.Write((byte)value); - } else if (type == typeof(ushort)) { - toc.Write((ushort)value); - } else if (type == typeof(uint)) { - toc.Write((uint)value); - } else if (type == typeof(ulong)) { - toc.Write((ulong)value); - } else if (type == typeof(float)) { - toc.Write((float)value); - } else { - throw new Exception("Not supported type!"); - } - - toc.Close(); - - var myStream = (MemoryStream)toc.BaseStream; - packet = myStream.ToArray(); - } - - public bool IsUtfEncrypted { get; set; } - public int Unk1 { get; set; } - public long UtfSize { get; set; } - public byte[] UtfPacket { get; set; } - - public byte[] CpkPacket { get; set; } - public byte[] TocPacket { get; set; } - public byte[] ItocPacket { get; set; } - public byte[] EtocPacket { get; set; } - public byte[] GtocPacket { get; set; } - - public ulong TocOffset, EtocOffset, ItocOffset, GtocOffset, ContentOffset; - } - - public class UTF { - public enum COLUMN_FLAGS : int { - STORAGE_MASK = 0xf0, - STORAGE_NONE = 0x00, - STORAGE_ZERO = 0x10, - STORAGE_CONSTANT = 0x30, - STORAGE_PERROW = 0x50, - - TYPE_MASK = 0x0f, - TYPE_DATA = 0x0b, - TYPE_STRING = 0x0a, - TYPE_FLOAT = 0x08, - TYPE_8BYTE2 = 0x07, - TYPE_8BYTE = 0x06, - TYPE_4BYTE2 = 0x05, - TYPE_4BYTE = 0x04, - TYPE_2BYTE2 = 0x03, - TYPE_2BYTE = 0x02, - TYPE_1BYTE2 = 0x01, - TYPE_1BYTE = 0x00, - } - - public List Columns; - public List Rows; - - Tools tools; - - public UTF(Tools tool) { - tools = tool; - } - - public bool ReadUTF(EndianReader br) { - var offset = br.BaseStream.Position; - - if (Tools.ReadCString(br, 4) != "@UTF") { - return false; - } - - TableSize = br.ReadInt32(); - RowsOffset = br.ReadInt32(); - StringsOffset = br.ReadInt32(); - DataOffset = br.ReadInt32(); - - // CPK Header & UTF Header are ignored, so add 8 to each offset - RowsOffset += (offset + 8); - StringsOffset += (offset + 8); - DataOffset += (offset + 8); - - TableName = br.ReadInt32(); - NumColumns = br.ReadInt16(); - RowLength = br.ReadInt16(); - NumRows = br.ReadInt32(); - - //read Columns - Columns = []; - - for (var i = 0; i < NumColumns; i++) { - var column = new COLUMN { - Flags = br.ReadByte() - }; - - if (column.Flags == 0) { - br.BaseStream.Seek(3, SeekOrigin.Current); - column.Flags = br.ReadByte(); - } - - column.Name = Tools.ReadCString(br, -1, br.ReadInt32() + StringsOffset); - Columns.Add(column); - } - - //read Rows - - Rows = []; - - for (var j = 0; j < NumRows; j++) { - br.BaseStream.Seek(RowsOffset + (j * RowLength), SeekOrigin.Begin); - - var currentEntry = new ROWS(); - - for (var i = 0; i < NumColumns; i++) { - var currentRow = new ROW(); - var storageFlag = (Columns[i].Flags & (int)COLUMN_FLAGS.STORAGE_MASK); - - switch (storageFlag) { - case (int)COLUMN_FLAGS.STORAGE_NONE: // 0x00 - case (int)COLUMN_FLAGS.STORAGE_ZERO: // 0x10 - case (int)COLUMN_FLAGS.STORAGE_CONSTANT: // 0x30 - currentEntry.Rows.Add(currentRow); - - continue; - } - - // 0x50 - currentRow.Type = Columns[i].Flags & (int)COLUMN_FLAGS.TYPE_MASK; - - currentRow.Position = br.BaseStream.Position; - - switch (currentRow.Type) { - case 0: - case 1: - currentRow.Uint8 = br.ReadByte(); - - break; - - case 2: - case 3: - currentRow.Uint16 = br.ReadUInt16(); - - break; - - case 4: - case 5: - currentRow.Uint32 = br.ReadUInt32(); - - break; - - case 6: - case 7: - currentRow.Uint64 = br.ReadUInt64(); - - break; - - case 8: - currentRow.Ufloat = br.ReadSingle(); - - break; - - case 0xA: - currentRow.Str = Tools.ReadCString(br, -1, br.ReadInt32() + StringsOffset); - - break; - - case 0xB: - var position = br.ReadInt32() + DataOffset; - currentRow.Position = position; - currentRow.Data = Tools.GetData(br, position, br.ReadInt32()); - - break; - - default: - throw new NotImplementedException(); - } - - currentEntry.Rows.Add(currentRow); - } - - Rows.Add(currentEntry); - } - - return true; - } - - public int TableSize { get; set; } - - public long RowsOffset { get; set; } - public long StringsOffset { get; set; } - public long DataOffset { get; set; } - public int TableName { get; set; } - public short NumColumns { get; set; } - public short RowLength { get; set; } - public int NumRows { get; set; } - } - - public class COLUMN { - public COLUMN() { } - - public byte Flags { get; set; } - public string Name { get; set; } - } - - public class ROWS { - public readonly List Rows = []; - } - - public class ROW { - public int Type { get; set; } = -1; - - public object GetValue() { - return Type switch { - 0 or 1 => Uint8, - 2 or 3 => Uint16, - 4 or 5 => Uint32, - 6 or 7 => Uint64, - 8 => Ufloat, - 0xA => Str, - 0xB => Data, - _ => null - }; - } - - public new Type GetType() { - return Type switch { - 0 or 1 => Uint8.GetType(), - 2 or 3 => Uint16.GetType(), - 4 or 5 => Uint32.GetType(), - 6 or 7 => Uint64.GetType(), - 8 => Ufloat.GetType(), - 0xA => Str.GetType(), - 0xB => Data.GetType(), - _ => null - }; - } - - //column based datatypes - public byte Uint8 { get; set; } - public ushort Uint16 { get; set; } - public uint Uint32 { get; set; } - public ulong Uint64 { get; set; } - public float Ufloat { get; set; } - public string Str { get; set; } - public byte[] Data { get; set; } - public long Position { get; set; } - } - - public class FileEntry { - public object DirName { get; set; } // string - public object FileName { get; set; } // string - - public object FileSize { get; set; } - public long FileSizePos { get; set; } - public Type FileSizeType { get; set; } - - public object ExtractSize { get; set; } // int - public long ExtractSizePos { get; set; } - public Type ExtractSizeType { get; set; } - - public ulong FileOffset { get; set; } - public long FileOffsetPos { get; set; } - public Type FileOffsetType { get; set; } - - public ulong Offset { get; set; } - public object ID { get; set; } // int - public object UserString { get; set; } // string - public ulong UpdateDateTime { get; set; } = 0; - public object LocalDir { get; set; } // string - public string TOCName { get; set; } - - public bool Encrypted { get; set; } - - public string FileType { get; set; } - } -} diff --git a/ShinRyuModManager-CE/ModLoadOrder/Dependency/CPKGen/Endian.cs b/ShinRyuModManager-CE/ModLoadOrder/Dependency/CPKGen/Endian.cs deleted file mode 100644 index 480676d..0000000 --- a/ShinRyuModManager-CE/ModLoadOrder/Dependency/CPKGen/Endian.cs +++ /dev/null @@ -1,155 +0,0 @@ -using System.Text; - -namespace CriPakTools { - public class EndianReader(Stream input, Encoding encoding, bool isLittleEndian) : BinaryReader(input, encoding) { - private readonly byte[] _buffer = new byte[8]; - - public EndianReader(Stream input, bool isLittleEndian) - : this(input, Encoding.UTF8, isLittleEndian) { } - - public bool IsLittleEndian { get; set; } = isLittleEndian; - - public override double ReadDouble() { - if (IsLittleEndian) - return base.ReadDouble(); - - FillMyBuffer(8); - - return BitConverter.ToDouble(_buffer.Take(8).Reverse().ToArray(), 0); - } - - public override short ReadInt16() { - if (IsLittleEndian) - return base.ReadInt16(); - - FillMyBuffer(2); - - return BitConverter.ToInt16(_buffer.Take(2).Reverse().ToArray(), 0); - } - - public override int ReadInt32() { - if (IsLittleEndian) - return base.ReadInt32(); - - FillMyBuffer(4); - - return BitConverter.ToInt32(_buffer.Take(4).Reverse().ToArray(), 0); - } - - public override long ReadInt64() { - if (IsLittleEndian) - return base.ReadInt64(); - - FillMyBuffer(8); - - return BitConverter.ToInt64(_buffer.Take(8).Reverse().ToArray(), 0); - } - - public override float ReadSingle() { - if (IsLittleEndian) - return base.ReadSingle(); - - FillMyBuffer(4); - - return BitConverter.ToSingle(_buffer.Take(4).Reverse().ToArray(), 0); - } - - public override ushort ReadUInt16() { - if (IsLittleEndian) - return base.ReadUInt16(); - - FillMyBuffer(2); - - return BitConverter.ToUInt16(_buffer.Take(2).Reverse().ToArray(), 0); - } - - public override uint ReadUInt32() { - if (IsLittleEndian) - return base.ReadUInt32(); - - FillMyBuffer(4); - - return BitConverter.ToUInt32(_buffer.Take(4).Reverse().ToArray(), 0); - } - - public override ulong ReadUInt64() { - if (IsLittleEndian) - return base.ReadUInt64(); - - FillMyBuffer(8); - - return BitConverter.ToUInt64(_buffer.Take(8).Reverse().ToArray(), 0); - } - - private void FillMyBuffer(int numBytes) { - var offset = 0; - int num2; - - if (numBytes == 1) { - num2 = BaseStream.ReadByte(); - - if (num2 == -1) { - throw new EndOfStreamException("Attempted to read past the end of the stream."); - } - - _buffer[0] = (byte)num2; - } else { - do { - num2 = BaseStream.Read(_buffer, offset, numBytes - offset); - - if (num2 == 0) { - throw new EndOfStreamException("Attempted to read past the end of the stream."); - } - - offset += num2; - } while (offset < numBytes); - } - } - } - - public class EndianWriter(Stream input, Encoding encoding, bool isLittleEndian) : BinaryWriter(input, encoding) { - public EndianWriter(Stream input, bool isLittleEndian) - : this(input, Encoding.UTF8, isLittleEndian) { } - - public bool IsLittleEndian { get; set; } = isLittleEndian; - - public void Write(T value) { - // A bit gross, but AOT and Trimming compatible - var someBytes = value switch { - bool b => BitConverter.GetBytes(b), - char c => BitConverter.GetBytes(c), - double d => BitConverter.GetBytes(d), - Half h => BitConverter.GetBytes(h), - short s => BitConverter.GetBytes(s), - int i => BitConverter.GetBytes(i), - long l => BitConverter.GetBytes(l), - float f => BitConverter.GetBytes(f), - ushort us => BitConverter.GetBytes(us), - uint ui => BitConverter.GetBytes(ui), - ulong ul => BitConverter.GetBytes(ul), - _ => throw new NotSupportedException($"Type {typeof(T)} is not supported.") - }; - - if (!IsLittleEndian) - someBytes = someBytes.Reverse().ToArray(); - - base.Write(someBytes); - } - - public void Write(FileEntry entry) { - if (entry.ExtractSizeType == typeof(byte)) { - Write((byte)entry.ExtractSize); - } else if (entry.ExtractSizeType == typeof(ushort)) { - Write((ushort)entry.ExtractSize); - } else if (entry.ExtractSizeType == typeof(uint)) { - Write((uint)entry.ExtractSize); - } else if (entry.ExtractSizeType == typeof(ulong)) { - Write((ulong)entry.ExtractSize); - } else if (entry.ExtractSizeType == typeof(float)) { - Write((float)entry.ExtractSize); - } else { - throw new Exception("Not supported type!"); - } - } - } -} diff --git a/ShinRyuModManager-CE/ModLoadOrder/Dependency/CPKGen/Program.cs b/ShinRyuModManager-CE/ModLoadOrder/Dependency/CPKGen/Program.cs deleted file mode 100644 index b3ea7b7..0000000 --- a/ShinRyuModManager-CE/ModLoadOrder/Dependency/CPKGen/Program.cs +++ /dev/null @@ -1,106 +0,0 @@ -using System.Diagnostics; -using Serilog; - -namespace CriPakTools { - public static class Program { - /*private static void Main(string[] args) { - Console.WriteLine("Yakuza CPK Repack - Based on CriPakTools\n"); - Modify(args[0], args[1], args[2]); - } - - public static byte[] FileToByteArray(string fileName) { - using var fs = File.OpenRead(fileName); - using var binaryReader = new BinaryReader(fs); - - var fileData = binaryReader.ReadBytes((int)fs.Length); - - return fileData; - }*/ - - public static void Modify(string inputCpk, string replaceDir, string outputCpk) { - GC.Collect(); - Console.WriteLine("Yakuza CPK Repack - Based on CriPakTools\n"); - - var cpk = new CPK(new Tools()); - cpk.ReadCPK(inputCpk); - - GC.Collect(); - - using var oldFile = new BinaryReader(new BufferedStream(File.OpenRead(inputCpk))); - - var files = Directory.GetFiles(replaceDir, "*."); - var filesNames = new HashSet(); - - foreach (var str in files.Select(Path.GetFileNameWithoutExtension)) - filesNames.Add(str); - - var fileInfo = new FileInfo(inputCpk); - var time = Stopwatch.StartNew(); - - using var newCpk = new BinaryWriter(new BufferedStream(File.OpenWrite(outputCpk))); - - var entries = cpk.FileTable.OrderBy(x => x.FileOffset).ToList(); - - foreach (var entry in entries) { - if (entry.FileType == "CONTENT") { - cpk.UpdateFileEntry(entry); - } else { - if (entry.FileType == "FILE" && (ulong)newCpk.BaseStream.Position < cpk.ContentOffset) { - var padLength = cpk.ContentOffset - (ulong)newCpk.BaseStream.Position; - - newCpk.Write(new byte[padLength], 0, (int)padLength); - } - - if (entry.FileSize == null || entry.FileName == null) { - throw new NullReferenceException("Critical properties of the file entry are not initialized."); - } - - if (!filesNames.Contains(entry.FileName.ToString())) { - oldFile.BaseStream.Seek((long)entry.FileOffset, SeekOrigin.Begin); - - entry.FileOffset = (ulong)newCpk.BaseStream.Position; - cpk.UpdateFileEntry(entry); - - - var chunk = ReadBytes(newCpk.BaseStream, int.Parse(entry.FileSize.ToString()!)); - newCpk.Write(chunk); - } else { - var newbie = File.ReadAllBytes(Path.Combine(replaceDir, entry.FileName.ToString()!)); - - entry.FileOffset = (ulong)newCpk.BaseStream.Position; - entry.FileSize = Convert.ChangeType(newbie.Length, entry.FileSizeType); - entry.ExtractSize = Convert.ChangeType(newbie.Length, entry.FileSizeType); - cpk.UpdateFileEntry(entry); - - newCpk.Write(newbie); - } - - if ((newCpk.BaseStream.Position % 0x800) > 0) { - var padding = (int)(0x800 - (newCpk.BaseStream.Position % 0x800)); - - newCpk.Write(new byte[padding], 0, padding); - } - } - } - - cpk.WriteCPK(newCpk); - cpk.WriteITOC(newCpk); - cpk.WriteTOC(newCpk); - cpk.WriteETOC(newCpk); - cpk.WriteGTOC(newCpk); - - Log.Information("Writing {FileName} took {TotalElapsedTime}", fileInfo.Name, time.Elapsed.TotalSeconds); - } - - private static byte[] ReadBytes(Stream stream, int count) { - var buffer = new byte[count]; - var bytesRead = stream.Read(buffer, 0, count); - - if (bytesRead != count) { - throw new EndOfStreamException(); - } - - return buffer; - } - } -} diff --git a/ShinRyuModManager-CE/ModLoadOrder/Dependency/CPKGen/Tools.cs b/ShinRyuModManager-CE/ModLoadOrder/Dependency/CPKGen/Tools.cs deleted file mode 100644 index df235c4..0000000 --- a/ShinRyuModManager-CE/ModLoadOrder/Dependency/CPKGen/Tools.cs +++ /dev/null @@ -1,72 +0,0 @@ -using System.Text; - -namespace CriPakTools { - public class Tools { - public static string ReadCString(BinaryReader br, int maxLength = -1, long lOffset = -1, Encoding enc = null) { - var max = maxLength == -1 ? 255 : maxLength; - var fTemp = br.BaseStream.Position; - var i = 0; - - string result; - - if (lOffset > -1) { - br.BaseStream.Seek(lOffset, SeekOrigin.Begin); - } - - do { - var bTemp = br.ReadByte(); - - if (bTemp == 0) - break; - - i += 1; - } while (i < max); - - if (maxLength == -1) - max = i + 1; - else - max = maxLength; - - if (lOffset > -1) { - br.BaseStream.Seek(lOffset, SeekOrigin.Begin); - - result = enc == null - ? Encoding.GetEncoding(932).GetString(br.ReadBytes(i)) - : enc.GetString(br.ReadBytes(i)); - - br.BaseStream.Seek(fTemp, SeekOrigin.Begin); - } else { - br.BaseStream.Seek(fTemp, SeekOrigin.Begin); - - result = enc == null - ? Encoding.GetEncoding(932).GetString(br.ReadBytes(i)) - : enc.GetString(br.ReadBytes(i)); - - br.BaseStream.Seek(fTemp + max, SeekOrigin.Begin); - } - - return result; - } - - public static void DeleteFileIfExists(string sPath) { - if (File.Exists(sPath)) - File.Delete(sPath); - } - - public static string GetPath(string input) { - return Path.GetDirectoryName(input) + Path.DirectorySeparatorChar + Path.GetFileNameWithoutExtension(input); - } - - public static byte[] GetData(BinaryReader br, long offset, int size) { - var backup = br.BaseStream.Position; - - br.BaseStream.Seek(offset, SeekOrigin.Begin); - - var result = br.ReadBytes(size); - - br.BaseStream.Seek(backup, SeekOrigin.Begin); - - return result; - } - } -} diff --git a/ShinRyuModManager-CE/ModLoadOrder/Mods/Mod.cs b/ShinRyuModManager-CE/ModLoadOrder/Mods/Mod.cs index 9d5ca09..5f8c38d 100644 --- a/ShinRyuModManager-CE/ModLoadOrder/Mods/Mod.cs +++ b/ShinRyuModManager-CE/ModLoadOrder/Mods/Mod.cs @@ -134,10 +134,10 @@ public void AddFiles(string path, string check) { case "se": case "speech": - cpkDataPath = GamePath.RemoveModPath(path); - if (GamePath.CurrentGame is Game.Yakuza5 or <= Game.YakuzaKiwami) { - RepackCpKs.Add(cpkDataPath + ".cpk"); + // Removed adding ".cpk" to se folder. Seemed incorrect + cpkDataPath = GamePath.RemoveModPath(path); + RepackCpKs.Add(cpkDataPath); } break; diff --git a/ShinRyuModManager-CE/ShinRyuModManager-CE.csproj b/ShinRyuModManager-CE/ShinRyuModManager-CE.csproj index ce944ff..1239fe1 100644 --- a/ShinRyuModManager-CE/ShinRyuModManager-CE.csproj +++ b/ShinRyuModManager-CE/ShinRyuModManager-CE.csproj @@ -35,6 +35,7 @@ + diff --git a/ShinRyuModManager-CE/UserInterface/Views/MainWindow.axaml.cs b/ShinRyuModManager-CE/UserInterface/Views/MainWindow.axaml.cs index 7401c5a..e8d3a92 100644 --- a/ShinRyuModManager-CE/UserInterface/Views/MainWindow.axaml.cs +++ b/ShinRyuModManager-CE/UserInterface/Views/MainWindow.axaml.cs @@ -155,6 +155,8 @@ private void ModDown_OnClick(object sender, RoutedEventArgs e) { } private async void ModSave_OnClick(object sender, RoutedEventArgs e) { + var progressWindow = new ProgressWindow("Applying mods. Please wait...", true); + try { if (DataContext is not MainWindowViewModel viewModel) return; @@ -169,8 +171,6 @@ private async void ModSave_OnClick(object sender, RoutedEventArgs e) { return; } - var progressWindow = new ProgressWindow("Applying mods. Please wait...", true); - progressWindow.Show(this); await Task.Yield(); @@ -188,11 +188,15 @@ await Task.Run(async () => { progressWindow.Close(); } else { + progressWindow.Close(); + _ = await MessageBoxWindow.Show(this, "Error", "Mod list is empty and was not saved."); } } catch (Exception ex) { Log.Fatal(ex, "ModSave failed!"); + progressWindow.Close(); + _ = await MessageBoxWindow.Show(this, "Fatal", $"An error has occurred.\nPlease check\"srmm_errors.txt\" for more info."); } }