From 47ce79939a63024f8015e7aea3773c2ea04a75dd Mon Sep 17 00:00:00 2001 From: Jordan Date: Mon, 18 Oct 2021 18:29:06 -0400 Subject: [PATCH 1/4] Update to the latest Cryptography library and other improvements. --- README.md | 23 ++-- cache/build.gradle.kts | 4 +- .../cache/exception/CompressionException.kt | 2 +- .../cache/exception/DatFileException.kt | 2 +- .../cache/exception/EndOfDatFileException.kt | 2 +- .../cache/exception/FileDataException.kt | 2 +- .../cache/exception/FileNotFoundException.kt | 2 +- .../cache/exception/IdxFileException.kt | 2 +- .../cache/exception/ProtocolException.kt | 2 +- .../runetopic/cache/extension/ByteBuffer.kt | 71 ++---------- .../com/runetopic/cache/extension/Int.kt | 8 -- .../com/runetopic/cache/extension/String.kt | 2 +- .../cache/hierarchy/ReferenceTable.kt | 104 +++++++++--------- .../com/runetopic/cache/store/Constants.kt | 35 +----- .../com/runetopic/cache/store/Js5Store.kt | 5 +- .../runetopic/cache/store/storage/IStorage.kt | 3 +- .../cache/store/storage/js5/Js5DiskStorage.kt | 49 +++++---- loader/build.gradle.kts | 2 +- .../runetopic/loader/extension/ByteBuffer.kt | 85 ++++++++++++++ .../com/runetopic/loader/extension/Int.kt | 8 ++ .../config/idk/IdentityKitEntryBuilder.kt | 6 +- .../index/config/inv/InventoryEntryBuilder.kt | 4 +- .../config/lighting/LightingEntryBuilder.kt | 4 +- .../config/mouseicon/MouseIconEntryBuilder.kt | 4 +- .../config/overlay/OverlayEntryBuilder.kt | 6 +- .../index/config/param/ParamEntryBuilder.kt | 6 +- .../index/config/skybox/SkyBoxEntryBuilder.kt | 4 +- .../index/config/struct/StructEntryBuilder.kt | 8 +- .../config/underlay/UnderlayEntryBuilder.kt | 6 +- .../loader/index/loc/LocEntryBuilder.kt | 2 +- .../loader/index/map/MapEntryBuilder.kt | 6 +- .../index/map/MapLocationEntryBuilder.kt | 6 +- .../loader/index/npc/NpcEntryBuilder.kt | 2 +- .../loader/index/obj/ObjEntryBuilder.kt | 2 +- .../index/particle/ParticleEntryBuilder.kt | 8 +- .../spotanim/SpotAnimationEntryBuilder.kt | 4 +- 36 files changed, 242 insertions(+), 249 deletions(-) delete mode 100644 cache/src/main/kotlin/com/runetopic/cache/extension/Int.kt create mode 100644 loader/src/main/kotlin/com/runetopic/loader/extension/ByteBuffer.kt create mode 100644 loader/src/main/kotlin/com/runetopic/loader/extension/Int.kt diff --git a/README.md b/README.md index 97e553b..1fec53e 100644 --- a/README.md +++ b/README.md @@ -9,28 +9,27 @@ A cache library written in Kotlin. - Java Version 16 # Supported -- RS2 (414-742) +- RS2 (414-772) +- RS3 (773-~788) - OSRS (1-current) # Features -- Thread-safe - - Cache Loading - - Definitions/Providers Loading +- Cache Reading +- Definitions/Providers Loading - Fast (Limited by I/O) # TODO - Cache Writing -- Flat file system for unpacking the cache files into a raw format that can be git versioned. -- Support for RS2 caches revision 743+. -- ~317 cache format support. -- RS3 caches. -- Testing +- Flat File System +- Ondemand Data Caching +- 317 and older support +- Tests # Implementation Just use cache if you do not require any of the revision specific loaders. ``` -cache = { module = "com.runetopic.cache:cache", version.ref "1.4.11-SNAPSHOT" } -loader = { module = "com.runetopic.cache:loader", version.ref "647.6.1-SNAPSHOT" } +cache = { module = "com.runetopic.cache:cache", version.ref "1.4.12-SNAPSHOT" } +loader = { module = "com.runetopic.cache:loader", version.ref "647.6.2-SNAPSHOT" } ``` ``` @@ -45,7 +44,7 @@ Index -> Group -> File ### Creating a new JS5 store ``` -val store = Js5Store(Path.of("/path/")) +val store = Js5Store(path = Path.of("/path/"), parallel = true) ``` ### Getting an index diff --git a/cache/build.gradle.kts b/cache/build.gradle.kts index 667b88f..3d5e525 100644 --- a/cache/build.gradle.kts +++ b/cache/build.gradle.kts @@ -4,7 +4,7 @@ plugins { signing } -version = "1.4.11-SNAPSHOT" +version = "1.4.12-SNAPSHOT" java { withJavadocJar() @@ -77,5 +77,5 @@ dependencies { implementation("com.michael-bull.kotlin-inline-logger:kotlin-inline-logger:1.0.3") implementation("org.slf4j:slf4j-simple:1.7.32") implementation("com.fasterxml.jackson.module:jackson-module-kotlin:2.13.+") - implementation("com.runetopic.cryptography:cryptography:1.0.4-SNAPSHOT") + implementation("com.runetopic.cryptography:cryptography:1.0.6-SNAPSHOT") } diff --git a/cache/src/main/kotlin/com/runetopic/cache/exception/CompressionException.kt b/cache/src/main/kotlin/com/runetopic/cache/exception/CompressionException.kt index 029dcc4..82cbed5 100644 --- a/cache/src/main/kotlin/com/runetopic/cache/exception/CompressionException.kt +++ b/cache/src/main/kotlin/com/runetopic/cache/exception/CompressionException.kt @@ -4,4 +4,4 @@ package com.runetopic.cache.exception * @author Tyler Telis * @email */ -class CompressionException(override val message: String): RuntimeException(message) \ No newline at end of file +internal class CompressionException(override val message: String): RuntimeException(message) \ No newline at end of file diff --git a/cache/src/main/kotlin/com/runetopic/cache/exception/DatFileException.kt b/cache/src/main/kotlin/com/runetopic/cache/exception/DatFileException.kt index c0ed181..b5e84e5 100644 --- a/cache/src/main/kotlin/com/runetopic/cache/exception/DatFileException.kt +++ b/cache/src/main/kotlin/com/runetopic/cache/exception/DatFileException.kt @@ -4,4 +4,4 @@ package com.runetopic.cache.exception * @author Tyler Telis * @email */ -class DatFileException(override val message: String): RuntimeException(message) \ No newline at end of file +internal class DatFileException(override val message: String): RuntimeException(message) \ No newline at end of file diff --git a/cache/src/main/kotlin/com/runetopic/cache/exception/EndOfDatFileException.kt b/cache/src/main/kotlin/com/runetopic/cache/exception/EndOfDatFileException.kt index 9e57c86..709969c 100644 --- a/cache/src/main/kotlin/com/runetopic/cache/exception/EndOfDatFileException.kt +++ b/cache/src/main/kotlin/com/runetopic/cache/exception/EndOfDatFileException.kt @@ -4,4 +4,4 @@ package com.runetopic.cache.exception * @author Tyler Telis * @email */ -class EndOfDatFileException(override val message: String): RuntimeException(message) \ No newline at end of file +internal class EndOfDatFileException(override val message: String): RuntimeException(message) \ No newline at end of file diff --git a/cache/src/main/kotlin/com/runetopic/cache/exception/FileDataException.kt b/cache/src/main/kotlin/com/runetopic/cache/exception/FileDataException.kt index e7032dd..2d646ce 100644 --- a/cache/src/main/kotlin/com/runetopic/cache/exception/FileDataException.kt +++ b/cache/src/main/kotlin/com/runetopic/cache/exception/FileDataException.kt @@ -4,4 +4,4 @@ package com.runetopic.cache.exception * @author Tyler Telis * @email */ -class FileDataException(override val message: String): RuntimeException(message) \ No newline at end of file +internal class FileDataException(override val message: String): RuntimeException(message) \ No newline at end of file diff --git a/cache/src/main/kotlin/com/runetopic/cache/exception/FileNotFoundException.kt b/cache/src/main/kotlin/com/runetopic/cache/exception/FileNotFoundException.kt index 9b865cb..fa7aebd 100644 --- a/cache/src/main/kotlin/com/runetopic/cache/exception/FileNotFoundException.kt +++ b/cache/src/main/kotlin/com/runetopic/cache/exception/FileNotFoundException.kt @@ -4,4 +4,4 @@ package com.runetopic.cache.exception * @author Tyler Telis * @email */ -class FileNotFoundException(override val message: String): RuntimeException(message) \ No newline at end of file +internal class FileNotFoundException(override val message: String): RuntimeException(message) \ No newline at end of file diff --git a/cache/src/main/kotlin/com/runetopic/cache/exception/IdxFileException.kt b/cache/src/main/kotlin/com/runetopic/cache/exception/IdxFileException.kt index 0958417..70e2bd7 100644 --- a/cache/src/main/kotlin/com/runetopic/cache/exception/IdxFileException.kt +++ b/cache/src/main/kotlin/com/runetopic/cache/exception/IdxFileException.kt @@ -4,4 +4,4 @@ package com.runetopic.cache.exception * @author Tyler Telis * @email */ -class IdxFileException(override val message: String): RuntimeException(message) \ No newline at end of file +internal class IdxFileException(override val message: String): RuntimeException(message) \ No newline at end of file diff --git a/cache/src/main/kotlin/com/runetopic/cache/exception/ProtocolException.kt b/cache/src/main/kotlin/com/runetopic/cache/exception/ProtocolException.kt index b4c32dd..97b9d4e 100644 --- a/cache/src/main/kotlin/com/runetopic/cache/exception/ProtocolException.kt +++ b/cache/src/main/kotlin/com/runetopic/cache/exception/ProtocolException.kt @@ -4,4 +4,4 @@ package com.runetopic.cache.exception * @author Tyler Telis * @email */ -class ProtocolException(override val message: String): RuntimeException(message) \ No newline at end of file +internal class ProtocolException(override val message: String): RuntimeException(message) \ No newline at end of file diff --git a/cache/src/main/kotlin/com/runetopic/cache/extension/ByteBuffer.kt b/cache/src/main/kotlin/com/runetopic/cache/extension/ByteBuffer.kt index d1c3c10..08df1bc 100644 --- a/cache/src/main/kotlin/com/runetopic/cache/extension/ByteBuffer.kt +++ b/cache/src/main/kotlin/com/runetopic/cache/extension/ByteBuffer.kt @@ -1,70 +1,13 @@ package com.runetopic.cache.extension -import com.runetopic.cache.store.Constants.cp1252Identifiers -import com.runetopic.cryptography.toWhirlpool import java.nio.ByteBuffer -import java.nio.charset.Charset -fun ByteBuffer.readCp1252Char(): Char { - var unsigned = get().toInt() and 0xff - require(unsigned != 0) { "Non cp1252 character 0x" + unsigned.toString(16) + " provided" } - if (unsigned in 128..159) { - var value: Int = cp1252Identifiers[unsigned - 128].code - if (value == 0) value = 63 - unsigned = value - } - return unsigned.toChar() -} +internal fun ByteBuffer.readUnsignedByte(): Int = get().toInt() and 0xFF +internal fun ByteBuffer.readUnsignedShort(): Int = short.toInt() and 0xFFFF +internal fun ByteBuffer.readUnsignedIntShortSmart(): Int = if (get(position()).toInt() < 0) int and Int.MAX_VALUE else readUnsignedShort() -fun ByteBuffer.readUnsignedSmart(): Int { - val peek = get(position()).toInt() and 0xFF - return if (peek < 128) readUnsignedByte() else (readUnsignedShort()) - 0x8000 -} - -fun ByteBuffer.readUnsignedSmartShort(): Int { - return if (get(position()).toInt() < 0) int and Int.MAX_VALUE else readUnsignedShort() -} - -fun ByteBuffer.readUnsignedByte(): Int = get().toInt() and 0xFF -fun ByteBuffer.readUnsignedShort(): Int = short.toInt() and 0xFFFF -fun ByteBuffer.readMedium(): Int = (get().toInt() and 0xFF) shl 16 or (get().toInt() and 0xFF shl 8) or (get().toInt() and 0xFF) -fun ByteBuffer.skip(amount: Int): ByteBuffer = position(position() + amount) - -fun ByteBuffer.readUnsignedIntSmartShortCompat(): Int { - var value = 0 - var i = readUnsignedSmart() - while (i == 32767) { - i = readUnsignedSmart() - value += 32767 - } - value += i - return value -} - -fun ByteBuffer.whirlpool(): ByteArray { - val position = position() - val data = ByteArray(limit()) - get(data) - position(position) - return data.toWhirlpool() -} - -fun ByteBuffer.readString(): String { - val mark = position() - var length = 0 - while (get().toInt() != 0) { - length++ - } - if (length == 0) return "" - val byteArray = ByteArray(length) - position(mark) - get(byteArray) - position(position() + 1) - return String(byteArray, Charset.defaultCharset()) -} - -fun ByteBuffer.remainingBytes(): ByteArray { - val b = ByteArray(remaining()) - get(b) - return b +internal fun ByteBuffer.remainingBytes(): ByteArray { + val bytes = ByteArray(remaining()) + get(bytes) + return bytes } \ No newline at end of file diff --git a/cache/src/main/kotlin/com/runetopic/cache/extension/Int.kt b/cache/src/main/kotlin/com/runetopic/cache/extension/Int.kt deleted file mode 100644 index b378b03..0000000 --- a/cache/src/main/kotlin/com/runetopic/cache/extension/Int.kt +++ /dev/null @@ -1,8 +0,0 @@ -package com.runetopic.cache.extension - -/** - * @author Jordan Abraham - */ -fun Int.toBoolean(): Boolean { - return this == 1 -} \ No newline at end of file diff --git a/cache/src/main/kotlin/com/runetopic/cache/extension/String.kt b/cache/src/main/kotlin/com/runetopic/cache/extension/String.kt index 1633a19..13f876f 100644 --- a/cache/src/main/kotlin/com/runetopic/cache/extension/String.kt +++ b/cache/src/main/kotlin/com/runetopic/cache/extension/String.kt @@ -1,6 +1,6 @@ package com.runetopic.cache.extension -fun String.nameHash(): Int { +internal fun String.nameHash(): Int { var hash = 0 this.forEach { element -> hash = element.toInt() + ((hash shl 5) - hash) } return hash diff --git a/cache/src/main/kotlin/com/runetopic/cache/hierarchy/ReferenceTable.kt b/cache/src/main/kotlin/com/runetopic/cache/hierarchy/ReferenceTable.kt index 92e68f7..d9abff3 100644 --- a/cache/src/main/kotlin/com/runetopic/cache/hierarchy/ReferenceTable.kt +++ b/cache/src/main/kotlin/com/runetopic/cache/hierarchy/ReferenceTable.kt @@ -4,8 +4,8 @@ import com.runetopic.cache.codec.Container import com.runetopic.cache.codec.decompress import com.runetopic.cache.exception.ProtocolException import com.runetopic.cache.extension.readUnsignedByte +import com.runetopic.cache.extension.readUnsignedIntShortSmart import com.runetopic.cache.extension.readUnsignedShort -import com.runetopic.cache.extension.readUnsignedSmartShort import com.runetopic.cache.hierarchy.index.Js5Index import com.runetopic.cache.hierarchy.index.group.Js5Group import com.runetopic.cache.hierarchy.index.group.file.File @@ -26,38 +26,8 @@ internal data class ReferenceTable( val idxFile: IIdxFile, val id: Int, val sector: Int, - val length: Int, + val length: Int ) { - override fun hashCode(): Int { - var hash = 7 - hash = 19 * hash + Objects.hashCode(this.idxFile) - hash = 19 * hash + this.id - hash = 19 * hash + this.sector - hash = 19 * hash + this.length - return hash - } - - override fun equals(other: Any?): Boolean { - when (other) { - null -> { - return false - } - else -> return when { - javaClass != other.javaClass -> false - else -> { - val referenceTable: ReferenceTable = other as ReferenceTable - when { - idxFile != referenceTable.idxFile -> false - id != referenceTable.id -> false - sector != referenceTable.sector -> false - this.length != referenceTable.length -> false - else -> true - } - } - } - } - } - fun exists(): Boolean = (length != 0 && sector != 0) fun loadIndex( @@ -76,7 +46,7 @@ internal data class ReferenceTable( else -> 0 } val hash = buffer.readUnsignedByte() - val count = if (protocol >= 7) buffer.readUnsignedSmartShort() else buffer.readUnsignedShort() + val count = if (protocol >= 7) buffer.readUnsignedIntShortSmart() else buffer.readUnsignedShort() val groupTables = mutableListOf() (0 until count).forEach { groupTables.add(datFile.readReferenceTable(idxFile.id(), idxFile.loadReferenceTable(it))) @@ -96,15 +66,14 @@ internal data class ReferenceTable( whirlpool: ByteArray, groupTables: List ): Js5Index { - val groupIds = IntArray(count) - val isNamed = (0x1 and hash) != 0 - val isUsingWhirlPool = (0x2 and hash) != 0 + val isUsingWhirlpool = (0x2 and hash) != 0 + val groupIds = IntArray(count) var lastGroupId = 0 var biggest = -1 (0 until count).forEach { - groupIds[it] = if (protocol >= 7) { buffer.readUnsignedSmartShort() } else { buffer.readUnsignedShort() } + groupIds[it] = if (protocol >= 7) { buffer.readUnsignedIntShortSmart() } else { buffer.readUnsignedShort() } .let { id -> lastGroupId += id; lastGroupId } if (groupIds[it] > biggest) biggest = groupIds[it] } @@ -112,9 +81,10 @@ internal data class ReferenceTable( val largestGroupId = biggest + 1 val groupNameHashes = groupNameHashes(largestGroupId, count, isNamed, groupIds, buffer) val groupCrcs = groupCrcs(largestGroupId, count, groupIds, buffer) - val groupWhirlpools = groupWhirlpools(largestGroupId, isUsingWhirlPool, count, buffer, groupIds) + val groupWhirlpools = groupWhirlpools(largestGroupId, isUsingWhirlpool, count, buffer, groupIds) val groupRevisions = groupRevisions(largestGroupId, count, groupIds, buffer) val groupFileIds = groupFileIds(largestGroupId, count, groupIds, buffer, protocol) + val fileIds = fileIds(largestGroupId, groupFileIds, count, groupIds, buffer, protocol) val fileNameHashes = fileNameHashes(largestGroupId, groupFileIds, count, groupIds, buffer, isNamed) @@ -122,14 +92,14 @@ internal data class ReferenceTable( (0 until count).forEach { val groupId = groupIds[it] groups[it] = (Js5Group( - id = groupId, - nameHash = groupNameHashes[groupId], - crc = groupCrcs[groupId], - whirlpool = groupWhirlpools[groupId], - revision = groupRevisions[groupId], - keys = intArrayOf(), - files = files(fileIds, fileNameHashes, groupTables[it], groupFileIds[it], it), - data = groupTables[it] + groupId, + groupNameHashes[groupId], + groupCrcs[groupId], + groupWhirlpools[groupId], + groupRevisions[groupId], + intArrayOf(),//TODO + groupFiles(fileIds, fileNameHashes, groupTables[it], groupFileIds[it], it), + groupTables[it] )) } return Js5Index(indexId, crc, whirlpool, compressionType, protocol, revision, isNamed, groups) @@ -144,7 +114,7 @@ internal data class ReferenceTable( ): IntArray { val groupFileIds = IntArray(largestGroupId) (0 until count).forEach { - groupFileIds[groupIds[it]] = if (protocol >= 7) buffer.readUnsignedSmartShort() else buffer.readUnsignedShort() + groupFileIds[groupIds[it]] = if (protocol >= 7) buffer.readUnsignedIntShortSmart() else buffer.readUnsignedShort() } return groupFileIds } @@ -153,7 +123,7 @@ internal data class ReferenceTable( largestGroupId: Int, count: Int, groupIds: IntArray, - buffer: ByteBuffer, + buffer: ByteBuffer ): IntArray { val revisions = IntArray(largestGroupId) (0 until count).forEach { @@ -167,7 +137,7 @@ internal data class ReferenceTable( usesWhirlpool: Boolean, count: Int, buffer: ByteBuffer, - groupIds: IntArray, + groupIds: IntArray ): Array { val whirlpools = Array(largestGroupId) { ByteArray(64) } if (usesWhirlpool.not()) return whirlpools @@ -184,7 +154,7 @@ internal data class ReferenceTable( largestGroupId: Int, count: Int, groupIds: IntArray, - buffer: ByteBuffer, + buffer: ByteBuffer ): IntArray { val crcs = IntArray(largestGroupId) (0 until count).forEach { @@ -198,7 +168,7 @@ internal data class ReferenceTable( count: Int, isNamed: Boolean, groupIds: IntArray, - buffer: ByteBuffer, + buffer: ByteBuffer ): IntArray { val nameHashes = IntArray(largestGroupId) { -1 } if (isNamed.not()) return nameHashes @@ -222,7 +192,7 @@ internal data class ReferenceTable( val groupId = groupIds[it] var currentFileId = 0 (0 until validFileIds[groupId]).forEach { fileId -> - if (protocol >= 7) { buffer.readUnsignedSmartShort() } else { buffer.readUnsignedShort() } + if (protocol >= 7) { buffer.readUnsignedIntShortSmart() } else { buffer.readUnsignedShort() } .let { i -> currentFileId += i; currentFileId } .also { fileIds[groupId][fileId] = currentFileId } } @@ -250,7 +220,7 @@ internal data class ReferenceTable( return fileNameHashes } - private fun files( + private fun groupFiles( fileIds: Array, fileNameHashes: Array, groupReferenceTableData: ByteArray, @@ -305,4 +275,32 @@ internal data class ReferenceTable( } return files } + + override fun hashCode(): Int { + var hash = 7 + hash = 19 * hash + Objects.hashCode(this.idxFile) + hash = 19 * hash + this.id + hash = 19 * hash + this.sector + hash = 19 * hash + this.length + return hash + } + + override fun equals(other: Any?): Boolean { + when (other) { + null -> return false + else -> return when { + javaClass != other.javaClass -> false + else -> { + val referenceTable: ReferenceTable = other as ReferenceTable + when { + idxFile != referenceTable.idxFile -> false + id != referenceTable.id -> false + sector != referenceTable.sector -> false + length != referenceTable.length -> false + else -> true + } + } + } + } + } } \ No newline at end of file diff --git a/cache/src/main/kotlin/com/runetopic/cache/store/Constants.kt b/cache/src/main/kotlin/com/runetopic/cache/store/Constants.kt index 9442957..90d0749 100644 --- a/cache/src/main/kotlin/com/runetopic/cache/store/Constants.kt +++ b/cache/src/main/kotlin/com/runetopic/cache/store/Constants.kt @@ -13,40 +13,7 @@ internal object Constants { const val SECTOR_SIZE = 520 const val MASTER_INDEX_ID = 255 - val cp1252Identifiers = charArrayOf( - '\u20ac', - '\u0000', - '\u201a', - '\u0192', - '\u201e', - '\u2026', - '\u2020', - '\u2021', - '\u02c6', - '\u2030', - '\u0160', - '\u2039', - '\u0152', - '\u0000', - '\u017d', - '\u0000', - '\u0000', - '\u2018', - '\u2019', - '\u201c', - '\u201d', - '\u2022', - '\u2013', - '\u2014', - '\u02dc', - '\u2122', - '\u0161', - '\u203a', - '\u0153', - '\u0000', - '\u017e', - '\u0178' - ) + val BZIP_HEADER = byteArrayOf( diff --git a/cache/src/main/kotlin/com/runetopic/cache/store/Js5Store.kt b/cache/src/main/kotlin/com/runetopic/cache/store/Js5Store.kt index 72fe2dd..5913961 100644 --- a/cache/src/main/kotlin/com/runetopic/cache/store/Js5Store.kt +++ b/cache/src/main/kotlin/com/runetopic/cache/store/Js5Store.kt @@ -15,9 +15,10 @@ import java.nio.file.Path * @author Jordan Abraham */ class Js5Store( - path: Path + path: Path, + parallel: Boolean = false ) : Closeable { - private var storage = Js5DiskStorage(path) + private var storage = Js5DiskStorage(path, parallel) private val indexes = arrayListOf() init { diff --git a/cache/src/main/kotlin/com/runetopic/cache/store/storage/IStorage.kt b/cache/src/main/kotlin/com/runetopic/cache/store/storage/IStorage.kt index 51402c4..abbe06b 100644 --- a/cache/src/main/kotlin/com/runetopic/cache/store/storage/IStorage.kt +++ b/cache/src/main/kotlin/com/runetopic/cache/store/storage/IStorage.kt @@ -4,7 +4,6 @@ import com.runetopic.cache.hierarchy.index.Index import com.runetopic.cache.store.Js5Store import java.io.Closeable import java.io.Flushable -import java.util.concurrent.CountDownLatch /** * @author Tyler Telis @@ -12,7 +11,7 @@ import java.util.concurrent.CountDownLatch */ internal interface IStorage: Closeable, Flushable { fun init(store: Js5Store) - fun store(indexId: Int, store: Js5Store, latch: CountDownLatch) + fun open(indexId: Int, store: Js5Store) fun loadReferenceTable(index: Index, groupId: Int): ByteArray fun loadMasterReferenceTable(groupId: Int): ByteArray fun loadReferenceTable(index: Index, groupName: String): ByteArray diff --git a/cache/src/main/kotlin/com/runetopic/cache/store/storage/js5/Js5DiskStorage.kt b/cache/src/main/kotlin/com/runetopic/cache/store/storage/js5/Js5DiskStorage.kt index d9cd48b..52faa47 100644 --- a/cache/src/main/kotlin/com/runetopic/cache/store/storage/js5/Js5DiskStorage.kt +++ b/cache/src/main/kotlin/com/runetopic/cache/store/storage/js5/Js5DiskStorage.kt @@ -2,7 +2,6 @@ package com.runetopic.cache.store.storage.js5 import com.github.michaelbull.logging.InlineLogger import com.runetopic.cache.codec.ContainerCodec -import com.runetopic.cache.extension.whirlpool import com.runetopic.cache.hierarchy.index.Index import com.runetopic.cache.hierarchy.index.Js5Index import com.runetopic.cache.store.Constants @@ -10,23 +9,24 @@ import com.runetopic.cache.store.Js5Store import com.runetopic.cache.store.storage.IStorage import com.runetopic.cache.store.storage.js5.impl.DatFile import com.runetopic.cache.store.storage.js5.impl.IdxFile +import com.runetopic.cryptography.toWhirlpool import java.io.FileNotFoundException -import java.nio.ByteBuffer import java.nio.file.Path import java.util.concurrent.CountDownLatch -import java.util.concurrent.ExecutorService import java.util.concurrent.Executors -import java.util.concurrent.TimeUnit import kotlin.io.path.ExperimentalPathApi import kotlin.io.path.exists /** * @author Tyler Telis * @email + * + * @author Jordan Abraham */ @OptIn(ExperimentalPathApi::class) internal class Js5DiskStorage( - private val path: Path + private val path: Path, + private val parallel: Boolean ) : IStorage { private var masterIdxFile: IIdxFile private var datFile: IDatFile @@ -51,36 +51,36 @@ internal class Js5DiskStorage( } override fun init(store: Js5Store) { - val cores = Runtime.getRuntime().availableProcessors() - var pool: ExecutorService? = null - if (cores > 4) { - pool = Executors.newFixedThreadPool(if (cores > 8) 4 else 2) + logger.debug { "Opening $path for js5 indexes." } + + if (parallel) { + val latch = CountDownLatch(masterIdxFile.validIndexCount()) + val threads = Runtime.getRuntime().availableProcessors() + val pool = Executors.newFixedThreadPool(if (threads >= 16) 8 else if (threads >= 8) 4 else 2) + (0 until masterIdxFile.validIndexCount()).forEach { + pool.execute { + open(it, store) + latch.countDown() + } + } + latch.await() + pool.shutdown() + } else { + (0 until masterIdxFile.validIndexCount()).forEach { open(it, store) } } - val latch = CountDownLatch(masterIdxFile.validIndexCount()) - - (0 until masterIdxFile.validIndexCount()).forEach { indexId -> - pool?.let { it.execute { store(indexId, store, latch) } } - ?: run { store(indexId, store, latch) } - } - - latch.await(60, TimeUnit.SECONDS) - pool?.shutdown() - logger.debug { "Loaded ${idxFiles.size} indexes." } + logger.debug { "Opened ${idxFiles.size} js5 indexes." } } - override fun store(indexId: Int, store: Js5Store, latch: CountDownLatch) { + override fun open(indexId: Int, store: Js5Store) { val indexTable = masterIdxFile.loadReferenceTable(indexId) idxFiles.add(getIdxFile(indexId)) if (indexTable.exists().not()) { store.addIndex(Js5Index.default(indexId)) - latch.countDown() return } - val indexDatTable = datFile.readReferenceTable(masterIdxFile.id(), indexTable) - store.addIndex(indexTable.loadIndex(datFile, getIdxFile(indexId), ByteBuffer.wrap(indexDatTable).whirlpool(), ContainerCodec.decompress(indexDatTable))) - latch.countDown() + store.addIndex(indexTable.loadIndex(datFile, getIdxFile(indexId), indexDatTable.toWhirlpool(), ContainerCodec.decompress(indexDatTable))) } override fun loadMasterReferenceTable(groupId: Int): ByteArray { @@ -97,6 +97,7 @@ internal class Js5DiskStorage( return datFile.readReferenceTable(index.getId(), getIdxFile(index.getId()).loadReferenceTable(group.getId())) } + @Synchronized private fun getIdxFile(id: Int): IdxFile { idxFiles.find { it.id() == id }?.let { return it } return IdxFile(id, Path.of("$path/${Constants.MAIN_FILE_IDX}${id}")) diff --git a/loader/build.gradle.kts b/loader/build.gradle.kts index bbe01b6..8d07076 100644 --- a/loader/build.gradle.kts +++ b/loader/build.gradle.kts @@ -4,7 +4,7 @@ plugins { signing } -version = "647.6.1-SNAPSHOT" +version = "647.6.2-SNAPSHOT" java { withJavadocJar() diff --git a/loader/src/main/kotlin/com/runetopic/loader/extension/ByteBuffer.kt b/loader/src/main/kotlin/com/runetopic/loader/extension/ByteBuffer.kt new file mode 100644 index 0000000..9e4ba86 --- /dev/null +++ b/loader/src/main/kotlin/com/runetopic/loader/extension/ByteBuffer.kt @@ -0,0 +1,85 @@ +package com.runetopic.loader.extension + +import java.nio.ByteBuffer +import java.nio.charset.Charset + +internal fun ByteBuffer.readCp1252Char(): Char { + var unsigned = get().toInt() and 0xff + require(unsigned != 0) { "Non cp1252 character 0x" + unsigned.toString(16) + " provided" } + if (unsigned in 128..159) { + var value: Int = cp1252Identifiers[unsigned - 128].code + if (value == 0) value = 63 + unsigned = value + } + return unsigned.toChar() +} + +internal fun ByteBuffer.readUnsignedSmart(): Int { + val peek = get(position()).toInt() and 0xFF + return if (peek < 128) readUnsignedByte() else (readUnsignedShort()) - 0x8000 +} + +internal fun ByteBuffer.readUnsignedByte(): Int = get().toInt() and 0xFF +internal fun ByteBuffer.readUnsignedShort(): Int = short.toInt() and 0xFFFF +internal fun ByteBuffer.readMedium(): Int = (get().toInt() and 0xFF) shl 16 or (get().toInt() and 0xFF shl 8) or (get().toInt() and 0xFF) +internal fun ByteBuffer.skip(amount: Int): ByteBuffer = position(position() + amount) + +internal fun ByteBuffer.readUnsignedIntSmartShortCompat(): Int { + var value = 0 + var i = readUnsignedSmart() + while (i == 32767) { + i = readUnsignedSmart() + value += 32767 + } + value += i + return value +} + +internal fun ByteBuffer.readString(): String { + val mark = position() + var length = 0 + while (get().toInt() != 0) { + length++ + } + if (length == 0) return "" + val byteArray = ByteArray(length) + position(mark) + get(byteArray) + position(position() + 1) + return String(byteArray, Charset.defaultCharset()) +} + +val cp1252Identifiers = charArrayOf( + '\u20ac', + '\u0000', + '\u201a', + '\u0192', + '\u201e', + '\u2026', + '\u2020', + '\u2021', + '\u02c6', + '\u2030', + '\u0160', + '\u2039', + '\u0152', + '\u0000', + '\u017d', + '\u0000', + '\u0000', + '\u2018', + '\u2019', + '\u201c', + '\u201d', + '\u2022', + '\u2013', + '\u2014', + '\u02dc', + '\u2122', + '\u0161', + '\u203a', + '\u0153', + '\u0000', + '\u017e', + '\u0178' +) \ No newline at end of file diff --git a/loader/src/main/kotlin/com/runetopic/loader/extension/Int.kt b/loader/src/main/kotlin/com/runetopic/loader/extension/Int.kt new file mode 100644 index 0000000..3a025f0 --- /dev/null +++ b/loader/src/main/kotlin/com/runetopic/loader/extension/Int.kt @@ -0,0 +1,8 @@ +package com.runetopic.loader.extension + +/** + * @author Jordan Abraham + */ +internal fun Int.toBoolean(): Boolean { + return this == 1 +} \ No newline at end of file diff --git a/loader/src/main/kotlin/com/runetopic/loader/index/config/idk/IdentityKitEntryBuilder.kt b/loader/src/main/kotlin/com/runetopic/loader/index/config/idk/IdentityKitEntryBuilder.kt index 39451dc..558b285 100644 --- a/loader/src/main/kotlin/com/runetopic/loader/index/config/idk/IdentityKitEntryBuilder.kt +++ b/loader/src/main/kotlin/com/runetopic/loader/index/config/idk/IdentityKitEntryBuilder.kt @@ -1,10 +1,10 @@ package com.runetopic.loader.index.config.idk -import com.runetopic.cache.extension.readUnsignedByte -import com.runetopic.cache.extension.readUnsignedShort -import com.runetopic.cache.extension.skip import com.runetopic.cache.store.Js5Store import com.runetopic.loader.IEntryBuilder +import com.runetopic.loader.extension.readUnsignedByte +import com.runetopic.loader.extension.readUnsignedShort +import com.runetopic.loader.extension.skip import java.nio.ByteBuffer /** diff --git a/loader/src/main/kotlin/com/runetopic/loader/index/config/inv/InventoryEntryBuilder.kt b/loader/src/main/kotlin/com/runetopic/loader/index/config/inv/InventoryEntryBuilder.kt index 40923e1..ac722b4 100644 --- a/loader/src/main/kotlin/com/runetopic/loader/index/config/inv/InventoryEntryBuilder.kt +++ b/loader/src/main/kotlin/com/runetopic/loader/index/config/inv/InventoryEntryBuilder.kt @@ -1,9 +1,9 @@ package com.runetopic.loader.index.config.inv -import com.runetopic.cache.extension.readUnsignedByte -import com.runetopic.cache.extension.readUnsignedShort import com.runetopic.cache.store.Js5Store import com.runetopic.loader.IEntryBuilder +import com.runetopic.loader.extension.readUnsignedByte +import com.runetopic.loader.extension.readUnsignedShort import java.nio.ByteBuffer /** diff --git a/loader/src/main/kotlin/com/runetopic/loader/index/config/lighting/LightingEntryBuilder.kt b/loader/src/main/kotlin/com/runetopic/loader/index/config/lighting/LightingEntryBuilder.kt index be8f386..05f5278 100644 --- a/loader/src/main/kotlin/com/runetopic/loader/index/config/lighting/LightingEntryBuilder.kt +++ b/loader/src/main/kotlin/com/runetopic/loader/index/config/lighting/LightingEntryBuilder.kt @@ -1,9 +1,9 @@ package com.runetopic.loader.index.config.lighting -import com.runetopic.cache.extension.readUnsignedByte -import com.runetopic.cache.extension.readUnsignedShort import com.runetopic.cache.store.Js5Store import com.runetopic.loader.IEntryBuilder +import com.runetopic.loader.extension.readUnsignedByte +import com.runetopic.loader.extension.readUnsignedShort import java.nio.ByteBuffer /** diff --git a/loader/src/main/kotlin/com/runetopic/loader/index/config/mouseicon/MouseIconEntryBuilder.kt b/loader/src/main/kotlin/com/runetopic/loader/index/config/mouseicon/MouseIconEntryBuilder.kt index e9fe6ee..04d2f47 100644 --- a/loader/src/main/kotlin/com/runetopic/loader/index/config/mouseicon/MouseIconEntryBuilder.kt +++ b/loader/src/main/kotlin/com/runetopic/loader/index/config/mouseicon/MouseIconEntryBuilder.kt @@ -1,9 +1,9 @@ package com.runetopic.loader.index.config.mouseicon -import com.runetopic.cache.extension.readUnsignedByte -import com.runetopic.cache.extension.readUnsignedShort import com.runetopic.cache.store.Js5Store import com.runetopic.loader.IEntryBuilder +import com.runetopic.loader.extension.readUnsignedByte +import com.runetopic.loader.extension.readUnsignedShort import java.nio.ByteBuffer /** diff --git a/loader/src/main/kotlin/com/runetopic/loader/index/config/overlay/OverlayEntryBuilder.kt b/loader/src/main/kotlin/com/runetopic/loader/index/config/overlay/OverlayEntryBuilder.kt index 68e5644..84c9999 100644 --- a/loader/src/main/kotlin/com/runetopic/loader/index/config/overlay/OverlayEntryBuilder.kt +++ b/loader/src/main/kotlin/com/runetopic/loader/index/config/overlay/OverlayEntryBuilder.kt @@ -1,10 +1,10 @@ package com.runetopic.loader.index.config.overlay -import com.runetopic.cache.extension.readMedium -import com.runetopic.cache.extension.readUnsignedByte -import com.runetopic.cache.extension.readUnsignedShort import com.runetopic.cache.store.Js5Store import com.runetopic.loader.IEntryBuilder +import com.runetopic.loader.extension.readMedium +import com.runetopic.loader.extension.readUnsignedByte +import com.runetopic.loader.extension.readUnsignedShort import java.nio.ByteBuffer /** diff --git a/loader/src/main/kotlin/com/runetopic/loader/index/config/param/ParamEntryBuilder.kt b/loader/src/main/kotlin/com/runetopic/loader/index/config/param/ParamEntryBuilder.kt index 5f80d7a..bde4439 100644 --- a/loader/src/main/kotlin/com/runetopic/loader/index/config/param/ParamEntryBuilder.kt +++ b/loader/src/main/kotlin/com/runetopic/loader/index/config/param/ParamEntryBuilder.kt @@ -1,10 +1,10 @@ package com.runetopic.loader.index.config.param -import com.runetopic.cache.extension.readCp1252Char -import com.runetopic.cache.extension.readString -import com.runetopic.cache.extension.readUnsignedByte import com.runetopic.cache.store.Js5Store import com.runetopic.loader.IEntryBuilder +import com.runetopic.loader.extension.readCp1252Char +import com.runetopic.loader.extension.readString +import com.runetopic.loader.extension.readUnsignedByte import java.nio.ByteBuffer /** diff --git a/loader/src/main/kotlin/com/runetopic/loader/index/config/skybox/SkyBoxEntryBuilder.kt b/loader/src/main/kotlin/com/runetopic/loader/index/config/skybox/SkyBoxEntryBuilder.kt index 9f730c9..3edbc07 100644 --- a/loader/src/main/kotlin/com/runetopic/loader/index/config/skybox/SkyBoxEntryBuilder.kt +++ b/loader/src/main/kotlin/com/runetopic/loader/index/config/skybox/SkyBoxEntryBuilder.kt @@ -1,9 +1,9 @@ package com.runetopic.loader.index.config.skybox -import com.runetopic.cache.extension.readUnsignedByte -import com.runetopic.cache.extension.readUnsignedShort import com.runetopic.cache.store.Js5Store import com.runetopic.loader.IEntryBuilder +import com.runetopic.loader.extension.readUnsignedByte +import com.runetopic.loader.extension.readUnsignedShort import java.nio.ByteBuffer /** diff --git a/loader/src/main/kotlin/com/runetopic/loader/index/config/struct/StructEntryBuilder.kt b/loader/src/main/kotlin/com/runetopic/loader/index/config/struct/StructEntryBuilder.kt index 58e7619..cf99574 100644 --- a/loader/src/main/kotlin/com/runetopic/loader/index/config/struct/StructEntryBuilder.kt +++ b/loader/src/main/kotlin/com/runetopic/loader/index/config/struct/StructEntryBuilder.kt @@ -1,11 +1,11 @@ package com.runetopic.loader.index.config.struct -import com.runetopic.cache.extension.readMedium -import com.runetopic.cache.extension.readString -import com.runetopic.cache.extension.readUnsignedByte -import com.runetopic.cache.extension.toBoolean import com.runetopic.cache.store.Js5Store import com.runetopic.loader.IEntryBuilder +import com.runetopic.loader.extension.readMedium +import com.runetopic.loader.extension.readString +import com.runetopic.loader.extension.readUnsignedByte +import com.runetopic.loader.extension.toBoolean import java.nio.ByteBuffer /** diff --git a/loader/src/main/kotlin/com/runetopic/loader/index/config/underlay/UnderlayEntryBuilder.kt b/loader/src/main/kotlin/com/runetopic/loader/index/config/underlay/UnderlayEntryBuilder.kt index e6a8201..66c40d5 100644 --- a/loader/src/main/kotlin/com/runetopic/loader/index/config/underlay/UnderlayEntryBuilder.kt +++ b/loader/src/main/kotlin/com/runetopic/loader/index/config/underlay/UnderlayEntryBuilder.kt @@ -1,10 +1,10 @@ package com.runetopic.loader.index.config.underlay -import com.runetopic.cache.extension.readMedium -import com.runetopic.cache.extension.readUnsignedByte -import com.runetopic.cache.extension.readUnsignedShort import com.runetopic.cache.store.Js5Store import com.runetopic.loader.IEntryBuilder +import com.runetopic.loader.extension.readMedium +import com.runetopic.loader.extension.readUnsignedByte +import com.runetopic.loader.extension.readUnsignedShort import java.nio.ByteBuffer /** diff --git a/loader/src/main/kotlin/com/runetopic/loader/index/loc/LocEntryBuilder.kt b/loader/src/main/kotlin/com/runetopic/loader/index/loc/LocEntryBuilder.kt index 005326b..573976a 100644 --- a/loader/src/main/kotlin/com/runetopic/loader/index/loc/LocEntryBuilder.kt +++ b/loader/src/main/kotlin/com/runetopic/loader/index/loc/LocEntryBuilder.kt @@ -1,8 +1,8 @@ package com.runetopic.loader.index.loc -import com.runetopic.cache.extension.* import com.runetopic.cache.store.Js5Store import com.runetopic.loader.IEntryBuilder +import com.runetopic.loader.extension.* import java.nio.ByteBuffer /** diff --git a/loader/src/main/kotlin/com/runetopic/loader/index/map/MapEntryBuilder.kt b/loader/src/main/kotlin/com/runetopic/loader/index/map/MapEntryBuilder.kt index 1e87eed..1038215 100644 --- a/loader/src/main/kotlin/com/runetopic/loader/index/map/MapEntryBuilder.kt +++ b/loader/src/main/kotlin/com/runetopic/loader/index/map/MapEntryBuilder.kt @@ -1,11 +1,11 @@ package com.runetopic.loader.index.map import com.runetopic.cache.codec.decompress -import com.runetopic.cache.extension.readUnsignedByte -import com.runetopic.cache.extension.readUnsignedShort -import com.runetopic.cache.extension.skip import com.runetopic.cache.store.Js5Store import com.runetopic.loader.IEntryBuilder +import com.runetopic.loader.extension.readUnsignedByte +import com.runetopic.loader.extension.readUnsignedShort +import com.runetopic.loader.extension.skip import com.runetopic.loader.util.vector.Vector3f import java.nio.ByteBuffer diff --git a/loader/src/main/kotlin/com/runetopic/loader/index/map/MapLocationEntryBuilder.kt b/loader/src/main/kotlin/com/runetopic/loader/index/map/MapLocationEntryBuilder.kt index ac08a38..b56b277 100644 --- a/loader/src/main/kotlin/com/runetopic/loader/index/map/MapLocationEntryBuilder.kt +++ b/loader/src/main/kotlin/com/runetopic/loader/index/map/MapLocationEntryBuilder.kt @@ -1,11 +1,11 @@ package com.runetopic.loader.index.map import com.runetopic.cache.codec.decompress -import com.runetopic.cache.extension.readUnsignedByte -import com.runetopic.cache.extension.readUnsignedIntSmartShortCompat -import com.runetopic.cache.extension.readUnsignedSmart import com.runetopic.cache.store.Js5Store import com.runetopic.loader.IEntryBuilder +import com.runetopic.loader.extension.readUnsignedByte +import com.runetopic.loader.extension.readUnsignedIntSmartShortCompat +import com.runetopic.loader.extension.readUnsignedSmart import java.nio.ByteBuffer import java.util.zip.ZipException diff --git a/loader/src/main/kotlin/com/runetopic/loader/index/npc/NpcEntryBuilder.kt b/loader/src/main/kotlin/com/runetopic/loader/index/npc/NpcEntryBuilder.kt index f9e95cd..c4477b5 100644 --- a/loader/src/main/kotlin/com/runetopic/loader/index/npc/NpcEntryBuilder.kt +++ b/loader/src/main/kotlin/com/runetopic/loader/index/npc/NpcEntryBuilder.kt @@ -1,8 +1,8 @@ package com.runetopic.loader.index.npc -import com.runetopic.cache.extension.* import com.runetopic.cache.store.Js5Store import com.runetopic.loader.IEntryBuilder +import com.runetopic.loader.extension.* import java.nio.ByteBuffer /** diff --git a/loader/src/main/kotlin/com/runetopic/loader/index/obj/ObjEntryBuilder.kt b/loader/src/main/kotlin/com/runetopic/loader/index/obj/ObjEntryBuilder.kt index 495e52b..7edbe8f 100644 --- a/loader/src/main/kotlin/com/runetopic/loader/index/obj/ObjEntryBuilder.kt +++ b/loader/src/main/kotlin/com/runetopic/loader/index/obj/ObjEntryBuilder.kt @@ -1,8 +1,8 @@ package com.runetopic.loader.index.obj -import com.runetopic.cache.extension.* import com.runetopic.cache.store.Js5Store import com.runetopic.loader.IEntryBuilder +import com.runetopic.loader.extension.* import java.nio.ByteBuffer /** diff --git a/loader/src/main/kotlin/com/runetopic/loader/index/particle/ParticleEntryBuilder.kt b/loader/src/main/kotlin/com/runetopic/loader/index/particle/ParticleEntryBuilder.kt index fea761e..f6536f1 100644 --- a/loader/src/main/kotlin/com/runetopic/loader/index/particle/ParticleEntryBuilder.kt +++ b/loader/src/main/kotlin/com/runetopic/loader/index/particle/ParticleEntryBuilder.kt @@ -1,11 +1,11 @@ package com.runetopic.loader.index.particle -import com.runetopic.cache.extension.readUnsignedByte -import com.runetopic.cache.extension.readUnsignedShort -import com.runetopic.cache.extension.skip -import com.runetopic.cache.extension.toBoolean import com.runetopic.cache.store.Js5Store import com.runetopic.loader.IEntryBuilder +import com.runetopic.loader.extension.readUnsignedByte +import com.runetopic.loader.extension.readUnsignedShort +import com.runetopic.loader.extension.skip +import com.runetopic.loader.extension.toBoolean import java.nio.ByteBuffer /** diff --git a/loader/src/main/kotlin/com/runetopic/loader/index/spotanim/SpotAnimationEntryBuilder.kt b/loader/src/main/kotlin/com/runetopic/loader/index/spotanim/SpotAnimationEntryBuilder.kt index 67b8819..ca955c1 100644 --- a/loader/src/main/kotlin/com/runetopic/loader/index/spotanim/SpotAnimationEntryBuilder.kt +++ b/loader/src/main/kotlin/com/runetopic/loader/index/spotanim/SpotAnimationEntryBuilder.kt @@ -1,9 +1,9 @@ package com.runetopic.loader.index.spotanim -import com.runetopic.cache.extension.readUnsignedByte -import com.runetopic.cache.extension.readUnsignedShort import com.runetopic.cache.store.Js5Store import com.runetopic.loader.IEntryBuilder +import com.runetopic.loader.extension.readUnsignedByte +import com.runetopic.loader.extension.readUnsignedShort import java.nio.ByteBuffer /** From 365adc5c02db0b51c1f69a392b4b371bd2945d86 Mon Sep 17 00:00:00 2001 From: Jordan Date: Mon, 18 Oct 2021 19:29:03 -0400 Subject: [PATCH 2/4] Separate out Js5Index from inside ReferenceTable. --- .../cache/hierarchy/RawReferenceTable.kt | 50 +++++++++ .../cache/hierarchy/ReferenceTable.kt | 106 +++--------------- .../cache/hierarchy/index/Js5Index.kt | 2 +- .../cache/hierarchy/index/RawIndex.kt | 30 +++++ .../cache/hierarchy/index/group/file/File.kt | 60 ++++++++++ .../cache/store/storage/js5/Js5DiskStorage.kt | 40 ++++++- 6 files changed, 198 insertions(+), 90 deletions(-) create mode 100644 cache/src/main/kotlin/com/runetopic/cache/hierarchy/RawReferenceTable.kt create mode 100644 cache/src/main/kotlin/com/runetopic/cache/hierarchy/index/RawIndex.kt diff --git a/cache/src/main/kotlin/com/runetopic/cache/hierarchy/RawReferenceTable.kt b/cache/src/main/kotlin/com/runetopic/cache/hierarchy/RawReferenceTable.kt new file mode 100644 index 0000000..6715999 --- /dev/null +++ b/cache/src/main/kotlin/com/runetopic/cache/hierarchy/RawReferenceTable.kt @@ -0,0 +1,50 @@ +package com.runetopic.cache.hierarchy + +/** + * @author Jordan Abraham + */ +data class RawReferenceTable( + val count: Int, + val isNamed: Boolean, + val groupIds: IntArray, + val groupTables: List, + val groupNameHashes: IntArray, + val groupCrcs: IntArray, + val groupWhirlpools: Array, + val groupRevisions: IntArray, + val groupFileIds: IntArray, + val fileIds: Array, + val fileNameHashes: Array +) { + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (javaClass != other?.javaClass) return false + + other as RawReferenceTable + + if (!groupIds.contentEquals(other.groupIds)) return false + if (groupTables != other.groupTables) return false + if (!groupNameHashes.contentEquals(other.groupNameHashes)) return false + if (!groupCrcs.contentEquals(other.groupCrcs)) return false + if (!groupWhirlpools.contentDeepEquals(other.groupWhirlpools)) return false + if (!groupRevisions.contentEquals(other.groupRevisions)) return false + if (!groupFileIds.contentEquals(other.groupFileIds)) return false + if (!fileIds.contentDeepEquals(other.fileIds)) return false + if (!fileNameHashes.contentDeepEquals(other.fileNameHashes)) return false + + return true + } + + override fun hashCode(): Int { + var result = groupIds.contentHashCode() + result = 31 * result + groupTables.hashCode() + result = 31 * result + groupNameHashes.contentHashCode() + result = 31 * result + groupCrcs.contentHashCode() + result = 31 * result + groupWhirlpools.contentDeepHashCode() + result = 31 * result + groupRevisions.contentHashCode() + result = 31 * result + groupFileIds.contentHashCode() + result = 31 * result + fileIds.contentDeepHashCode() + result = 31 * result + fileNameHashes.contentDeepHashCode() + return result + } +} \ No newline at end of file diff --git a/cache/src/main/kotlin/com/runetopic/cache/hierarchy/ReferenceTable.kt b/cache/src/main/kotlin/com/runetopic/cache/hierarchy/ReferenceTable.kt index d9abff3..81387d2 100644 --- a/cache/src/main/kotlin/com/runetopic/cache/hierarchy/ReferenceTable.kt +++ b/cache/src/main/kotlin/com/runetopic/cache/hierarchy/ReferenceTable.kt @@ -1,20 +1,15 @@ package com.runetopic.cache.hierarchy import com.runetopic.cache.codec.Container -import com.runetopic.cache.codec.decompress import com.runetopic.cache.exception.ProtocolException import com.runetopic.cache.extension.readUnsignedByte import com.runetopic.cache.extension.readUnsignedIntShortSmart import com.runetopic.cache.extension.readUnsignedShort -import com.runetopic.cache.hierarchy.index.Js5Index -import com.runetopic.cache.hierarchy.index.group.Js5Group -import com.runetopic.cache.hierarchy.index.group.file.File -import com.runetopic.cache.hierarchy.index.group.file.Js5File +import com.runetopic.cache.hierarchy.index.RawIndex import com.runetopic.cache.store.storage.js5.IDatFile import com.runetopic.cache.store.storage.js5.IIdxFile import java.nio.ByteBuffer import java.util.* -import java.util.zip.ZipException /** * @author Tyler Telis @@ -35,7 +30,7 @@ internal data class ReferenceTable( idxFile: IIdxFile, whirlpool: ByteArray, decompressed: Container - ): Js5Index { + ): RawIndex { val buffer = ByteBuffer.wrap(decompressed.data) val crc = decompressed.crc val compressionType = decompressed.compression @@ -51,21 +46,16 @@ internal data class ReferenceTable( (0 until count).forEach { groupTables.add(datFile.readReferenceTable(idxFile.id(), idxFile.loadReferenceTable(it))) } - return loadIndexContents(idxFile.id(), buffer, crc, compressionType, revision, protocol, hash, count, whirlpool, groupTables) + return RawIndex(idxFile.id(), crc, whirlpool, compressionType, protocol, revision, buildRawReferenceTable(buffer, protocol, hash, count, groupTables)) } - private fun loadIndexContents( - indexId: Int, + private fun buildRawReferenceTable( buffer: ByteBuffer, - crc: Int, - compressionType: Int, - revision: Int, protocol: Int, hash: Int, count: Int, - whirlpool: ByteArray, groupTables: List - ): Js5Index { + ): RawReferenceTable { val isNamed = (0x1 and hash) != 0 val isUsingWhirlpool = (0x2 and hash) != 0 @@ -84,25 +74,21 @@ internal data class ReferenceTable( val groupWhirlpools = groupWhirlpools(largestGroupId, isUsingWhirlpool, count, buffer, groupIds) val groupRevisions = groupRevisions(largestGroupId, count, groupIds, buffer) val groupFileIds = groupFileIds(largestGroupId, count, groupIds, buffer, protocol) - val fileIds = fileIds(largestGroupId, groupFileIds, count, groupIds, buffer, protocol) val fileNameHashes = fileNameHashes(largestGroupId, groupFileIds, count, groupIds, buffer, isNamed) - - val groups = hashMapOf() - (0 until count).forEach { - val groupId = groupIds[it] - groups[it] = (Js5Group( - groupId, - groupNameHashes[groupId], - groupCrcs[groupId], - groupWhirlpools[groupId], - groupRevisions[groupId], - intArrayOf(),//TODO - groupFiles(fileIds, fileNameHashes, groupTables[it], groupFileIds[it], it), - groupTables[it] - )) - } - return Js5Index(indexId, crc, whirlpool, compressionType, protocol, revision, isNamed, groups) + return RawReferenceTable( + count, + isNamed, + groupIds, + groupTables, + groupNameHashes, + groupCrcs, + groupWhirlpools, + groupRevisions, + groupFileIds, + fileIds, + fileNameHashes + ) } private fun groupFileIds( @@ -220,62 +206,6 @@ internal data class ReferenceTable( return fileNameHashes } - private fun groupFiles( - fileIds: Array, - fileNameHashes: Array, - groupReferenceTableData: ByteArray, - count: Int, - groupId: Int - ): Map { - if (groupReferenceTableData.isEmpty()) return hashMapOf(Pair(0, Js5File.DEFAULT)) - - val src: ByteArray = try { - groupReferenceTableData.decompress() - } catch (exception: ZipException) { - groupReferenceTableData - } - - if (count == 1) { - return hashMapOf(Pair(0, Js5File(fileIds[groupId][0], fileNameHashes[groupId][0], src))) - } - - var position = src.size - val chunks = src[--position].toInt() and 0xFF - position -= chunks * (count * 4) - val buffer = ByteBuffer.wrap(src) - buffer.position(position) - val filesSizes = IntArray(count) - (0 until chunks).forEach { _ -> - var read = 0 - (0 until count).forEach { - read += buffer.int - filesSizes[it] += read - } - } - val filesDatas = Array(count) { byteArrayOf() } - (0 until count).forEach { - filesDatas[it] = ByteArray(filesSizes[it]) - filesSizes[it] = 0 - } - buffer.position(position) - var offset = 0 - (0 until chunks).forEach { _ -> - var read = 0 - (0 until count).forEach { - read += buffer.int - System.arraycopy(src, offset, filesDatas[it], filesSizes[it], read) - offset += read - filesSizes[it] += read - } - } - - val files = hashMapOf() - (0 until count).forEach { - files[it] = Js5File(fileIds[groupId][it], fileNameHashes[groupId][it], filesDatas[it]) - } - return files - } - override fun hashCode(): Int { var hash = 7 hash = 19 * hash + Objects.hashCode(this.idxFile) diff --git a/cache/src/main/kotlin/com/runetopic/cache/hierarchy/index/Js5Index.kt b/cache/src/main/kotlin/com/runetopic/cache/hierarchy/index/Js5Index.kt index 4ebefc3..411a051 100644 --- a/cache/src/main/kotlin/com/runetopic/cache/hierarchy/index/Js5Index.kt +++ b/cache/src/main/kotlin/com/runetopic/cache/hierarchy/index/Js5Index.kt @@ -18,7 +18,7 @@ class Js5Index( private val protocol: Int, private val revision: Int, private val isNamed: Boolean, - private val groups: Map + private val groups: Map ): Index { override fun getId(): Int = id override fun getCRC(): Int = crc diff --git a/cache/src/main/kotlin/com/runetopic/cache/hierarchy/index/RawIndex.kt b/cache/src/main/kotlin/com/runetopic/cache/hierarchy/index/RawIndex.kt new file mode 100644 index 0000000..315ddff --- /dev/null +++ b/cache/src/main/kotlin/com/runetopic/cache/hierarchy/index/RawIndex.kt @@ -0,0 +1,30 @@ +package com.runetopic.cache.hierarchy.index + +import com.runetopic.cache.hierarchy.RawReferenceTable +import com.runetopic.cache.hierarchy.index.group.Group + +/** + * @author Jordan Abraham + */ +class RawIndex( + private val id: Int, + private val crc: Int, + private val whirlpool: ByteArray, + private val compression: Int, + private val protocol: Int, + private val revision: Int, + val referenceTable: RawReferenceTable +): Index { + override fun getId(): Int = id + override fun getCRC(): Int = crc + override fun getWhirlpool(): ByteArray = whirlpool + override fun getCompression(): Int = compression + override fun getProtocol(): Int = protocol + override fun getRevision(): Int = revision + override fun getIsNamed(): Boolean = referenceTable.isNamed + + override fun getGroups(): Collection = throw RuntimeException() + override fun getGroup(groupId: Int): Group = throw RuntimeException() + override fun getGroup(groupName: String): Group = throw RuntimeException() + override fun expand(): Int = throw RuntimeException() +} \ No newline at end of file diff --git a/cache/src/main/kotlin/com/runetopic/cache/hierarchy/index/group/file/File.kt b/cache/src/main/kotlin/com/runetopic/cache/hierarchy/index/group/file/File.kt index b88120b..d6c5115 100644 --- a/cache/src/main/kotlin/com/runetopic/cache/hierarchy/index/group/file/File.kt +++ b/cache/src/main/kotlin/com/runetopic/cache/hierarchy/index/group/file/File.kt @@ -1,5 +1,9 @@ package com.runetopic.cache.hierarchy.index.group.file +import com.runetopic.cache.codec.decompress +import java.nio.ByteBuffer +import java.util.zip.ZipException + /** * @author Jordan Abraham */ @@ -11,4 +15,60 @@ interface File: Comparable { override fun compareTo(other: File): Int { return getId().compareTo(other.getId()) } +} + +fun groupFiles( + fileIds: Array, + fileNameHashes: Array, + groupReferenceTableData: ByteArray, + count: Int, + groupId: Int +): Map { + if (groupReferenceTableData.isEmpty()) return hashMapOf(Pair(0, Js5File.DEFAULT)) + + val src: ByteArray = try { + groupReferenceTableData.decompress() + } catch (exception: ZipException) { + groupReferenceTableData + } + + if (count == 1) { + return hashMapOf(Pair(0, Js5File(fileIds[groupId][0], fileNameHashes[groupId][0], src))) + } + + var position = src.size + val chunks = src[--position].toInt() and 0xFF + position -= chunks * (count * 4) + val buffer = ByteBuffer.wrap(src) + buffer.position(position) + val filesSizes = IntArray(count) + (0 until chunks).forEach { _ -> + var read = 0 + (0 until count).forEach { + read += buffer.int + filesSizes[it] += read + } + } + val filesDatas = Array(count) { byteArrayOf() } + (0 until count).forEach { + filesDatas[it] = ByteArray(filesSizes[it]) + filesSizes[it] = 0 + } + buffer.position(position) + var offset = 0 + (0 until chunks).forEach { _ -> + var read = 0 + (0 until count).forEach { + read += buffer.int + System.arraycopy(src, offset, filesDatas[it], filesSizes[it], read) + offset += read + filesSizes[it] += read + } + } + + val files = hashMapOf() + (0 until count).forEach { + files[it] = Js5File(fileIds[groupId][it], fileNameHashes[groupId][it], filesDatas[it]) + } + return files } \ No newline at end of file diff --git a/cache/src/main/kotlin/com/runetopic/cache/store/storage/js5/Js5DiskStorage.kt b/cache/src/main/kotlin/com/runetopic/cache/store/storage/js5/Js5DiskStorage.kt index 52faa47..54f5945 100644 --- a/cache/src/main/kotlin/com/runetopic/cache/store/storage/js5/Js5DiskStorage.kt +++ b/cache/src/main/kotlin/com/runetopic/cache/store/storage/js5/Js5DiskStorage.kt @@ -4,6 +4,8 @@ import com.github.michaelbull.logging.InlineLogger import com.runetopic.cache.codec.ContainerCodec import com.runetopic.cache.hierarchy.index.Index import com.runetopic.cache.hierarchy.index.Js5Index +import com.runetopic.cache.hierarchy.index.group.Js5Group +import com.runetopic.cache.hierarchy.index.group.file.groupFiles import com.runetopic.cache.store.Constants import com.runetopic.cache.store.Js5Store import com.runetopic.cache.store.storage.IStorage @@ -80,7 +82,43 @@ internal class Js5DiskStorage( return } val indexDatTable = datFile.readReferenceTable(masterIdxFile.id(), indexTable) - store.addIndex(indexTable.loadIndex(datFile, getIdxFile(indexId), indexDatTable.toWhirlpool(), ContainerCodec.decompress(indexDatTable))) + + val rawIndex = indexTable.loadIndex(datFile, getIdxFile(indexId), indexDatTable.toWhirlpool(), ContainerCodec.decompress(indexDatTable)) + val rawReferenceTable = rawIndex.referenceTable + + val groups = hashMapOf() + (0 until rawReferenceTable.count).forEach { + val groupId = rawReferenceTable.groupIds[it] + groups[it] = (Js5Group( + groupId, + rawReferenceTable.groupNameHashes[groupId], + rawReferenceTable.groupCrcs[groupId], + rawReferenceTable.groupWhirlpools[groupId], + rawReferenceTable.groupRevisions[groupId], + intArrayOf(),//TODO + groupFiles( + rawReferenceTable.fileIds, + rawReferenceTable.fileNameHashes, + rawReferenceTable.groupTables[it], + rawReferenceTable.groupFileIds[it], + it + ), + rawReferenceTable.groupTables[it] + )) + } + + store.addIndex( + Js5Index( + indexId, + rawIndex.getCRC(), + rawIndex.getWhirlpool(), + rawIndex.getCompression(), + rawIndex.getProtocol(), + rawIndex.getRevision(), + rawIndex.getIsNamed(), + groups + ) + ) } override fun loadMasterReferenceTable(groupId: Int): ByteArray { From 3af3457b070df0532e426cd3b8404d3e910f2af1 Mon Sep 17 00:00:00 2001 From: Jordan Date: Mon, 18 Oct 2021 19:30:07 -0400 Subject: [PATCH 3/4] Bump version and update README. --- README.md | 2 +- cache/build.gradle.kts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 1fec53e..bc580d9 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ A cache library written in Kotlin. # Implementation Just use cache if you do not require any of the revision specific loaders. ``` -cache = { module = "com.runetopic.cache:cache", version.ref "1.4.12-SNAPSHOT" } +cache = { module = "com.runetopic.cache:cache", version.ref "1.4.13-SNAPSHOT" } loader = { module = "com.runetopic.cache:loader", version.ref "647.6.2-SNAPSHOT" } ``` diff --git a/cache/build.gradle.kts b/cache/build.gradle.kts index 3d5e525..6769b27 100644 --- a/cache/build.gradle.kts +++ b/cache/build.gradle.kts @@ -4,7 +4,7 @@ plugins { signing } -version = "1.4.12-SNAPSHOT" +version = "1.4.13-SNAPSHOT" java { withJavadocJar() From beef5b9b782580cef5be736b19effb74a01d9a17 Mon Sep 17 00:00:00 2001 From: Jordan Date: Mon, 18 Oct 2021 22:19:43 -0400 Subject: [PATCH 4/4] Rework the cache heiarchy structure and bump version and update README. --- README.md | 4 +- cache/build.gradle.kts | 2 +- .../cache/hierarchy/RawReferenceTable.kt | 50 ---- .../cache/hierarchy/ReferenceTable.kt | 189 ------------- .../runetopic/cache/hierarchy/index/Index.kt | 71 ++++- .../cache/hierarchy/index/Js5Index.kt | 39 --- .../cache/hierarchy/index/RawIndex.kt | 30 --- .../cache/hierarchy/index/group/Group.kt | 68 ++++- .../cache/hierarchy/index/group/Js5Group.kt | 36 --- .../cache/hierarchy/index/group/file/File.kt | 83 ++---- .../hierarchy/index/group/file/Js5File.kt | 21 -- .../com/runetopic/cache/store/Js5Store.kt | 18 +- .../cache/store/storage/js5/Js5DiskStorage.kt | 51 +--- .../store/storage/js5/Js5IndexFactory.kt | 248 ++++++++++++++++++ loader/build.gradle.kts | 2 +- .../config/idk/IdentityKitEntryBuilder.kt | 4 +- .../index/config/inv/InventoryEntryBuilder.kt | 4 +- .../config/lighting/LightingEntryBuilder.kt | 4 +- .../config/mouseicon/MouseIconEntryBuilder.kt | 4 +- .../config/overlay/OverlayEntryBuilder.kt | 4 +- .../index/config/param/ParamEntryBuilder.kt | 4 +- .../index/config/skybox/SkyBoxEntryBuilder.kt | 4 +- .../index/config/struct/StructEntryBuilder.kt | 4 +- .../config/underlay/UnderlayEntryBuilder.kt | 4 +- .../loader/index/loc/LocEntryBuilder.kt | 2 +- .../loader/index/map/MapEntryBuilder.kt | 2 +- .../index/map/MapLocationEntryBuilder.kt | 2 +- .../loader/index/npc/NpcEntryBuilder.kt | 2 +- .../loader/index/obj/ObjEntryBuilder.kt | 2 +- .../index/particle/ParticleEntryBuilder.kt | 4 +- .../spotanim/SpotAnimationEntryBuilder.kt | 2 +- 31 files changed, 428 insertions(+), 536 deletions(-) delete mode 100644 cache/src/main/kotlin/com/runetopic/cache/hierarchy/RawReferenceTable.kt delete mode 100644 cache/src/main/kotlin/com/runetopic/cache/hierarchy/index/Js5Index.kt delete mode 100644 cache/src/main/kotlin/com/runetopic/cache/hierarchy/index/RawIndex.kt delete mode 100644 cache/src/main/kotlin/com/runetopic/cache/hierarchy/index/group/Js5Group.kt delete mode 100644 cache/src/main/kotlin/com/runetopic/cache/hierarchy/index/group/file/Js5File.kt create mode 100644 cache/src/main/kotlin/com/runetopic/cache/store/storage/js5/Js5IndexFactory.kt diff --git a/README.md b/README.md index bc580d9..d71c47e 100644 --- a/README.md +++ b/README.md @@ -28,8 +28,8 @@ A cache library written in Kotlin. # Implementation Just use cache if you do not require any of the revision specific loaders. ``` -cache = { module = "com.runetopic.cache:cache", version.ref "1.4.13-SNAPSHOT" } -loader = { module = "com.runetopic.cache:loader", version.ref "647.6.2-SNAPSHOT" } +cache = { module = "com.runetopic.cache:cache", version.ref "1.4.15-SNAPSHOT" } +loader = { module = "com.runetopic.cache:loader", version.ref "647.6.3-SNAPSHOT" } ``` ``` diff --git a/cache/build.gradle.kts b/cache/build.gradle.kts index 6769b27..c2863e9 100644 --- a/cache/build.gradle.kts +++ b/cache/build.gradle.kts @@ -4,7 +4,7 @@ plugins { signing } -version = "1.4.13-SNAPSHOT" +version = "1.4.15-SNAPSHOT" java { withJavadocJar() diff --git a/cache/src/main/kotlin/com/runetopic/cache/hierarchy/RawReferenceTable.kt b/cache/src/main/kotlin/com/runetopic/cache/hierarchy/RawReferenceTable.kt deleted file mode 100644 index 6715999..0000000 --- a/cache/src/main/kotlin/com/runetopic/cache/hierarchy/RawReferenceTable.kt +++ /dev/null @@ -1,50 +0,0 @@ -package com.runetopic.cache.hierarchy - -/** - * @author Jordan Abraham - */ -data class RawReferenceTable( - val count: Int, - val isNamed: Boolean, - val groupIds: IntArray, - val groupTables: List, - val groupNameHashes: IntArray, - val groupCrcs: IntArray, - val groupWhirlpools: Array, - val groupRevisions: IntArray, - val groupFileIds: IntArray, - val fileIds: Array, - val fileNameHashes: Array -) { - override fun equals(other: Any?): Boolean { - if (this === other) return true - if (javaClass != other?.javaClass) return false - - other as RawReferenceTable - - if (!groupIds.contentEquals(other.groupIds)) return false - if (groupTables != other.groupTables) return false - if (!groupNameHashes.contentEquals(other.groupNameHashes)) return false - if (!groupCrcs.contentEquals(other.groupCrcs)) return false - if (!groupWhirlpools.contentDeepEquals(other.groupWhirlpools)) return false - if (!groupRevisions.contentEquals(other.groupRevisions)) return false - if (!groupFileIds.contentEquals(other.groupFileIds)) return false - if (!fileIds.contentDeepEquals(other.fileIds)) return false - if (!fileNameHashes.contentDeepEquals(other.fileNameHashes)) return false - - return true - } - - override fun hashCode(): Int { - var result = groupIds.contentHashCode() - result = 31 * result + groupTables.hashCode() - result = 31 * result + groupNameHashes.contentHashCode() - result = 31 * result + groupCrcs.contentHashCode() - result = 31 * result + groupWhirlpools.contentDeepHashCode() - result = 31 * result + groupRevisions.contentHashCode() - result = 31 * result + groupFileIds.contentHashCode() - result = 31 * result + fileIds.contentDeepHashCode() - result = 31 * result + fileNameHashes.contentDeepHashCode() - return result - } -} \ No newline at end of file diff --git a/cache/src/main/kotlin/com/runetopic/cache/hierarchy/ReferenceTable.kt b/cache/src/main/kotlin/com/runetopic/cache/hierarchy/ReferenceTable.kt index 81387d2..d4d63cc 100644 --- a/cache/src/main/kotlin/com/runetopic/cache/hierarchy/ReferenceTable.kt +++ b/cache/src/main/kotlin/com/runetopic/cache/hierarchy/ReferenceTable.kt @@ -1,14 +1,6 @@ package com.runetopic.cache.hierarchy -import com.runetopic.cache.codec.Container -import com.runetopic.cache.exception.ProtocolException -import com.runetopic.cache.extension.readUnsignedByte -import com.runetopic.cache.extension.readUnsignedIntShortSmart -import com.runetopic.cache.extension.readUnsignedShort -import com.runetopic.cache.hierarchy.index.RawIndex -import com.runetopic.cache.store.storage.js5.IDatFile import com.runetopic.cache.store.storage.js5.IIdxFile -import java.nio.ByteBuffer import java.util.* /** @@ -25,187 +17,6 @@ internal data class ReferenceTable( ) { fun exists(): Boolean = (length != 0 && sector != 0) - fun loadIndex( - datFile: IDatFile, - idxFile: IIdxFile, - whirlpool: ByteArray, - decompressed: Container - ): RawIndex { - val buffer = ByteBuffer.wrap(decompressed.data) - val crc = decompressed.crc - val compressionType = decompressed.compression - val protocol = buffer.readUnsignedByte() - val revision = when { - protocol < 5 || protocol > 7 -> throw ProtocolException("Unhandled protocol $protocol when reading index $this") - protocol >= 6 -> buffer.int - else -> 0 - } - val hash = buffer.readUnsignedByte() - val count = if (protocol >= 7) buffer.readUnsignedIntShortSmart() else buffer.readUnsignedShort() - val groupTables = mutableListOf() - (0 until count).forEach { - groupTables.add(datFile.readReferenceTable(idxFile.id(), idxFile.loadReferenceTable(it))) - } - return RawIndex(idxFile.id(), crc, whirlpool, compressionType, protocol, revision, buildRawReferenceTable(buffer, protocol, hash, count, groupTables)) - } - - private fun buildRawReferenceTable( - buffer: ByteBuffer, - protocol: Int, - hash: Int, - count: Int, - groupTables: List - ): RawReferenceTable { - val isNamed = (0x1 and hash) != 0 - val isUsingWhirlpool = (0x2 and hash) != 0 - - val groupIds = IntArray(count) - var lastGroupId = 0 - var biggest = -1 - (0 until count).forEach { - groupIds[it] = if (protocol >= 7) { buffer.readUnsignedIntShortSmart() } else { buffer.readUnsignedShort() } - .let { id -> lastGroupId += id; lastGroupId } - if (groupIds[it] > biggest) biggest = groupIds[it] - } - - val largestGroupId = biggest + 1 - val groupNameHashes = groupNameHashes(largestGroupId, count, isNamed, groupIds, buffer) - val groupCrcs = groupCrcs(largestGroupId, count, groupIds, buffer) - val groupWhirlpools = groupWhirlpools(largestGroupId, isUsingWhirlpool, count, buffer, groupIds) - val groupRevisions = groupRevisions(largestGroupId, count, groupIds, buffer) - val groupFileIds = groupFileIds(largestGroupId, count, groupIds, buffer, protocol) - val fileIds = fileIds(largestGroupId, groupFileIds, count, groupIds, buffer, protocol) - val fileNameHashes = fileNameHashes(largestGroupId, groupFileIds, count, groupIds, buffer, isNamed) - return RawReferenceTable( - count, - isNamed, - groupIds, - groupTables, - groupNameHashes, - groupCrcs, - groupWhirlpools, - groupRevisions, - groupFileIds, - fileIds, - fileNameHashes - ) - } - - private fun groupFileIds( - largestGroupId: Int, - count: Int, - groupIds: IntArray, - buffer: ByteBuffer, - protocol: Int - ): IntArray { - val groupFileIds = IntArray(largestGroupId) - (0 until count).forEach { - groupFileIds[groupIds[it]] = if (protocol >= 7) buffer.readUnsignedIntShortSmart() else buffer.readUnsignedShort() - } - return groupFileIds - } - - private fun groupRevisions( - largestGroupId: Int, - count: Int, - groupIds: IntArray, - buffer: ByteBuffer - ): IntArray { - val revisions = IntArray(largestGroupId) - (0 until count).forEach { - revisions[groupIds[it]] = buffer.int - } - return revisions - } - - private fun groupWhirlpools( - largestGroupId: Int, - usesWhirlpool: Boolean, - count: Int, - buffer: ByteBuffer, - groupIds: IntArray - ): Array { - val whirlpools = Array(largestGroupId) { ByteArray(64) } - if (usesWhirlpool.not()) return whirlpools - - (0 until count).forEach { - val whirlpool = ByteArray(64) - buffer.get(whirlpool) - whirlpools[groupIds[it]] = whirlpool - } - return whirlpools - } - - private fun groupCrcs( - largestGroupId: Int, - count: Int, - groupIds: IntArray, - buffer: ByteBuffer - ): IntArray { - val crcs = IntArray(largestGroupId) - (0 until count).forEach { - crcs[groupIds[it]] = buffer.int - } - return crcs - } - - private fun groupNameHashes( - largestGroupId: Int, - count: Int, - isNamed: Boolean, - groupIds: IntArray, - buffer: ByteBuffer - ): IntArray { - val nameHashes = IntArray(largestGroupId) { -1 } - if (isNamed.not()) return nameHashes - - (0 until count).forEach { - nameHashes[groupIds[it]] = buffer.int - } - return nameHashes - } - - private fun fileIds( - largestGroupId: Int, - validFileIds: IntArray, - count: Int, - groupIds: IntArray, - buffer: ByteBuffer, - protocol: Int - ): Array { - val fileIds = Array(largestGroupId) { IntArray(validFileIds[it]) } - (0 until count).forEach { - val groupId = groupIds[it] - var currentFileId = 0 - (0 until validFileIds[groupId]).forEach { fileId -> - if (protocol >= 7) { buffer.readUnsignedIntShortSmart() } else { buffer.readUnsignedShort() } - .let { i -> currentFileId += i; currentFileId } - .also { fileIds[groupId][fileId] = currentFileId } - } - } - return fileIds - } - - private fun fileNameHashes( - largestGroupId: Int, - validFileIds: IntArray, - count: Int, - groupIds: IntArray, - buffer: ByteBuffer, - isNamed: Boolean - ): Array { - val fileNameHashes = Array(largestGroupId) { IntArray(validFileIds[it]) } - if (isNamed) { - (0 until count).forEach { - val groupId = groupIds[it] - (0 until validFileIds[groupId]).forEach { fileId -> - fileNameHashes[groupId][fileId] = buffer.int - } - } - } - return fileNameHashes - } - override fun hashCode(): Int { var hash = 7 hash = 19 * hash + Objects.hashCode(this.idxFile) diff --git a/cache/src/main/kotlin/com/runetopic/cache/hierarchy/index/Index.kt b/cache/src/main/kotlin/com/runetopic/cache/hierarchy/index/Index.kt index 6f843e4..b803e0e 100644 --- a/cache/src/main/kotlin/com/runetopic/cache/hierarchy/index/Index.kt +++ b/cache/src/main/kotlin/com/runetopic/cache/hierarchy/index/Index.kt @@ -1,26 +1,69 @@ package com.runetopic.cache.hierarchy.index +import com.runetopic.cache.extension.nameHash import com.runetopic.cache.hierarchy.index.group.Group /** + * @author Tyler Telis + * @email + * * @author Jordan Abraham */ -interface Index: Comparable { - fun getId(): Int - fun getCRC(): Int - fun getWhirlpool(): ByteArray - fun getCompression(): Int - fun getProtocol(): Int - fun getRevision(): Int - fun getIsNamed(): Boolean - fun getGroups(): Collection - fun getGroup(groupId: Int): Group - fun getGroup(groupName: String): Group - fun expand(): Int +data class Index( + val id: Int, + val crc: Int, + val whirlpool: ByteArray, + val compression: Int, + val protocol: Int, + val revision: Int, + val isNamed: Boolean, + private val groups: Map +): Comparable { + @JvmName("getGroups") + fun groups(): Collection = groups.values + + @JvmName("getGroup") + fun group(groupId: Int): Group = groups[groupId] ?: Group.DEFAULT + + @JvmName("getGroup") + fun group(groupName: String): Group = groups.values.find { it.nameHash == groupName.nameHash() } ?: Group.DEFAULT + + fun expand(): Int = groups.values.last().files().size + (groups.values.last().id shl 8) fun use(block: (Index) -> Unit) = block.invoke(this) - override fun compareTo(other: Index): Int { - return getId().compareTo(other.getId()) + override fun compareTo(other: Index): Int = this.id.compareTo(other.id) + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (javaClass != other?.javaClass) return false + + other as Index + + if (id != other.id) return false + if (crc != other.crc) return false + if (!whirlpool.contentEquals(other.whirlpool)) return false + if (compression != other.compression) return false + if (protocol != other.protocol) return false + if (revision != other.revision) return false + if (isNamed != other.isNamed) return false + if (groups != other.groups) return false + + return true + } + + override fun hashCode(): Int { + var result = id + result = 31 * result + crc + result = 31 * result + whirlpool.contentHashCode() + result = 31 * result + compression + result = 31 * result + protocol + result = 31 * result + revision + result = 31 * result + isNamed.hashCode() + result = 31 * result + groups.hashCode() + return result + } + + internal companion object { + fun default(indexId: Int): Index = Index(indexId, 0, ByteArray(64), -1, -1, 0, false, hashMapOf()) } } \ No newline at end of file diff --git a/cache/src/main/kotlin/com/runetopic/cache/hierarchy/index/Js5Index.kt b/cache/src/main/kotlin/com/runetopic/cache/hierarchy/index/Js5Index.kt deleted file mode 100644 index 411a051..0000000 --- a/cache/src/main/kotlin/com/runetopic/cache/hierarchy/index/Js5Index.kt +++ /dev/null @@ -1,39 +0,0 @@ -package com.runetopic.cache.hierarchy.index - -import com.runetopic.cache.extension.nameHash -import com.runetopic.cache.hierarchy.index.group.Group -import com.runetopic.cache.hierarchy.index.group.Js5Group - -/** - * @author Tyler Telis - * @email - * - * @author Jordan Abraham - */ -class Js5Index( - private val id: Int, - private val crc: Int, - private val whirlpool: ByteArray, - private val compression: Int, - private val protocol: Int, - private val revision: Int, - private val isNamed: Boolean, - private val groups: Map -): Index { - override fun getId(): Int = id - override fun getCRC(): Int = crc - override fun getWhirlpool(): ByteArray = whirlpool - override fun getCompression(): Int = compression - override fun getProtocol(): Int = protocol - override fun getRevision(): Int = revision - override fun getIsNamed(): Boolean = isNamed - - override fun getGroups(): Collection = groups.values - override fun getGroup(groupId: Int): Group = groups[groupId] ?: Js5Group.DEFAULT - override fun getGroup(groupName: String): Group = groups.values.find { it.getNameHash() == groupName.nameHash() } ?: Js5Group.DEFAULT - override fun expand(): Int = groups.values.last().getFiles().size + (groups.values.last().getId() shl 8) - - internal companion object { - fun default(indexId: Int): Js5Index = Js5Index(indexId, 0, ByteArray(64), -1, -1, 0, false, hashMapOf()) - } -} \ No newline at end of file diff --git a/cache/src/main/kotlin/com/runetopic/cache/hierarchy/index/RawIndex.kt b/cache/src/main/kotlin/com/runetopic/cache/hierarchy/index/RawIndex.kt deleted file mode 100644 index 315ddff..0000000 --- a/cache/src/main/kotlin/com/runetopic/cache/hierarchy/index/RawIndex.kt +++ /dev/null @@ -1,30 +0,0 @@ -package com.runetopic.cache.hierarchy.index - -import com.runetopic.cache.hierarchy.RawReferenceTable -import com.runetopic.cache.hierarchy.index.group.Group - -/** - * @author Jordan Abraham - */ -class RawIndex( - private val id: Int, - private val crc: Int, - private val whirlpool: ByteArray, - private val compression: Int, - private val protocol: Int, - private val revision: Int, - val referenceTable: RawReferenceTable -): Index { - override fun getId(): Int = id - override fun getCRC(): Int = crc - override fun getWhirlpool(): ByteArray = whirlpool - override fun getCompression(): Int = compression - override fun getProtocol(): Int = protocol - override fun getRevision(): Int = revision - override fun getIsNamed(): Boolean = referenceTable.isNamed - - override fun getGroups(): Collection = throw RuntimeException() - override fun getGroup(groupId: Int): Group = throw RuntimeException() - override fun getGroup(groupName: String): Group = throw RuntimeException() - override fun expand(): Int = throw RuntimeException() -} \ No newline at end of file diff --git a/cache/src/main/kotlin/com/runetopic/cache/hierarchy/index/group/Group.kt b/cache/src/main/kotlin/com/runetopic/cache/hierarchy/index/group/Group.kt index a25ad83..565b50c 100644 --- a/cache/src/main/kotlin/com/runetopic/cache/hierarchy/index/group/Group.kt +++ b/cache/src/main/kotlin/com/runetopic/cache/hierarchy/index/group/Group.kt @@ -3,20 +3,60 @@ package com.runetopic.cache.hierarchy.index.group import com.runetopic.cache.hierarchy.index.group.file.File /** + * @author Tyler Telis + * @email + * * @author Jordan Abraham */ -interface Group: Comparable { - fun getId(): Int - fun getNameHash(): Int - fun getCRC(): Int - fun getWhirlpool(): ByteArray - fun getRevision(): Int - fun getKeys(): IntArray - fun getFiles(): Collection - fun getData(): ByteArray - fun getFile(fileId: Int): File - - override fun compareTo(other: Group): Int { - return getId().compareTo(other.getId()) +data class Group( + val id: Int, + val nameHash: Int, + val crc: Int, + val whirlpool: ByteArray, + val revision: Int, + val keys: IntArray, + private val files: Map, + val data: ByteArray +): Comparable { + + @JvmName("getFiles") + fun files(): Collection = files.values + + @JvmName("getFile") + fun file(fileId: Int): File = files[fileId] ?: File.DEFAULT + + override fun compareTo(other: Group): Int = this.id.compareTo(other.id) + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (javaClass != other?.javaClass) return false + + other as Group + + if (id != other.id) return false + if (nameHash != other.nameHash) return false + if (crc != other.crc) return false + if (!whirlpool.contentEquals(other.whirlpool)) return false + if (revision != other.revision) return false + if (!keys.contentEquals(other.keys)) return false + if (files != other.files) return false + if (!data.contentEquals(other.data)) return false + + return true + } + + override fun hashCode(): Int { + var result = id + result = 31 * result + nameHash + result = 31 * result + crc + result = 31 * result + whirlpool.contentHashCode() + result = 31 * result + revision + result = 31 * result + keys.contentHashCode() + result = 31 * result + files.hashCode() + result = 31 * result + data.contentHashCode() + return result + } + + internal companion object { + val DEFAULT = Group(-1, -1, -1, byteArrayOf(), -1, intArrayOf(), mapOf(), byteArrayOf()) } -} \ No newline at end of file +} diff --git a/cache/src/main/kotlin/com/runetopic/cache/hierarchy/index/group/Js5Group.kt b/cache/src/main/kotlin/com/runetopic/cache/hierarchy/index/group/Js5Group.kt deleted file mode 100644 index 1f38373..0000000 --- a/cache/src/main/kotlin/com/runetopic/cache/hierarchy/index/group/Js5Group.kt +++ /dev/null @@ -1,36 +0,0 @@ -package com.runetopic.cache.hierarchy.index.group - -import com.runetopic.cache.hierarchy.index.group.file.File -import com.runetopic.cache.hierarchy.index.group.file.Js5File - -/** - * @author Tyler Telis - * @email - * - * @author Jordan Abraham - */ -class Js5Group( - private val id: Int, - private val nameHash: Int, - private val crc: Int, - private val whirlpool: ByteArray, - private val revision: Int, - private val keys: IntArray, - private val files: Map, - private val data: ByteArray -): Group { - override fun getId(): Int = id - override fun getNameHash(): Int = nameHash - override fun getCRC(): Int = crc - override fun getWhirlpool(): ByteArray = whirlpool - override fun getRevision(): Int = revision - override fun getKeys(): IntArray = keys - override fun getFiles(): Collection = files.values - override fun getData(): ByteArray = data - - override fun getFile(fileId: Int): File = files[fileId] ?: Js5File.DEFAULT - - internal companion object { - val DEFAULT = Js5Group(-1, -1, -1, byteArrayOf(), -1, intArrayOf(), mapOf(), byteArrayOf()) - } -} diff --git a/cache/src/main/kotlin/com/runetopic/cache/hierarchy/index/group/file/File.kt b/cache/src/main/kotlin/com/runetopic/cache/hierarchy/index/group/file/File.kt index d6c5115..652c5af 100644 --- a/cache/src/main/kotlin/com/runetopic/cache/hierarchy/index/group/file/File.kt +++ b/cache/src/main/kotlin/com/runetopic/cache/hierarchy/index/group/file/File.kt @@ -1,74 +1,39 @@ package com.runetopic.cache.hierarchy.index.group.file -import com.runetopic.cache.codec.decompress -import java.nio.ByteBuffer -import java.util.zip.ZipException - /** + * @author Tyler Telis + * @email + * * @author Jordan Abraham */ -interface File: Comparable { - fun getId(): Int - fun getNameHash(): Int - fun getData(): ByteArray +data class File( + val id: Int, + val nameHash: Int, + val data: ByteArray +): Comparable { - override fun compareTo(other: File): Int { - return getId().compareTo(other.getId()) - } -} + override fun compareTo(other: File): Int = this.id.compareTo(other.id) + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (javaClass != other?.javaClass) return false -fun groupFiles( - fileIds: Array, - fileNameHashes: Array, - groupReferenceTableData: ByteArray, - count: Int, - groupId: Int -): Map { - if (groupReferenceTableData.isEmpty()) return hashMapOf(Pair(0, Js5File.DEFAULT)) + other as File - val src: ByteArray = try { - groupReferenceTableData.decompress() - } catch (exception: ZipException) { - groupReferenceTableData - } + if (id != other.id) return false + if (nameHash != other.nameHash) return false + if (!data.contentEquals(other.data)) return false - if (count == 1) { - return hashMapOf(Pair(0, Js5File(fileIds[groupId][0], fileNameHashes[groupId][0], src))) + return true } - var position = src.size - val chunks = src[--position].toInt() and 0xFF - position -= chunks * (count * 4) - val buffer = ByteBuffer.wrap(src) - buffer.position(position) - val filesSizes = IntArray(count) - (0 until chunks).forEach { _ -> - var read = 0 - (0 until count).forEach { - read += buffer.int - filesSizes[it] += read - } - } - val filesDatas = Array(count) { byteArrayOf() } - (0 until count).forEach { - filesDatas[it] = ByteArray(filesSizes[it]) - filesSizes[it] = 0 - } - buffer.position(position) - var offset = 0 - (0 until chunks).forEach { _ -> - var read = 0 - (0 until count).forEach { - read += buffer.int - System.arraycopy(src, offset, filesDatas[it], filesSizes[it], read) - offset += read - filesSizes[it] += read - } + override fun hashCode(): Int { + var result = id + result = 31 * result + nameHash + result = 31 * result + data.contentHashCode() + return result } - val files = hashMapOf() - (0 until count).forEach { - files[it] = Js5File(fileIds[groupId][it], fileNameHashes[groupId][it], filesDatas[it]) + internal companion object { + val DEFAULT = File(-1, -1, byteArrayOf(0)) } - return files } \ No newline at end of file diff --git a/cache/src/main/kotlin/com/runetopic/cache/hierarchy/index/group/file/Js5File.kt b/cache/src/main/kotlin/com/runetopic/cache/hierarchy/index/group/file/Js5File.kt deleted file mode 100644 index b60027d..0000000 --- a/cache/src/main/kotlin/com/runetopic/cache/hierarchy/index/group/file/Js5File.kt +++ /dev/null @@ -1,21 +0,0 @@ -package com.runetopic.cache.hierarchy.index.group.file - -/** - * @author Tyler Telis - * @email - * - * @author Jordan Abraham - */ -class Js5File( - private val id: Int, - private val nameHash: Int, - private val data: ByteArray -): File { - override fun getId(): Int = id - override fun getNameHash(): Int = nameHash - override fun getData(): ByteArray = data - - internal companion object { - val DEFAULT = Js5File(-1, -1, byteArrayOf(0)) - } -} \ No newline at end of file diff --git a/cache/src/main/kotlin/com/runetopic/cache/store/Js5Store.kt b/cache/src/main/kotlin/com/runetopic/cache/store/Js5Store.kt index 5913961..28bc176 100644 --- a/cache/src/main/kotlin/com/runetopic/cache/store/Js5Store.kt +++ b/cache/src/main/kotlin/com/runetopic/cache/store/Js5Store.kt @@ -23,21 +23,21 @@ class Js5Store( init { storage.init(this) - indexes.sortWith(compareBy { it.getId() }) + indexes.sortWith(compareBy { it.id }) } @Synchronized internal fun addIndex(index: Index) { - indexes.forEach { i -> require(index.getId() != i.getId()) { "Index with Id={${index.getId()}} already exists." } } + indexes.forEach { i -> require(index.id != i.id) { "Index with Id={${index.id}} already exists." } } indexes.add(index) } - fun index(indexId: Int): Index = indexes.find { it.getId() == indexId }!! + fun index(indexId: Int): Index = indexes.find { it.id == indexId }!! fun indexReferenceTableSize(indexId: Int): Int { var size = 0 index(indexId).use { index -> - index.getGroups().forEach { size += storage.loadReferenceTable(index, it.getId()).size } + index.groups().forEach { size += storage.loadReferenceTable(index, it.id).size } } return size } @@ -60,8 +60,8 @@ class Js5Store( fun checksumsWithoutRSA(): ByteArray { val header = ByteBuffer.allocate(indexes.size * 8) indexes.forEach { - header.putInt(it.getCRC()) - header.putInt(it.getRevision()) + header.putInt(it.crc) + header.putInt(it.revision) } return header.array() } @@ -71,9 +71,9 @@ class Js5Store( header.position(5) header.put(indexes.size.toByte()) indexes.forEach { - header.putInt(it.getCRC()) - header.putInt(it.getRevision()) - header.put(it.getWhirlpool()) + header.putInt(it.crc) + header.putInt(it.revision) + header.put(it.whirlpool) } val headerPosition = header.position() val headerArray = header.array() diff --git a/cache/src/main/kotlin/com/runetopic/cache/store/storage/js5/Js5DiskStorage.kt b/cache/src/main/kotlin/com/runetopic/cache/store/storage/js5/Js5DiskStorage.kt index 54f5945..24b2c30 100644 --- a/cache/src/main/kotlin/com/runetopic/cache/store/storage/js5/Js5DiskStorage.kt +++ b/cache/src/main/kotlin/com/runetopic/cache/store/storage/js5/Js5DiskStorage.kt @@ -3,9 +3,6 @@ package com.runetopic.cache.store.storage.js5 import com.github.michaelbull.logging.InlineLogger import com.runetopic.cache.codec.ContainerCodec import com.runetopic.cache.hierarchy.index.Index -import com.runetopic.cache.hierarchy.index.Js5Index -import com.runetopic.cache.hierarchy.index.group.Js5Group -import com.runetopic.cache.hierarchy.index.group.file.groupFiles import com.runetopic.cache.store.Constants import com.runetopic.cache.store.Js5Store import com.runetopic.cache.store.storage.IStorage @@ -78,47 +75,11 @@ internal class Js5DiskStorage( idxFiles.add(getIdxFile(indexId)) if (indexTable.exists().not()) { - store.addIndex(Js5Index.default(indexId)) + store.addIndex(Index.default(indexId)) return } val indexDatTable = datFile.readReferenceTable(masterIdxFile.id(), indexTable) - - val rawIndex = indexTable.loadIndex(datFile, getIdxFile(indexId), indexDatTable.toWhirlpool(), ContainerCodec.decompress(indexDatTable)) - val rawReferenceTable = rawIndex.referenceTable - - val groups = hashMapOf() - (0 until rawReferenceTable.count).forEach { - val groupId = rawReferenceTable.groupIds[it] - groups[it] = (Js5Group( - groupId, - rawReferenceTable.groupNameHashes[groupId], - rawReferenceTable.groupCrcs[groupId], - rawReferenceTable.groupWhirlpools[groupId], - rawReferenceTable.groupRevisions[groupId], - intArrayOf(),//TODO - groupFiles( - rawReferenceTable.fileIds, - rawReferenceTable.fileNameHashes, - rawReferenceTable.groupTables[it], - rawReferenceTable.groupFileIds[it], - it - ), - rawReferenceTable.groupTables[it] - )) - } - - store.addIndex( - Js5Index( - indexId, - rawIndex.getCRC(), - rawIndex.getWhirlpool(), - rawIndex.getCompression(), - rawIndex.getProtocol(), - rawIndex.getRevision(), - rawIndex.getIsNamed(), - groups - ) - ) + store.addIndex(loadIndex(datFile, getIdxFile(indexId), indexDatTable.toWhirlpool(), ContainerCodec.decompress(indexDatTable))) } override fun loadMasterReferenceTable(groupId: Int): ByteArray { @@ -126,13 +87,13 @@ internal class Js5DiskStorage( } override fun loadReferenceTable(index: Index, groupId: Int): ByteArray { - return datFile.readReferenceTable(index.getId(), getIdxFile(index.getId()).loadReferenceTable(groupId)) + return datFile.readReferenceTable(index.id, getIdxFile(index.id).loadReferenceTable(groupId)) } override fun loadReferenceTable(index: Index, groupName: String): ByteArray { - val group = index.getGroup(groupName) - if (group.getData().isEmpty()) return group.getData() - return datFile.readReferenceTable(index.getId(), getIdxFile(index.getId()).loadReferenceTable(group.getId())) + val group = index.group(groupName) + if (group.data.isEmpty()) return group.data + return datFile.readReferenceTable(index.id, getIdxFile(index.id).loadReferenceTable(group.id)) } @Synchronized diff --git a/cache/src/main/kotlin/com/runetopic/cache/store/storage/js5/Js5IndexFactory.kt b/cache/src/main/kotlin/com/runetopic/cache/store/storage/js5/Js5IndexFactory.kt new file mode 100644 index 0000000..b189a18 --- /dev/null +++ b/cache/src/main/kotlin/com/runetopic/cache/store/storage/js5/Js5IndexFactory.kt @@ -0,0 +1,248 @@ +package com.runetopic.cache.store.storage.js5 + +import com.runetopic.cache.codec.Container +import com.runetopic.cache.codec.decompress +import com.runetopic.cache.exception.ProtocolException +import com.runetopic.cache.extension.readUnsignedByte +import com.runetopic.cache.extension.readUnsignedIntShortSmart +import com.runetopic.cache.extension.readUnsignedShort +import com.runetopic.cache.hierarchy.index.Index +import com.runetopic.cache.hierarchy.index.group.Group +import com.runetopic.cache.hierarchy.index.group.file.File +import java.nio.ByteBuffer +import java.util.zip.ZipException + +/** + * @author Jordan Abraham + */ +internal fun loadIndex( + datFile: IDatFile, + idxFile: IIdxFile, + whirlpool: ByteArray, + decompressed: Container +): Index { + val buffer = ByteBuffer.wrap(decompressed.data) + val crc = decompressed.crc + val compression = decompressed.compression + val protocol = buffer.readUnsignedByte() + val revision = when { + protocol < 5 || protocol > 7 -> throw ProtocolException("Unhandled protocol $protocol") + protocol >= 6 -> buffer.int + else -> 0 + } + val hash = buffer.readUnsignedByte() + val count = if (protocol >= 7) buffer.readUnsignedIntShortSmart() else buffer.readUnsignedShort() + + val groupTables = mutableListOf() + (0 until count).forEach { + groupTables.add(datFile.readReferenceTable(idxFile.id(), idxFile.loadReferenceTable(it))) + } + + val isNamed = (0x1 and hash) != 0 + val isUsingWhirlpool = (0x2 and hash) != 0 + + val groupIds = IntArray(count) + var lastGroupId = 0 + var biggest = -1 + (0 until count).forEach { + groupIds[it] = if (protocol >= 7) { buffer.readUnsignedIntShortSmart() } else { buffer.readUnsignedShort() } + .let { id -> lastGroupId += id; lastGroupId } + if (groupIds[it] > biggest) biggest = groupIds[it] + } + + val largestGroupId = biggest + 1 + val groupNameHashes = groupNameHashes(largestGroupId, count, isNamed, groupIds, buffer) + val groupCrcs = groupCrcs(largestGroupId, count, groupIds, buffer) + val groupWhirlpools = groupWhirlpools(largestGroupId, isUsingWhirlpool, count, buffer, groupIds) + val groupRevisions = groupRevisions(largestGroupId, count, groupIds, buffer) + val groupFileIds = groupFileIds(largestGroupId, count, groupIds, buffer, protocol) + val fileIds = fileIds(largestGroupId, groupFileIds, count, groupIds, buffer, protocol) + val fileNameHashes = fileNameHashes(largestGroupId, groupFileIds, count, groupIds, buffer, isNamed) + + val groups = hashMapOf() + (0 until count).forEach { + val groupId = groupIds[it] + groups[it] = (Group( + groupId, + groupNameHashes[groupId], + groupCrcs[groupId], + groupWhirlpools[groupId], + groupRevisions[groupId], + intArrayOf(),//TODO + groupFiles(fileIds, fileNameHashes, groupTables[it], groupFileIds[it], it), + groupTables[it] + )) + } + return Index(idxFile.id(), crc, whirlpool, compression, protocol, revision, isNamed, groups) +} + +private fun groupFileIds( + largestGroupId: Int, + count: Int, + groupIds: IntArray, + buffer: ByteBuffer, + protocol: Int +): IntArray { + val groupFileIds = IntArray(largestGroupId) + (0 until count).forEach { + groupFileIds[groupIds[it]] = if (protocol >= 7) buffer.readUnsignedIntShortSmart() else buffer.readUnsignedShort() + } + return groupFileIds +} + +private fun groupRevisions( + largestGroupId: Int, + count: Int, + groupIds: IntArray, + buffer: ByteBuffer +): IntArray { + val revisions = IntArray(largestGroupId) + (0 until count).forEach { + revisions[groupIds[it]] = buffer.int + } + return revisions +} + +private fun groupWhirlpools( + largestGroupId: Int, + usesWhirlpool: Boolean, + count: Int, + buffer: ByteBuffer, + groupIds: IntArray +): Array { + val whirlpools = Array(largestGroupId) { ByteArray(64) } + if (usesWhirlpool.not()) return whirlpools + + (0 until count).forEach { + val whirlpool = ByteArray(64) + buffer.get(whirlpool) + whirlpools[groupIds[it]] = whirlpool + } + return whirlpools +} + +private fun groupCrcs( + largestGroupId: Int, + count: Int, + groupIds: IntArray, + buffer: ByteBuffer +): IntArray { + val crcs = IntArray(largestGroupId) + (0 until count).forEach { + crcs[groupIds[it]] = buffer.int + } + return crcs +} + +private fun groupNameHashes( + largestGroupId: Int, + count: Int, + isNamed: Boolean, + groupIds: IntArray, + buffer: ByteBuffer +): IntArray { + val nameHashes = IntArray(largestGroupId) { -1 } + if (isNamed.not()) return nameHashes + + (0 until count).forEach { + nameHashes[groupIds[it]] = buffer.int + } + return nameHashes +} + +private fun fileIds( + largestGroupId: Int, + validFileIds: IntArray, + count: Int, + groupIds: IntArray, + buffer: ByteBuffer, + protocol: Int +): Array { + val fileIds = Array(largestGroupId) { IntArray(validFileIds[it]) } + (0 until count).forEach { + val groupId = groupIds[it] + var currentFileId = 0 + (0 until validFileIds[groupId]).forEach { fileId -> + if (protocol >= 7) { buffer.readUnsignedIntShortSmart() } else { buffer.readUnsignedShort() } + .let { i -> currentFileId += i; currentFileId } + .also { fileIds[groupId][fileId] = currentFileId } + } + } + return fileIds +} + +private fun fileNameHashes( + largestGroupId: Int, + validFileIds: IntArray, + count: Int, + groupIds: IntArray, + buffer: ByteBuffer, + isNamed: Boolean +): Array { + val fileNameHashes = Array(largestGroupId) { IntArray(validFileIds[it]) } + if (isNamed) { + (0 until count).forEach { + val groupId = groupIds[it] + (0 until validFileIds[groupId]).forEach { fileId -> + fileNameHashes[groupId][fileId] = buffer.int + } + } + } + return fileNameHashes +} + +internal fun groupFiles( + fileIds: Array, + fileNameHashes: Array, + groupReferenceTableData: ByteArray, + count: Int, + groupId: Int +): Map { + if (groupReferenceTableData.isEmpty()) return hashMapOf(Pair(0, File.DEFAULT)) + + val src: ByteArray = try { + groupReferenceTableData.decompress() + } catch (exception: ZipException) { + groupReferenceTableData + } + + if (count == 1) { + return hashMapOf(Pair(0, File(fileIds[groupId][0], fileNameHashes[groupId][0], src))) + } + + var position = src.size + val chunks = src[--position].toInt() and 0xFF + position -= chunks * (count * 4) + val buffer = ByteBuffer.wrap(src) + buffer.position(position) + val filesSizes = IntArray(count) + (0 until chunks).forEach { _ -> + var read = 0 + (0 until count).forEach { + read += buffer.int + filesSizes[it] += read + } + } + val filesDatas = Array(count) { byteArrayOf() } + (0 until count).forEach { + filesDatas[it] = ByteArray(filesSizes[it]) + filesSizes[it] = 0 + } + buffer.position(position) + var offset = 0 + (0 until chunks).forEach { _ -> + var read = 0 + (0 until count).forEach { + read += buffer.int + System.arraycopy(src, offset, filesDatas[it], filesSizes[it], read) + offset += read + filesSizes[it] += read + } + } + + val files = hashMapOf() + (0 until count).forEach { + files[it] = File(fileIds[groupId][it], fileNameHashes[groupId][it], filesDatas[it]) + } + return files +} \ No newline at end of file diff --git a/loader/build.gradle.kts b/loader/build.gradle.kts index 8d07076..79a244a 100644 --- a/loader/build.gradle.kts +++ b/loader/build.gradle.kts @@ -4,7 +4,7 @@ plugins { signing } -version = "647.6.2-SNAPSHOT" +version = "647.6.3-SNAPSHOT" java { withJavadocJar() diff --git a/loader/src/main/kotlin/com/runetopic/loader/index/config/idk/IdentityKitEntryBuilder.kt b/loader/src/main/kotlin/com/runetopic/loader/index/config/idk/IdentityKitEntryBuilder.kt index 558b285..c9d1a66 100644 --- a/loader/src/main/kotlin/com/runetopic/loader/index/config/idk/IdentityKitEntryBuilder.kt +++ b/loader/src/main/kotlin/com/runetopic/loader/index/config/idk/IdentityKitEntryBuilder.kt @@ -18,8 +18,8 @@ internal class IdentityKitEntryBuilder: IEntryBuilder { @OptIn(ExperimentalStdlibApi::class) override fun build(store: Js5Store) { identityKitTypes = buildSet { - store.index(2).getGroup(3).getFiles().forEach { - add(read(ByteBuffer.wrap(it.getData()), IdentityKitEntryType(it.getId()))) + store.index(2).group(3).files().forEach { + add(read(ByteBuffer.wrap(it.data), IdentityKitEntryType(it.id))) } } } diff --git a/loader/src/main/kotlin/com/runetopic/loader/index/config/inv/InventoryEntryBuilder.kt b/loader/src/main/kotlin/com/runetopic/loader/index/config/inv/InventoryEntryBuilder.kt index ac722b4..a256fae 100644 --- a/loader/src/main/kotlin/com/runetopic/loader/index/config/inv/InventoryEntryBuilder.kt +++ b/loader/src/main/kotlin/com/runetopic/loader/index/config/inv/InventoryEntryBuilder.kt @@ -16,8 +16,8 @@ internal class InventoryEntryBuilder: IEntryBuilder { @OptIn(ExperimentalStdlibApi::class) override fun build(store: Js5Store) { inventoryTypes = buildSet { - store.index(2).getGroup(5).getFiles().forEach { - add(read(ByteBuffer.wrap(it.getData()), InventoryEntryType(it.getId()))) + store.index(2).group(5).files().forEach { + add(read(ByteBuffer.wrap(it.data), InventoryEntryType(it.id))) } } } diff --git a/loader/src/main/kotlin/com/runetopic/loader/index/config/lighting/LightingEntryBuilder.kt b/loader/src/main/kotlin/com/runetopic/loader/index/config/lighting/LightingEntryBuilder.kt index 05f5278..c3b053f 100644 --- a/loader/src/main/kotlin/com/runetopic/loader/index/config/lighting/LightingEntryBuilder.kt +++ b/loader/src/main/kotlin/com/runetopic/loader/index/config/lighting/LightingEntryBuilder.kt @@ -17,8 +17,8 @@ internal class LightingEntryBuilder : IEntryBuilder { @OptIn(ExperimentalStdlibApi::class) override fun build(store: Js5Store) { lightings = buildSet { - store.index(2).getGroup(31).getFiles().forEach { - add(read(ByteBuffer.wrap(it.getData()), LightingEntryType(it.getId()))) + store.index(2).group(31).files().forEach { + add(read(ByteBuffer.wrap(it.data), LightingEntryType(it.id))) } } } diff --git a/loader/src/main/kotlin/com/runetopic/loader/index/config/mouseicon/MouseIconEntryBuilder.kt b/loader/src/main/kotlin/com/runetopic/loader/index/config/mouseicon/MouseIconEntryBuilder.kt index 04d2f47..e954e31 100644 --- a/loader/src/main/kotlin/com/runetopic/loader/index/config/mouseicon/MouseIconEntryBuilder.kt +++ b/loader/src/main/kotlin/com/runetopic/loader/index/config/mouseicon/MouseIconEntryBuilder.kt @@ -16,8 +16,8 @@ internal class MouseIconEntryBuilder: IEntryBuilder { @OptIn(ExperimentalStdlibApi::class) override fun build(store: Js5Store) { mouseIconTypes = buildSet { - store.index(2).getGroup(33).getFiles().forEach { - add(read(ByteBuffer.wrap(it.getData()), MouseIconEntryType(it.getId()))) + store.index(2).group(33).files().forEach { + add(read(ByteBuffer.wrap(it.data), MouseIconEntryType(it.id))) } } } diff --git a/loader/src/main/kotlin/com/runetopic/loader/index/config/overlay/OverlayEntryBuilder.kt b/loader/src/main/kotlin/com/runetopic/loader/index/config/overlay/OverlayEntryBuilder.kt index 84c9999..4e7c489 100644 --- a/loader/src/main/kotlin/com/runetopic/loader/index/config/overlay/OverlayEntryBuilder.kt +++ b/loader/src/main/kotlin/com/runetopic/loader/index/config/overlay/OverlayEntryBuilder.kt @@ -17,8 +17,8 @@ class OverlayEntryBuilder: IEntryBuilder { @OptIn(ExperimentalStdlibApi::class) override fun build(store: Js5Store) { overlays = buildSet { - store.index(2).getGroup(4).getFiles().forEach { - add(read(ByteBuffer.wrap(it.getData()), OverlayEntryType(it.getId()))) + store.index(2).group(4).files().forEach { + add(read(ByteBuffer.wrap(it.data), OverlayEntryType(it.id))) } } } diff --git a/loader/src/main/kotlin/com/runetopic/loader/index/config/param/ParamEntryBuilder.kt b/loader/src/main/kotlin/com/runetopic/loader/index/config/param/ParamEntryBuilder.kt index bde4439..5d2db63 100644 --- a/loader/src/main/kotlin/com/runetopic/loader/index/config/param/ParamEntryBuilder.kt +++ b/loader/src/main/kotlin/com/runetopic/loader/index/config/param/ParamEntryBuilder.kt @@ -18,8 +18,8 @@ internal class ParamEntryBuilder: IEntryBuilder { @OptIn(ExperimentalStdlibApi::class) override fun build(store: Js5Store) { paramTypes = buildSet { - store.index(2).getGroup(11).getFiles().forEach { - add(read(ByteBuffer.wrap(it.getData()), ParamEntryType(it.getId()))) + store.index(2).group(11).files().forEach { + add(read(ByteBuffer.wrap(it.data), ParamEntryType(it.id))) } } } diff --git a/loader/src/main/kotlin/com/runetopic/loader/index/config/skybox/SkyBoxEntryBuilder.kt b/loader/src/main/kotlin/com/runetopic/loader/index/config/skybox/SkyBoxEntryBuilder.kt index 3edbc07..a7139a5 100644 --- a/loader/src/main/kotlin/com/runetopic/loader/index/config/skybox/SkyBoxEntryBuilder.kt +++ b/loader/src/main/kotlin/com/runetopic/loader/index/config/skybox/SkyBoxEntryBuilder.kt @@ -16,8 +16,8 @@ internal class SkyBoxEntryBuilder: IEntryBuilder { @OptIn(ExperimentalStdlibApi::class) override fun build(store: Js5Store) { skyBoxTypes = buildSet { - store.index(2).getGroup(29).getFiles().forEach { - add(read(ByteBuffer.wrap(it.getData()), SkyBoxEntryType(it.getId()))) + store.index(2).group(29).files().forEach { + add(read(ByteBuffer.wrap(it.data), SkyBoxEntryType(it.id))) } } } diff --git a/loader/src/main/kotlin/com/runetopic/loader/index/config/struct/StructEntryBuilder.kt b/loader/src/main/kotlin/com/runetopic/loader/index/config/struct/StructEntryBuilder.kt index cf99574..1a128f0 100644 --- a/loader/src/main/kotlin/com/runetopic/loader/index/config/struct/StructEntryBuilder.kt +++ b/loader/src/main/kotlin/com/runetopic/loader/index/config/struct/StructEntryBuilder.kt @@ -19,8 +19,8 @@ internal class StructEntryBuilder: IEntryBuilder { @OptIn(ExperimentalStdlibApi::class) override fun build(store: Js5Store) { structTypes = buildSet { - store.index(2).getGroup(26).getFiles().forEach { - add(read(ByteBuffer.wrap(it.getData()), StructEntryType(it.getId()))) + store.index(2).group(26).files().forEach { + add(read(ByteBuffer.wrap(it.data), StructEntryType(it.id))) } } } diff --git a/loader/src/main/kotlin/com/runetopic/loader/index/config/underlay/UnderlayEntryBuilder.kt b/loader/src/main/kotlin/com/runetopic/loader/index/config/underlay/UnderlayEntryBuilder.kt index 66c40d5..5564372 100644 --- a/loader/src/main/kotlin/com/runetopic/loader/index/config/underlay/UnderlayEntryBuilder.kt +++ b/loader/src/main/kotlin/com/runetopic/loader/index/config/underlay/UnderlayEntryBuilder.kt @@ -17,8 +17,8 @@ class UnderlayEntryBuilder: IEntryBuilder { @OptIn(ExperimentalStdlibApi::class) override fun build(store: Js5Store) { underlays = buildSet { - store.index(2).getGroup(1).getFiles().forEach { - add(read(ByteBuffer.wrap(it.getData()), UnderlayEntryType(it.getId()))) + store.index(2).group(1).files().forEach { + add(read(ByteBuffer.wrap(it.data), UnderlayEntryType(it.id))) } } } diff --git a/loader/src/main/kotlin/com/runetopic/loader/index/loc/LocEntryBuilder.kt b/loader/src/main/kotlin/com/runetopic/loader/index/loc/LocEntryBuilder.kt index 573976a..1125bab 100644 --- a/loader/src/main/kotlin/com/runetopic/loader/index/loc/LocEntryBuilder.kt +++ b/loader/src/main/kotlin/com/runetopic/loader/index/loc/LocEntryBuilder.kt @@ -18,7 +18,7 @@ internal class LocEntryBuilder : IEntryBuilder { mapTypes = buildSet { store.index(16).use { index -> (0 until index.expand()).forEach { - add(read(ByteBuffer.wrap(index.getGroup(it ushr 8).getFile(it and 0xFF).getData()), LocEntryType(it))) + add(read(ByteBuffer.wrap(index.group(it ushr 8).file(it and 0xFF).data), LocEntryType(it))) } } } diff --git a/loader/src/main/kotlin/com/runetopic/loader/index/map/MapEntryBuilder.kt b/loader/src/main/kotlin/com/runetopic/loader/index/map/MapEntryBuilder.kt index 1038215..d301281 100644 --- a/loader/src/main/kotlin/com/runetopic/loader/index/map/MapEntryBuilder.kt +++ b/loader/src/main/kotlin/com/runetopic/loader/index/map/MapEntryBuilder.kt @@ -24,7 +24,7 @@ internal class MapEntryBuilder : IEntryBuilder { (0..Short.MAX_VALUE).forEach { regionId -> val regionX: Int = regionId shr 8 val regionY: Int = regionId and 0xFF - it.getGroup("m${regionX}_${regionY}").getData().let { data -> + it.group("m${regionX}_${regionY}").data.let { data -> if (data.isEmpty()) return@forEach add(read(ByteBuffer.wrap(data.decompress()), MapEntryType(regionId, regionX, regionY))) } diff --git a/loader/src/main/kotlin/com/runetopic/loader/index/map/MapLocationEntryBuilder.kt b/loader/src/main/kotlin/com/runetopic/loader/index/map/MapLocationEntryBuilder.kt index b56b277..c7f803b 100644 --- a/loader/src/main/kotlin/com/runetopic/loader/index/map/MapLocationEntryBuilder.kt +++ b/loader/src/main/kotlin/com/runetopic/loader/index/map/MapLocationEntryBuilder.kt @@ -24,7 +24,7 @@ internal class MapLocationEntryBuilder : IEntryBuilder { (0..Short.MAX_VALUE).forEach { regionId -> val regionX: Int = regionId shr 8 val regionY: Int = regionId and 0xFF - it.getGroup("l${regionX}_${regionY}").getData().let { data -> + it.group("l${regionX}_${regionY}").data.let { data -> if (data.isEmpty()) return@forEach try { add(read(ByteBuffer.wrap(data.decompress()), MapLocationEntryType(regionId, regionX, regionY))) diff --git a/loader/src/main/kotlin/com/runetopic/loader/index/npc/NpcEntryBuilder.kt b/loader/src/main/kotlin/com/runetopic/loader/index/npc/NpcEntryBuilder.kt index c4477b5..7e6d090 100644 --- a/loader/src/main/kotlin/com/runetopic/loader/index/npc/NpcEntryBuilder.kt +++ b/loader/src/main/kotlin/com/runetopic/loader/index/npc/NpcEntryBuilder.kt @@ -17,7 +17,7 @@ class NpcEntryBuilder: IEntryBuilder { npcs = buildSet { store.index(18).use { index -> (0 until index.expand()).forEach { - add(read(ByteBuffer.wrap(index.getGroup(it ushr 8).getFile(it and 0xFF).getData()), NpcEntryType(it))) + add(read(ByteBuffer.wrap(index.group(it ushr 8).file(it and 0xFF).data), NpcEntryType(it))) } } } diff --git a/loader/src/main/kotlin/com/runetopic/loader/index/obj/ObjEntryBuilder.kt b/loader/src/main/kotlin/com/runetopic/loader/index/obj/ObjEntryBuilder.kt index 7edbe8f..19126cb 100644 --- a/loader/src/main/kotlin/com/runetopic/loader/index/obj/ObjEntryBuilder.kt +++ b/loader/src/main/kotlin/com/runetopic/loader/index/obj/ObjEntryBuilder.kt @@ -17,7 +17,7 @@ internal class ObjEntryBuilder : IEntryBuilder { objs = buildSet { store.index(19).use { index -> (0 until index.expand()).forEach { - add(read(ByteBuffer.wrap(index.getGroup(it ushr 8).getFile(it and 0xFF).getData()), ObjEntryType(it))) + add(read(ByteBuffer.wrap(index.group(it ushr 8).file(it and 0xFF).data), ObjEntryType(it))) } } } diff --git a/loader/src/main/kotlin/com/runetopic/loader/index/particle/ParticleEntryBuilder.kt b/loader/src/main/kotlin/com/runetopic/loader/index/particle/ParticleEntryBuilder.kt index f6536f1..a005501 100644 --- a/loader/src/main/kotlin/com/runetopic/loader/index/particle/ParticleEntryBuilder.kt +++ b/loader/src/main/kotlin/com/runetopic/loader/index/particle/ParticleEntryBuilder.kt @@ -18,8 +18,8 @@ internal class ParticleEntryBuilder : IEntryBuilder { @OptIn(ExperimentalStdlibApi::class) override fun build(store: Js5Store) { particles = buildSet { - store.index(27).getGroup(0).getFiles().forEach { - add(read(ByteBuffer.wrap(it.getData()), ParticleEntryType(it.getId()))) + store.index(27).group(0).files().forEach { + add(read(ByteBuffer.wrap(it.data), ParticleEntryType(it.id))) } } } diff --git a/loader/src/main/kotlin/com/runetopic/loader/index/spotanim/SpotAnimationEntryBuilder.kt b/loader/src/main/kotlin/com/runetopic/loader/index/spotanim/SpotAnimationEntryBuilder.kt index ca955c1..bfadeea 100644 --- a/loader/src/main/kotlin/com/runetopic/loader/index/spotanim/SpotAnimationEntryBuilder.kt +++ b/loader/src/main/kotlin/com/runetopic/loader/index/spotanim/SpotAnimationEntryBuilder.kt @@ -18,7 +18,7 @@ internal class SpotAnimationEntryBuilder: IEntryBuilder spotAnimations = buildSet { store.index(21).use { index -> (0 until index.expand()).forEach { - add(read(ByteBuffer.wrap(index.getGroup(it ushr 8).getFile(it and 0xFF).getData()), SpotAnimationEntryType(it))) + add(read(ByteBuffer.wrap(index.group(it ushr 8).file(it and 0xFF).data), SpotAnimationEntryType(it))) } } }