Cache loading improvements.

README.md

@@ -29,7 +29,7 @@ A cache library written in Kotlin.
 # Implementation
 Just use cache if you do not require any of the revision specific loaders.
 ```
-cache = { module = "com.runetopic.cache:cache", version.ref "1.4.8-SNAPSHOT" }
+cache = { module = "com.runetopic.cache:cache", version.ref "1.4.10-SNAPSHOT" }
 loader = { module = "com.runetopic.cache:loader", version.ref "647.6.1-SNAPSHOT" }
 ```
 

cache/build.gradle.kts

@@ -4,7 +4,7 @@ plugins {
     signing
 }
 
-version = "1.4.8-SNAPSHOT"
+version = "1.4.10-SNAPSHOT"
 
 java {
     withJavadocJar()
@@ -77,5 +77,5 @@ dependencies {
     implementation("com.michael-bull.kotlin-inline-logger:kotlin-inline-logger:1.0.3")
     implementation("org.slf4j:slf4j-simple:1.7.32")
     implementation("com.fasterxml.jackson.module:jackson-module-kotlin:2.13.+")
-    implementation("com.runetopic.cryptography:cryptography:1.0.2-SNAPSHOT")
+    implementation("com.runetopic.cryptography:cryptography:1.0.4-SNAPSHOT")
 }

cache/src/main/kotlin/com/runetopic/cache/codec/ContainerCodec.kt

@@ -4,7 +4,7 @@ import com.runetopic.cache.codec.CodecType.*
 import com.runetopic.cache.exception.CompressionException
 import com.runetopic.cache.extension.readUnsignedShort
 import com.runetopic.cache.extension.remainingBytes
-import com.runetopic.cryptography.ext.fromXTEA
+import com.runetopic.cryptography.fromXTEA
 import java.nio.ByteBuffer
 import java.util.zip.CRC32
 

cache/src/main/kotlin/com/runetopic/cache/extension/ByteBuffer.kt

@@ -1,7 +1,7 @@
 package com.runetopic.cache.extension
 
 import com.runetopic.cache.store.Constants.cp1252Identifiers
-import com.runetopic.cryptography.ext.toWhirlpool
+import com.runetopic.cryptography.toWhirlpool
 import java.nio.ByteBuffer
 import java.nio.charset.Charset
 

cache/src/main/kotlin/com/runetopic/cache/hierarchy/ReferenceTable.kt

@@ -1,6 +1,6 @@
 package com.runetopic.cache.hierarchy
 
-import com.runetopic.cache.codec.ContainerCodec
+import com.runetopic.cache.codec.Container
 import com.runetopic.cache.codec.decompress
 import com.runetopic.cache.exception.ProtocolException
 import com.runetopic.cache.extension.readUnsignedByte
@@ -59,31 +59,49 @@ internal data class ReferenceTable(
 
     fun exists(): Boolean = (length != 0 && sector != 0)
 
-    fun loadIndex(datFile: IDatFile, idxFile: IIdxFile, whirlpool: ByteArray, data: ByteArray): Js5Index {
-        val container = ContainerCodec.decompress(data)
-        val buffer = ByteBuffer.wrap(container.data)
+    fun loadIndex(
+        datFile: IDatFile,
+        idxFile: IIdxFile,
+        whirlpool: ByteArray,
+        decompressed: Container
+    ): Js5Index {
+        val buffer = ByteBuffer.wrap(decompressed.data)
+        val crc = decompressed.crc
+        val compressionType = decompressed.compression
         val protocol = buffer.readUnsignedByte()
-        var revision = 0
-
-        if (protocol < 5 || protocol > 6) {
-            throw ProtocolException("Unhandled protocol $protocol when reading index $this")
+        val revision = when {
+            protocol < 5 || protocol > 6 -> throw ProtocolException("Unhandled protocol $protocol when reading index $this")
+            protocol == 6 -> buffer.int
+            else -> 0
         }
-
-        if (protocol >= 6) {
-            revision = buffer.int
+        val hash = buffer.readUnsignedByte()
+        val count = buffer.readUnsignedShort()
+        val groupTables = mutableListOf<ByteArray>()
+        (0 until count).forEach {
+            groupTables.add(datFile.readReferenceTable(idxFile.id(), idxFile.loadReferenceTable(it)))
         }
+        return loadIndexContents(idxFile.id(), buffer, crc, compressionType, revision, protocol, hash, count, whirlpool, groupTables)
+    }
+
+    private fun loadIndexContents(
+        indexId: Int,
+        buffer: ByteBuffer,
+        crc: Int,
+        compressionType: Int,
+        revision: Int,
+        protocol: Int,
+        hash: Int,
+        count: Int,
+        whirlpool: ByteArray,
+        groupTables: List<ByteArray>
+    ): Js5Index {
+        val groupIds = IntArray(count)
 
-        val hash = buffer.readUnsignedByte()
         val isNamed = (0x1 and hash) != 0
         val isUsingWhirlPool = (0x2 and hash) != 0
 
-        val count = buffer.readUnsignedShort()
-
         var lastGroupId = 0
         var biggest = -1
-
-        val groupIds = IntArray(count)
-
         (0 until count).forEach {
             groupIds[it] = buffer.readUnsignedShort().let { id -> lastGroupId += id; lastGroupId }
             if (groupIds[it] > biggest) biggest = groupIds[it]
@@ -101,19 +119,18 @@ internal data class ReferenceTable(
         val groups = hashMapOf<Int, Js5Group>()
         (0 until count).forEach {
             val groupId = groupIds[it]
-            val groupReferenceTable = datFile.readReferenceTable(idxFile.id(), idxFile.loadReferenceTable(it))
             groups[it] = (Js5Group(
                 id = groupId,
                 nameHash = groupNameHashes[groupId],
                 crc = groupCrcs[groupId],
                 whirlpool = groupWhirlpools[groupId],
                 revision = groupRevisions[groupId],
                 keys = intArrayOf(),
-                files = files(fileIds, fileNameHashes, groupReferenceTable, groupFileIds[it], it),
-                data = groupReferenceTable
+                files = files(fileIds, fileNameHashes, groupTables[it], groupFileIds[it], it),
+                data = groupTables[it]
             ))
         }
-        return Js5Index(idxFile.id(), container.crc, whirlpool, container.compression, protocol, revision, isNamed, groups)
+        return Js5Index(indexId, crc, whirlpool, compressionType, protocol, revision, isNamed, groups)
     }
 
     private fun groupFileIds(
@@ -195,8 +212,8 @@ internal data class ReferenceTable(
         count: Int,
         groupIds: IntArray,
         buffer: ByteBuffer
-    ): Array<Array<Int>> {
-        val fileIds = Array(largestGroupId) { Array(validFileIds[it]) { -1 } }
+    ): Array<IntArray> {
+        val fileIds = Array(largestGroupId) { IntArray(validFileIds[it]) }
         (0 until count).forEach {
             val groupId = groupIds[it]
             var currentFileId = 0
@@ -216,8 +233,8 @@ internal data class ReferenceTable(
         groupIds: IntArray,
         buffer: ByteBuffer,
         isNamed: Boolean
-    ): Array<Array<Int>> {
-        val fileNameHashes = Array(largestGroupId) { Array(validFileIds[it]) { -1 } }
+    ): Array<IntArray> {
+        val fileNameHashes = Array(largestGroupId) { IntArray(validFileIds[it]) }
         if (isNamed) {
             (0 until count).forEach {
                 val groupId = groupIds[it]
@@ -230,8 +247,8 @@ internal data class ReferenceTable(
     }
 
     private fun files(
-        fileIds: Array<Array<Int>>,
-        fileNameHashes: Array<Array<Int>>,
+        fileIds: Array<IntArray>,
+        fileNameHashes: Array<IntArray>,
         groupReferenceTableData: ByteArray,
         count: Int,
         groupId: Int
@@ -249,7 +266,7 @@ internal data class ReferenceTable(
         }
 
         var position = src.size
-        val chunks: Int = src[--position].toInt() and 0xFF
+        val chunks = src[--position].toInt() and 0xFF
         position -= chunks * (count * 4)
         val buffer = ByteBuffer.wrap(src)
         buffer.position(position)

cache/src/main/kotlin/com/runetopic/cache/store/Js5Store.kt

@@ -2,7 +2,7 @@ package com.runetopic.cache.store
 
 import com.runetopic.cache.hierarchy.index.Index
 import com.runetopic.cache.store.storage.js5.Js5DiskStorage
-import com.runetopic.cryptography.ext.toWhirlpool
+import com.runetopic.cryptography.toWhirlpool
 import java.io.Closeable
 import java.math.BigInteger
 import java.nio.ByteBuffer
@@ -21,15 +21,17 @@ class Js5Store(
     private val indexes = arrayListOf<Index>()
 
     init {
-        this.storage.init(this)
+        storage.init(this)
+        indexes.sortWith(compareBy { it.getId() })
     }
 
+    @Synchronized
     internal fun addIndex(index: Index) {
         indexes.forEach { i -> require(index.getId() != i.getId()) { "Index with Id={${index.getId()}} already exists." } }
-        this.indexes.add(index)
+        indexes.add(index)
     }
 
-    fun index(indexId: Int): Index = this.indexes.find { it.getId() == indexId }!!
+    fun index(indexId: Int): Index = indexes.find { it.getId() == indexId }!!
 
     fun indexReferenceTableSize(indexId: Int): Int {
         var size = 0
@@ -77,12 +79,12 @@ class Js5Store(
 
         val whirlpool = ByteBuffer.allocate(64 + 1)
         whirlpool.put(1)
-        whirlpool.put(ByteBuffer.wrap(headerArray, 5, headerPosition - 5).array().toWhirlpool())
-        val rsa = BigInteger(whirlpool.array()).modPow(exponent, modulus).toByteArray()
+        whirlpool.put(headerArray.copyInto(ByteArray(headerPosition - 5), 0, 5, headerPosition).toWhirlpool())
 
-        val checksums = ByteBuffer.allocate(headerPosition.plus(rsa.size))
+        val rsa = BigInteger(whirlpool.array()).modPow(exponent, modulus).toByteArray()
+        val checksums = ByteBuffer.allocate(headerPosition + rsa.size)
         checksums.put(0)
-        checksums.putInt((headerPosition.plus(rsa.size)) - 5)
+        checksums.putInt((headerPosition + rsa.size) - 5)
         checksums.put(headerArray, 5, headerPosition - 5)
         checksums.put(rsa)
         return checksums.array()

cache/src/main/kotlin/com/runetopic/cache/store/storage/IStorage.kt

@@ -1,18 +1,18 @@
 package com.runetopic.cache.store.storage
 
-import com.runetopic.cache.hierarchy.ReferenceTable
 import com.runetopic.cache.hierarchy.index.Index
 import com.runetopic.cache.store.Js5Store
 import java.io.Closeable
 import java.io.Flushable
+import java.util.concurrent.CountDownLatch
 
 /**
  * @author Tyler Telis
  * @email <[email protected]>
  */
 internal interface IStorage: Closeable, Flushable {
     fun init(store: Js5Store)
-    fun loadIndex(table: ReferenceTable, indexId: Int, whirlpool: ByteArray, referenceTable: ByteArray): Index
+    fun store(indexId: Int, store: Js5Store, latch: CountDownLatch)
     fun loadReferenceTable(index: Index, groupId: Int): ByteArray
     fun loadMasterReferenceTable(groupId: Int): ByteArray
     fun loadReferenceTable(index: Index, groupName: String): ByteArray

cache/src/main/kotlin/com/runetopic/cache/store/storage/js5/Js5DiskStorage.kt

@@ -1,28 +1,30 @@
 package com.runetopic.cache.store.storage.js5
 
 import com.github.michaelbull.logging.InlineLogger
+import com.runetopic.cache.codec.ContainerCodec
 import com.runetopic.cache.extension.whirlpool
-import com.runetopic.cache.hierarchy.ReferenceTable
 import com.runetopic.cache.hierarchy.index.Index
 import com.runetopic.cache.hierarchy.index.Js5Index
 import com.runetopic.cache.store.Constants
 import com.runetopic.cache.store.Js5Store
 import com.runetopic.cache.store.storage.IStorage
 import com.runetopic.cache.store.storage.js5.impl.DatFile
 import com.runetopic.cache.store.storage.js5.impl.IdxFile
-import java.io.File
 import java.io.FileNotFoundException
 import java.nio.ByteBuffer
 import java.nio.file.Path
 import java.util.concurrent.CountDownLatch
 import java.util.concurrent.ExecutorService
 import java.util.concurrent.Executors
 import java.util.concurrent.TimeUnit
+import kotlin.io.path.ExperimentalPathApi
+import kotlin.io.path.exists
 
 /**
  * @author Tyler Telis
  * @email <[email protected]>
  */
+@OptIn(ExperimentalPathApi::class)
 internal class Js5DiskStorage(
     private val path: Path
 ) : IStorage {
@@ -32,13 +34,13 @@ internal class Js5DiskStorage(
     private val logger = InlineLogger()
 
     init {
-        val masterIndexFile = File("${path}/${Constants.MAIN_FILE_255}")
+        val masterIndexFile = Path.of("${path}/${Constants.MAIN_FILE_255}")
 
         if (masterIndexFile.exists().not()) {
             throw FileNotFoundException("Missing ${Constants.MAIN_FILE_255} in directory ${path}/${Constants.MAIN_FILE_255}")
         }
 
-        val datFile = File("${path}/${Constants.MAIN_FILE_DAT}")
+        val datFile = Path.of("${path}/${Constants.MAIN_FILE_DAT}")
 
         if (datFile.exists().not()) {
             throw FileNotFoundException("Missing ${Constants.MAIN_FILE_DAT} in directory ${path}/${Constants.MAIN_FILE_DAT}")
@@ -56,34 +58,29 @@ internal class Js5DiskStorage(
         }
         val latch = CountDownLatch(masterIdxFile.validIndexCount())
 
-        (0 until masterIdxFile.validIndexCount()).forEach {
-            val referenceTable = masterIdxFile.loadReferenceTable(it)
-            idxFiles.add(getIdxFile(it))
-
-            if (referenceTable.exists().not()) {
-                store.addIndex(Js5Index.default(it))
-                latch.countDown()
-                return@forEach
-            }
-            val datTable = datFile.readReferenceTable(masterIdxFile.id(), referenceTable)
-
-            pool?.let { service ->
-                service.execute {
-                    store.addIndex(loadIndex(referenceTable, it, ByteBuffer.wrap(datTable).whirlpool(), datTable))
-                    latch.countDown()
-                }
-            } ?: run {
-                store.addIndex(loadIndex(referenceTable, it, ByteBuffer.wrap(datTable).whirlpool(), datTable))
-                latch.countDown()
-            }
+        (0 until masterIdxFile.validIndexCount()).forEach { indexId ->
+            pool?.let { it.execute { store(indexId, store, latch) } }
+                ?: run { store(indexId, store, latch) }
         }
+
         latch.await(60, TimeUnit.SECONDS)
         pool?.shutdown()
-        logger.debug { "Loaded ${idxFiles.size} indices." }
+        logger.debug { "Loaded ${idxFiles.size} indexes." }
     }
 
-    override fun loadIndex(table: ReferenceTable, indexId: Int, whirlpool: ByteArray, referenceTable: ByteArray): Js5Index {
-        return table.loadIndex(datFile, getIdxFile(indexId), whirlpool, referenceTable)
+    override fun store(indexId: Int, store: Js5Store, latch: CountDownLatch) {
+        val indexTable = masterIdxFile.loadReferenceTable(indexId)
+        idxFiles.add(getIdxFile(indexId))
+
+        if (indexTable.exists().not()) {
+            store.addIndex(Js5Index.default(indexId))
+            latch.countDown()
+            return
+        }
+
+        val indexDatTable = datFile.readReferenceTable(masterIdxFile.id(), indexTable)
+        store.addIndex(indexTable.loadIndex(datFile, getIdxFile(indexId), ByteBuffer.wrap(indexDatTable).whirlpool(), ContainerCodec.decompress(indexDatTable)))
+        latch.countDown()
     }
 
     override fun loadMasterReferenceTable(groupId: Int): ByteArray {
@@ -102,7 +99,7 @@ internal class Js5DiskStorage(
 
     private fun getIdxFile(id: Int): IdxFile {
         idxFiles.find { it.id() == id }?.let { return it }
-        return IdxFile(id, File("$path/${Constants.MAIN_FILE_IDX}${id}"))
+        return IdxFile(id, Path.of("$path/${Constants.MAIN_FILE_IDX}${id}"))
     }
 
     override fun close() {

cache/src/main/kotlin/com/runetopic/cache/store/storage/js5/impl/DatFile.kt

@@ -5,18 +5,18 @@ import com.runetopic.cache.exception.EndOfDatFileException
 import com.runetopic.cache.hierarchy.ReferenceTable
 import com.runetopic.cache.store.Constants.SECTOR_SIZE
 import com.runetopic.cache.store.storage.js5.IDatFile
-import java.io.File
 import java.io.RandomAccessFile
 import java.nio.ByteBuffer
+import java.nio.file.Path
 
 /**
  * @author Tyler Telis
  * @email <[email protected]>
  */
 internal class DatFile(
-    file: File
+    path: Path
 ): IDatFile {
-    private val datFile: RandomAccessFile = RandomAccessFile(file, "rw")
+    private val datFile: RandomAccessFile = RandomAccessFile(path.toFile(), "rw")
 
     @Synchronized
     override fun readReferenceTable(id: Int, referenceTable: ReferenceTable): ByteArray {
@@ -107,7 +107,7 @@ internal class DatFile(
         currentId: Int,
     ) {
         if (referenceTableId != currentReferenceTableId || currentPart != part || id != currentId) {
-           throw DatFileException("DataFile mismatch Id={${currentId}} != {${id}}, ReferenceTableId={${currentReferenceTableId}} != {${referenceTableId}}, CurrentPart={${currentPart}} != {${part}}")
+            throw DatFileException("DataFile mismatch Id={${currentId}} != {${id}}, ReferenceTableId={${currentReferenceTableId}} != {${referenceTableId}}, CurrentPart={${currentPart}} != {${part}}")
         }
     }