Merge pull request #19 from runetopic/development

Replace Synchronized annotations by making the lists Synchronized ins…

Author: ultraviolet-jordan

README.md

@@ -28,7 +28,7 @@ A cache library written in Kotlin.
 # Implementation
 Just use cache if you do not require any of the revision specific loaders.
 ```
-cache = { module = "com.runetopic.cache:cache", version.ref "1.4.15-SNAPSHOT" }
+cache = { module = "com.runetopic.cache:cache", version.ref "1.4.16-SNAPSHOT" }
 loader = { module = "com.runetopic.cache:loader", version.ref "647.6.3-SNAPSHOT" }
 ```
 

cache/build.gradle.kts

@@ -4,7 +4,7 @@ plugins {
     signing
 }
 
-version = "1.4.15-SNAPSHOT"
+version = "1.4.16-SNAPSHOT"
 
 java {
     withJavadocJar()

cache/src/main/kotlin/com/runetopic/cache/codec/ContainerCodec.kt

@@ -2,6 +2,7 @@ package com.runetopic.cache.codec
 
 import com.runetopic.cache.codec.CodecType.*
 import com.runetopic.cache.exception.CompressionException
+import com.runetopic.cache.extension.readUnsignedByte
 import com.runetopic.cache.extension.readUnsignedShort
 import com.runetopic.cache.extension.remainingBytes
 import com.runetopic.cryptography.fromXTEA
@@ -19,7 +20,7 @@ internal object ContainerCodec {
     fun decompress(data: ByteArray, keys: IntArray = intArrayOf()): Container {
         val buffer = ByteBuffer.wrap(data)
 
-        val compression = buffer.get().toInt() and 0xFF
+        val compression = buffer.readUnsignedByte()
         val length = buffer.int
 
         if (length < 0 || length > 2000000) {

cache/src/main/kotlin/com/runetopic/cache/extension/ByteBuffer.kt

@@ -4,6 +4,7 @@ import java.nio.ByteBuffer
 
 internal fun ByteBuffer.readUnsignedByte(): Int = get().toInt() and 0xFF
 internal fun ByteBuffer.readUnsignedShort(): Int = short.toInt() and 0xFFFF
+internal fun ByteBuffer.readUnsignedMedium(): Int = ((get().toInt() and 0xFF shl 16) or (get().toInt() and 0xFF shl 8) or (get().toInt() and 0xFF))
 internal fun ByteBuffer.readUnsignedIntShortSmart(): Int = if (get(position()).toInt() < 0) int and Int.MAX_VALUE else readUnsignedShort()
 
 internal fun ByteBuffer.remainingBytes(): ByteArray {

cache/src/main/kotlin/com/runetopic/cache/store/Js5Store.kt

@@ -7,6 +7,7 @@ import java.io.Closeable
 import java.math.BigInteger
 import java.nio.ByteBuffer
 import java.nio.file.Path
+import java.util.concurrent.CopyOnWriteArrayList
 
 /**
  * @author Tyler Telis
@@ -19,14 +20,13 @@ class Js5Store(
     parallel: Boolean = false
 ) : Closeable {
     private var storage = Js5DiskStorage(path, parallel)
-    private val indexes = arrayListOf<Index>()
+    private val indexes = CopyOnWriteArrayList<Index>()
 
     init {
         storage.init(this)
         indexes.sortWith(compareBy { it.id })
     }
 
-    @Synchronized
     internal fun addIndex(index: Index) {
         indexes.forEach { i -> require(index.id != i.id) { "Index with Id={${index.id}} already exists." } }
         indexes.add(index)

cache/src/main/kotlin/com/runetopic/cache/store/storage/js5/Js5DiskStorage.kt

@@ -11,6 +11,7 @@ import com.runetopic.cache.store.storage.js5.impl.IdxFile
 import com.runetopic.cryptography.toWhirlpool
 import java.io.FileNotFoundException
 import java.nio.file.Path
+import java.util.concurrent.CopyOnWriteArrayList
 import java.util.concurrent.CountDownLatch
 import java.util.concurrent.Executors
 import kotlin.io.path.ExperimentalPathApi
@@ -29,7 +30,7 @@ internal class Js5DiskStorage(
 ) : IStorage {
     private var masterIdxFile: IIdxFile
     private var datFile: IDatFile
-    private var idxFiles: ArrayList<IdxFile> = arrayListOf()
+    private var idxFiles = CopyOnWriteArrayList<IdxFile>()
     private val logger = InlineLogger()
 
     init {
@@ -79,7 +80,7 @@ internal class Js5DiskStorage(
             return
         }
         val indexDatTable = datFile.readReferenceTable(masterIdxFile.id(), indexTable)
-        store.addIndex(loadIndex(datFile, getIdxFile(indexId), indexDatTable.toWhirlpool(), ContainerCodec.decompress(indexDatTable)))
+        store.addIndex(decode(datFile, getIdxFile(indexId), indexDatTable.toWhirlpool(), ContainerCodec.decompress(indexDatTable)))
     }
 
     override fun loadMasterReferenceTable(groupId: Int): ByteArray {
@@ -96,7 +97,6 @@ internal class Js5DiskStorage(
         return datFile.readReferenceTable(index.id, getIdxFile(index.id).loadReferenceTable(group.id))
     }
 
-    @Synchronized
     private fun getIdxFile(id: Int): IdxFile {
         idxFiles.find { it.id() == id }?.let { return it }
         return IdxFile(id, Path.of("$path/${Constants.MAIN_FILE_IDX}${id}"))

cache/src/main/kotlin/com/runetopic/cache/store/storage/js5/Js5IndexFactory.kt renamed to cache/src/main/kotlin/com/runetopic/cache/store/storage/js5/Js5IndexDecoder.kt

@@ -15,15 +15,13 @@ import java.util.zip.ZipException
 /**
  * @author Jordan Abraham
  */
-internal fun loadIndex(
+internal fun decode(
     datFile: IDatFile,
     idxFile: IIdxFile,
     whirlpool: ByteArray,
     decompressed: Container
 ): Index {
     val buffer = ByteBuffer.wrap(decompressed.data)
-    val crc = decompressed.crc
-    val compression = decompressed.compression
     val protocol = buffer.readUnsignedByte()
     val revision = when {
         protocol < 5 || protocol > 7 -> throw ProtocolException("Unhandled protocol $protocol")
@@ -41,23 +39,15 @@ internal fun loadIndex(
     val isNamed = (0x1 and hash) != 0
     val isUsingWhirlpool = (0x2 and hash) != 0
 
-    val groupIds = IntArray(count)
-    var lastGroupId = 0
-    var biggest = -1
-    (0 until count).forEach {
-        groupIds[it] = if (protocol >= 7) { buffer.readUnsignedIntShortSmart() } else { buffer.readUnsignedShort() }
-            .let { id -> lastGroupId += id; lastGroupId }
-        if (groupIds[it] > biggest) biggest = groupIds[it]
-    }
-
-    val largestGroupId = biggest + 1
-    val groupNameHashes = groupNameHashes(largestGroupId, count, isNamed, groupIds, buffer)
-    val groupCrcs = groupCrcs(largestGroupId, count, groupIds, buffer)
-    val groupWhirlpools = groupWhirlpools(largestGroupId, isUsingWhirlpool, count, buffer, groupIds)
-    val groupRevisions = groupRevisions(largestGroupId, count, groupIds, buffer)
-    val groupFileIds = groupFileIds(largestGroupId, count, groupIds, buffer, protocol)
-    val fileIds = fileIds(largestGroupId, groupFileIds, count, groupIds, buffer, protocol)
-    val fileNameHashes = fileNameHashes(largestGroupId, groupFileIds, count, groupIds, buffer, isNamed)
+    val groupIds = decodeGroupIds(count, buffer, protocol)
+    val maxGroupId = (groupIds.maxOrNull() ?: -1) + 1
+    val groupNameHashes = decodeGroupNameHashes(maxGroupId, count, isNamed, groupIds, buffer)
+    val groupCrcs = decodeGroupCrcs(maxGroupId, count, groupIds, buffer)
+    val groupWhirlpools = decodeGroupWhirlpools(maxGroupId, isUsingWhirlpool, count, buffer, groupIds)
+    val groupRevisions = decodeGroupRevisions(maxGroupId, count, groupIds, buffer)
+    val groupFileIds = decodeGroupFileIds(maxGroupId, count, groupIds, buffer, protocol)
+    val fileIds = decodeFileIds(maxGroupId, groupFileIds, count, groupIds, buffer, protocol)
+    val fileNameHashes = decodeFileNameHashes(maxGroupId, groupFileIds, count, groupIds, buffer, isNamed)
 
     val groups = hashMapOf<Int, Group>()
     (0 until count).forEach {
@@ -69,48 +59,62 @@ internal fun loadIndex(
             groupWhirlpools[groupId],
             groupRevisions[groupId],
             intArrayOf(),//TODO
-            groupFiles(fileIds, fileNameHashes, groupTables[it], groupFileIds[it], it),
+            decodeFiles(fileIds, fileNameHashes, groupTables[it], groupFileIds[it], it),
             groupTables[it]
         ))
     }
-    return Index(idxFile.id(), crc, whirlpool, compression, protocol, revision, isNamed, groups)
+    return Index(idxFile.id(), decompressed.crc, whirlpool, decompressed.compression, protocol, revision, isNamed, groups)
+}
+
+private fun decodeGroupIds(
+    count: Int,
+    buffer: ByteBuffer,
+    protocol: Int
+): IntArray {
+    val groupIds = IntArray(count)
+    var offset = 0
+    (0 until count).forEach {
+        groupIds[it] = if (protocol >= 7) { buffer.readUnsignedIntShortSmart() } else { buffer.readUnsignedShort() }
+            .let { id -> offset += id; offset }
+    }
+    return groupIds
 }
 
-private fun groupFileIds(
-    largestGroupId: Int,
+private fun decodeGroupFileIds(
+    maxGroupId: Int,
     count: Int,
     groupIds: IntArray,
     buffer: ByteBuffer,
     protocol: Int
 ): IntArray {
-    val groupFileIds = IntArray(largestGroupId)
+    val groupFileIds = IntArray(maxGroupId)
     (0 until count).forEach {
         groupFileIds[groupIds[it]] = if (protocol >= 7) buffer.readUnsignedIntShortSmart() else buffer.readUnsignedShort()
     }
     return groupFileIds
 }
 
-private fun groupRevisions(
-    largestGroupId: Int,
+private fun decodeGroupRevisions(
+    maxGroupId: Int,
     count: Int,
     groupIds: IntArray,
     buffer: ByteBuffer
 ): IntArray {
-    val revisions = IntArray(largestGroupId)
+    val revisions = IntArray(maxGroupId)
     (0 until count).forEach {
         revisions[groupIds[it]] = buffer.int
     }
     return revisions
 }
 
-private fun groupWhirlpools(
-    largestGroupId: Int,
+private fun decodeGroupWhirlpools(
+    maxGroupId: Int,
     usesWhirlpool: Boolean,
     count: Int,
     buffer: ByteBuffer,
     groupIds: IntArray
 ): Array<ByteArray> {
-    val whirlpools = Array(largestGroupId) { ByteArray(64) }
+    val whirlpools = Array(maxGroupId) { ByteArray(64) }
     if (usesWhirlpool.not()) return whirlpools
 
     (0 until count).forEach {
@@ -121,27 +125,27 @@ private fun groupWhirlpools(
     return whirlpools
 }
 
-private fun groupCrcs(
-    largestGroupId: Int,
+private fun decodeGroupCrcs(
+    maxGroupId: Int,
     count: Int,
     groupIds: IntArray,
     buffer: ByteBuffer
 ): IntArray {
-    val crcs = IntArray(largestGroupId)
+    val crcs = IntArray(maxGroupId)
     (0 until count).forEach {
         crcs[groupIds[it]] = buffer.int
     }
     return crcs
 }
 
-private fun groupNameHashes(
-    largestGroupId: Int,
+private fun decodeGroupNameHashes(
+    maxGroupId: Int,
     count: Int,
     isNamed: Boolean,
     groupIds: IntArray,
     buffer: ByteBuffer
 ): IntArray {
-    val nameHashes = IntArray(largestGroupId) { -1 }
+    val nameHashes = IntArray(maxGroupId) { -1 }
     if (isNamed.not()) return nameHashes
 
     (0 until count).forEach {
@@ -150,36 +154,36 @@ private fun groupNameHashes(
     return nameHashes
 }
 
-private fun fileIds(
-    largestGroupId: Int,
+private fun decodeFileIds(
+    maxGroupId: Int,
     validFileIds: IntArray,
     count: Int,
     groupIds: IntArray,
     buffer: ByteBuffer,
     protocol: Int
 ): Array<IntArray> {
-    val fileIds = Array(largestGroupId) { IntArray(validFileIds[it]) }
+    val fileIds = Array(maxGroupId) { IntArray(validFileIds[it]) }
     (0 until count).forEach {
         val groupId = groupIds[it]
-        var currentFileId = 0
+        var offset = 0
         (0 until validFileIds[groupId]).forEach { fileId ->
             if (protocol >= 7) { buffer.readUnsignedIntShortSmart() } else { buffer.readUnsignedShort() }
-                .let { i -> currentFileId += i; currentFileId }
-                .also { fileIds[groupId][fileId] = currentFileId }
+                .let { i -> offset += i; offset }
+                .also { fileIds[groupId][fileId] = offset }
         }
     }
     return fileIds
 }
 
-private fun fileNameHashes(
-    largestGroupId: Int,
+private fun decodeFileNameHashes(
+    maxGroupId: Int,
     validFileIds: IntArray,
     count: Int,
     groupIds: IntArray,
     buffer: ByteBuffer,
     isNamed: Boolean
 ): Array<IntArray> {
-    val fileNameHashes = Array(largestGroupId) { IntArray(validFileIds[it]) }
+    val fileNameHashes = Array(maxGroupId) { IntArray(validFileIds[it]) }
     if (isNamed) {
         (0 until count).forEach {
             val groupId = groupIds[it]
@@ -191,7 +195,7 @@ private fun fileNameHashes(
     return fileNameHashes
 }
 
-internal fun groupFiles(
+internal fun decodeFiles(
     fileIds: Array<IntArray>,
     fileNameHashes: Array<IntArray>,
     groupReferenceTableData: ByteArray,