diff --git a/README.md b/README.md index 2779aa9..1c81622 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ repositories { } dependencies { - implementation("me.alex_s168:blitz:0.19") + implementation("me.alex_s168:blitz:0.20") } ``` @@ -235,7 +235,11 @@ val json = """ """ println(JSON.parse(json)!!.obj["b"]!!.obj["1"]!!.num) ``` -### Either -No example yet -### Tree -No example yet \ No newline at end of file +### Features without examples +- `Either` +- `Tree` +- `ByteVec` +- `BlitzHashMap` +- `Dense16x16BoolMap` +- `DenseIx16x16BoolMap` +- `SlicedIntKeyMap` \ No newline at end of file diff --git a/build.gradle.kts b/build.gradle.kts index 6199d3b..8bcfeb7 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -5,7 +5,7 @@ plugins { } group = "me.alex_s168" -version = "0.19" +version = "0.20" repositories { mavenCentral() diff --git a/src/main/kotlin/blitz/collections/BitVec.kt b/src/main/kotlin/blitz/collections/BitVec.kt index c0576c8..8b9c000 100644 --- a/src/main/kotlin/blitz/collections/BitVec.kt +++ b/src/main/kotlin/blitz/collections/BitVec.kt @@ -7,6 +7,7 @@ import kotlin.math.ceil // TODO: make it hybrid to a real bitset if a lot of elements +@Deprecated(message = "slow") class BitVec private constructor( private val byteVec: ByteVec ): Vec { @@ -51,6 +52,10 @@ class BitVec private constructor( byteVec[index] = value.toByte() } + override fun clear() { + byteVec.clear() + } + companion object { // TODO: implement better fun from(bytes: ByteArray): BitVec = diff --git a/src/main/kotlin/blitz/collections/BlitzHashMap.kt b/src/main/kotlin/blitz/collections/BlitzHashMap.kt index 1999e14..1a4df6f 100644 --- a/src/main/kotlin/blitz/collections/BlitzHashMap.kt +++ b/src/main/kotlin/blitz/collections/BlitzHashMap.kt @@ -37,11 +37,10 @@ class BlitzHashMap( val key: K, ): Index - override val contents: Contents> - get() = buckets + override fun contents(): Contents> = buckets .map { bucketSrc.contents(it) } .reduce { acc, pairs -> acc + pairs } - val bucketStats - get() = Contents(buckets.map { bucketSrc.contents(it).count() }) + fun bucketStats() = + Contents(buckets.map { bucketSrc.contents(it).count() }) } \ No newline at end of file diff --git a/src/main/kotlin/blitz/collections/BlitzMap.kt b/src/main/kotlin/blitz/collections/BlitzMap.kt index c4077de..1f1e377 100644 --- a/src/main/kotlin/blitz/collections/BlitzMap.kt +++ b/src/main/kotlin/blitz/collections/BlitzMap.kt @@ -4,7 +4,7 @@ interface BlitzMap { fun index(key: K): I operator fun get(index: I): V? operator fun set(index: I, value: V?) - val contents: Contents> + fun contents(): Contents> } fun BlitzMap.remove(index: I) = diff --git a/src/main/kotlin/blitz/collections/ByteVec.kt b/src/main/kotlin/blitz/collections/ByteVec.kt index 59dbf14..ce9ff0c 100644 --- a/src/main/kotlin/blitz/collections/ByteVec.kt +++ b/src/main/kotlin/blitz/collections/ByteVec.kt @@ -1,10 +1,23 @@ package blitz.collections -class ByteVec(initCap: Int = 0): Vec, ByteBatchSequence { +import kotlin.contracts.ExperimentalContracts +import kotlin.contracts.contract + +class ByteVec(private val initCap: Int = 0): Vec, ByteBatchSequence { override var size = 0 private var cap = initCap private var array = ByteArray(initCap) + override fun clear() { + size = 0 + if (array.size <= initCap) { + cap = array.size + } else { + cap = initCap + array = ByteArray(initCap) + } + } + fun copyAsArray(): ByteArray = array.copyOfRange(0, size) @@ -29,10 +42,58 @@ class ByteVec(initCap: Int = 0): Vec, ByteBatchSequence { size -- } + fun tryPopPack(dest: ByteArray, destOff: Int = 0): Int { + val can = kotlin.math.min(size, dest.size - destOff) + copyIntoArray(dest, destOff, size - can) + reserve(-can) + size -= can + return can + } + fun popBack(dest: ByteArray, destOff: Int = 0) { - copyIntoArray(dest, destOff, size - dest.size) - reserve(-dest.size) - size -= dest.size + val destCopySize = dest.size - destOff + require(size >= destCopySize) + copyIntoArray(dest, destOff, size - destCopySize) + reserve(-destCopySize) + size -= destCopySize + } + + @OptIn(ExperimentalContracts::class) + inline fun consumePopBack(batching: ByteArray, fn: (ByteArray, Int) -> Unit) { + contract { + callsInPlace(fn) + } + + while (true) { + val rem = tryPopPack(batching) + if (rem == 0) break + + fn(batching, rem) + } + } + + inline fun consumePopBack(batching: ByteArray, fn: (Byte) -> Unit) = + consumePopBack(batching) { batch, count -> + repeat(count) { + fn(batch[it]) + } + } + + @OptIn(ExperimentalContracts::class) + inline fun consumePopBackSlicedBatches(batching: ByteArray, fn: (ByteArray) -> Unit) { + contract { + callsInPlace(fn) + } + + while (true) { + val rem = tryPopPack(batching) + if (rem == 0) break + + if (rem == batching.size) + fn(batching) + else + fn(batching.copyOf(rem)) + } } override fun get(index: Int): Byte = diff --git a/src/main/kotlin/blitz/collections/Dense16x16BoolMap.kt b/src/main/kotlin/blitz/collections/Dense16x16BoolMap.kt new file mode 100644 index 0000000..12a4192 --- /dev/null +++ b/src/main/kotlin/blitz/collections/Dense16x16BoolMap.kt @@ -0,0 +1,197 @@ +package blitz.collections + +import kotlin.contracts.ExperimentalContracts +import kotlin.contracts.contract + +@JvmInline +@OptIn(ExperimentalUnsignedTypes::class) +value class Dense16x16BoolMap( + val packed: UShortArray = UShortArray(16) +) { + fun fillRowsWith(value: UShort) { + repeat(16) { + packed[it] = value + } + } + + fun clear() { + fillRowsWith(0u) + } + + fun getPackedBytes(dest: ByteArray, destOff: Int = 0) { + require(dest.size - destOff >= 32) + packed.forEachIndexed { index, uShort -> + val didx = destOff + index * 2 + dest[didx] = (uShort and 0xFFu).toByte() + dest[didx + 1] = ((uShort.toInt() ushr 8) and 0xFF).toByte() + } + } + + fun fromPackedBytes(src: ByteArray, srcOff: Int = 0) { + require(src.size - srcOff >= 32) + + repeat(16) { uShortIdx -> + val sidx = srcOff + uShortIdx * 2 + val uShort = ((src[sidx].toInt() shl 8) or (src[sidx].toInt())).toUShort() + packed[uShortIdx] = uShort + } + } + + fun appendFrom(other: Dense16x16BoolMap) { + other.packed.forEachIndexed { index, otherUShort -> + val old = packed[index] + packed[index] = old or otherUShort + } + } + + @OptIn(ExperimentalContracts::class) + inline fun forEachSetRow(fn: (Int) -> Unit) { + contract { + callsInPlace(fn) + } + + repeat(16) { + if (anyInRow(it)) { + fn(it) + } + } + } + + @OptIn(ExperimentalContracts::class) + fun forEachSet(fn: (Int, Int) -> Unit) { + contract { + callsInPlace(fn) + } + + forEachSetRow { rowId -> + val row = packed[rowId] + val lo = row and 0xFFu + val hi = (row.toInt() shr 8) and 0xFF + + if (lo > 0u) { + var acc = lo.toInt() + repeat(8) { + val v = acc and 1 + acc = acc shr 1 + + if (v > 0) fn(rowId, it) + } + } + + if (hi > 0) { + var acc = hi + repeat(8) { + val v = acc and 1 + acc = acc shr 1 + + if (v > 0) fn(rowId, 8 + it) + } + } + } + } + + @OptIn(ExperimentalContracts::class) + inline fun getSetPosList(dest: MutableList = mutableListOf(), crossinline mapfn: (Int, Int) -> T): MutableList { + contract { + callsInPlace(mapfn) + } + + forEachSet { x, y -> + dest.add(mapfn(x, y)) + } + + return dest + } + + fun packedSetPosList(dest: ByteVec = ByteVec(16)): ByteVec { + forEachSet { x, y -> + val packed = packPos(x, y) + dest.pushBack(packed.toByte()) + } + return dest + } + + companion object { + fun packPos(row: Int, col: Int): UByte = + ((row shl 4) or col).toUByte() + + @OptIn(ExperimentalContracts::class) + inline fun unpackPos(packed: UByte, fn: (Int, Int) -> T): T { + contract { + callsInPlace(fn) + } + + return fn((packed.toInt() shr 4) and 0xF, packed.toInt() and 0xF) + } + + inline fun unpackPos(packed: Byte, fn: (Int, Int) -> T): T = + unpackPos(packed.toUByte(), fn) + + inline fun forEachPackedPos(packed: Iterator, fn: (Int, Int) -> Unit) { + for (it in packed) { + unpackPos(it, fn) + } + } + + inline fun forEachPackedPos(packed: Iterable, fn: (Int, Int) -> Unit) = + forEachPackedPos(packed.iterator(), fn) + + inline fun forEachPackedPos(packed: Sequence, fn: (Int, Int) -> Unit) = + forEachPackedPos(packed.iterator(), fn) + + inline fun consumeAllPackedPos(vec: ByteVec, batching: ByteArray, fn: (Int, Int) -> Unit) { + vec.consumePopBack(batching) { it -> + unpackPos(it, fn) + } + } + } + + fun anyInRow(row: Int) = + packed[row] > 0u + + fun setRow(row: Int, value: UShort) { + packed[row] = value + } + + fun clearRow(row: Int) { + setRow(row, 0u) + } + + fun countSetInRow(row: Int) = + packed[row].countOneBits() + + fun columnMask(col: Int) = + 1 shl col + + operator fun get(row: Int, col: Int) = + (packed[row].toInt() and columnMask(col)) > 0 + + operator fun get(packedRowCol: UByte) = + unpackPos(packedRowCol) { x, y -> get(x, y) } + + operator fun get(packedRowCol: Byte) = + unpackPos(packedRowCol) { x, y -> get(x, y) } + + fun set(row: Int, col: Int) { + packed[row] = (packed[row].toInt() or columnMask(col)).toUShort() + } + + fun unset(row: Int, col: Int) { + val mask = columnMask(col).inv() + packed[row] = (packed[row].toInt() and mask).toUShort() + } + + operator fun set(row: Int, col: Int, value: Boolean) { + if (value) { + set(row, col) + } else { + unset(row, col) + } + } + + operator fun set(packedRowCol: Byte, value: Boolean) = + unpackPos(packedRowCol) { x, y -> set(x, y, value) } + + operator fun set(packedRowCol: UByte, value: Boolean) = + unpackPos(packedRowCol) { x, y -> set(x, y, value) } +} \ No newline at end of file diff --git a/src/main/kotlin/blitz/collections/DenseIx16x16BoolMap.kt b/src/main/kotlin/blitz/collections/DenseIx16x16BoolMap.kt new file mode 100644 index 0000000..a6f8fd2 --- /dev/null +++ b/src/main/kotlin/blitz/collections/DenseIx16x16BoolMap.kt @@ -0,0 +1,143 @@ +package blitz.collections + +import blitz.Endian +import blitz.toBytes +import blitz.toInt +import blitz.toShort +import kotlin.contracts.ExperimentalContracts +import kotlin.contracts.contract + +class DenseIx16x16BoolMap { + val backing = SlicedIntKeyMap() + + operator fun get(y: Int) = + backing[y] + + operator fun get(x: Int, y: Int, z: Int): Boolean = + backing[y]?.get(x, z) ?: false + + fun getOrCreateLayer(y: Int) = + @OptIn(ExperimentalUnsignedTypes::class) + backing.computeIfAbsent(y) { Dense16x16BoolMap() } + + operator fun set(x: Int, y: Int, z: Int, value: Boolean) { + getOrCreateLayer(y)[x, z] = value + } + + @OptIn(ExperimentalContracts::class) + inline fun forEachSet(fn: (Int, Int, Int) -> Unit) { + contract { + callsInPlace(fn) + } + + val layerBytes = ByteVec() + val buf32 = ByteArray(32) + backing.forEachSet { y, layer -> + layer.packedSetPosList(layerBytes) + Dense16x16BoolMap.consumeAllPackedPos(layerBytes, buf32) { x, z -> + fn(x, y, z) + } + } + } + + inline fun getSetAsSequence(crossinline convertIndex: (Int, Int, Int) -> T) = + sequence { + forEachSet { x, y, z -> + yield(convertIndex(x, y, z)) + } + } + + /** + * base: 4 + * per contained layer: (4 or 2) + 1 + (0 to 256) bytes + * only recommended if very few positions per layer + */ + fun serializeByPositions(yPosAsWord: Boolean, unbufferedConsumer: (ByteArray) -> Unit) { + val layerBytes = ByteVec() + val buf32 = ByteArray(32) + + backing.countSet().toBytes(Endian.LITTLE).also(unbufferedConsumer) + + backing.forEachSet { y, layer -> + if (yPosAsWord) { + y.toShort().toBytes(Endian.LITTLE).also(unbufferedConsumer) + } else { + y.toBytes(Endian.LITTLE).also(unbufferedConsumer) + } + + layer.packedSetPosList(layerBytes) + layerBytes.size.toUByte().toBytes().also(unbufferedConsumer) + layerBytes.consumePopBackSlicedBatches(buf32, unbufferedConsumer) + } + } + + /** + * base: 4 + * per contained layer: (4 or 2) + 32 bytes + * should almost always be used + */ + fun serializeByLayers(yPosAsWord: Boolean, unbufferedConsumer: (ByteArray) -> Unit) { + backing.countSet().toBytes(Endian.LITTLE).also(unbufferedConsumer) + + val buf32 = ByteArray(32) + backing.forEachSet { y, layer -> + if (yPosAsWord) { + y.toShort().toBytes(Endian.LITTLE).also(unbufferedConsumer) + } else { + y.toBytes(Endian.LITTLE).also(unbufferedConsumer) + } + + layer.getPackedBytes(buf32) + unbufferedConsumer(buf32) + } + } + + companion object { + fun deserializeByPositions(yPosAsWord: Boolean, appendTo: DenseIx16x16BoolMap = DenseIx16x16BoolMap(), unbufferedProvider: (Int) -> ByteArray): DenseIx16x16BoolMap { + val count = unbufferedProvider(4).toInt(Endian.LITTLE) + + repeat(count) { + val (ypos, layerByteCount) = if (yPosAsWord) { + val byteArr = unbufferedProvider(3) + byteArr.toShort(Endian.LITTLE).toInt() to byteArr.last() + } else { + val byteArr = unbufferedProvider(5) + byteArr.toInt(Endian.LITTLE) to byteArr.last() + } + + val layer = appendTo.getOrCreateLayer(ypos) + + val packedPositions = unbufferedProvider(layerByteCount.toInt()) + packedPositions.forEach { + Dense16x16BoolMap.unpackPos(it, layer::set) + } + } + + return appendTo + } + + fun deserializeByLayers(yPosAsWord: Boolean, appendTo: DenseIx16x16BoolMap = DenseIx16x16BoolMap(), unbufferedProvider: (Int) -> ByteArray): DenseIx16x16BoolMap { + val count = unbufferedProvider(4).toInt(Endian.LITTLE) + + @OptIn(ExperimentalUnsignedTypes::class) + val tempLayer = Dense16x16BoolMap() + repeat(count) { + val ypos = if (yPosAsWord) { + unbufferedProvider(2).toShort(Endian.LITTLE).toInt() + } else { + unbufferedProvider(4).toInt(Endian.LITTLE) + } + + val layer = appendTo.getOrCreateLayer(ypos) + + val bytes = unbufferedProvider(32) + tempLayer.fromPackedBytes(bytes) + + layer.appendFrom(tempLayer) + tempLayer.clear() + } + + return appendTo + } + } +} \ No newline at end of file diff --git a/src/main/kotlin/blitz/collections/ListOps.kt b/src/main/kotlin/blitz/collections/ListOps.kt index 376d792..57f1fc8 100644 --- a/src/main/kotlin/blitz/collections/ListOps.kt +++ b/src/main/kotlin/blitz/collections/ListOps.kt @@ -27,4 +27,7 @@ fun MutableList.removeLastInto(count: Int, dest: MutableList = mutable } fun MutableList.addFront(value: T) = - add(0, value) \ No newline at end of file + add(0, value) + +fun Iterable.countNotNull() = + count { it != null } \ No newline at end of file diff --git a/src/main/kotlin/blitz/collections/SlicedIntKeyMap.kt b/src/main/kotlin/blitz/collections/SlicedIntKeyMap.kt new file mode 100644 index 0000000..20e729b --- /dev/null +++ b/src/main/kotlin/blitz/collections/SlicedIntKeyMap.kt @@ -0,0 +1,104 @@ +package blitz.collections + +import kotlin.contracts.ExperimentalContracts +import kotlin.contracts.InvocationKind +import kotlin.contracts.contract + +class SlicedIntKeyMap: BlitzMap { + val vec = ArrayList() + var vecOffset: Int? = null + + override fun index(key: Int) = key + + override fun get(index: Int): V? { + if (vecOffset == null) return null + + if (index < vecOffset!!) return null + + return vec.getOrNull(index - vecOffset!!) + } + + /** @return Changed */ + @OptIn(ExperimentalContracts::class) + fun setIfNotPresent(index: Int, value: (Int) -> V?): Boolean { + contract { + callsInPlace(value, InvocationKind.AT_MOST_ONCE) + } + + if (vecOffset == null) { + vecOffset = index + vec.add(value(index)) + return true + } else if (index < vecOffset!!) { + // prepend + + val diff = vecOffset!! - index + repeat(diff) { + vec.add(0, null) + } + vecOffset = index + vec[0] = value(index) + return true + } + + val offPlusSize = vecOffset!! + vec.size + if (index >= offPlusSize) { + // append + + val diff = index - offPlusSize + repeat(diff + 1) { + vec.add(null) + } + vec[vec.size - 1] = value(index) + return true + } + + return false + } + + @OptIn(ExperimentalContracts::class) + fun computeIfAbsent(index: Int, fn: (Int) -> V): V { + contract { + callsInPlace(fn, InvocationKind.AT_MOST_ONCE) + } + + var value: V? = null + setIfNotPresent(index) { + fn(it).also { value = it } + } + return vec[index - vecOffset!!] ?: let { + val v = value ?: fn(index) + vec[index - vecOffset!!] = v + v + } + } + + override fun set(index: Int, value: V?) { + if (!setIfNotPresent(index) { value }) { + vec[index - vecOffset!!] = value + } + } + + override fun contents(): Contents> = + vec.withIndex() + .mapNotNull { (id, v) -> v?.let { id + vecOffset!! to it } } + .contents + + fun countSet(): Int = + vec.countNotNull() + + @OptIn(ExperimentalContracts::class) + inline fun forEachSet(fn: (Int, V) -> Unit) { + contract { + callsInPlace(fn) + } + + if (vecOffset == null) return + + repeat(vec.size) { ko -> + val v = vec[ko] + + v?.let { fn(ko + vecOffset!!, it) } + } + } +} \ No newline at end of file diff --git a/src/main/kotlin/blitz/collections/Vec.kt b/src/main/kotlin/blitz/collections/Vec.kt index 075ce8f..54c493d 100644 --- a/src/main/kotlin/blitz/collections/Vec.kt +++ b/src/main/kotlin/blitz/collections/Vec.kt @@ -28,4 +28,6 @@ interface Vec: IndexableSequence { } operator fun set(index: Int, value: T) + + fun clear() } \ No newline at end of file diff --git a/src/test/kotlin/map.kt b/src/test/kotlin/map.kt index 9221715..65561c0 100644 --- a/src/test/kotlin/map.kt +++ b/src/test/kotlin/map.kt @@ -1,5 +1,7 @@ +import blitz.collections.DenseIx16x16BoolMap import blitz.collections.I2HashMap import blitz.collections.I2HashMapKey +import blitz.collections.contents import kotlin.test.Test class Maps { @@ -10,7 +12,20 @@ class Maps { a[a.index(I2HashMapKey(320, 23))] = "bye" a[a.index(I2HashMapKey(320, 25))] = "bye2" a[a.index(I2HashMapKey(32, 344))] = "bye3" - println(a.contents) - println(a.bucketStats) + println(a.contents()) + println(a.bucketStats()) + } + + @Test + /** test for: DenseIx16x16BoolMap, SlicedIntKeyMap, Dense16x16BoolMap */ + fun denseI16x16() { + val a = DenseIx16x16BoolMap() + a[1, 0, 1] = true + a[2, 0, 2] = true + a[3, -1, 4] = true + a[6, 1, 3] = true + require(a[1, 0, 1]) + require(!a[1, -1, 1]) + println(a.getSetAsSequence(::Triple).contents) } } \ No newline at end of file