dense bool maps that are shaped like minecraft chunks for no reason at all

src/main/kotlin/blitz/collections/BitVec.kt

@@ -7,6 +7,7 @@ import kotlin.math.ceil
 
 // TODO: make it hybrid to a real bitset if a lot of elements
 
+@Deprecated(message = "slow")
 class BitVec private constructor(
     private val byteVec: ByteVec
 ): Vec<Boolean> {
@@ -51,6 +52,10 @@ class BitVec private constructor(
         byteVec[index] = value.toByte()
     }
 
+    override fun clear() {
+        byteVec.clear()
+    }
+
     companion object {
         // TODO: implement better
         fun from(bytes: ByteArray): BitVec =

src/main/kotlin/blitz/collections/BlitzHashMap.kt

@@ -37,11 +37,10 @@ class BlitzHashMap<K, V>(
         val key: K,
     ): Index<K,V>
 
-    override val contents: Contents<Pair<K, V>>
-        get() = buckets
+    override fun contents(): Contents<Pair<K, V>> = buckets
             .map { bucketSrc.contents(it) }
             .reduce { acc, pairs -> acc + pairs }
 
-    val bucketStats
-        get() = Contents(buckets.map { bucketSrc.contents(it).count() })
+    fun bucketStats() =
+        Contents(buckets.map { bucketSrc.contents(it).count() })
 }

src/main/kotlin/blitz/collections/BlitzMap.kt

@@ -4,7 +4,7 @@ interface BlitzMap<K,V,I> {
     fun index(key: K): I
     operator fun get(index: I): V?
     operator fun set(index: I, value: V?)
-    val contents: Contents<Pair<K,V>>
+    fun contents(): Contents<Pair<K,V>>
 }
 
 fun <K,V,I> BlitzMap<K,V,I>.remove(index: I) =

src/main/kotlin/blitz/collections/ByteVec.kt

@@ -1,10 +1,23 @@
 package blitz.collections
 
-class ByteVec(initCap: Int = 0): Vec<Byte>, ByteBatchSequence {
+import kotlin.contracts.ExperimentalContracts
+import kotlin.contracts.contract
+
+class ByteVec(private val initCap: Int = 0): Vec<Byte>, ByteBatchSequence {
     override var size = 0
     private var cap = initCap
     private var array = ByteArray(initCap)
 
+    override fun clear() {
+        size = 0
+        if (array.size <= initCap) {
+            cap = array.size
+        } else {
+            cap = initCap
+            array = ByteArray(initCap)
+        }
+    }
+
     fun copyAsArray(): ByteArray =
         array.copyOfRange(0, size)
 
@@ -29,10 +42,58 @@ class ByteVec(initCap: Int = 0): Vec<Byte>, ByteBatchSequence {
             size --
         }
 
+    fun tryPopPack(dest: ByteArray, destOff: Int = 0): Int {
+        val can = kotlin.math.min(size, dest.size - destOff)
+        copyIntoArray(dest, destOff, size - can)
+        reserve(-can)
+        size -= can
+        return can
+    }
+
     fun popBack(dest: ByteArray, destOff: Int = 0) {
-        copyIntoArray(dest, destOff, size - dest.size)
-        reserve(-dest.size)
-        size -= dest.size
+        val destCopySize = dest.size - destOff
+        require(size >= destCopySize)
+        copyIntoArray(dest, destOff, size - destCopySize)
+        reserve(-destCopySize)
+        size -= destCopySize
+    }
+
+    @OptIn(ExperimentalContracts::class)
+    inline fun consumePopBack(batching: ByteArray, fn: (ByteArray, Int) -> Unit) {
+        contract {
+            callsInPlace(fn)
+        }
+
+        while (true) {
+            val rem = tryPopPack(batching)
+            if (rem == 0) break
+
+            fn(batching, rem)
+        }
+    }
+
+    inline fun consumePopBack(batching: ByteArray, fn: (Byte) -> Unit) =
+        consumePopBack(batching) { batch, count ->
+            repeat(count) {
+                fn(batch[it])
+            }
+        }
+
+    @OptIn(ExperimentalContracts::class)
+    inline fun consumePopBackSlicedBatches(batching: ByteArray, fn: (ByteArray) -> Unit) {
+        contract {
+            callsInPlace(fn)
+        }
+
+        while (true) {
+            val rem = tryPopPack(batching)
+            if (rem == 0) break
+
+            if (rem == batching.size)
+                fn(batching)
+            else
+                fn(batching.copyOf(rem))
+        }
     }
 
     override fun get(index: Int): Byte =

src/main/kotlin/blitz/collections/Dense16x16BoolMap.kt

@@ -0,0 +1,197 @@
+package blitz.collections
+
+import kotlin.contracts.ExperimentalContracts
+import kotlin.contracts.contract
+
+@JvmInline
+@OptIn(ExperimentalUnsignedTypes::class)
+value class Dense16x16BoolMap(
+    val packed: UShortArray = UShortArray(16)
+) {
+    fun fillRowsWith(value: UShort) {
+        repeat(16) {
+            packed[it] = value
+        }
+    }
+
+    fun clear() {
+        fillRowsWith(0u)
+    }
+
+    fun getPackedBytes(dest: ByteArray, destOff: Int = 0) {
+        require(dest.size - destOff >= 32)
+        packed.forEachIndexed { index, uShort ->
+            val didx = destOff + index * 2
+            dest[didx] = (uShort and 0xFFu).toByte()
+            dest[didx + 1] = ((uShort.toInt() ushr 8) and 0xFF).toByte()
+        }
+    }
+
+    fun fromPackedBytes(src: ByteArray, srcOff: Int = 0) {
+        require(src.size - srcOff >= 32)
+
+        repeat(16) { uShortIdx ->
+            val sidx = srcOff + uShortIdx * 2
+            val uShort = ((src[sidx].toInt() shl 8) or (src[sidx].toInt())).toUShort()
+            packed[uShortIdx] = uShort
+        }
+    }
+
+    fun appendFrom(other: Dense16x16BoolMap) {
+        other.packed.forEachIndexed { index, otherUShort ->
+            val old = packed[index]
+            packed[index] = old or otherUShort
+        }
+    }
+
+    @OptIn(ExperimentalContracts::class)
+    inline fun forEachSetRow(fn: (Int) -> Unit) {
+        contract {
+            callsInPlace(fn)
+        }
+
+        repeat(16) {
+            if (anyInRow(it)) {
+                fn(it)
+            }
+        }
+    }
+
+    @OptIn(ExperimentalContracts::class)
+    fun forEachSet(fn: (Int, Int) -> Unit) {
+        contract {
+            callsInPlace(fn)
+        }
+
+        forEachSetRow { rowId ->
+            val row = packed[rowId]
+            val lo = row and 0xFFu
+            val hi = (row.toInt() shr 8) and 0xFF
+
+            if (lo > 0u) {
+                var acc = lo.toInt()
+                repeat(8) {
+                    val v = acc and 1
+                    acc = acc shr 1
+
+                    if (v > 0) fn(rowId, it)
+                }
+            }
+
+            if (hi > 0) {
+                var acc = hi
+                repeat(8) {
+                    val v = acc and 1
+                    acc = acc shr 1
+
+                    if (v > 0) fn(rowId, 8 + it)
+                }
+            }
+        }
+    }
+
+    @OptIn(ExperimentalContracts::class)
+    inline fun <T> getSetPosList(dest: MutableList<T> = mutableListOf(), crossinline mapfn: (Int, Int) -> T): MutableList<T> {
+        contract {
+            callsInPlace(mapfn)
+        }
+
+        forEachSet { x, y ->
+            dest.add(mapfn(x, y))
+        }
+
+        return dest
+    }
+
+    fun packedSetPosList(dest: ByteVec = ByteVec(16)): ByteVec {
+        forEachSet { x, y ->
+            val packed = packPos(x, y)
+            dest.pushBack(packed.toByte())
+        }
+        return dest
+    }
+
+    companion object {
+        fun packPos(row: Int, col: Int): UByte =
+            ((row shl 4) or col).toUByte()
+
+        @OptIn(ExperimentalContracts::class)
+        inline fun <T> unpackPos(packed: UByte, fn: (Int, Int) -> T): T {
+            contract {
+                callsInPlace(fn)
+            }
+
+            return fn((packed.toInt() shr 4) and 0xF, packed.toInt() and 0xF)
+        }
+
+        inline fun <T> unpackPos(packed: Byte, fn: (Int, Int) -> T): T =
+            unpackPos(packed.toUByte(), fn)
+
+        inline fun forEachPackedPos(packed: Iterator<Byte>, fn: (Int, Int) -> Unit) {
+            for (it in packed) {
+                unpackPos(it, fn)
+            }
+        }
+
+        inline fun forEachPackedPos(packed: Iterable<Byte>, fn: (Int, Int) -> Unit) =
+            forEachPackedPos(packed.iterator(), fn)
+
+        inline fun forEachPackedPos(packed: Sequence<Byte>, fn: (Int, Int) -> Unit) =
+            forEachPackedPos(packed.iterator(), fn)
+
+        inline fun consumeAllPackedPos(vec: ByteVec, batching: ByteArray, fn: (Int, Int) -> Unit) {
+            vec.consumePopBack(batching) { it ->
+                unpackPos(it, fn)
+            }
+        }
+    }
+
+    fun anyInRow(row: Int) =
+        packed[row] > 0u
+
+    fun setRow(row: Int, value: UShort) {
+        packed[row] = value
+    }
+
+    fun clearRow(row: Int) {
+        setRow(row, 0u)
+    }
+
+    fun countSetInRow(row: Int) =
+        packed[row].countOneBits()
+
+    fun columnMask(col: Int) =
+        1 shl col
+
+    operator fun get(row: Int, col: Int) =
+        (packed[row].toInt() and columnMask(col)) > 0
+
+    operator fun get(packedRowCol: UByte) =
+        unpackPos(packedRowCol) { x, y -> get(x, y) }
+
+    operator fun get(packedRowCol: Byte) =
+        unpackPos(packedRowCol) { x, y -> get(x, y) }
+
+    fun set(row: Int, col: Int) {
+        packed[row] = (packed[row].toInt() or columnMask(col)).toUShort()
+    }
+
+    fun unset(row: Int, col: Int) {
+        val mask = columnMask(col).inv()
+        packed[row] = (packed[row].toInt() and mask).toUShort()
+    }
+
+    operator fun set(row: Int, col: Int, value: Boolean) {
+        if (value) {
+            set(row, col)
+        } else {
+            unset(row, col)
+        }
+    }
+
+    operator fun set(packedRowCol: Byte, value: Boolean) =
+        unpackPos(packedRowCol) { x, y -> set(x, y, value) }
+
+    operator fun set(packedRowCol: UByte, value: Boolean) =
+        unpackPos(packedRowCol) { x, y -> set(x, y, value) }
+}

src/main/kotlin/blitz/collections/DenseIx16x16BoolMap.kt

@@ -0,0 +1,143 @@
+package blitz.collections
+
+import blitz.Endian
+import blitz.toBytes
+import blitz.toInt
+import blitz.toShort
+import kotlin.contracts.ExperimentalContracts
+import kotlin.contracts.contract
+
+class DenseIx16x16BoolMap {
+    val backing = SlicedIntKeyMap<Dense16x16BoolMap>()
+
+    operator fun get(y: Int) =
+        backing[y]
+
+    operator fun get(x: Int, y: Int, z: Int): Boolean =
+        backing[y]?.get(x, z) ?: false
+
+    fun getOrCreateLayer(y: Int) =
+        @OptIn(ExperimentalUnsignedTypes::class)
+        backing.computeIfAbsent(y) { Dense16x16BoolMap() }
+
+    operator fun set(x: Int, y: Int, z: Int, value: Boolean) {
+        getOrCreateLayer(y)[x, z] = value
+    }
+
+    @OptIn(ExperimentalContracts::class)
+    inline fun forEachSet(fn: (Int, Int, Int) -> Unit) {
+        contract {
+            callsInPlace(fn)
+        }
+
+        val layerBytes = ByteVec()
+        val buf32 = ByteArray(32)
+        backing.forEachSet { y, layer ->
+            layer.packedSetPosList(layerBytes)
+            Dense16x16BoolMap.consumeAllPackedPos(layerBytes, buf32) { x, z ->
+                fn(x, y, z)
+            }
+        }
+    }
+
+    inline fun <T> getSetAsSequence(crossinline convertIndex: (Int, Int, Int) -> T) =
+        sequence {
+            forEachSet { x, y, z ->
+                yield(convertIndex(x, y, z))
+            }
+        }
+
+    /**
+     * base: 4
+     * per contained layer: (4 or 2) + 1 + (0 to 256) bytes
+     * only recommended if very few positions per layer
+     */
+    fun serializeByPositions(yPosAsWord: Boolean, unbufferedConsumer: (ByteArray) -> Unit) {
+        val layerBytes = ByteVec()
+        val buf32 = ByteArray(32)
+
+        backing.countSet().toBytes(Endian.LITTLE).also(unbufferedConsumer)
+
+        backing.forEachSet { y, layer ->
+            if (yPosAsWord) {
+                y.toShort().toBytes(Endian.LITTLE).also(unbufferedConsumer)
+            } else {
+                y.toBytes(Endian.LITTLE).also(unbufferedConsumer)
+            }
+
+            layer.packedSetPosList(layerBytes)
+            layerBytes.size.toUByte().toBytes().also(unbufferedConsumer)
+            layerBytes.consumePopBackSlicedBatches(buf32, unbufferedConsumer)
+        }
+    }
+
+    /**
+     * base: 4
+     * per contained layer: (4 or 2) + 32 bytes
+     * should almost always be used
+     */
+    fun serializeByLayers(yPosAsWord: Boolean, unbufferedConsumer: (ByteArray) -> Unit) {
+        backing.countSet().toBytes(Endian.LITTLE).also(unbufferedConsumer)
+
+        val buf32 = ByteArray(32)
+        backing.forEachSet { y, layer ->
+            if (yPosAsWord) {
+                y.toShort().toBytes(Endian.LITTLE).also(unbufferedConsumer)
+            } else {
+                y.toBytes(Endian.LITTLE).also(unbufferedConsumer)
+            }
+
+            layer.getPackedBytes(buf32)
+            unbufferedConsumer(buf32)
+        }
+    }
+
+    companion object {
+        fun deserializeByPositions(yPosAsWord: Boolean, appendTo: DenseIx16x16BoolMap = DenseIx16x16BoolMap(), unbufferedProvider: (Int) -> ByteArray): DenseIx16x16BoolMap {
+            val count = unbufferedProvider(4).toInt(Endian.LITTLE)
+
+            repeat(count) {
+                val (ypos, layerByteCount) = if (yPosAsWord) {
+                    val byteArr = unbufferedProvider(3)
+                    byteArr.toShort(Endian.LITTLE).toInt() to byteArr.last()
+                } else {
+                    val byteArr = unbufferedProvider(5)
+                    byteArr.toInt(Endian.LITTLE) to byteArr.last()
+                }
+
+                val layer = appendTo.getOrCreateLayer(ypos)
+
+                val packedPositions = unbufferedProvider(layerByteCount.toInt())
+                packedPositions.forEach {
+                    Dense16x16BoolMap.unpackPos(it, layer::set)
+                }
+            }
+
+            return appendTo
+        }
+
+        fun deserializeByLayers(yPosAsWord: Boolean, appendTo: DenseIx16x16BoolMap = DenseIx16x16BoolMap(), unbufferedProvider: (Int) -> ByteArray): DenseIx16x16BoolMap {
+            val count = unbufferedProvider(4).toInt(Endian.LITTLE)
+
+            @OptIn(ExperimentalUnsignedTypes::class)
+            val tempLayer = Dense16x16BoolMap()
+            repeat(count) {
+                val ypos = if (yPosAsWord) {
+                    unbufferedProvider(2).toShort(Endian.LITTLE).toInt()
+                } else {
+                    unbufferedProvider(4).toInt(Endian.LITTLE)
+                }
+
+                val layer = appendTo.getOrCreateLayer(ypos)
+
+                val bytes = unbufferedProvider(32)
+                tempLayer.fromPackedBytes(bytes)
+
+                layer.appendFrom(tempLayer)
+                tempLayer.clear()
+            }
+
+            return appendTo
+        }
+    }
+}

src/main/kotlin/blitz/collections/ListOps.kt

@@ -27,4 +27,7 @@ fun <T> MutableList<T>.removeLastInto(count: Int, dest: MutableList<T> = mutable
 }
 
 fun <T> MutableList<T>.addFront(value: T) =
-    add(0, value)
+    add(0, value)
+
+fun <T: Any> Iterable<T?>.countNotNull() =
+    count { it != null }

src/main/kotlin/blitz/collections/SlicedIntKeyMap.kt

@@ -0,0 +1,104 @@
+package blitz.collections
+
+import kotlin.contracts.ExperimentalContracts
+import kotlin.contracts.InvocationKind
+import kotlin.contracts.contract
+
+class SlicedIntKeyMap<V: Any>: BlitzMap<Int,V, Int> {
+    val vec = ArrayList<V?>()
+    var vecOffset: Int? = null
+
+    override fun index(key: Int) = key
+
+    override fun get(index: Int): V? {
+        if (vecOffset == null) return null
+
+        if (index < vecOffset!!) return null
+
+        return vec.getOrNull(index - vecOffset!!)
+    }
+
+    /** @return Changed */
+    @OptIn(ExperimentalContracts::class)
+    fun setIfNotPresent(index: Int, value: (Int) -> V?): Boolean {
+        contract {
+            callsInPlace(value, InvocationKind.AT_MOST_ONCE)
+        }
+
+        if (vecOffset == null) {
+            vecOffset = index
+            vec.add(value(index))
+            return true
+        } else if (index < vecOffset!!) {
+            // prepend
+
+            val diff = vecOffset!! - index
+            repeat(diff) {
+                vec.add(0, null)
+            }
+            vecOffset = index
+            vec[0] = value(index)
+            return true
+        }
+
+        val offPlusSize = vecOffset!! + vec.size
+        if (index >= offPlusSize) {
+            // append
+
+            val diff = index - offPlusSize
+            repeat(diff + 1) {
+                vec.add(null)
+            }
+            vec[vec.size - 1] = value(index)
+            return true
+        }
+
+        return false
+    }
+
+    @OptIn(ExperimentalContracts::class)
+    fun computeIfAbsent(index: Int, fn: (Int) -> V): V {
+        contract {
+            callsInPlace(fn, InvocationKind.AT_MOST_ONCE)
+        }
+
+        var value: V? = null
+        setIfNotPresent(index) {
+            fn(it).also { value = it }
+        }
+        return vec[index - vecOffset!!] ?: let {
+            val v = value ?: fn(index)
+            vec[index - vecOffset!!] = v
+            v
+        }
+    }
+
+    override fun set(index: Int, value: V?) {
+        if (!setIfNotPresent(index) { value }) {
+            vec[index - vecOffset!!] = value
+        }
+    }
+
+    override fun contents(): Contents<Pair<Int, V>> =
+        vec.withIndex()
+            .mapNotNull { (id, v) -> v?.let { id + vecOffset!! to it } }
+            .contents
+
+    fun countSet(): Int =
+        vec.countNotNull()
+
+    @OptIn(ExperimentalContracts::class)
+    inline fun forEachSet(fn: (Int, V) -> Unit) {
+        contract {
+            callsInPlace(fn)
+        }
+
+        if (vecOffset == null) return
+
+        repeat(vec.size) { ko ->
+            val v = vec[ko]
+
+            v?.let { fn(ko + vecOffset!!, it) }
+        }
+    }
+}

src/main/kotlin/blitz/collections/Vec.kt

@@ -28,4 +28,6 @@ interface Vec<T>: IndexableSequence<T> {
     }
 
     operator fun set(index: Int, value: T)
+
+    fun clear()
 }