From b8bd109182905da594a5c281f0fa422fb1c4fdef Mon Sep 17 00:00:00 2001 From: "alexander.nutz" Date: Fri, 29 Mar 2024 16:27:00 +0100 Subject: [PATCH] expression parsing, sorted list, tree, other shit; bump to 0.10 --- README.md | 52 ++++++- build.gradle.kts | 2 +- src/main/kotlin/blitz/codeerrors/Errors.kt | 40 +----- .../kotlin/blitz/collections/BatchedUtils.kt | 8 -- .../kotlin/blitz/collections/Constructors.kt | 4 + .../kotlin/blitz/collections/IterUtils.kt | 45 +++++- src/main/kotlin/blitz/collections/MapEmpty.kt | 7 + src/main/kotlin/blitz/collections/Search.kt | 53 ++++++++ .../kotlin/blitz/collections/SequenceBase.kt | 1 + .../kotlin/blitz/collections/SortedList.kt | 128 ++++++++++++++++++ .../blitz/collections/SynchronizedList.kt | 79 +++++++++++ src/main/kotlin/blitz/collections/Tree.kt | 84 ++++++++++++ .../kotlin/blitz/collections/Ungettable.kt | 27 ++++ src/main/kotlin/blitz/ice/Cooled.kt | 20 +++ src/main/kotlin/blitz/ice/Freezable.kt | 10 ++ src/main/kotlin/blitz/logic/Bools.kt | 36 +++++ src/main/kotlin/blitz/parse/Operator.kt | 7 + src/main/kotlin/blitz/parse/ShuntingYard.kt | 62 +++++++++ src/main/kotlin/blitz/parse/Token.kt | 17 +++ src/main/kotlin/blitz/parse/Tokenize.kt | 91 +++++++++++++ src/main/kotlin/blitz/str/Flatten.kt | 25 ++++ 21 files changed, 748 insertions(+), 50 deletions(-) create mode 100644 src/main/kotlin/blitz/collections/Constructors.kt create mode 100644 src/main/kotlin/blitz/collections/MapEmpty.kt create mode 100644 src/main/kotlin/blitz/collections/Search.kt create mode 100644 src/main/kotlin/blitz/collections/SortedList.kt create mode 100644 src/main/kotlin/blitz/collections/SynchronizedList.kt create mode 100644 src/main/kotlin/blitz/collections/Tree.kt create mode 100644 src/main/kotlin/blitz/collections/Ungettable.kt create mode 100644 src/main/kotlin/blitz/ice/Cooled.kt create mode 100644 src/main/kotlin/blitz/ice/Freezable.kt create mode 100644 src/main/kotlin/blitz/logic/Bools.kt create mode 100644 src/main/kotlin/blitz/parse/Operator.kt create mode 100644 src/main/kotlin/blitz/parse/ShuntingYard.kt create mode 100644 src/main/kotlin/blitz/parse/Token.kt create mode 100644 src/main/kotlin/blitz/parse/Tokenize.kt create mode 100644 src/main/kotlin/blitz/str/Flatten.kt diff --git a/README.md b/README.md index 8d8280e..2488016 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ repositories { } dependencies { - implementation("me.alex_s168:blitz:0.9") + implementation("me.alex_s168:blitz:0.10") } ``` @@ -138,5 +138,55 @@ Errors.print(config, errors) Output: ![img.png](img.png) +### Split with nesting +```kotlin +val inp = "arr[int], long, long, arr[double], int" +val split = inp.splitWithNesting(',', nestUp = '[', nestDown = ']') +println(split) +``` +### Expression parsing +```kotlin +val input = "sin(max(2, 3) / (3 * pi))".iterator() +val tok = input.tokenize( + ignore = listOf(' ', '\n'), + operators = listOf( + Operator('+', 2, true), + Operator('-', 2, true), + + Operator('*', 3, true), + Operator('/', 3, true), + Operator('%', 3, true), + + Operator('~', 10, false), + ) + ) +val rpn = tok.shuntingYard().collect() +if (input.hasNext()) + error("Unexpected token(s): ${input.collectToString()}") +println(rpn) +// outputs: +// [ +// Token(type=NUMBER, value=2, op=null), +// Token(type=NUMBER, value=3, op=null), +// Token(type=IDENT, value=max, op=null), +// Token(type=NUMBER, value=3, op=null), +// Token(type=IDENT, value=pi, op=null), +// Token(type=OPERATOR, value=null, op=Operator(symbol=*, precedence=3, leftAssociative=true)), +// Token(type=OPERATOR, value=null, op=Operator(symbol=/, precedence=3, leftAssociative=true)), +// Token(type=IDENT, value=sin, op=null) +// ] +``` +### SortedList +```kotlin +val list = SortedArrayList { it } +list.add(1) +list.add(9) +list.add(5) +println(list.contents) +// outputs: [1, 5, 9] +``` ### Either No example yet + +### Tree +No example yet \ No newline at end of file diff --git a/build.gradle.kts b/build.gradle.kts index 7229f57..07fa9b9 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -5,7 +5,7 @@ plugins { } group = "me.alex_s168" -version = "0.9" +version = "0.10" repositories { mavenCentral() diff --git a/src/main/kotlin/blitz/codeerrors/Errors.kt b/src/main/kotlin/blitz/codeerrors/Errors.kt index 8a9aae8..770a720 100644 --- a/src/main/kotlin/blitz/codeerrors/Errors.kt +++ b/src/main/kotlin/blitz/codeerrors/Errors.kt @@ -4,7 +4,6 @@ import blitz.collections.inBounds import blitz.str.ColoredChar import blitz.str.MutMultiColoredMultiLineString import blitz.str.MutMultiLineString -import blitz.str.MutString import blitz.term.AnsiiMode import blitz.term.Terminal @@ -143,7 +142,7 @@ object Errors { byCol.filter { it.isLongDesc }.forEach { val msgLines = MutMultiLineString.from(it.message, fill = ' ') - msg.set(row, nextCol, msgLines, Terminal.COLORS.WHITE.brighter.fg) + msg[row, nextCol, msgLines] = Terminal.COLORS.WHITE.brighter.fg row ++ } @@ -153,41 +152,4 @@ object Errors { Terminal.errln("================================================================================", config.styles[worst]!! + Terminal.STYLES.BOLD) } } -} - -fun main() { - val source = Errors.Source("main.kt", MutMultiLineString.from(""" - fn main() { - return 1 + 0 - } - """.trimIndent(), ' ')) - - val errors = listOf( - Errors.Error( - "cannot return integer from function with return type void", - Errors.Error.Level.ERROR, - Errors.Location(source, 1, 11, 5) - ), - Errors.Error( - "return is deprecated. use yeet instead", - Errors.Error.Level.WARN, - Errors.Location(source, 1, 4, 6) - ), - Errors.Error( - "useless addition", - Errors.Error.Level.INFO, - Errors.Location(source, 1, 13, 3), - isHint = true - ), - Errors.Error( - "Visit https://www.example.com/doc/yeet for more information", - Errors.Error.Level.INFO, - Errors.Location(source, 1, 0, 0), - isLongDesc = true - ) - ) - - val config = Errors.PrintConfig() - - Errors.print(config, errors) } \ No newline at end of file diff --git a/src/main/kotlin/blitz/collections/BatchedUtils.kt b/src/main/kotlin/blitz/collections/BatchedUtils.kt index 5cff024..f337239 100644 --- a/src/main/kotlin/blitz/collections/BatchedUtils.kt +++ b/src/main/kotlin/blitz/collections/BatchedUtils.kt @@ -7,12 +7,4 @@ fun ByteBatchSequence.stringify(batch: Int = 64): Sequence { iter.nextBytes(batch).decodeToString() else null } -} - -fun Sequence.flatten(): String { - val out = StringBuilder() - forEach { - out.append(it) - } - return out.toString() } \ No newline at end of file diff --git a/src/main/kotlin/blitz/collections/Constructors.kt b/src/main/kotlin/blitz/collections/Constructors.kt new file mode 100644 index 0000000..28946a7 --- /dev/null +++ b/src/main/kotlin/blitz/collections/Constructors.kt @@ -0,0 +1,4 @@ +package blitz.collections + +fun > SortedArrayList(sorter: (T) -> C) = + SortedList(ArrayList(), sorter) \ No newline at end of file diff --git a/src/main/kotlin/blitz/collections/IterUtils.kt b/src/main/kotlin/blitz/collections/IterUtils.kt index bcf56f3..a51608d 100644 --- a/src/main/kotlin/blitz/collections/IterUtils.kt +++ b/src/main/kotlin/blitz/collections/IterUtils.kt @@ -7,4 +7,47 @@ fun Iterator.mapModifier(fn: (T) -> R): Iterator = override fun hasNext(): Boolean = this@mapModifier.hasNext() - } \ No newline at end of file + } + +fun generateIterator(fn: () -> T?): Iterator = + object: Iterator { + var calc = false + var nx: T? = null + + override fun hasNext(): Boolean { + if (!calc) { + nx = fn() + calc = true + } + return nx != null + } + + override fun next(): T { + if (!hasNext()) + throw Exception("iterator end") + calc = false + return nx!! + } + } + +/** + * Can't explain this function. Look at the source of [blitz.parse.tokenize] as an example + */ +fun Iterator.funnyMap(fn: (UnGettableIterator) -> R?): Iterator { + val iter = asUnGettable() + return generateIterator { fn(iter) } +} + +fun Iterator.collect(to: MutableList = mutableListOf()): MutableList { + forEachRemaining { + to.add(it) + } + return to +} + +fun Iterator.collect(to: Vec): Vec { + forEachRemaining { + to.pushBack(it) + } + return to +} \ No newline at end of file diff --git a/src/main/kotlin/blitz/collections/MapEmpty.kt b/src/main/kotlin/blitz/collections/MapEmpty.kt new file mode 100644 index 0000000..637a867 --- /dev/null +++ b/src/main/kotlin/blitz/collections/MapEmpty.kt @@ -0,0 +1,7 @@ +package blitz.collections + +fun Collection.nullIfEmpty(): Collection? = + ifEmpty { null } + +fun String.nullIfEmpty(): String? = + ifEmpty { null } \ No newline at end of file diff --git a/src/main/kotlin/blitz/collections/Search.kt b/src/main/kotlin/blitz/collections/Search.kt new file mode 100644 index 0000000..a971c57 --- /dev/null +++ b/src/main/kotlin/blitz/collections/Search.kt @@ -0,0 +1,53 @@ +package blitz.collections + +import kotlin.math.max + +object Search { + /** + * Find a element by comparing. + * Only works on sorted lists + */ + fun > findElementComparing( + list: List, + element: T, + comparer: (T) -> C, + ): Int { + var rev = max(0, list.size / 2 - 1) + var prev = 0 + + while (true) { + // outside of array: + if (rev < 0) + break + + // outside of array: + if (rev >= list.size) + break + + val cmp = comparer(element) + .compareTo(comparer(list[rev])) + + // found: + if (cmp == 0) + return rev + + // in between of two existing values: + if (cmp > 0 && prev < 0) + break + + // in between of two existing values: + if (cmp < 0 && prev > 0) + break + + if (cmp < 0) + rev -- + + else if (cmp > 0) + rev ++ + + prev = cmp + } + + return -1 + } +} \ No newline at end of file diff --git a/src/main/kotlin/blitz/collections/SequenceBase.kt b/src/main/kotlin/blitz/collections/SequenceBase.kt index 031bde1..9612768 100644 --- a/src/main/kotlin/blitz/collections/SequenceBase.kt +++ b/src/main/kotlin/blitz/collections/SequenceBase.kt @@ -4,6 +4,7 @@ interface IndexableSequence: Sequence { operator fun get(index: Int): T } +// TODO: rename to map fun IndexableSequence.modifier(mod: (Sequence) -> Sequence) = object : IndexableSequence { val other = mod(this@modifier) diff --git a/src/main/kotlin/blitz/collections/SortedList.kt b/src/main/kotlin/blitz/collections/SortedList.kt new file mode 100644 index 0000000..b3d990f --- /dev/null +++ b/src/main/kotlin/blitz/collections/SortedList.kt @@ -0,0 +1,128 @@ +package blitz.collections + +import kotlin.math.max + +/** + * A mutable list that uses a different mutable list as underlying data structure. + * The list guarantees all elements to stay sorted unless manually inserted. + */ +class SortedList>( + private val al: MutableList, + private val sorter: (T) -> C +): MutableList, RandomAccess { + override val size: Int = + al.size + + override fun clear() = + al.clear() + + override fun addAll(elements: Collection): Boolean { + elements.forEach(::add) + return true + } + + override fun addAll(index: Int, elements: Collection): Boolean = + al.addAll(index, elements) + + override fun add(index: Int, element: T) = + al.add(index, element) + + override fun add(element: T): Boolean { + var rev = max(0, al.size / 2 - 1) + var prev = 0 + while (true) { + if (rev < 0) { + al.add(0, element) + return true + } + if (rev >= al.size) { + al.add(element) + return true + } + val cmp = sorter(element) + .compareTo(sorter(al[rev])) + if (cmp == 0) { + al.add(rev, element) + return true + } + if (cmp > 0 && prev < 0) { + al.add(rev - 1, element) + return true + } + if (cmp < 0 && prev > 0) { + al.add(rev, element) + return true + } + if (cmp < 0) + rev -- + else if (cmp > 0) + rev ++ + prev = cmp + } + } + + override fun get(index: Int): T = + al[index] + + override fun isEmpty(): Boolean = + al.isEmpty() + + override fun iterator(): MutableIterator = + al.iterator() + + override fun listIterator(): MutableListIterator = + al.listIterator() + + override fun listIterator(index: Int): MutableListIterator = + al.listIterator(index) + + override fun removeAt(index: Int): T = + al.removeAt(index) + + override fun subList(fromIndex: Int, toIndex: Int): MutableList = + al.subList(fromIndex, toIndex) + + override fun set(index: Int, element: T): T = + al.set(index, element) + + override fun retainAll(elements: Collection): Boolean = + al.retainAll(elements) + + override fun removeAll(elements: Collection): Boolean { + var rem = false + for (x in elements) { + if (remove(x)) + rem = true + } + return rem + } + + override fun remove(element: T): Boolean { + val i = Search.findElementComparing(al, element, sorter) + if (i == -1) + return false + removeAt(i) + return true + } + + override fun lastIndexOf(element: T): Int = + al.lastIndexOf(element) + + override fun indexOf(element: T): Int = + al.indexOf(element) + + override fun containsAll(elements: Collection): Boolean = + elements.all { contains(it) } + + override fun contains(element: T): Boolean = + Search.findElementComparing(al, element, sorter) != -1 + + override fun toString(): String = + al.toString() + + inline fun sorted(): SortedList = + this + + inline fun sortedDescending(): List = + this.reversed() +} \ No newline at end of file diff --git a/src/main/kotlin/blitz/collections/SynchronizedList.kt b/src/main/kotlin/blitz/collections/SynchronizedList.kt new file mode 100644 index 0000000..6fd24ad --- /dev/null +++ b/src/main/kotlin/blitz/collections/SynchronizedList.kt @@ -0,0 +1,79 @@ +package blitz.collections + +import blitz.async.Lock + +/** + * Wraps any MutableList. + * Concurrent by synchronizing. + */ +class SynchronizedList>( + private val innerList: L +): MutableList, RandomAccess { + private val innerLock = Lock() + + fun access(block: (L) -> R): R = + innerLock.use { block(innerList) } + + override val size: Int + get() = innerList.size + + override fun clear() = + access { it.clear() } + + override fun addAll(elements: Collection): Boolean = + access { it.addAll(elements) } + + override fun addAll(index: Int, elements: Collection): Boolean = + access { it.addAll(index, elements) } + + override fun add(index: Int, element: T) = + access { it.add(index, element) } + + override fun add(element: T): Boolean = + access { it.add(element) } + + override fun get(index: Int): T = + access { it.get(index) } + + override fun isEmpty(): Boolean = + access { it.isEmpty() } + + override fun iterator(): MutableIterator = + access { it.iterator() } + + override fun listIterator(): MutableListIterator = + access { it.listIterator() } + + override fun listIterator(index: Int): MutableListIterator = + access { it.listIterator(index) } + + override fun removeAt(index: Int): T = + access { it.removeAt(index) } + + override fun subList(fromIndex: Int, toIndex: Int): MutableList = + access { it.subList(fromIndex, toIndex) } + + override fun set(index: Int, element: T): T = + access { it.set(index, element) } + + override fun retainAll(elements: Collection): Boolean = + access { it.retainAll(elements) } + + override fun removeAll(elements: Collection): Boolean = + access { it.removeAll(elements) } + + override fun remove(element: T): Boolean = + access { it.remove(element) } + + override fun lastIndexOf(element: T): Int = + access { it.lastIndexOf(element) } + + override fun indexOf(element: T): Int = + access { it.indexOf(element) } + + override fun containsAll(elements: Collection): Boolean = + access { it.containsAll(elements) } + + override fun contains(element: T): Boolean = + access { it.contains(element) } +} \ No newline at end of file diff --git a/src/main/kotlin/blitz/collections/Tree.kt b/src/main/kotlin/blitz/collections/Tree.kt new file mode 100644 index 0000000..a70aa2a --- /dev/null +++ b/src/main/kotlin/blitz/collections/Tree.kt @@ -0,0 +1,84 @@ +package blitz.collections + +class Tree { + var root: Node? = null + + operator fun plusAssign(value: T) { + val node = Node() + node.value = value + root?.children?.add(node) + } + + fun traverse(traverser: TreeTraverser) = + root?.traverse(traverser) + + fun traverse(process: (Node) -> Boolean) = + root?.traverse(process) + + fun updateParents() { + traverse { node -> + node.children.forEach { it.parent = node } + true + } + } + + override fun toString(): String { + return root.toString() + } +} + +class Node( + var value: T? = null, + var children: MutableList> = mutableListOf() +) { + var parent: Node? = null + + fun traverse(traverser: TreeTraverser) = + traverser.traverse(this) + + fun traverse(process: (Node) -> Boolean) { + val trav = object : TreeTraverser { + override fun process(node: Node): Boolean { + return process(node) + } + } + traverse(trav) + } + + operator fun plusAssign(value: T) { + val node = Node() + node.value = value + children.add(node) + } + + fun toLines(): List> { + val lines = mutableListOf>() + lines += 0 to value.toString() + children.forEach { child -> + child.toLines().forEach { (indent, line) -> + lines += indent + 1 to line + } + } + return lines + } + + override fun toString(): String { + val lines = toLines() + val sb = StringBuilder() + lines.forEach { (indent, line) -> + sb.append(" ".repeat(indent * 2)) + sb.append(line) + sb.append("\n") + } + return sb.toString() + } +} + +interface TreeTraverser { + fun process(node: Node): Boolean /* true if continue, false if stop */ + + fun traverse(tree: Node) { + if (!process(tree)) return + tree.children.forEach { traverse(it) } + } +} \ No newline at end of file diff --git a/src/main/kotlin/blitz/collections/Ungettable.kt b/src/main/kotlin/blitz/collections/Ungettable.kt new file mode 100644 index 0000000..905d14e --- /dev/null +++ b/src/main/kotlin/blitz/collections/Ungettable.kt @@ -0,0 +1,27 @@ +package blitz.collections + +interface UnGettableIterator: Iterator { + /** + * Undo the previous next() operation. Only possible once per next()! + */ + fun unGet() +} + +fun Iterator.asUnGettable(): UnGettableIterator = + object: UnGettableIterator { + private var prev: T? = null + private var unget = false + + override fun unGet() { + if (prev == null) + throw Exception("Cannot unGet because there was no previous") + unget = true + } + + override fun hasNext(): Boolean = + unget || this@asUnGettable.hasNext() + + override fun next(): T = + if (unget) prev!!.also { unget = false } + else this@asUnGettable.next().also { prev = it } + } \ No newline at end of file diff --git a/src/main/kotlin/blitz/ice/Cooled.kt b/src/main/kotlin/blitz/ice/Cooled.kt new file mode 100644 index 0000000..57a8971 --- /dev/null +++ b/src/main/kotlin/blitz/ice/Cooled.kt @@ -0,0 +1,20 @@ +package blitz.ice + +class Cooled(private val of: T): Freezable { + private var frozen = false + + override fun freeze() { + frozen = true + } + + override fun isFrozen(): Boolean { + return frozen + } + + fun getOrNull(): T? = + if (isFrozen()) null else of + + fun use(block: (T) -> R): R? = + if (isFrozen()) null + else block(of) +} \ No newline at end of file diff --git a/src/main/kotlin/blitz/ice/Freezable.kt b/src/main/kotlin/blitz/ice/Freezable.kt new file mode 100644 index 0000000..96a1aad --- /dev/null +++ b/src/main/kotlin/blitz/ice/Freezable.kt @@ -0,0 +1,10 @@ +package blitz.ice + +interface Freezable { + fun freeze() + fun isFrozen(): Boolean +} + +inline fun Freezable.map(block: (Freezable) -> R): R? = + if (isFrozen()) null + else block(this) \ No newline at end of file diff --git a/src/main/kotlin/blitz/logic/Bools.kt b/src/main/kotlin/blitz/logic/Bools.kt new file mode 100644 index 0000000..30d8d40 --- /dev/null +++ b/src/main/kotlin/blitz/logic/Bools.kt @@ -0,0 +1,36 @@ +package blitz.logic + +/** + * Execute a block if the boolean is true. + */ +fun Boolean.then(block: () -> Unit): Boolean { + if (this) { + block() + } + + return this +} + +/** + * Execute a block if the boolean is false. + */ +fun Boolean.otherwise(block: () -> Unit): Boolean { + if (!this) { + block() + } + + return this +} + +/** + * Execute a block if the boolean is true, otherwise execute another block. + */ +fun Boolean.then(block: () -> Unit, otherwise: () -> Unit): Boolean { + if (this) { + block() + } else { + otherwise() + } + + return this +} \ No newline at end of file diff --git a/src/main/kotlin/blitz/parse/Operator.kt b/src/main/kotlin/blitz/parse/Operator.kt new file mode 100644 index 0000000..3f2b4c7 --- /dev/null +++ b/src/main/kotlin/blitz/parse/Operator.kt @@ -0,0 +1,7 @@ +package blitz.parse + +data class Operator( + val symbol: Char, + val precedence: Int = 0, + val leftAssociative: Boolean = true, +) \ No newline at end of file diff --git a/src/main/kotlin/blitz/parse/ShuntingYard.kt b/src/main/kotlin/blitz/parse/ShuntingYard.kt new file mode 100644 index 0000000..cb5eaff --- /dev/null +++ b/src/main/kotlin/blitz/parse/ShuntingYard.kt @@ -0,0 +1,62 @@ +package blitz.parse + +fun Iterator.shuntingYard(): Iterator { + val iter = this + val opStack = mutableListOf() + return sequence { + while (iter.hasNext()) { + val tk0 = iter.next() + when (tk0.type) { + Token.Type.NUMBER -> yield(tk0) + Token.Type.IDENT -> opStack.add(tk0) + Token.Type.OPERATOR -> { + while (true) { + if (opStack.isEmpty()) + break + val oo = opStack.last() + if (oo.type == Token.Type.PAREN_OPEN) + break + if (!((oo.op!!.precedence > tk0.op!!.precedence) || (tk0.op.precedence == oo.op.precedence && tk0.op.leftAssociative))) + break + opStack.removeLast() + yield(oo) + } + opStack.add(tk0) + } + Token.Type.COMMA -> { + while (true) { + if (opStack.isEmpty()) + break + val oo = opStack.last() + if (oo.type == Token.Type.PAREN_OPEN) + break + opStack.removeLast() + } + } + Token.Type.PAREN_OPEN -> opStack.add(tk0) + Token.Type.PAREN_CLOSE -> { + while (true) { + if (opStack.isEmpty()) + throw Exception("Unexpected closing parenthesis!") + val oo = opStack.last() + if (oo.type == Token.Type.PAREN_OPEN) + break + opStack.removeLast() + yield(oo) + } + if (opStack.removeLastOrNull()?.type != Token.Type.PAREN_OPEN) + throw Exception("Unexpected closing parenthesis!") + if (opStack.lastOrNull()?.type == Token.Type.IDENT) + yield(opStack.removeLast()) + } + Token.Type.SEPARATOR -> continue + } + } + while (opStack.isNotEmpty()) { + val oo = opStack.removeLast() + if (oo.type in listOf(Token.Type.PAREN_OPEN, Token.Type.PAREN_CLOSE)) + throw Exception("Mismatched parenthesis") + yield(oo) + } + }.iterator() +} \ No newline at end of file diff --git a/src/main/kotlin/blitz/parse/Token.kt b/src/main/kotlin/blitz/parse/Token.kt new file mode 100644 index 0000000..7081188 --- /dev/null +++ b/src/main/kotlin/blitz/parse/Token.kt @@ -0,0 +1,17 @@ +package blitz.parse + +data class Token( + val type: Type, + val value: String? = null, + val op: Operator? = null, +) { + enum class Type { + IDENT, + NUMBER, + PAREN_OPEN, + PAREN_CLOSE, + OPERATOR, + SEPARATOR, + COMMA, + } +} \ No newline at end of file diff --git a/src/main/kotlin/blitz/parse/Tokenize.kt b/src/main/kotlin/blitz/parse/Tokenize.kt new file mode 100644 index 0000000..f8bc883 --- /dev/null +++ b/src/main/kotlin/blitz/parse/Tokenize.kt @@ -0,0 +1,91 @@ +package blitz.parse + +import blitz.collections.funnyMap + +fun Iterator.tokenize( + operators: Collection = emptyList(), + ignore: Collection = emptyList(), +): Iterator = + funnyMap { + val builder = StringBuilder() + var ident = false + var num = false + var paren: Char? = null + var op: Operator? = null + var sep = false + var comma = false + while (it.hasNext()) { + when (val c = it.next()) { + in 'a'..'z', + in 'A'..'Z', + '_' -> { + if (num) { + it.unGet() + break + } else { + builder.append(c) + ident = true + } + } + in '0'..'9' -> { + builder.append(c) + if (!ident) + num = true + } + '.' -> { + if (num) + builder.append(it) + else { + it.unGet() + break + } + } + '(', + ')' -> { + if (!ident && !num) + paren = c + else + it.unGet() + break + } + in ignore -> { + if (!ident && !num) + sep = true + else + it.unGet() + break + } + ',' -> { + if (!ident && !num) + comma = true + else + it.unGet() + break + } + else -> { + val oo = operators.firstOrNull { o -> o.symbol == c } + if (oo != null) + op = oo + else + it.unGet() + break + } + } + } + if (ident) + Token(Token.Type.IDENT, builder.toString()) + else if (num) + Token(Token.Type.NUMBER, builder.toString()) + else if (paren == '(') + Token(Token.Type.PAREN_OPEN) + else if (paren == ')') + Token(Token.Type.PAREN_CLOSE) + else if (op != null) + Token(Token.Type.OPERATOR, op = op) + else if (sep) + Token(Token.Type.SEPARATOR) + else if (comma) + Token(Token.Type.COMMA) + else + null + } diff --git a/src/main/kotlin/blitz/str/Flatten.kt b/src/main/kotlin/blitz/str/Flatten.kt new file mode 100644 index 0000000..907f8b9 --- /dev/null +++ b/src/main/kotlin/blitz/str/Flatten.kt @@ -0,0 +1,25 @@ +package blitz.str + +fun Sequence.flattenToString(): String { + val out = StringBuilder() + forEach { + out.append(it) + } + return out.toString() +} + +fun Iterable.flattenToString(): String { + val out = StringBuilder() + forEach { + out.append(it) + } + return out.toString() +} + +fun Iterator.collectToString(): String { + val out = StringBuilder() + forEachRemaining { + out.append(it) + } + return out.toString() +} \ No newline at end of file