|
2 | 2 |
|
3 | 3 | package kotlinx.coroutines.internal |
4 | 4 |
|
| 5 | +import kotlinx.atomicfu.* |
| 6 | +import kotlinx.coroutines.* |
| 7 | +import kotlin.coroutines.* |
| 8 | +import kotlin.experimental.* |
| 9 | +import kotlin.jvm.* |
| 10 | + |
5 | 11 | /** @suppress **This is unstable API and it is subject to change.** */ |
6 | | -public expect open class LockFreeLinkedListNode() { |
| 12 | +internal open class LockFreeLinkedListNode { |
7 | 13 | /** |
8 | | - * Try putting `node` into a list. |
| 14 | + * Try putting this node into a list. |
9 | 15 | * |
10 | 16 | * Returns: |
11 | 17 | * - The new head of the list if the operation succeeded. |
12 | 18 | * - The head of the list if someone else concurrently added this node to the list, |
13 | 19 | * but no other modifications to the list were made. |
14 | | - * - Some garbage if the list was already edited. |
15 | 20 | */ |
16 | | - public fun attachToList(node: LockFreeLinkedListHead): LockFreeLinkedListNode |
| 21 | + fun attachToList(head: LockFreeLinkedListHead): LockFreeLinkedListHead { |
| 22 | + val newAddress = head.addLastWithoutModifying(this, permissionsBitmask = 0) |
| 23 | + assert { newAddress != null } |
| 24 | + return if (_address.compareAndSet(null, newAddress)) { |
| 25 | + head |
| 26 | + } else { |
| 27 | + _address.value!!.segment.head |
| 28 | + } |
| 29 | + } |
17 | 30 |
|
18 | 31 | /** |
19 | 32 | * Remove this node from the list. |
20 | 33 | */ |
21 | | - public open fun remove() |
| 34 | + open fun remove() { |
| 35 | + _address.value?.let { |
| 36 | + val segment = it.segment |
| 37 | + segment.clearSlot(it.index) |
| 38 | + } |
| 39 | + } |
| 40 | + |
| 41 | + private val _address = atomic<Address?>(null) |
| 42 | + |
| 43 | + val address: Address get() = _address.value!! |
| 44 | + |
| 45 | + internal fun trySetAddress(address: Address) = this._address.compareAndSet(null, address) |
22 | 46 | } |
23 | 47 |
|
24 | 48 | /** @suppress **This is unstable API and it is subject to change.** */ |
25 | | -public expect open class LockFreeLinkedListHead() { |
| 49 | +internal open class LockFreeLinkedListHead { |
| 50 | + private val head = LockFreeLinkedListSegment( |
| 51 | + id = 0, |
| 52 | + prev = null, |
| 53 | + pointers = 2, |
| 54 | + head = this, |
| 55 | + ) |
| 56 | + private val tail = atomic(head) |
| 57 | + private val nextElement = atomic(0L) |
| 58 | + |
| 59 | + /** |
| 60 | + * The list of bits that are forbidden from entering the list. |
| 61 | + * |
| 62 | + * TODO: we can store this in the extra bits in [head], there's enough space for that there, and it's never removed. |
| 63 | + */ |
| 64 | + private val forbiddenBits: AtomicInt = atomic(0) |
| 65 | + |
26 | 66 | /** |
27 | 67 | * Iterates over all non-removed elements in this list, skipping every node until (and including) [startAfter]. |
28 | 68 | */ |
29 | | - public inline fun forEach(startAfter: LockFreeLinkedListNode? = null, block: (LockFreeLinkedListNode) -> Unit) |
| 69 | + inline fun forEach( |
| 70 | + forbidBitmask: Byte = 0, |
| 71 | + startAfter: LockFreeLinkedListNode? = null, |
| 72 | + block: (LockFreeLinkedListNode) -> Unit |
| 73 | + ) { |
| 74 | + forbiddenBits.update { it or forbidBitmask.toInt() } |
| 75 | + val startAddress = startAfter?.address |
| 76 | + var segment: LockFreeLinkedListSegment? = startAddress?.segment ?: head |
| 77 | + var startIndex: Int = startAddress?.index?.let { it + 1 } ?: 0 |
| 78 | + while (segment != null) { |
| 79 | + segment.forEach(forbidBitmask = forbidBitmask, startIndex = startIndex, block = block) |
| 80 | + segment = segment.next |
| 81 | + startIndex = 0 |
| 82 | + } |
| 83 | + } |
30 | 84 |
|
31 | 85 | /** |
32 | | - * Closes the list for anything that requests the permission [forbiddenElementsBit]. |
33 | | - * Only a single permission can be forbidden at a time, but this isn't checked. |
| 86 | + * Adds the [node] to the end of the list if every bit in [permissionsBitmask] is still allowed in the list, |
| 87 | + * and then sets the [node]'s address to the new address. |
34 | 88 | */ |
35 | | - public fun close(forbiddenElementsBit: Int) |
| 89 | + fun addLast(node: LockFreeLinkedListNode, permissionsBitmask: Byte): Boolean { |
| 90 | + val address = addLastWithoutModifying(node, permissionsBitmask) ?: return false |
| 91 | + val success = node.trySetAddress(address) |
| 92 | + assert { success } |
| 93 | + return true |
| 94 | + } |
36 | 95 |
|
37 | 96 | /** |
38 | 97 | * Adds the [node] to the end of the list if every bit in [permissionsBitmask] is still allowed in the list. |
| 98 | + * As opposed to [addLast], doesn't modify the [node]'s address. |
39 | 99 | */ |
40 | | - public fun addLast(node: LockFreeLinkedListNode, permissionsBitmask: Int): Boolean |
| 100 | + fun addLastWithoutModifying(node: LockFreeLinkedListNode, permissionsBitmask: Byte): Address? { |
| 101 | + /** First, avoid modifying the list at all if it was already closed for elements like ours. */ |
| 102 | + if (permissionsBitmask and forbiddenBits.value.toByte() != 0.toByte()) return null |
| 103 | + /** Obtain the place from which the desired segment will certainly be reachable. */ |
| 104 | + val curTail = tail.value |
| 105 | + /** Allocate a place for our element. */ |
| 106 | + val index = nextElement.getAndIncrement() |
| 107 | + /** Find or create a segment where the node can be stored. */ |
| 108 | + val createNewSegment = ::createSegment // can't just pass the function, as the compiler crashes (KT-67332) |
| 109 | + val segmentId = index / SEGMENT_SIZE |
| 110 | + val segment = tail.findSegmentAndMoveForward(id = segmentId, curTail, createNewSegment).segment |
| 111 | + assert { segment.id == segmentId } |
| 112 | + val indexInSegment = (index % SEGMENT_SIZE).toInt() |
| 113 | + /** Double-check that it's still not forbidden for the node to enter the list. */ |
| 114 | + if (permissionsBitmask and forbiddenBits.value.toByte() != 0.toByte()) return null |
| 115 | + /** Now we know that the list was still not closed at some point *even after the segment* was created. |
| 116 | + * Because [forbiddenBits] is set before [forEach] traverses the list, this means that [forEach] is guaranteed |
| 117 | + * to observe the new segment and either break the cell where [node] wants to arrive or process the [node]. |
| 118 | + * In any case, we have linearizable behavior. */ |
| 119 | + return if (segment.tryAdd(node, permissionsBitmask = permissionsBitmask, indexInSegment = indexInSegment)) { |
| 120 | + Address(segment, indexInSegment) |
| 121 | + } else { |
| 122 | + null |
| 123 | + } |
| 124 | + } |
41 | 125 | } |
| 126 | + |
| 127 | +internal open class LockFreeLinkedListSegment( |
| 128 | + id: Long, |
| 129 | + prev: LockFreeLinkedListSegment?, |
| 130 | + pointers: Int, |
| 131 | + /** Used only during promoting of a single node to a list to ensure wait-freedom of the promotion operation. |
| 132 | + * Without this, promotion can't be implemented without a (possibly bounded) spin loop: once the node is committed |
| 133 | + * to be part of some list, the other threads can't do anything until that one thread sets the state to be the |
| 134 | + * head of the list. */ |
| 135 | + @JvmField val head: LockFreeLinkedListHead, |
| 136 | +) : Segment<LockFreeLinkedListSegment>(id = id, prev = prev, pointers = pointers) |
| 137 | +{ |
| 138 | + /** Each cell is a [LockFreeLinkedListNode], a [BrokenForSomeElements], or `null`. */ |
| 139 | + private val cells = atomicArrayOfNulls<Any>(SEGMENT_SIZE) |
| 140 | + |
| 141 | + override val numberOfSlots: Int get() = SEGMENT_SIZE |
| 142 | + |
| 143 | + fun clearSlot(index: Int) { |
| 144 | + cells[index].value = null |
| 145 | + onSlotCleaned() |
| 146 | + } |
| 147 | + |
| 148 | + inline fun forEach(forbidBitmask: Byte, startIndex: Int, block: (LockFreeLinkedListNode) -> Unit) { |
| 149 | + for (i in startIndex until SEGMENT_SIZE) { |
| 150 | + val node = breakCellOrGetValue(forbidBitmask, i) |
| 151 | + if (node != null) block(node) |
| 152 | + } |
| 153 | + } |
| 154 | + |
| 155 | + private fun breakCellOrGetValue(forbidBitmask: Byte, index: Int): LockFreeLinkedListNode? { |
| 156 | + while (true) { |
| 157 | + val value = cells[index].value |
| 158 | + if (value is BrokenForSomeElements?) { |
| 159 | + val newForbiddenBits = value.forbiddenBits or forbidBitmask |
| 160 | + if (newForbiddenBits == value.forbiddenBits |
| 161 | + || cells[index].compareAndSet(value, BrokenForSomeElements.fromBitmask(newForbiddenBits))) |
| 162 | + return null |
| 163 | + } else { |
| 164 | + return value as LockFreeLinkedListNode |
| 165 | + } |
| 166 | + } |
| 167 | + } |
| 168 | + |
| 169 | + /** |
| 170 | + * Adds the [node] to the array of cells if the slot wasn't broken. |
| 171 | + */ |
| 172 | + fun tryAdd(node: LockFreeLinkedListNode, permissionsBitmask: Byte, indexInSegment: Int): Boolean { |
| 173 | + if (cells[indexInSegment].compareAndSet(null, node)) return true |
| 174 | + cells[indexInSegment].loop { value -> |
| 175 | + // This means that some elements are forbidden from entering the list. |
| 176 | + value as BrokenForSomeElements |
| 177 | + // Are *we* forbidden from entering the list? |
| 178 | + if (value.forbiddenBits and permissionsBitmask != 0.toByte()) { |
| 179 | + cells[indexInSegment].value = BrokenForSomeElements.FULLY_BROKEN |
| 180 | + onSlotCleaned() |
| 181 | + return false |
| 182 | + } |
| 183 | + // We aren't forbidden. Let's try entering it. |
| 184 | + if (cells[indexInSegment].compareAndSet(value, node)) return true |
| 185 | + } |
| 186 | + } |
| 187 | + |
| 188 | + override fun onCancellation(index: Int, cause: Throwable?, context: CoroutineContext) { |
| 189 | + throw UnsupportedOperationException("Cancellation is not supported on LockFreeLinkedList") |
| 190 | + } |
| 191 | +} |
| 192 | + |
| 193 | +internal class Address(@JvmField val segment: LockFreeLinkedListSegment, @JvmField val index: Int) |
| 194 | + |
| 195 | +private fun createSegment(id: Long, prev: LockFreeLinkedListSegment): LockFreeLinkedListSegment = |
| 196 | + LockFreeLinkedListSegment( |
| 197 | + id = id, |
| 198 | + prev = prev, |
| 199 | + pointers = 0, |
| 200 | + head = prev.head |
| 201 | + ) |
| 202 | + |
| 203 | +private const val SEGMENT_SIZE = 8 |
| 204 | + |
| 205 | +@JvmInline |
| 206 | +private value class BrokenForSomeElements private constructor(val forbiddenBits: Byte) { |
| 207 | + companion object { |
| 208 | + fun fromBitmask(forbiddenBits: Byte): BrokenForSomeElements? = when (forbiddenBits) { |
| 209 | + 0.toByte() -> null // no one is forbidden |
| 210 | + else -> BrokenForSomeElements(forbiddenBits) |
| 211 | + } |
| 212 | + |
| 213 | + val FULLY_BROKEN = BrokenForSomeElements(255.toByte()) |
| 214 | + } |
| 215 | +} |
| 216 | + |
| 217 | +private val BrokenForSomeElements?.forbiddenBits get() = this?.forbiddenBits ?: 0 |
0 commit comments