r.0.9.1.source-code.Projection.kt Maven / Gradle / Ivy
The newest version!
package se.wollan.tolr
import kotlinx.coroutines.*
import kotlinx.coroutines.channels.Channel
import kotlinx.coroutines.channels.ReceiveChannel
import kotlinx.coroutines.channels.SendChannel
import kotlinx.coroutines.flow.MutableSharedFlow
import kotlinx.coroutines.flow.first
import kotlinx.coroutines.selects.SelectClause1
import kotlinx.coroutines.selects.select
import org.slf4j.Logger
import java.util.concurrent.ConcurrentHashMap
interface Projection {
suspend fun value(): T
}
internal interface ProjectionFactory {
fun createProjection(
id: ProjectionId,
filter: RecordTypePattern,
initial: T,
step: suspend (T, LogRecord) -> T
): Projection
fun requireTheseRecordsInProjections(records: List)
}
internal class ProjectionFactoryImpl(
coroutineScope: CoroutineScope,
private val recordRepo: LogRecordRepo,
private val logger: Logger
) : ProjectionFactory {
private val runningProjectionIds = ConcurrentHashMap.newKeySet()
private val requiredProjectionTimestamps = ConcurrentHashMap()
// we create a supervisor scope so failures in one projection doesn't affect others or rest of application
private val projectionFactoryScope = CoroutineScope(
coroutineScope.coroutineContext + SupervisorJob(coroutineScope.coroutineContext[Job])
)
override fun createProjection(
id: ProjectionId,
filter: RecordTypePattern,
initial: T,
step: suspend (T, LogRecord) -> T
): Projection {
val added = runningProjectionIds.add(id)
require(added) { "Projection $id is already running." }
requiredProjectionTimestamps.putIfAbsent(filter, NodeTimestamp.initial)
val projection = TimestampOrderGuardProjectionDecorator(
ProjectionSupervisor(
filter = filter,
supervisorScope = projectionFactoryScope,
initial = initial,
step = step,
recordRepo = recordRepo,
logger = logger,
id = id,
onProjectionException = ::onProjectionException,
requiredTimestamp = ::getRequiredTimestampFor
),
initial
)
logger.debug("projection $id started")
return projection
}
override fun requireTheseRecordsInProjections(records: List) {
if (records.isEmpty())
return
requiredProjectionTimestamps.replaceAll { filter, oldTs ->
val newTs = records.filter { filter.matches(it.type) }.maxOfOrNull { it.nodeTimestamp }
if (newTs == null) oldTs else maxOf(oldTs, newTs)
}
}
private fun onProjectionException(id: ProjectionId) {
runningProjectionIds.remove(id)
}
private fun getRequiredTimestampFor(filter: RecordTypePattern): NodeTimestamp =
requiredProjectionTimestamps[filter] ?: NodeTimestamp.initial // initial-case should never happen
}
private class ProjectionSupervisor(
private val filter: RecordTypePattern,
supervisorScope: CoroutineScope,
private val initial: T,
step: suspend (T, LogRecord) -> T,
recordRepo: LogRecordRepo,
logger: Logger,
id: ProjectionId,
onProjectionException: (ProjectionId) -> Unit,
private val requiredTimestamp: (RecordTypePattern) -> NodeTimestamp,
) : Projection> {
@Volatile
private lateinit var activeProjection: ProjectionChannel
@Volatile
private var candidateProjection: ProjectionChannel? = null
@Volatile
private var projectionException: Throwable? = null
init {
val context = CoroutineExceptionHandler { _, exception ->
projectionException = exception
activeProjection.notifyNewException()
onProjectionException(id)
logger.error(
"Projection $id aborted due to ${exception.javaClass}, will be propagated on next value() call.",
exception
)
}
// It's important that we only launch one coroutine job of the supervisor scope and rest as children.
// That way failure in one job will cancel all jobs within this projection but not outside of it.
// UNDISPATCHED so activeProjection is initialized directly.
supervisorScope.launch(context, CoroutineStart.UNDISPATCHED) {
activeProjection =
ProjectionChannel(
coroutineScope = this,
initial = initial,
step = step,
recordRepo = recordRepo,
filter = filter,
getProjectionException = ::getProjectionException,
requiredTimestamp = ::getRequiredTimestamp
)
while (true) {
select {
activeProjection.onControlReceive { controlMessage ->
when (controlMessage) {
ControlMessage.StartProcessingAppendOrderRecords -> {
logger.debug("first active $id projection database read complete")
}
ControlMessage.RecordOutOfOrder -> {
val restart = candidateProjection != null
candidateProjection?.cancel()
candidateProjection =
ProjectionChannel(
coroutineScope = this@launch,
initial = initial,
step = step,
recordRepo = recordRepo,
filter = filter,
getProjectionException = ::getProjectionException,
requiredTimestamp = ::getRequiredTimestamp
)
logger.debug("active of $id received out of order record -> ${if (restart) "restarted" else "started"} candidate projection")
}
}
}
val currentCandidateProjection = candidateProjection
currentCandidateProjection?.onControlReceive { controlMessage ->
when (controlMessage) {
ControlMessage.StartProcessingAppendOrderRecords -> {
val oldActiveProjection = activeProjection
activeProjection = currentCandidateProjection
candidateProjection = null
oldActiveProjection.cancel()
logger.debug("candidate of $id caught up and swapped in!")
}
ControlMessage.RecordOutOfOrder -> {
// we will let active out-of order messages handle restarts to not restart twice on same record
}
}
}
}
}
}
}
override suspend fun value(): Pair = activeProjection.value()
private fun getProjectionException(): Throwable? = projectionException
private fun getRequiredTimestamp(): NodeTimestamp = requiredTimestamp(filter)
}
private sealed interface StreamEntry {
data class Page(val page: LogPage) : StreamEntry // node-timestamp order
data class NewRecord(val record: LogRecord) : StreamEntry // append-order (could contain older timestamps!)
}
private class ProjectionChannel(
coroutineScope: CoroutineScope,
initial: T,
step: suspend (T, LogRecord) -> T,
recordRepo: LogRecordRepo,
filter: RecordTypePattern,
getProjectionException: () -> Throwable?,
requiredTimestamp: () -> NodeTimestamp
) : Projection> {
// low capacity as we don't want control to lag too far behind
private val controlChannel: Channel = Channel(capacity = 1)
private val projectionChannel: Channel = Channel(capacity = 2)
private val projectionStateMachine: ProjectionStateMachine
init {
startStreamingRecords(projectionChannel, controlChannel, coroutineScope, recordRepo, filter)
projectionStateMachine = ProjectionStateMachine(
projectionChannel,
coroutineScope,
initial,
step,
getProjectionException,
requiredTimestamp
)
}
override suspend fun value(): Pair = projectionStateMachine.value()
val onControlReceive: SelectClause1
get() = controlChannel.onReceive
fun cancel() {
projectionChannel.cancel()
controlChannel.cancel()
}
fun notifyNewException() {
projectionStateMachine.notifyNewException()
}
}
private enum class ControlMessage {
StartProcessingAppendOrderRecords,
RecordOutOfOrder
}
private fun startStreamingRecords(
recordStreamChannel: SendChannel,
controlChannel: SendChannel,
coroutineScope: CoroutineScope,
recordRepo: LogRecordRepo,
filter: RecordTypePattern
) {
coroutineScope.launch {
val preStreamRowId = recordRepo.getLargestRowId()
val newlyAppendedInStream = mutableSetOf() // records that have rowId > pre-stream rowId
// start listen for new records *before* getting existing to not miss anything
val newRecordsChannel = recordRepo.insertChannel.startReceiving(filter)
try {
var highestTimestamp: NodeTimestamp? = null
// send existing records
recordRepo.listByType(filter).collect { page ->
recordStreamChannel.send(StreamEntry.Page(page))
newlyAppendedInStream.addAll(page.listTimestampsAppendedAfter(preStreamRowId))
// last record has highest ts in a page
page.records.lastOrNull()?.let {
highestTimestamp = it.nodeTimestamp
}
}
// notify that we are now going from timestamp-order to append-order
controlChannel.send(ControlMessage.StartProcessingAppendOrderRecords)
// send new incoming records continuously
for (newRecord in newRecordsChannel) {
if (newlyAppendedInStream.contains(newRecord.nodeTimestamp))
continue
val ts = highestTimestamp
if (ts == null || newRecord.nodeTimestamp > ts) {
// appended record was in ts order, let's pass it through!
recordStreamChannel.send(StreamEntry.NewRecord(newRecord))
highestTimestamp = newRecord.nodeTimestamp
} else {
// record wasn't in ts-order, let someone else decide what to do!
controlChannel.send(ControlMessage.RecordOutOfOrder)
}
}
} finally {
recordRepo.insertChannel.stopReceiving(newRecordsChannel)
}
}
}
private class ProjectionStateMachine(
recordStream: ReceiveChannel,
coroutineScope: CoroutineScope,
initial: T,
step: suspend (T, LogRecord) -> T,
private val getProjectionException: () -> Throwable?,
private val requiredTimestamp: () -> NodeTimestamp,
) : Projection> {
@Volatile
private var current: Pair = NodeTimestamp.initial to initial
@Volatile
private var allPagesProjected = false
private val newRecordPing = MutableSharedFlow(replay = 1)
init {
coroutineScope.launch {
for (entry in recordStream) {
when (entry) {
is StreamEntry.Page -> {
for (record in entry.page.records) {
current = record.nodeTimestamp to step(current.second, record)
}
if (entry.page.next == null)
allPagesProjected = true
}
is StreamEntry.NewRecord -> {
current = entry.record.nodeTimestamp to step(current.second, entry.record)
}
}
// release all waiting value-fetchers so they can re-evaluate
newRecordPing.emit(Unit)
}
}
}
override suspend fun value(): Pair {
while (true) {
getProjectionException()?.let { throw it }
val current = current
// Check requirements:
// - All pages read. Means all db records in db are projected -> we don't regress ts-wise compared to previous app-run.
// - Last ts >= min-ts-requirement. This ensures all local records are projected as this min-ts is set when appending.
if (allPagesProjected && current.first >= requiredTimestamp())
return current
// Wait until something changes, where something being new incoming records
// as [requiredTs] can only change for the worse (higher value) which won't help us.
// Timeout is just a safety net, shouldn't really be needed as we have replay > 0 for race conditions.
withTimeoutOrNull(500) { newRecordPing.first() }
}
}
fun notifyNewException() {
// just an optimization to not having to wait for the timeout above
newRecordPing.tryEmit(Unit)
}
}
private class TimestampOrderGuardProjectionDecorator(
private val inner: Projection>, initial: T
) : Projection {
@Volatile
private var highestTimestampValue: Pair = NodeTimestamp.initial to initial
override suspend fun value(): T {
val next = inner.value()
return synchronized(this) {
val highest = highestTimestampValue
if (next.first < highest.first)
return highest.second // edge case where things got OOO during swap
highestTimestampValue = next
next.second
}
}
}