com.avito.android.runner.devices.internal.kubernetes.ReservationDeploymentFactoryImpl.kt Maven / Gradle / Ivy

Go to download
package com.avito.android.runner.devices.internal.kubernetes

import com.avito.android.runner.devices.model.ReservationData
import com.avito.instrumentation.reservation.request.Device
import com.avito.k8s.toValidKubernetesName
import com.avito.logger.LoggerFactory
import com.avito.logger.create
import com.fkorotkov.kubernetes.apps.metadata
import com.fkorotkov.kubernetes.apps.newDeployment
import com.fkorotkov.kubernetes.apps.selector
import com.fkorotkov.kubernetes.apps.spec
import com.fkorotkov.kubernetes.apps.template
import com.fkorotkov.kubernetes.metadata
import com.fkorotkov.kubernetes.newContainer
import com.fkorotkov.kubernetes.newEnvVar
import com.fkorotkov.kubernetes.newToleration
import com.fkorotkov.kubernetes.newTopologySpreadConstraint
import com.fkorotkov.kubernetes.resources
import com.fkorotkov.kubernetes.securityContext
import com.fkorotkov.kubernetes.spec
import io.fabric8.kubernetes.api.model.LabelSelector
import io.fabric8.kubernetes.api.model.PodSpec
import io.fabric8.kubernetes.api.model.Quantity
import io.fabric8.kubernetes.api.model.apps.Deployment

internal class ReservationDeploymentFactoryImpl(
    private val configurationName: String,
    private val projectName: String,
    private val buildId: String,
    private val buildType: String,
    private val deploymentNameGenerator: DeploymentNameGenerator,
    private val useLegacyExtensionsV1Beta: Boolean,
    loggerFactory: LoggerFactory
) : ReservationDeploymentFactory {

    private val logger = loggerFactory.create()

    init {
        val prefix = { reason: String -> "Can't create configuration, precondition failed: $reason" }
        require(configurationName.isNotBlank()) { prefix.invoke("configurationName is blank; used to label pods") }
        require(buildId.isNotBlank()) { prefix.invoke("buildId is blank, client can't distinguish reservations") }
    }

    override fun createDeployment(namespace: String, reservation: ReservationData): Deployment {
        logger.info("Creating deployment for configuration: $configurationName")
        val deploymentName = deploymentNameGenerator.generateName(namespace)
        logger.info("Deployment name will be: $deploymentName")

        return when (val device = reservation.device) {
            is Device.LocalEmulator -> throw IllegalStateException(
                "Local emulator $device is unsupported in kubernetes reservation"
            )
            is Device.CloudEmulator -> {
                logger.info("Creating ${reservation.count} replicas of cloud emulator deployment: $device")
                getCloudEmulatorDeployment(
                    emulator = device,
                    deploymentName = deploymentName,
                    count = reservation.count
                )
            }
            is Device.MockEmulator -> throw IllegalStateException(
                "Mock emulator ${reservation.device} is unsupported in kubernetes reservation"
            )
        }
    }

    private fun getCloudEmulatorDeployment(
        emulator: Device.CloudEmulator,
        deploymentName: String,
        count: Int
    ): Deployment {
        val deploymentMatchLabels = deviceMatchLabels(emulator)
        return deviceDeployment(
            deploymentMatchLabels = deploymentMatchLabels,
            deploymentName = deploymentName,
            count = count
        ) {

            containers = listOf(
                newContainer {
                    name = emulator.name.toValidKubernetesName()
                    image = emulator.image

                    securityContext {
                        privileged = true
                    }

                    resources {
                        limits = mutableMapOf().apply {
                            if (!emulator.cpuCoresLimit.isNullOrBlank()) {
                                plusAssign("cpu" to Quantity(emulator.cpuCoresLimit))
                            }
                            if (!emulator.memoryLimit.isNullOrBlank()) {
                                plusAssign("memory" to Quantity(emulator.memoryLimit))
                            }
                        }

                        requests = mutableMapOf().apply {
                            if (!emulator.cpuCoresRequest.isNullOrBlank()) {
                                plusAssign("cpu" to Quantity(emulator.cpuCoresRequest))
                            }
                            if (!emulator.memoryRequest.isNullOrBlank()) {
                                plusAssign("memory" to Quantity(emulator.memoryRequest))
                            }
                        }
                    }

                    env = listOf(
                        // used to start from prepared snapshot, see `prepare_snapshot.sh`
                        newEnvVar {
                            name = "SNAPSHOT_ENABLED"
                            value = "true"
                        },
                        // run headless
                        newEnvVar {
                            name = "WINDOW"
                            value = "false"
                        }
                    )
                }
            )

            tolerations = listOf(
                newToleration {
                    key = "dedicated"
                    operator = "Equal"
                    value = "android"
                    effect = "NoSchedule"
                }
            )

            topologySpreadConstraints = listOf(
                newTopologySpreadConstraint {
                    labelSelector = LabelSelector().apply {
                        matchLabels = deploymentMatchLabels.plus("deploymentName" to deploymentName)
                    }
                    maxSkew = 1
                    topologyKey = "kubernetes.io/hostname"
                    whenUnsatisfiable = "ScheduleAnyway"
                },
            )
        }
    }

    private fun deviceDeployment(
        deploymentMatchLabels: Map,
        deploymentName: String,
        count: Int,
        block: PodSpec.() -> Unit
    ): Deployment {
        val deploymentSpecificationsMatchLabels = deploymentMatchLabels
            .plus("deploymentName" to deploymentName)

        return newDeployment {
            apiVersion = if (useLegacyExtensionsV1Beta) "extensions/v1beta1" else "apps/v1"
            metadata {
                name = deploymentName
                labels = deploymentMatchLabels
                finalizers = listOf(
                    // Remove all dependencies (replicas) in foreground after removing deployment
                    "foregroundDeletion"
                )
            }
            spec {
                replicas = count
                selector {
                    matchLabels = deploymentSpecificationsMatchLabels
                }

                template {
                    metadata {
                        labels = deploymentSpecificationsMatchLabels
                    }
                    spec(block)
                }
            }
        }
    }

    private fun deviceMatchLabels(
        device: Device
    ): Map {
        return mapOf(

            /**
             * used to distinguish different strategies for clearing leaked kubernetes deployments
             * for incorrectly finished builds
             * see [com.avito.ci.DeploymentEnvironment]
             */
            "type" to buildType,
            "id" to buildId, // teamcity_build_id or local synthetic
            "project" to projectName,
            "instrumentationConfiguration" to configurationName,
            "device" to device.description
        )
    }
}