ai.platon.pulsar.protocol.browser.driver.BrowserManager.kt Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of pulsar-protocol Show documentation
There is a newer version: 2.0.2
package ai.platon.pulsar.protocol.browser.driver

import ai.platon.pulsar.browser.driver.chrome.common.ChromeOptions
import ai.platon.pulsar.browser.driver.chrome.common.LauncherOptions
import ai.platon.pulsar.common.*
import ai.platon.pulsar.common.config.ImmutableConfig
import ai.platon.pulsar.skeleton.context.PulsarContexts
import ai.platon.pulsar.skeleton.crawl.fetch.driver.*
import ai.platon.pulsar.skeleton.crawl.fetch.privacy.BrowserId
import ai.platon.pulsar.protocol.browser.BrowserLaunchException
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.ConcurrentLinkedDeque
import java.util.concurrent.atomic.AtomicBoolean

open class BrowserManager(
    val conf: ImmutableConfig
): AutoCloseable {
    private val logger = getLogger(this)
    private var registered = AtomicBoolean()
    private val closed = AtomicBoolean()
    private val browserFactory = BrowserFactory()
    private val _browsers = ConcurrentHashMap()
    private val historicalBrowsers = ConcurrentLinkedDeque()
    private val closedBrowsers = ConcurrentLinkedDeque()
    /**
     * The active browsers
     * */
    val browsers: Map = _browsers
    /**
     * Launch a browser. If the browser with the id is already launched, return the existing one.
     * */
    @Throws(BrowserLaunchException::class)
    fun launch(browserId: BrowserId, driverSettings: WebDriverSettings, capabilities: Map): Browser {
        registerAsClosableIfNecessary()

        val launcherOptions = LauncherOptions(driverSettings)
        if (driverSettings.isSupervised) {
            launcherOptions.supervisorProcess = driverSettings.supervisorProcess
            launcherOptions.supervisorProcessArgs.addAll(driverSettings.supervisorProcessArgs)
        }

        val launchOptions = driverSettings.createChromeOptions(capabilities)
        return launchIfAbsent(browserId, launcherOptions, launchOptions)
    }
    /**
     * Find an existing browser.
     * */
    @Synchronized
    fun findBrowser(browserId: BrowserId) = browsers[browserId]

    /**
     * Close a browser.
     * */
    @Synchronized
    fun closeBrowser(browserId: BrowserId) {
        val browser = _browsers.remove(browserId)
        if (browser is AbstractBrowser) {
            kotlin.runCatching { browser.close() }.onFailure { warnForClose(this, it) }
            closedBrowsers.add(browser)
        }
    }

    @Synchronized
    fun destroyBrowserForcibly(browserId: BrowserId) {
        historicalBrowsers.filter { browserId == it.id }.forEach { browser ->
            kotlin.runCatching { browser.destroyForcibly() }.onFailure { warnInterruptible(this, it) }
            closedBrowsers.add(browser)
        }
    }

    @Synchronized
    fun closeBrowser(browser: Browser) {
        closeBrowser(browser.id)
    }

    @Synchronized
    fun closeDriver(driver: WebDriver) {
        kotlin.runCatching { driver.close() }.onFailure { warnForClose(this, it) }
    }

    @Synchronized
    fun findLeastValuableDriver(): WebDriver? {
        val drivers = browsers.values.flatMap { it.drivers.values }
        return findLeastValuableDriver(drivers)
    }

    @Synchronized
    fun closeLeastValuableDriver() {
        val driver = findLeastValuableDriver()
        if (driver != null) {
            closeDriver(driver)
        }
    }

    /**
     * Destroy the zombie browsers forcibly, kill the associated browser processes,
     * release all allocated resources, regardless of whether the browser is closed or not.
     * */
    @Synchronized
    fun destroyZombieBrowsersForcibly() {
        val zombieBrowsers = historicalBrowsers - browsers.values.toSet() - closedBrowsers
        if (zombieBrowsers.isNotEmpty()) {
            logger.warn("There are {} zombie browsers, cleaning them ...", zombieBrowsers.size)
            zombieBrowsers.forEach { browser ->
                logger.info("Closing zombie browser | {}", browser.id)
                kotlin.runCatching { browser.destroyForcibly() }.onFailure { warnInterruptible(this, it) }
            }
        }
    }

    private fun findLeastValuableDriver(drivers: Iterable): WebDriver? {
        return drivers.filterIsInstance()
            .filter { !it.isReady && !it.isWorking }
            .minByOrNull { it.lastActiveTime }
    }

    fun maintain() {
        browsers.values.forEach {
            require(it is AbstractBrowser)
            it.emit(BrowserEvents.willMaintain)
            it.emit(BrowserEvents.maintain)
            it.emit(BrowserEvents.didMaintain)
        }
    }

    @Synchronized
    override fun close() {
        if (closed.compareAndSet(false, true)) {
            _browsers.values.forEach { browser ->
                require(browser is AbstractBrowser)
                kotlin.runCatching { browser.close() }.onFailure { warnForClose(this, it) }
            }
            _browsers.clear()
        }
    }

    @Throws(BrowserLaunchException::class)
    private fun launchIfAbsent(
        browserId: BrowserId, launcherOptions: LauncherOptions, launchOptions: ChromeOptions
    ): Browser {
        val browser = _browsers[browserId]
        if (browser != null) {
            return browser
        }

        synchronized(browserFactory) {
            val browser1 = browserFactory.launch(browserId, launcherOptions, launchOptions)
            _browsers[browserId] = browser1
            historicalBrowsers.add(browser1)

            return browser1
        }
    }

    private fun registerAsClosableIfNecessary() {
        if (registered.compareAndSet(false, true)) {
            // Actually, it's safe to register multiple times, the manager will be closed only once, and the browsers
            // will be closed in the manager's close function.
            PulsarContexts.registerClosable(this, -100)
        }
    }
}