ai.platon.pulsar.examples.sites.language.ThaiCrawler.kt Maven / Gradle / Ivy

Go to download

Show more of this group Show more artifacts with this name
Show all versions of pulsar-examples Show documentation

The newest version!

package ai.platon.pulsar.examples.sites.language

import ai.platon.pulsar.skeleton.context.PulsarContexts
import ai.platon.pulsar.skeleton.crawl.event.impl.CloseMaskLayerHandler

fun main() {
    val portalUrl = "https://shopee.co.th/กระเป๋าเป้ผู้ชาย-cat.49.1037.10297?page=1"
    val args = """
        -i 1s -ii 1s -ol ".shopee-search-item-result__item a" -sc 10
    """.trimIndent()
    val closeMaskLayerSelector = ".language-selection button:nth-child(1)"

    val session = PulsarContexts.createSession()
    val options = session.options(args)

    val closeMaskLayerHandler = CloseMaskLayerHandler(closeMaskLayerSelector)
    options.event.browseEventHandlers.onDocumentActuallyReady.addLast(closeMaskLayerHandler)

    session.loadOutPages(portalUrl, options)
}