ai.platon.pulsar.skeleton.crawl.schedule.NewsFetchSchedule.kt Maven / Gradle / Ivy

Go to download
/**
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package ai.platon.pulsar.skeleton.crawl.schedule

import ai.platon.pulsar.skeleton.common.message.MiscMessageWriter
import ai.platon.pulsar.common.config.AppConstants
import ai.platon.pulsar.common.config.ImmutableConfig
import ai.platon.pulsar.persist.WebPage
import ai.platon.pulsar.persist.metadata.Mark
import org.slf4j.LoggerFactory
import java.time.Duration
import java.time.Instant
import java.time.temporal.ChronoUnit

/**
 * This class implements an adaptive re-fetch algorithm.
 *
 * NOTE: values of DEC_FACTOR and INC_FACTOR higher than 0.4f may destabilize
 * the algorithm, so that the fetch interval either increases or decreases
 * infinitely, with little relevance to the page changes
 *
 * @author Vincent Zhang
 */
class NewsFetchSchedule(
        conf: ImmutableConfig,
        messageWriter: MiscMessageWriter? = null
) : AdaptiveFetchSchedule(conf, messageWriter) {
    private val LOG = LoggerFactory.getLogger(NewsFetchSchedule::class.java)

    override fun setFetchSchedule(page: WebPage, m: ModifyInfo) {
        var time = m.modifiedTime
        if (time.isBefore(AppConstants.TCP_IP_STANDARDIZED_TIME)) {
            time = m.fetchTime
        }

        var interval = Duration.ofDays(365 * 10.toLong())
        if (page.isSeed) {
            interval = adjustSeedFetchInterval(page, m.fetchTime, time)
        } else {
            page.marks.put(Mark.INACTIVE, AppConstants.YES_STRING)
        }

        m.fetchTime = time
        updateRefetchTime(page, interval, m)
    }

    private fun adjustSeedFetchInterval(page: WebPage, fetchTime: Instant, modifiedTime: Instant): Duration {
        var time: Instant? = modifiedTime
        val publishTime = page.contentPublishTime
        if (publishTime.isAfter(time)) {
            time = publishTime
        }
        val days = ChronoUnit.DAYS.between(time, fetchTime)
        if (days > 7) {
            messageWriter?.reportFetchSchedule(page, false)
            return Duration.ofHours(1)
        }
        return MIN_INTERVAL
    }
}