Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance. Project price only 1 $
You can buy this project and download/modify it how often you want.
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.flink.runtime.healthmanager.plugins.policies;
import org.apache.flink.annotation.VisibleForTesting;
import org.apache.flink.configuration.ConfigOption;
import org.apache.flink.configuration.ConfigOptions;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.runtime.healthmanager.HealthMonitor;
import org.apache.flink.runtime.healthmanager.plugins.Policy;
import org.apache.flink.util.StringUtils;
import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonCreator;
import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonIgnore;
import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonInclude;
import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonProperty;
import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.JsonParser;
import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.JsonProcessingException;
import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.DeserializationFeature;
import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.JavaType;
import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
import java.util.List;
import java.util.TimeZone;
import java.util.stream.Collectors;
/**
* TimedSuspendPolicy which use different policy in different period. currently we support to resume job config
* to original settings in given period.
*/
public class TimedSuspendPolicy implements Policy {
private static final Logger LOGGER = LoggerFactory.getLogger(TimedSuspendPolicy.class);
public static final long MS_OF_DAY = 86400_000L;
public static final long MS_OF_HOUR = 3600_000L;
public static final ConfigOption SUSPEND_PERIOD = ConfigOptions.key("healthmonitor.autoscale.suspend.periods").noDefaultValue();
public static final ConfigOption TIME_ZONE = ConfigOptions.key("healthmonitor.autoscale.suspend.periods.timezone")
.defaultValue("Asia/Shanghai")
.withDeprecatedKeys("blink.job.timeZone");
public static final String DATE_FORMAT = "yyyy-MM-dd HH:mm";
public static final String TIME_FORMAT = "HH:mm";
private SimpleDateFormat dateFormatter = new SimpleDateFormat(DATE_FORMAT);
private SimpleDateFormat timeFormatter = new SimpleDateFormat(TIME_FORMAT);
private long nextReloadTime;
private List fixedRanges;
private List periodicRanges;
private Clock systemClock = Clock.SYSTEM_CLOCK;
private Clock zonedClock;
private HealthMonitor monitor;
private String definedDetectors;
private String definedResolvers;
@Override
public void open(HealthMonitor monitor) {
this.monitor = monitor;
Configuration config = monitor.getConfig();
List suspendRanges =
SuspendRangeDescriptor.fromJson(config.getString(SUSPEND_PERIOD), true);
if (suspendRanges == null) {
suspendRanges = Collections.EMPTY_LIST;
}
TimeZone timeZone = TimeZone.getTimeZone(config.getString(TIME_ZONE));
zonedClock = () -> systemClock.now() + timeZone.getRawOffset();
timeFormatter.setTimeZone(TimeZone.getTimeZone("UTC"));
dateFormatter.setTimeZone(TimeZone.getTimeZone("UTC"));
suspendRanges.forEach(suspendRangeDescriptor -> suspendRangeDescriptor.parseTime(dateFormatter, timeFormatter));
fixedRanges = suspendRanges.stream().filter(range -> range.policy == TimeRangeType.SPECIFIED_TIME && range.startTs >= 0 && range.endTs >= 0).collect(Collectors.toList());
periodicRanges = suspendRanges.stream().filter(range -> range.policy == TimeRangeType.EVERY_DAY_TIME && range.startTs >= 0 && range.endTs >= 0).collect(Collectors.toList());
Collections.sort(fixedRanges);
Collections.sort(periodicRanges);
LOGGER.info("Fix Period:{}", fixedRanges);
LOGGER.info("Periodic Period:{}", periodicRanges);
}
@Override
public void apply(HealthMonitor monitor) {
long now = zonedClock.now();
long dateTs = now / MS_OF_DAY * MS_OF_DAY;
long dailyTs = now % MS_OF_DAY;
long nextFixedTrigger = -1;
boolean suspended = false;
SuspendRangeDescriptor descriptor = null;
for (SuspendRangeDescriptor range : fixedRanges) {
if (range.startTs > now) {
nextFixedTrigger = range.startTs;
break;
} else if (range.endTs > now) {
suspended = true;
descriptor = range;
nextReloadTime = range.endTs;
break;
}
}
long nextDailyTrigger = -1;
if (!suspended) {
for (SuspendRangeDescriptor range : periodicRanges) {
if (range.startTs <= dailyTs) {
if (range.endTs < range.startTs) {
suspended = true;
descriptor = range;
nextReloadTime = dateTs + MS_OF_DAY + range.endTs;
break;
} else if (range.endTs > dailyTs) {
suspended = true;
descriptor = range;
nextReloadTime = dateTs + range.endTs;
break;
}
} else if (range.startTs > dailyTs) {
nextDailyTrigger = dateTs + range.startTs;
break;
}
}
}
if (!suspended) {
if (nextDailyTrigger == -1 && periodicRanges.size() > 0) {
// trigger suspend in next day.
nextDailyTrigger = dateTs + MS_OF_DAY + periodicRanges.get(0).startTs;
}
if (nextDailyTrigger > 0) {
nextReloadTime = nextDailyTrigger;
if (nextFixedTrigger > 0) {
nextReloadTime = Math.min(nextFixedTrigger, nextDailyTrigger);
}
} else {
nextReloadTime = nextFixedTrigger;
}
}
if (descriptor != null) {
LOGGER.info("Auto scale is suspended currently ,using {}", descriptor.plan);
definedDetectors = monitor.getConfig().getString(HealthMonitor.DETECTOR_CLASSES);
definedResolvers = monitor.getConfig().getString(HealthMonitor.RESOLVER_CLASSES);
if (descriptor.plan == PlanType.ORIGINAL) {
FixedPolicy fixedPolicy = new FixedPolicy();
fixedPolicy.apply(monitor);
} else {
// do not load any policy.
EmptyPolicy emptyPolicy = new EmptyPolicy();
emptyPolicy.apply(monitor);
}
} else {
LOGGER.info("Auto scale is enabled currently");
PolicyUtils.loadBuiltInPolicy(monitor.getConfig()).apply(monitor);
}
LOGGER.info("Next switch time {}", nextReloadTime);
}
@Override
public boolean reloadPlugin() {
long now = zonedClock.now();
if (nextReloadTime > 0 && now >= nextReloadTime) {
if (definedResolvers != null) {
monitor.getConfig().setString(HealthMonitor.RESOLVER_CLASSES, definedResolvers);
} else {
monitor.getConfig().remove(HealthMonitor.RESOLVER_CLASSES);
}
if (definedDetectors != null) {
monitor.getConfig().setString(HealthMonitor.DETECTOR_CLASSES, definedResolvers);
} else {
monitor.getConfig().remove(HealthMonitor.DETECTOR_CLASSES);
}
return true;
}
return false;
}
public long getNextReloadTime() {
return nextReloadTime;
}
public List getFixedRanges() {
return fixedRanges;
}
public List getPeriodicRanges() {
return periodicRanges;
}
public void setSystemClock(Clock clock) {
this.systemClock = clock;
}
/**
* Type of TimeRange to suspend auto scale.
*/
enum TimeRangeType {
EVERY_DAY_TIME,
SPECIFIED_TIME
}
/**
* Which Plan we need to use when suspending auto scale.
*/
enum PlanType {
ORIGINAL,
CURRENT
}
/**
* Suspend Time Range Descriptor.
*/
@VisibleForTesting
@JsonInclude(JsonInclude.Include.NON_DEFAULT)
@JsonIgnoreProperties(ignoreUnknown = true)
public static class SuspendRangeDescriptor implements Comparable {
public TimeRangeType policy;
public PlanType plan;
public String startTime;
public String endTime;
@JsonIgnore
public long startTs = -1;
@JsonIgnore
public long endTs = -1;
@JsonCreator
public SuspendRangeDescriptor(
@JsonProperty("policy") TimeRangeType policy,
@JsonProperty("plan") PlanType plan,
@JsonProperty("startTime") String startTime,
@JsonProperty("endTime") String endTime) throws ParseException {
this.policy = policy;
this.plan = plan;
this.startTime = startTime;
this.endTime = endTime;
}
public void parseTime(SimpleDateFormat dateFormatter, SimpleDateFormat timeFormatter) {
try {
if (policy == TimeRangeType.SPECIFIED_TIME) {
this.startTs = getDateTs(dateFormatter, startTime);
this.endTs = getDateTs(dateFormatter, endTime);
} else if (policy == TimeRangeType.EVERY_DAY_TIME) {
this.startTs = getTimeTs(timeFormatter, startTime);
this.endTs = getTimeTs(timeFormatter, endTime);
}
} catch (Throwable e) {
LOGGER.warn("Fail to parse time in policy", e);
}
}
@Override
public int compareTo(SuspendRangeDescriptor o) {
return startTs > o.startTs ? 1 : -1;
}
public static final long getDateTs(SimpleDateFormat dateFormatter, String dateStr) throws ParseException {
Date date = dateFormatter.parse(dateStr);
return date.getTime();
}
public static final long getTimeTs(SimpleDateFormat timeFormatter, String timeStr) throws ParseException {
Date date = timeFormatter.parse(timeStr);
return date.getTime();
}
public static List fromJson(String json, boolean ignoreUnknownProperties) {
if (StringUtils.isNullOrWhitespaceOnly(json)) {
return null;
}
try {
ObjectMapper objectMapper = new ObjectMapper();
objectMapper.configure(JsonParser.Feature.ALLOW_COMMENTS, true);
if (ignoreUnknownProperties) {
objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
} else {
objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true);
}
JavaType javaType =
objectMapper.getTypeFactory().constructParametricType(ArrayList.class, SuspendRangeDescriptor.class);
return objectMapper.readValue(json, javaType);
} catch (IOException e) {
// DON'T add the json string to error message to spam logging
// e will contain the json string
throw new IllegalArgumentException("error converting from JSON string.", e);
}
}
public static String toJson(List value) throws JsonProcessingException {
return new ObjectMapper().writerWithDefaultPrettyPrinter().writeValueAsString(value);
}
}
@VisibleForTesting
public Clock getZonedClock() {
return zonedClock;
}
/**
* Interface to define clock.
*/
interface Clock {
long now();
Clock SYSTEM_CLOCK = () -> System.currentTimeMillis();
}
}
