com.alibaba.ververica.connectors.odps.sink.OdpsWriter Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of ververica-connector-odps Show documentation
There is a newer version: 1.17-vvr-8.0.8
Show newest version
/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.alibaba.ververica.connectors.odps.sink;

import org.apache.flink.types.Row;

import com.alibaba.ververica.connectors.odps.OdpsConf;
import com.alibaba.ververica.connectors.odps.util.OdpsUtils;
import com.aliyun.odps.PartitionSpec;
import com.aliyun.odps.commons.util.RetryStrategy;
import com.aliyun.odps.data.RecordWriter;
import com.aliyun.odps.tunnel.TableTunnel;
import com.aliyun.odps.tunnel.io.TunnelBufferedWriter;
import org.apache.commons.lang3.concurrent.BasicThreadFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.io.Serializable;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.TimeUnit;

import static java.lang.Thread.sleep;

/**
 * Base odps writer to write record into odps table.
 */
public abstract class OdpsWriter implements Serializable {

	private static final Logger LOGGER = LoggerFactory.getLogger(OdpsWriter.class);

	protected long batchCount = Long.MAX_VALUE;
	protected int retryTimes = 6;
	protected long sleepTime = 4000;
	protected final long flushIntervalMs;
	protected final OdpsConf odpsConf;
	protected final String table;

	protected transient TableTunnel tableTunnel;
	protected transient ScheduledExecutorService flusher;
	private transient volatile Throwable flushException = null;
	private volatile boolean flushError = false;

	public OdpsWriter(long flushIntervalMs, OdpsConf odpsConf, String table) {
		this.flushIntervalMs = flushIntervalMs;
		this.odpsConf = odpsConf;
		this.table = table;
	}

	/**
	 * Open the session on the server side.
	 */
	public void open() {
		this.tableTunnel = OdpsUtils.createTableTunnel(odpsConf);
		if (flushIntervalMs > 0) {
			scheduleFlusher();
		}
	}

	/**
	 * Get the upload session of the ODPS tunnel.
	 *
	 * @param partitionKey Partition Key
	 * @return Tunnel Upload Session.
	 */
	abstract TableTunnel.UploadSession getUploadSession(String partitionKey);

	abstract RecordWriter getRecordWriter(String partitionKey);

	/**
	 * Get the dynamic partition key, the key can be null.
	 *
	 * @param row Current row
	 * @return partition key eg "pt='20190101'" or null
	 */
	abstract String getPartitionKey(Row row);

	/**
	 * Flush the data in buffer to the ODPS tunnel.
	 *
	 * @param commit If commit is true, trigger flush, commit and then rebuild the writer
	 * and upload session. If commit is false, trigger flush only.
	 * @throws IOException
	 */
	abstract void flush(boolean commit) throws IOException;

	public void close() {
		if (flusher != null) {
			flusher.shutdownNow();
			while (!flusher.isTerminated()) {
				try {
					sleep(10);
				} catch (Throwable t) {
					LOGGER.error("Exception Happened In Sleep Method", t);
					//ignore
				}
			}
			flusher = null;
		}
	}

	abstract void checkAndFlush(String partitionKey);

	public long getRetryTimeout() {
		return retryTimes * sleepTime;
	}

	public boolean flushError() {
		return flushError && null != flushException;
	}

	public Throwable getFlushException() {
		return flushException;
	}

	/**
	 * Start flusher that will flush buffer automatically.
	 */
	protected void scheduleFlusher() {
		flusher = new ScheduledThreadPoolExecutor(
				1,
				new BasicThreadFactory.Builder().namingPattern("OdpsOutputFormat.buffer.flusher")
						.daemon(true)
						.build());
		flusher.scheduleAtFixedRate(() -> {
			try {
				flush(false);
			} catch (Throwable e) {
				LOGGER.error("Sync sink buffer to ODPS failed!", e);
				flushException = e;
				flushError = true;
			}
		}, flushIntervalMs, flushIntervalMs, TimeUnit.MILLISECONDS);
	}

	protected RecordWriter createRecordWriter(TableTunnel.UploadSession uploadSession) {
		try {
			RecordWriter recordWriter = uploadSession.openBufferedWriter(true);
			RetryStrategy retryStrategy = new RetryStrategy(retryTimes, (int) (sleepTime / 1000),
					RetryStrategy.BackoffStrategy.EXPONENTIAL_BACKOFF);
			((TunnelBufferedWriter) recordWriter).setRetryStrategy(retryStrategy);
			return recordWriter;
		} catch (Throwable e) {
			LOGGER.warn("Fail to create odps writer! ", e);
			throw new RuntimeException("Fail to create odps writer! ", e);
		}
	}

	protected TableTunnel.UploadSession createUploadSession(PartitionSpec partitionSpec) {
		try {
			String project = odpsConf.getProject();
			TableTunnel.UploadSession newUploadSession = tableTunnel.createUploadSession(project, table, partitionSpec);
			LOGGER.info("Created upload session id {} for partition {}.", newUploadSession.getId(), partitionSpec);
			return newUploadSession;
		} catch (Throwable e) {
			LOGGER.error("Fail to create uploadSession for partition {}! ", partitionSpec, e);
			throw new RuntimeException("Fail to create uploadSession! ", e);
		}
	}
}