org.apache.flume.sink.AvroSink Maven / Gradle / Ivy
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.flume.sink;
import java.util.List;
import java.util.Properties;
import org.apache.flume.Channel;
import org.apache.flume.ChannelException;
import org.apache.flume.Context;
import org.apache.flume.Event;
import org.apache.flume.EventDeliveryException;
import org.apache.flume.FlumeException;
import org.apache.flume.Sink;
import org.apache.flume.Transaction;
import org.apache.flume.api.RpcClient;
import org.apache.flume.api.RpcClientConfigurationConstants;
import org.apache.flume.api.RpcClientFactory;
import org.apache.flume.conf.Configurable;
import org.apache.flume.instrumentation.SinkCounter;
import org.apache.flume.source.AvroSource;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
/**
*
* A {@link Sink} implementation that can send events to an RPC server (such as
* Flume's {@link AvroSource}).
*
*
* This sink forms one half of Flume's tiered collection support. Events sent to
* this sink are transported over the network to the hostname / port pair using
* the RPC implementation encapsulated in {@link RpcClient}.
* The destination is an instance of Flume's {@link AvroSource}, which
* allows Flume agents to forward to other Flume agents, forming a tiered
* collection infrastructure. Of course, nothing prevents one from using this
* sink to speak to other custom built infrastructure that implements the same
* RPC protocol.
*
*
* Events are taken from the configured {@link Channel} in batches of the
* configured batch-size. The batch size has no theoretical limits
* although all events in the batch must fit in memory. Generally, larger
* batches are far more efficient, but introduce a slight delay (measured in
* millis) in delivery. The batch behavior is such that underruns (i.e. batches
* smaller than the configured batch size) are possible. This is a compromise
* made to maintain low latency of event delivery. If the channel returns a null
* event, meaning it is empty, the batch is immediately sent, regardless of
* size. Batch underruns are tracked in the metrics. Empty batches do not incur
* an RPC roundtrip.
*
*
* Configuration options
*
*
*
* Parameter
* Description
* Unit (data type)
* Default
*
*
* hostname
* The hostname to which events should be sent.
* Hostname or IP (String)
* none (required)
*
*
* port
* The port to which events should be sent on hostname.
* TCP port (int)
* none (required)
*
*
* batch-size
* The maximum number of events to send per RPC.
* events (int)
* 100
*
*
* connect-timeout
* Maximum time to wait for the first Avro handshake and RPC request
* milliseconds (long)
* 20000
*
*
* request-timeout
* Maximum time to wait RPC requests after the first
* milliseconds (long)
* 20000
*
*
*
* Metrics
*
*
* TODO
*
*/
public class AvroSink extends AbstractRpcSink {
private static final Logger logger = LoggerFactory.getLogger(AvroSink.class);
@Override
protected RpcClient initializeRpcClient(Properties props) {
logger.info("Attempting to create Avro Rpc client.");
return RpcClientFactory.getInstance(props);
}
}