com.teambytes.logback.flume.FlumeLogstashV1Appender Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of logback-flume-appender_2.11 Show documentation
logback-flume-appender
The newest version!
package com.teambytes.logback.flume;

import ch.qos.logback.classic.spi.ILoggingEvent;
import ch.qos.logback.core.Layout;
import ch.qos.logback.core.UnsynchronizedAppenderBase;
import org.apache.commons.lang.StringUtils;
import org.apache.flume.Event;
import org.apache.flume.FlumeException;
import org.apache.flume.event.EventBuilder;

import java.net.InetAddress;
import java.net.UnknownHostException;
import java.nio.charset.Charset;
import java.util.*;

public class FlumeLogstashV1Appender extends UnsynchronizedAppenderBase {

  protected static final Charset UTF_8 = Charset.forName("UTF-8");

  private FlumeAvroManager flumeManager;

  private String flumeAgents;

  private String flumeProperties;

  private Long reportingWindow;

  private Integer batchSize;

  private Integer reporterMaxThreadPoolSize;

  private Integer reporterMaxQueueSize;

  private Map additionalAvroHeaders;

  private String application;

  protected Layout layout;

  private String hostname;

  private String type;

  public void setType(String type) {
    this.type = type;
  }

  public void setHostname(String hostname) {
    this.hostname = hostname;
  }

  public void setApplication(String application) {
    this.application = application;
  }

  public void setLayout(Layout layout) {
    this.layout = layout;
  }

  public void setFlumeAgents(String flumeAgents) {
    this.flumeAgents = flumeAgents;
  }

  public void setFlumeProperties(String flumeProperties) {
    this.flumeProperties = flumeProperties;
  }

  public void setAdditionalAvroHeaders(String additionalHeaders) {
    this.additionalAvroHeaders = extractProperties(additionalHeaders);
  }

  public void setBatchSize(String batchSizeStr) {
    try {
      this.batchSize = Integer.parseInt(batchSizeStr);
    } catch (NumberFormatException nfe) {
      addWarn("Cannot set the batchSize to " + batchSizeStr, nfe);
    }
  }

  public void setReportingWindow(String reportingWindowStr) {
    try {
      this.reportingWindow = Long.parseLong(reportingWindowStr);
    } catch (NumberFormatException nfe) {
      addWarn("Cannot set the reportingWindow to " + reportingWindowStr, nfe);
    }
  }


  public void setReporterMaxThreadPoolSize(String reporterMaxThreadPoolSizeStr) {
    try {
      this.reporterMaxThreadPoolSize = Integer.parseInt(reporterMaxThreadPoolSizeStr);
    } catch (NumberFormatException nfe) {
      addWarn("Cannot set the reporterMaxThreadPoolSize to " + reporterMaxThreadPoolSizeStr, nfe);
    }
  }

  public void setReporterMaxQueueSize(String reporterMaxQueueSizeStr) {
    try {
      this.reporterMaxQueueSize = Integer.parseInt(reporterMaxQueueSizeStr);
    } catch (NumberFormatException nfe) {
      addWarn("Cannot set the reporterMaxQueueSize to " + reporterMaxQueueSizeStr, nfe);
    }
  }

  @Override
  public void start() {
    if (layout == null) {
      addWarn("Layout was not defined, will only log the message, no stack traces or custom layout");
    }
    if (StringUtils.isEmpty(application)) {
      application = resolveApplication();
    }

    if (StringUtils.isNotEmpty(flumeAgents)) {
      String[] agentConfigs = flumeAgents.split(",");

      List agents = new ArrayList(agentConfigs.length);
      for (String conf : agentConfigs) {
        RemoteFlumeAgent agent = RemoteFlumeAgent.fromString(conf.trim());
        if (agent != null) {
          agents.add(agent);
        } else {
          addWarn("Cannot build a Flume agent config for '" + conf + "'");
        }
      }
      Properties overrides = new Properties();
      overrides.putAll(extractProperties(flumeProperties));
      flumeManager = FlumeAvroManager.create(agents, overrides,
              batchSize, reportingWindow, reporterMaxThreadPoolSize, reporterMaxQueueSize, this);
    } else {
      addError("Cannot configure a flume agent with an empty configuration");
    }
    super.start();

  }

  private Map extractProperties(String propertiesAsString) {
    final Map props = new HashMap();
    if (StringUtils.isNotEmpty(propertiesAsString)) {
      final String[] segments = propertiesAsString.split(";");
      for (final String segment : segments) {
        final String[] pair = segment.split("=");
        if (pair.length == 2) {
          final String key = pair[0].trim();
          final String value = pair[1].trim();
          if (StringUtils.isNotEmpty(key) && StringUtils.isNotEmpty(value)) {
            props.put(key, value);
          } else {
            addWarn("Empty key or value not accepted: " + segment);
          }
        } else {
          addWarn("Not a valid {key}:{value} format: " + segment);
        }
      }
    } else {
      addInfo("Not overriding any flume agent properties");
    }

    return props;
  }

  @Override
  public void stop() {
    try {
      if (flumeManager != null) {
        flumeManager.stop();
      }
    } catch (FlumeException fe) {
      addWarn(fe.getMessage(), fe);
    }
  }

  @Override
  protected void append(ILoggingEvent eventObject) {

    if (flumeManager != null) {
      try {
        String body = layout != null ? layout.doLayout(eventObject) : eventObject.getFormattedMessage();
        Map headers = new HashMap();
        if(additionalAvroHeaders != null) {
          headers.putAll(additionalAvroHeaders);
        }
        headers.putAll(extractHeaders(eventObject));

        Event event = EventBuilder.withBody(body.trim(), UTF_8, headers);

        flumeManager.send(event);
      } catch (Exception e) {
        addError(e.getLocalizedMessage(), e);
      }
    }

  }

  private Map extractHeaders(ILoggingEvent eventObject) {
    Map headers = new HashMap(10);
    headers.put("timestamp", Long.toString(eventObject.getTimeStamp()));
    headers.put("type", eventObject.getLevel().toString());
    headers.put("logger", eventObject.getLoggerName());
    headers.put("message", eventObject.getMessage());
    headers.put("level", eventObject.getLevel().toString());
    try {
      headers.put("host", resolveHostname());
    } catch (UnknownHostException e) {
      addWarn(e.getMessage());
    }
    headers.put("thread", eventObject.getThreadName());
    if (StringUtils.isNotEmpty(application)) {
      headers.put("application", application);
    }

    if (StringUtils.isNotEmpty(type)) {
      headers.put("type", type);
    }

    return headers;
  }

  private String resolveHostname() throws UnknownHostException {
    return hostname != null ? hostname : InetAddress.getLocalHost().getHostName();
  }

  private String resolveApplication() {
    return System.getProperty("application.name");
  }
}