org.appenders.log4j2.elasticsearch.jest.BufferedJestHttpObjectFactory Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of log4j2-elasticsearch-jest Show documentation
Log4j2 Appender plugin pushing logs in batches to Elasticsearch (2.x/5.x/6.x) clusters
The newest version!
package org.appenders.log4j2.elasticsearch.jest;

/*-
 * #%L
 * %%
 * Copyright (C) 2018 Rafal Foltynski
 * %%
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * 
 *      http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 * #L%
 */


import io.searchbox.action.AbstractDocumentTargetedAction;
import io.searchbox.client.JestClient;
import io.searchbox.client.JestClientFactory;
import io.searchbox.client.JestResult;
import io.searchbox.client.JestResultHandler;
import io.searchbox.core.Bulk;
import io.searchbox.core.DocumentResult;
import org.apache.logging.log4j.core.config.ConfigurationException;
import org.apache.logging.log4j.core.config.Node;
import org.apache.logging.log4j.core.config.plugins.Plugin;
import org.apache.logging.log4j.core.config.plugins.PluginBuilderFactory;
import org.apache.logging.log4j.core.config.plugins.PluginElement;
import org.appenders.log4j2.elasticsearch.BatchOperations;
import org.appenders.log4j2.elasticsearch.ClientObjectFactory;
import org.appenders.log4j2.elasticsearch.ClientProvider;
import org.appenders.log4j2.elasticsearch.FailoverPolicy;
import org.appenders.log4j2.elasticsearch.GenericItemSourcePool;
import org.appenders.log4j2.elasticsearch.ItemSourceFactory;
import org.appenders.log4j2.elasticsearch.JacksonMixIn;
import org.appenders.log4j2.elasticsearch.JacksonMixInPlugin;
import org.appenders.log4j2.elasticsearch.PooledItemSourceFactory;
import org.appenders.log4j2.elasticsearch.failover.FailedItemOps;
import org.appenders.log4j2.elasticsearch.jest.failover.BufferedHttpFailedItemOps;
import org.appenders.log4j2.elasticsearch.metrics.Measured;
import org.appenders.log4j2.elasticsearch.metrics.MetricsRegistry;

import java.util.function.Function;

import static org.appenders.core.logging.InternalLogging.getLogger;

@Plugin(name = BufferedJestHttpObjectFactory.PLUGIN_NAME, category = Node.CATEGORY, elementType = ClientObjectFactory.ELEMENT_TYPE, printObject = true)
public class BufferedJestHttpObjectFactory extends JestHttpObjectFactory {

    public static final String PLUGIN_NAME = "JestBufferedHttp";

    private volatile State state = State.STOPPED;

    private final PooledItemSourceFactory itemSourceFactory;

    private final JacksonMixIn[] mixIns;

    protected BufferedJestHttpObjectFactory(Builder builder) {
        super(builder);
        this.itemSourceFactory = builder.pooledItemSourceFactory;
        this.mixIns = builder.mixIns;
    }

    @Override
    public Function createFailureHandler(FailoverPolicy failover) {
        return bulk -> {

            final long start = System.currentTimeMillis();
            final BufferedBulk bufferedBulk = (BufferedBulk)bulk;

            metrics.batchFailed();
            final int batchSize = bufferedBulk.getActions().size();
            metrics.itemsFailed(batchSize);

            getLogger().warn(String.format("Batch of %s items failed. Redirecting to %s", batchSize, failover.getClass().getName()));

            try {
                bufferedBulk.getActions().stream()
                        .map(item -> failedItemOps.createItem((BufferedIndex) item))
                        .forEach(failover::deliver);
                return true;
            } catch (Exception e) {
                getLogger().error("Unable to execute failover", e);
                return false;
            } finally {
                metrics.failoverTookMs(System.currentTimeMillis() - start);
            }

        };
    }

    @Override
    public BatchOperations createBatchOperations() {
        if (dataStreamsEnabled) {
            return new BufferedBulkOperations(itemSourceFactory, mixIns, true);
        }
        return new BufferedBulkOperations(itemSourceFactory, mixIns, mappingType);
    }

    protected JestResultHandler createResultHandler(Bulk bulk, Function failureHandler) {
        return new JestResultHandler() {

            @Override
            public void completed(JestResult result) {

                backoffPolicy.deregister(bulk);

                if (result.isSucceeded()) {
                    metrics.itemsDelivered(getBatchSize(bulk));
                } else {
                    getLogger().warn(result.getErrorMessage());
                    // TODO: filter only failed items when retry is ready.
                    // failing whole bulk for now
                    failureHandler.apply(bulk);
                }
                ((BufferedBulk)bulk).completed();
            }

            @Override
            public void failed(Exception ex) {
                getLogger().warn(ex.getMessage(), ex);
                backoffPolicy.deregister(bulk);
                failureHandler.apply(bulk);
                ((BufferedBulk)bulk).completed();
            }

        };
    }

    @Override
    int getBatchSize(final Bulk bulk) {
        return ((BufferedBulk)bulk).getActions().size();
    }

    @PluginBuilderFactory
    public static Builder newBuilder() {
        return new Builder();
    }

    // visible for testing
    ClientProvider getClientProvider(WrappedHttpClientConfig.Builder clientConfigBuilder) {
        return new JestClientProvider(clientConfigBuilder) {
            @Override
            public JestClient createClient() {
                WrappedHttpClientConfig wrappedHttpClientConfig = clientConfigBuilder.build();
                JestClientFactory jestClientFactory = new BufferedJestClientFactory(wrappedHttpClientConfig);
                return jestClientFactory.getObject();
            }
        };
    }

    @Override
    public void register(MetricsRegistry registry) {
        super.register(registry);
        Measured.of(itemSourceFactory).register(registry);
    }

    @Override
    public void deregister() {
        super.deregister();
        Measured.of(itemSourceFactory).deregister();
    }

    public static class Builder extends JestHttpObjectFactory.Builder {

        @PluginElement(ItemSourceFactory.ELEMENT_TYPE)
        protected PooledItemSourceFactory pooledItemSourceFactory;

        @PluginElement(JacksonMixInPlugin.ELEMENT_TYPE)
        private JacksonMixIn[] mixIns = new JacksonMixIn[0];

        public Builder() {
            metricsFactory.configure(GenericItemSourcePool.metricConfigs(false));
        }

        @Override
        public BufferedJestHttpObjectFactory build() {

            validate();

            resolveLazyProperties(); // FIXME: this is just wrong.. but should be solved once Log4j2 annotations are pushed to Plugins

            return new BufferedJestHttpObjectFactory(this);
        }

        protected void validate() {

            super.validate();

            if (pooledItemSourceFactory == null) {
                throw new ConfigurationException("No PooledItemSourceFactory configured for BufferedJestHttpObjectFactory");
            }
        }

        protected FailedItemOps> failedItemOps() {
            return new BufferedHttpFailedItemOps();
        }

        public Builder withItemSourceFactory(PooledItemSourceFactory pooledItemSourceFactory) {
            this.pooledItemSourceFactory = pooledItemSourceFactory;
            return this;
        }

        public Builder withMixIns(JacksonMixIn[] mixIns) {
            this.mixIns = mixIns;
            return this;
        }

    }

    // ==========
    // LIFECYCLE
    // ==========

    @Override
    public void start() {
        super.start();
        if (!itemSourceFactory.isStarted()) {
            itemSourceFactory.start();
        }
        state = State.STARTED;
    }

    @Override
    public void stop() {
        super.stop();
        if (!itemSourceFactory.isStopped()) {
            itemSourceFactory.stop();
        }
        state = State.STOPPED;
    }

    @Override
    public boolean isStarted() {
        return state == State.STARTED;
    }

    @Override
    public boolean isStopped() {
        return state == State.STOPPED;
    }

}