at.molindo.esi4j.rebuild.scrutineer.ScrutineerRebuildProcessor Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of esi4j Show documentation
There is a newer version: 3.0.2
/**
 * Copyright 2010 Molindo GmbH
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package at.molindo.esi4j.rebuild.scrutineer;

import org.apache.commons.lang.SystemUtils;
import org.elasticsearch.client.Client;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import at.molindo.esi4j.core.Esi4JOperation;
import at.molindo.esi4j.core.internal.InternalIndex;
import at.molindo.esi4j.mapping.TypeMapping;
import at.molindo.esi4j.rebuild.Esi4JRebuildProcessor;
import at.molindo.esi4j.rebuild.Esi4JRebuildSession;
import at.molindo.esi4j.rebuild.util.BulkIndexHelper;

import com.aconex.scrutineer.IdAndVersion;
import com.aconex.scrutineer.IdAndVersionFactory;
import com.aconex.scrutineer.IdAndVersionStream;
import com.aconex.scrutineer.IdAndVersionStreamVerifierListener;
import com.aconex.scrutineer.elasticsearch.ElasticSearchDownloader;
import com.aconex.scrutineer.elasticsearch.ElasticSearchIdAndVersionStream;
import com.aconex.scrutineer.elasticsearch.ElasticSearchSorter;
import com.aconex.scrutineer.elasticsearch.IdAndVersionDataReaderFactory;
import com.aconex.scrutineer.elasticsearch.IdAndVersionDataWriterFactory;
import com.aconex.scrutineer.elasticsearch.IteratorFactory;
import com.fasterxml.sort.DataReaderFactory;
import com.fasterxml.sort.DataWriterFactory;
import com.fasterxml.sort.SortConfig;
import com.fasterxml.sort.Sorter;
import com.fasterxml.sort.util.NaturalComparator;

public class ScrutineerRebuildProcessor implements Esi4JRebuildProcessor {

	private static final int DEFAULT_SORT_MEM = 256 * 1024 * 1024;
	private static final int DEFAULT_BATCH_SIZE = 1000;

	private static final Logger log = LoggerFactory.getLogger(ScrutineerRebuildProcessor.class);

	@Override
	public boolean isSupported(Esi4JRebuildSession rebuildSession) {
		return rebuildSession.isOrdered();
	}

	@Override
	public void rebuild(InternalIndex index, final Esi4JRebuildSession rebuildSession) {
		index.updateMapping(rebuildSession.getType());

		index.execute(new Esi4JOperation() {

			@Override
			public Void execute(Client client, String indexName, Esi4JOperation.OperationContext context) {

				final TypeMapping mapping = context.findTypeMapping(rebuildSession.getType());

				ModuleIdAndVersionStream moduleStream = new ModuleIdAndVersionStream(rebuildSession,
						DEFAULT_BATCH_SIZE, mapping);

				IdAndVersionFactory factory = new MappedObjectIdAndVersionFactory(mapping);

				ElasticSearchSorter elasticSearchSorter = new ElasticSearchSorter(createSorter(factory));
				IteratorFactory iteratorFactory = new IteratorFactory(factory);
				String workingDirectory = SystemUtils.getJavaIoTmpDir().getAbsolutePath();

				ElasticSearchIdAndVersionStream esStream = new ElasticSearchIdAndVersionStream(
						new ElasticSearchDownloader(client, indexName, "_type:" + mapping.getTypeAlias(), factory),
						elasticSearchSorter, iteratorFactory, workingDirectory);

				long start = System.currentTimeMillis();

				BulkIndexHelper h = new BulkIndexHelper().setMaxRunning(2);
				h.setResponseHandler(new BulkIndexHelper.IResponseHandler() {

					@Override
					public void handle(String id, String type) {
						if ("delete".equals(type)) {
							onDelete(mapping.getTypeClass(), mapping.toId(id));
						} else {
							onIndex(mapping.getTypeClass(), mapping.toId(id));
						}
					}

				});

				VerifierListener listener = new VerifierListener(client, indexName, context, mapping, h,
						DEFAULT_BATCH_SIZE);

				try {
					verify(moduleStream, esStream, listener);
				} finally {
					/*
					 * TODO let IdAndVersionStreamVerifier do that as it does
					 * with streams
					 */
					listener.close();
				}

				try {
					h.await();

					long seconds = (System.currentTimeMillis() - start) / 1000;

					// logging
					StringBuilder logMsg = new StringBuilder("finished indexing of ").append(h.getSucceeded())
							.append(" objects of type ").append(rebuildSession.getType().getName()).append(" in ")
							.append(seconds).append(" seconds");

					if (h.getFailed() > 0) {
						logMsg.append(" (").append(h.getFailed()).append(" failed)");
						log.warn(logMsg.toString());
					} else {
						log.info(logMsg.toString());
					}

				} catch (InterruptedException e) {
					log.error("awaiting completion of indexing has been interrupted", e);
				}

				return null;
			}
		});
	}

	private Sorter createSorter(IdAndVersionFactory factory) {
		SortConfig sortConfig = new SortConfig().withMaxMemoryUsage(DEFAULT_SORT_MEM);
		DataReaderFactory dataReaderFactory = new IdAndVersionDataReaderFactory(factory);
		DataWriterFactory dataWriterFactory = new IdAndVersionDataWriterFactory();
		return new Sorter(sortConfig, dataReaderFactory, dataWriterFactory,
				new NaturalComparator());
	}

	private void verify(IdAndVersionStream primaryStream, IdAndVersionStream secondaryStream,
			IdAndVersionStreamVerifierListener listener) {
		new IdAndVersionStreamVerifier().verify(primaryStream, secondaryStream, listener);
	}

	protected void onIndex(Class type, Object id) {
	}

	protected void onDelete(Class type, Object id) {
	}

}