com.thematchbox.river.actions.AbstractIndexAction Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of AbstractRiver Show documentation
Show all versions of AbstractRiver Show documentation
This project contains an abstract implementation of an ElasticSearch River and is used as a basis for custom river implementations.
package com.thematchbox.river.actions;
/* Copyright 2015 theMatchBox
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
import com.thematchbox.river.data.ContentDelegator;
import com.thematchbox.river.data.PersistentObject;
import com.thematchbox.river.sessions.SessionDelegator;
import com.thematchbox.river.sessions.SessionDelegatorFactory;
import com.thematchbox.river.sessions.SessionException;
import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
import org.elasticsearch.action.admin.indices.mapping.delete.DeleteMappingRequest;
import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse;
import org.elasticsearch.action.bulk.BulkItemResponse;
import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.index.IndexRequestBuilder;
import org.elasticsearch.client.Client;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
public abstract class AbstractIndexAction, T extends PersistentObject, D extends SessionDelegator, F extends SessionDelegatorFactory> implements IndexAction {
public static final Logger logger = LoggerFactory.getLogger(AbstractIndexAction.class);
private int batchSize;
private Class clazz;
private ContentDelegator contentDelegator;
private F sessionDelegatorFactory;
protected AbstractIndexAction(int batchSize, Class clazz, ContentDelegator contentDelegator, F sessionDelegatorFactory) {
this.batchSize = batchSize;
this.clazz = clazz;
this.contentDelegator = contentDelegator;
this.sessionDelegatorFactory = sessionDelegatorFactory;
}
protected Class getClazz() {
return clazz;
}
protected abstract List getIdsToReindex(Client client, IndexJob indexJob, D sessionDelegator) throws SessionException;
protected abstract boolean deleteFirst();
@Override
public void execute(Client client, IndexJob indexJob, IndexFeedback feedback) {
String indexName = indexJob.indexKey.indexName;
String indexType = indexJob.indexKey.indexType;
try {
D sessionDelegator = sessionDelegatorFactory.create();
try {
//
// First load all ids
//
List ids;
try {
sessionDelegator.openSession();
ids = getIdsToReindex(client, indexJob, sessionDelegator);
} finally {
sessionDelegator.closeSession();
}
feedback.init(ids.size());
// Delete entire index if requested
boolean hasIndex = IndexTools.hasIndex(client, indexName);
boolean hasIndexAndType = hasIndex && IndexTools.hasType(client, indexName, indexType);
if (deleteFirst()) {
if (hasIndexAndType) {
client.admin().indices().deleteMapping(new DeleteMappingRequest(indexName).types(indexType)).actionGet();
hasIndexAndType = false;
}
} else {
if (ids.isEmpty()) {
feedback.setIgnore(true);
return;
}
}
// Create empty index (to make sure correct settings are used)
if (!hasIndex) {
CreateIndexResponse response = IndexTools.createEmptyIndex(client, indexName);
if (!response.isAcknowledged()) {
logger.error("Failed to create index " + indexName);
}
}
if (!hasIndexAndType) {
XContentBuilder indexMapping = contentDelegator.getMapping(indexType);
if (indexMapping != null) {
PutMappingResponse response = IndexTools.createMapping(client, indexName, indexType, indexMapping);
if (!response.isAcknowledged()) {
logger.error("Failed to create index mapping for " + indexName + "/" + indexType);
}
}
}
//
// Some objects are returned multiple times because of table joining. IDs are checked to avoid multiple index insertions.
//
//
int from = 0;
S previousID = null;
while (ids != null && from < ids.size()) {
try {
int to = Math.min(ids.size(), from + batchSize);
List subList = ids.subList(from, to);
sessionDelegator.openSession();
//noinspection unchecked
List objects = sessionDelegator.getObjects(clazz, subList);
Collections.sort(objects, new Comparator() {
@Override
public int compare(T o1, T o2) {
S s1 = o1.getId();
S s2 = o2.getId();
return s1.compareTo(s2);
}
});
BulkRequestBuilder bulkRequest = client.prepareBulk();
for (T t : objects) {
try {
// This code prevents that the same object is indexed multiple times. In case of the eager fetching of some class members,
// the same object can be returned more than once. This is a side-effect of the sql join query that is executed behind the
// scenes for eager fetching.
if (t.getId().equals(previousID)) {
continue;
}
previousID = t.getId();
XContentBuilder xContentBuilder = contentDelegator.getXContentBuilder(t);
IndexRequestBuilder indexRequestBuilder = client.prepareIndex(indexName, indexType, t.getId().toString()).setSource(xContentBuilder);
bulkRequest.add(indexRequestBuilder);
} catch (Exception e) {
feedback.incrementFailCount(1);
logger.error(e.getMessage(), e);
}
}
BulkResponse bulkResponse = bulkRequest.execute().actionGet();
boolean hasFailures = false;
for (BulkItemResponse response : bulkResponse.getItems()) {
if (response.isFailed()) {
hasFailures = true;
feedback.incrementFailCount(1);
} else {
feedback.incrementSuccessCount(1);
}
}
if (hasFailures) {
logger.error(bulkResponse.buildFailureMessage());
}
} finally {
from += batchSize;
sessionDelegator.closeSession();
}
}
} catch (SessionException e) {
sessionDelegator.reset();
throw e;
}
} catch (SessionException | IOException e) {
logger.error(e.getMessage(), e);
}
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy