Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance.
Project price only 1 $
You can buy this project and download/modify it how often you want.
com.v5analytics.simpleorm.AccumuloSimpleOrmSession Maven / Gradle / Ivy
package com.v5analytics.simpleorm;
import org.apache.accumulo.core.client.*;
import org.apache.accumulo.core.client.Scanner;
import org.apache.accumulo.core.client.security.tokens.PasswordToken;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.iterators.user.RowDeletingIterator;
import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.core.security.ColumnVisibility;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.io.Text;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.*;
import static com.google.common.base.Preconditions.checkNotNull;
public class AccumuloSimpleOrmSession extends SimpleOrmSession {
private static final Logger LOGGER = LoggerFactory.getLogger(AccumuloSimpleOrmSession.class);
public static final String ACCUMULO_INSTANCE_NAME = "simpleOrm.accumulo.instanceName";
public static final String ACCUMULO_USER = "simpleOrm.accumulo.username";
public static final String ACCUMULO_PASSWORD = "simpleOrm.accumulo.password";
public static final String ZK_SERVER_NAMES = "simpleOrm.accumulo.zookeeperServerNames";
private static final String ROW_DELETING_ITERATOR_NAME = RowDeletingIterator.class.getSimpleName();
private static final int ROW_DELETING_ITERATOR_PRIORITY = 7;
private Connector connector;
private BatchWriterConfig batchWriterConfig = new BatchWriterConfig();
private final Set initializedTables = new HashSet<>();
private final Map batchWriters = new HashMap<>();
private String tablePrefix;
public AccumuloSimpleOrmSession() {
}
public AccumuloSimpleOrmSession(Map properties) {
setTablePrefix(properties);
}
public void init(Map properties) {
try {
String zkServerNames;
if (properties.get(ZK_SERVER_NAMES) instanceof Collection) {
zkServerNames = StringUtils.join((Collection) properties.get(ZK_SERVER_NAMES), ",");
} else {
zkServerNames = (String) properties.get(ZK_SERVER_NAMES);
}
checkNotNull(zkServerNames, "Could not find config: " + ZK_SERVER_NAMES);
String accumuloInstanceName = (String) properties.get(ACCUMULO_INSTANCE_NAME);
checkNotNull(accumuloInstanceName, "Could not find config: " + ACCUMULO_INSTANCE_NAME);
ZooKeeperInstance zk = new ZooKeeperInstance(accumuloInstanceName, zkServerNames);
String username = (String) properties.get(ACCUMULO_USER);
String password = (String) properties.get(ACCUMULO_PASSWORD);
connector = zk.getConnector(username, new PasswordToken(password));
setTablePrefix(properties);
} catch (Exception e) {
throw new SimpleOrmException("Failed to init", e);
}
}
@Override
public SimpleOrmContext createContext(String... authorizations) {
return new AccumuloSimpleOrmContext(new Authorizations(authorizations));
}
@Override
public void close() {
for (BatchWriter writer : this.batchWriters.values()) {
try {
writer.close();
} catch (MutationsRejectedException e) {
throw new SimpleOrmException("Could not close batch writer", e);
}
}
}
private void setTablePrefix(Map properties) {
tablePrefix = (String) properties.get(TABLE_PREFIX);
if (tablePrefix == null) {
tablePrefix = "";
}
}
@Override
public Iterable findAll(final Class rowClass, SimpleOrmContext context) {
try {
ModelMetadata modelMetadata = ModelMetadata.getModelMetadata(rowClass);
final Scanner scanner = createScanner(getTableName(modelMetadata), (AccumuloSimpleOrmContext) context);
return scannerToRows(scanner, modelMetadata);
} catch (TableNotFoundException e) {
throw new SimpleOrmException("Could not find all", e);
}
}
@Override
public void delete(final Class rowClass, String id, SimpleOrmContext context) {
ModelMetadata modelMetadata = ModelMetadata.getModelMetadata(rowClass);
LOGGER.trace("deleteRow called with parameters: tableName=?, id=?", getTableName(modelMetadata), id);
// In most instances (e.g., when reading is not necessary), the
// RowDeletingIterator gives better performance than the deleting
// mutation. This is due to the fact that Deleting mutations marks each
// entry with a delete marker. Using the iterator marks a whole row with
// a single mutation.
try {
BatchWriter writer = connector.createBatchWriter(getTableName(modelMetadata), batchWriterConfig);
try {
Mutation mutation = new Mutation(id);
mutation.put(new byte[0], new byte[0], RowDeletingIterator.DELETE_ROW_VALUE.get());
writer.addMutation(mutation);
writer.flush();
} catch (AccumuloException ae) {
throw new SimpleOrmException("Could not delete", ae);
} finally {
writer.close();
}
} catch (Exception mre) {
throw new SimpleOrmException("Could not delete: " + rowClass.getName() + " id " + id, mre);
}
}
@Override
public String getTablePrefix() {
return tablePrefix;
}
@Override
public Iterable getTableList(SimpleOrmContext simpleOrmContext) {
return this.connector.tableOperations().list();
}
@Override
public void deleteTable(String table, SimpleOrmContext simpleOrmContext) {
try {
this.connector.tableOperations().delete(table);
} catch (Exception e) {
throw new SimpleOrmException("Could not delete table: " + table, e);
}
}
@Override
public void clearTable(String table, SimpleOrmContext simpleOrmContext) {
try {
this.connector.tableOperations().deleteRows(table, null, null);
} catch (Exception e) {
throw new SimpleOrmException("Could not clear table: " + table, e);
}
}
private Iterable scannerToRows(final Scanner scanner, final ModelMetadata modelMetadata) {
return new Iterable() {
@Override
public Iterator iterator() {
final RowIterator rowIterator = new RowIterator(scanner);
return new Iterator() {
@Override
public boolean hasNext() {
return rowIterator.hasNext();
}
@Override
public T next() {
Iterator> row = rowIterator.next();
return createObjectFromRow(modelMetadata, row);
}
@Override
public void remove() {
rowIterator.remove();
}
};
}
};
}
@Override
public T findById(Class rowClass, String id, SimpleOrmContext context) {
try {
ModelMetadata modelMetadata = ModelMetadata.getModelMetadata(rowClass);
final Scanner scanner = createScanner(getTableName(modelMetadata), (AccumuloSimpleOrmContext) context);
try {
scanner.setRange(new Range(id));
Iterator rows = scannerToRows(scanner, modelMetadata).iterator();
if (!rows.hasNext()) {
return null;
}
T result = rows.next();
if (rows.hasNext()) {
throw new SimpleOrmException("Too many rows returned for a single row query (rowKey: " + id + ")");
}
return result;
} finally {
scanner.close();
}
} catch (TableNotFoundException e) {
throw new SimpleOrmException("Could not find by id", e);
}
}
@Override
public Iterable findByIdStartsWith(Class rowClass, String idPrefix, SimpleOrmContext context) {
try {
ModelMetadata modelMetadata = ModelMetadata.getModelMetadata(rowClass);
final Scanner scanner = createScanner(getTableName(modelMetadata), (AccumuloSimpleOrmContext) context);
scanner.setRange(Range.prefix(idPrefix));
return scannerToRows(scanner, modelMetadata);
} catch (TableNotFoundException e) {
throw new SimpleOrmException("Could not find by id starts with", e);
}
}
@Override
public void alterVisibility(T obj, String currentVisibility, String newVisibility, SimpleOrmContext context) {
try {
ModelMetadata modelMetadata = ModelMetadata.getModelMetadata(obj);
final Scanner scanner = createScanner(getTableName(modelMetadata), (AccumuloSimpleOrmContext) context);
scanner.setRange(new Range(modelMetadata.getId(obj)));
RowIterator rowIterator = new RowIterator(scanner);
BatchWriter writer = getBatchWriter(getTableName(modelMetadata));
ColumnVisibility newColumnVisibility = new ColumnVisibility(newVisibility);
while (rowIterator.hasNext()) {
Iterator> row = rowIterator.next();
alterVisibilityRow(writer, row, currentVisibility, newColumnVisibility);
}
writer.flush();
} catch (Throwable ex) {
throw new SimpleOrmException("Could not alterVisibility", ex);
}
}
private void alterVisibilityRow(BatchWriter writer, Iterator> row, String currentVisibility, ColumnVisibility newVisibility) throws MutationsRejectedException {
Mutation mAdd = null;
Mutation mDelete = null;
boolean hasItems = false;
while (row.hasNext()) {
Map.Entry column = row.next();
if (mAdd == null) {
mAdd = new Mutation(column.getKey().getRow());
mDelete = new Mutation(column.getKey().getRow());
}
if (column.getKey().getColumnVisibility().toString().equals(currentVisibility)) {
hasItems = true;
mAdd.put(
column.getKey().getColumnFamily(),
column.getKey().getColumnQualifier(),
newVisibility,
column.getKey().getTimestamp(),
column.getValue()
);
mDelete.putDelete(
column.getKey().getColumnFamily(),
column.getKey().getColumnQualifier()
);
}
}
if (hasItems) {
writer.addMutation(mAdd);
writer.addMutation(mDelete);
}
}
@Override
public void save(T obj, String visibility, SimpleOrmContext context) {
try {
ModelMetadata modelMetadata = ModelMetadata.getModelMetadata(obj);
ModelMetadata.Type modelMetadataType = modelMetadata.getTypeFromObject(obj);
ensureTableIsInitialized(getTableName(modelMetadata));
BatchWriter writer = getBatchWriter(getTableName(modelMetadata));
ColumnVisibility columnVisibility = new ColumnVisibility(visibility);
writer.addMutation(getMutationForObject(modelMetadata, modelMetadataType, obj, columnVisibility));
writer.flush();
} catch (MutationsRejectedException e) {
throw new SimpleOrmException("Error occurred when writing mutation", e);
}
}
@Override
public void saveMany(Collection objs, String visibility, SimpleOrmContext context) {
try {
if (objs.size() == 0) {
return;
}
BatchWriter writer = null;
ModelMetadata modelMetadata = null;
ColumnVisibility columnVisibility = new ColumnVisibility(visibility);
for (T obj : objs) {
if (modelMetadata == null) {
modelMetadata = ModelMetadata.getModelMetadata(obj);
ensureTableIsInitialized(getTableName(modelMetadata));
}
if (writer == null) {
writer = getBatchWriter(getTableName(modelMetadata));
}
writer.addMutation(getMutationForObject(modelMetadata, modelMetadata.getTypeFromObject(obj), obj, columnVisibility));
}
if (writer != null) {
writer.flush();
}
} catch (MutationsRejectedException e) {
throw new SimpleOrmException("Error occurred when writing mutation", e);
}
}
public String getTableName(Class rowClass) {
ModelMetadata modelMetadata = ModelMetadata.getModelMetadata(rowClass);
return getTableName(modelMetadata);
}
private String getTableName(ModelMetadata modelMetadata) {
return tablePrefix + modelMetadata.getTableName();
}
public Mutation getMutationForObject(T obj, String visibility) {
ModelMetadata modelMetadata = ModelMetadata.getModelMetadata(obj);
ModelMetadata.Type type = modelMetadata.getTypeFromObject(obj);
return getMutationForObject(modelMetadata, type, obj, new ColumnVisibility(visibility));
}
private Mutation getMutationForObject(ModelMetadata modelMetadata, ModelMetadata.Type type, T obj, ColumnVisibility columnVisibility) {
String rowKey = modelMetadata.getId(obj);
Mutation mutation = new Mutation(rowKey);
for (Map.Entry> columnFamilyFields : type.getFields().entrySet()) {
Text columnFamilyName = new Text(columnFamilyFields.getKey());
for (Map.Entry columnField : columnFamilyFields.getValue().entrySet()) {
Text columnName = new Text(columnField.getKey());
byte[] value = columnField.getValue().get(obj);
if (value == null) {
continue;
}
mutation.put(columnFamilyName, columnName, columnVisibility, new Value(value));
}
}
return mutation;
}
private BatchWriter getBatchWriter(String tableName) {
try {
synchronized (batchWriters) {
BatchWriter writer = batchWriters.get(tableName);
if (writer == null) {
writer = connector.createBatchWriter(tableName, batchWriterConfig);
batchWriters.put(tableName, writer);
}
return writer;
}
} catch (TableNotFoundException e) {
throw new SimpleOrmException("Could not find table: " + tableName, e);
}
}
private T createObjectFromRow(ModelMetadata modelMetadata, Iterator> row) {
try {
boolean first = true;
String discriminatorValue;
Iterator columns;
if (modelMetadata.getDiscriminatorColumnName() == null) {
discriminatorValue = ModelMetadata.DEFAULT_DISCRIMINATOR;
columns = ColumnData.createIterator(row);
} else {
RowData rowData = new RowData(row);
byte[] discriminatorValueBytes = rowData.getColumnValue(modelMetadata.getDiscriminatorColumnFamily(), modelMetadata.getDiscriminatorColumnName());
checkNotNull(discriminatorValueBytes, "Could not find discriminatorValue " + modelMetadata.getDiscriminatorColumnFamily() + "." + modelMetadata.getDiscriminatorColumnName());
discriminatorValue = new String(discriminatorValueBytes);
columns = rowData.createIterator();
}
ModelMetadata.Type type = modelMetadata.getType(discriminatorValue);
T result = type.newInstance();
while (columns.hasNext()) {
ColumnData column = columns.next();
if (first) {
modelMetadata.setIdField(result, column.getRowKey());
}
String columnFamily = column.getColumnFamily();
String columnName = column.getColumnName();
ModelMetadata.Field field = type.getFieldForColumn(columnFamily, columnName);
if (field != null) {
field.set(result, column.getValue());
}
first = false;
}
return result;
} catch (Exception e) {
throw new SimpleOrmException("Could not create class: " + modelMetadata.toString(), e);
}
}
private static class RowData {
private String rowKey;
private final Map> data = new HashMap<>();
public RowData(Iterator> row) {
while (row.hasNext()) {
Map.Entry column = row.next();
if (rowKey == null) {
rowKey = column.getKey().getRow().toString();
}
addColumn(column);
}
}
private void addColumn(Map.Entry column) {
Map columnFamilyData = getColumnFamilyData(column.getKey().getColumnFamily().toString());
columnFamilyData.put(column.getKey().getColumnQualifier().toString(), column.getValue().get());
}
private Map getColumnFamilyData(String columnFamilyName) {
Map columnFamilyData = data.get(columnFamilyName);
if (columnFamilyData == null) {
columnFamilyData = new HashMap<>();
data.put(columnFamilyName, columnFamilyData);
}
return columnFamilyData;
}
public byte[] getColumnValue(String columnFamily, String columnName) {
return getColumnFamilyData(columnFamily).get(columnName);
}
public Iterator createIterator() {
List results = new ArrayList<>();
for (Map.Entry> columnFamily : data.entrySet()) {
String columnFamilyName = columnFamily.getKey();
for (Map.Entry column : columnFamily.getValue().entrySet()) {
String columnName = column.getKey();
byte[] value = column.getValue();
results.add(new ColumnData(rowKey, columnFamilyName, columnName, value));
}
}
return results.iterator();
}
}
private static class ColumnData {
private final String rowKey;
private final String columnFamily;
private final String columnName;
private final byte[] value;
private ColumnData(String rowKey, String columnFamily, String columnName, byte[] value) {
this.rowKey = rowKey;
this.columnFamily = columnFamily;
this.columnName = columnName;
this.value = value;
}
public String getRowKey() {
return rowKey;
}
public String getColumnFamily() {
return columnFamily;
}
public String getColumnName() {
return columnName;
}
public byte[] getValue() {
return value;
}
public static Iterator createIterator(final Iterator> row) {
return new Iterator() {
@Override
public boolean hasNext() {
return row.hasNext();
}
@Override
public ColumnData next() {
Map.Entry column = row.next();
String rowKey = column.getKey().getRow().toString();
String columnFamily = column.getKey().getColumnFamily().toString();
String columnName = column.getKey().getColumnQualifier().toString();
byte[] value = column.getValue().get();
return new ColumnData(rowKey, columnFamily, columnName, value);
}
@Override
public void remove() {
throw new SimpleOrmException("Not supported");
}
};
}
}
private Scanner createScanner(String tableName, AccumuloSimpleOrmContext context) throws TableNotFoundException {
ensureTableIsInitialized(tableName);
Scanner scanner = connector.createScanner(tableName, context.getAuthorizations());
IteratorSetting iteratorSetting = new IteratorSetting(
100,
RowDeletingIterator.class.getSimpleName(),
RowDeletingIterator.class
);
scanner.addScanIterator(iteratorSetting);
return scanner;
}
private void ensureTableIsInitialized(String tableName) {
try {
if (initializedTables.contains(tableName)) {
return;
}
if (!connector.tableOperations().list().contains(tableName)) {
LOGGER.info("creating table: " + tableName);
try {
connector.tableOperations().create(tableName);
} catch (TableExistsException e) {
// This sometimes happens. Just ignore since the table is present.
}
}
IteratorSetting is = new IteratorSetting(ROW_DELETING_ITERATOR_PRIORITY, ROW_DELETING_ITERATOR_NAME, RowDeletingIterator.class);
if (!connector.tableOperations().listIterators(tableName).containsKey(ROW_DELETING_ITERATOR_NAME)) {
try {
connector.tableOperations().attachIterator(tableName, is);
} catch (AccumuloException e) {
absorbIteratorNameConflictException(e);
}
}
initializedTables.add(tableName);
} catch (Exception e) {
throw new SimpleOrmException("Could not initialize table", e);
}
}
private void absorbIteratorNameConflictException(AccumuloException ex) throws AccumuloException {
// Like the TableExistsException, this sometimes happens.
// The exception inspection here depends on the implementation of
// org.apache.accumulo.core.client.impl.TableOperationsHelper#checkIteratorConflicts(), and is therefore
// fragile.
boolean isErrorOk = false;
Throwable cause = ex.getCause();
if (cause instanceof IllegalArgumentException) {
if (cause.getMessage().contains("iterator name conflict")) {
isErrorOk = true;
}
}
if (!isErrorOk) {
throw ex;
}
}
}
