Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance.
Project price only 1 $
You can buy this project and download/modify it how often you want.
com.hazelcast.jet.impl.connector.HazelcastWriters Maven / Gradle / Ivy
/*
* Copyright (c) 2008-2018, Hazelcast, Inc. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.hazelcast.jet.impl.connector;
import com.hazelcast.cache.ICache;
import com.hazelcast.client.config.ClientConfig;
import com.hazelcast.core.ExecutionCallback;
import com.hazelcast.core.HazelcastInstance;
import com.hazelcast.core.HazelcastInstanceNotActiveException;
import com.hazelcast.core.IList;
import com.hazelcast.core.IMap;
import com.hazelcast.jet.core.AbstractProcessor;
import com.hazelcast.jet.core.Processor;
import com.hazelcast.jet.core.ProcessorMetaSupplier;
import com.hazelcast.jet.core.ProcessorSupplier;
import com.hazelcast.jet.core.processor.SinkProcessors;
import com.hazelcast.jet.function.DistributedBiConsumer;
import com.hazelcast.jet.function.DistributedBiFunction;
import com.hazelcast.jet.function.DistributedBinaryOperator;
import com.hazelcast.jet.function.DistributedConsumer;
import com.hazelcast.jet.function.DistributedFunction;
import com.hazelcast.jet.impl.SerializationConstants;
import com.hazelcast.map.EntryBackupProcessor;
import com.hazelcast.map.EntryProcessor;
import com.hazelcast.nio.ObjectDataInput;
import com.hazelcast.nio.ObjectDataOutput;
import com.hazelcast.nio.serialization.IdentifiedDataSerializable;
import java.io.IOException;
import java.util.AbstractMap;
import java.util.AbstractSet;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
import java.util.stream.Stream;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import static com.hazelcast.client.HazelcastClient.newHazelcastClient;
import static com.hazelcast.jet.core.ProcessorMetaSupplier.preferLocalParallelismOne;
import static com.hazelcast.jet.function.DistributedFunctions.noopConsumer;
import static com.hazelcast.jet.impl.util.ExceptionUtil.sneakyThrow;
import static com.hazelcast.jet.impl.util.Util.callbackOf;
import static com.hazelcast.jet.impl.util.Util.tryIncrement;
import static java.util.stream.Collectors.toList;
/**
* This is private API. Check out the {@link SinkProcessors} class for
* public factory methods.
*/
public final class HazelcastWriters {
private HazelcastWriters() {
}
@Nonnull
@SuppressWarnings("unchecked")
public static ProcessorMetaSupplier mergeMapP(
@Nonnull String name,
@Nullable ClientConfig clientConfig,
@Nonnull DistributedFunction toKeyFn,
@Nonnull DistributedFunction toValueFn,
@Nonnull DistributedBinaryOperator mergeFn
) {
return updateMapP(name, clientConfig, toKeyFn, (V oldValue, T item) -> {
V newValue = toValueFn.apply(item);
if (oldValue == null) {
return newValue;
}
return mergeFn.apply(oldValue, newValue);
});
}
@Nonnull
@SuppressWarnings("unchecked")
public static ProcessorMetaSupplier updateMapP(
@Nonnull String name,
@Nullable ClientConfig clientConfig,
@Nonnull DistributedFunction toKeyFn,
@Nonnull DistributedBiFunction updateFn
) {
boolean isLocal = clientConfig == null;
return preferLocalParallelismOne(new HazelcastWriterSupplier<>(
serializableConfig(clientConfig),
index -> new ArrayList<>(),
ArrayList::add,
instance -> {
IMap map = instance.getMap(name);
Map tmpMap = new HashMap<>();
ApplyFnEntryProcessor entryProcessor = new ApplyFnEntryProcessor<>(tmpMap, updateFn);
return buffer -> {
try {
if (buffer.isEmpty()) {
return;
}
for (Object object : buffer) {
T item = (T) object;
K key = toKeyFn.apply(item);
// on duplicate key, we'll flush immediately
if (tmpMap.containsKey(key)) {
map.executeOnKeys(tmpMap.keySet(), entryProcessor);
tmpMap.clear();
}
tmpMap.put(key, item);
}
map.executeOnKeys(tmpMap.keySet(), entryProcessor);
tmpMap.clear();
} catch (HazelcastInstanceNotActiveException e) {
handleInstanceNotActive(instance, e, isLocal);
}
buffer.clear();
};
},
noopConsumer()
));
}
@Nonnull
@SuppressWarnings("unchecked")
public static ProcessorMetaSupplier updateMapP(
@Nonnull String name,
@Nullable ClientConfig clientConfig,
@Nonnull DistributedFunction toKeyFn,
@Nonnull DistributedFunction> toEntryProcessorFn
) {
boolean isLocal = clientConfig == null;
return preferLocalParallelismOne(new EntryProcessorWriterSupplier<>(
name,
serializableConfig(clientConfig),
toKeyFn,
toEntryProcessorFn,
isLocal
)
);
}
@Nonnull
@SuppressWarnings("unchecked")
public static ProcessorMetaSupplier writeMapP(@Nonnull String name, @Nullable ClientConfig clientConfig) {
boolean isLocal = clientConfig == null;
return preferLocalParallelismOne(new HazelcastWriterSupplier<>(
serializableConfig(clientConfig),
index -> new ArrayMap(),
ArrayMap::add,
instance -> {
IMap map = instance.getMap(name);
return buffer -> {
try {
map.putAll(buffer);
} catch (HazelcastInstanceNotActiveException e) {
handleInstanceNotActive(instance, e, isLocal);
}
buffer.clear();
};
},
noopConsumer()
));
}
@Nonnull
public static ProcessorMetaSupplier writeCacheP(@Nonnull String name, @Nullable ClientConfig clientConfig) {
boolean isLocal = clientConfig == null;
return preferLocalParallelismOne(new HazelcastWriterSupplier<>(
serializableConfig(clientConfig),
index -> new ArrayMap(),
ArrayMap::add,
CacheFlush.flushToCache(name, isLocal),
noopConsumer()
));
}
@Nonnull
public static ProcessorMetaSupplier writeListP(@Nonnull String name, @Nullable ClientConfig clientConfig) {
boolean isLocal = clientConfig == null;
return preferLocalParallelismOne(new HazelcastWriterSupplier<>(
serializableConfig(clientConfig),
index -> new ArrayList<>(),
ArrayList::add,
instance -> {
IList list = instance.getList(name);
return buffer -> {
try {
list.addAll(buffer);
} catch (HazelcastInstanceNotActiveException e) {
handleInstanceNotActive(instance, e, isLocal);
}
buffer.clear();
};
},
noopConsumer()
));
}
private static void handleInstanceNotActive(
HazelcastInstance instance, HazelcastInstanceNotActiveException e, boolean isLocal
) {
if (isLocal) {
// if we are writing to a local instance, we can safely ignore this exception
// as the job will eventually restart on its own.
instance.getLoggingService().getLogger(HazelcastWriters.class).fine(
"Ignoring HazelcastInstanceNotActiveException from local cluster as the job will be" +
" restarted automatically.", e);
return;
}
throw e;
}
private static SerializableClientConfig serializableConfig(ClientConfig clientConfig) {
return clientConfig != null ? new SerializableClientConfig(clientConfig) : null;
}
/**
* Wrapper class needed to conceal the JCache API while
* serializing/deserializing other lambdas
*/
private static class CacheFlush {
static DistributedFunction> flushToCache(
String name, boolean isLocal
) {
return instance -> {
ICache cache = instance.getCacheManager().getCache(name);
return buffer -> {
try {
cache.putAll(buffer);
} catch (HazelcastInstanceNotActiveException e) {
handleInstanceNotActive(instance, e, isLocal);
}
buffer.clear();
};
};
}
}
private static final class ArrayMap extends AbstractMap {
private final List> entries;
private final ArraySet set = new ArraySet();
ArrayMap() {
entries = new ArrayList<>();
}
@Override @Nonnull
public Set> entrySet() {
return set;
}
public void add(Map.Entry entry) {
entries.add(entry);
}
private class ArraySet extends AbstractSet> {
@Override @Nonnull
public Iterator> iterator() {
return entries.iterator();
}
@Override
public int size() {
return entries.size();
}
}
@Override
public String toString() {
return entries.toString();
}
}
private static final class EntryProcessorWriter extends AbstractProcessor {
private static final int MAX_PARALLEL_ASYNC_OPS = 1000;
private final AtomicInteger numConcurrentOps = new AtomicInteger();
private final IMap map;
private final DistributedFunction toKeyFn;
private final DistributedFunction> toEntryProcessorFn;
private final AtomicReference lastError = new AtomicReference<>();
private final HazelcastInstance instance;
private final ExecutionCallback callback = callbackOf(
response -> numConcurrentOps.decrementAndGet(),
exception -> {
numConcurrentOps.decrementAndGet();
if (exception != null) {
lastError.compareAndSet(null, exception);
}
});
private final boolean isLocal;
private EntryProcessorWriter(HazelcastInstance instance, String name,
DistributedFunction toKeyFn,
DistributedFunction> toEntryProcessorFn,
boolean isLocal) {
this.instance = instance;
this.map = instance.getMap(name);
this.toKeyFn = toKeyFn;
this.toEntryProcessorFn = toEntryProcessorFn;
this.isLocal = isLocal;
}
@Override
public boolean isCooperative() {
return false;
}
@Override
public boolean tryProcess() {
checkError();
return true;
}
@Override
protected boolean tryProcess(int ordinal, @Nonnull Object object) throws Exception {
checkError();
if (!tryIncrement(numConcurrentOps, 1, MAX_PARALLEL_ASYNC_OPS)) {
return false;
}
try {
T item = (T) object;
EntryProcessor entryProcessor = toEntryProcessorFn.apply(item);
K key = toKeyFn.apply(item);
map.submitToKey(key, entryProcessor, callback);
return true;
} catch (HazelcastInstanceNotActiveException e) {
handleInstanceNotActive(instance, e, isLocal);
return false;
}
}
@Override
public boolean complete() {
return ensureAllWritten();
}
@Override
public boolean saveToSnapshot() {
return ensureAllWritten();
}
private boolean ensureAllWritten() {
boolean allWritten = numConcurrentOps.get() == 0;
checkError();
return allWritten;
}
private void checkError() {
Throwable t = lastError.get();
if (t != null) {
throw sneakyThrow(t);
}
}
}
private static final class EntryProcessorWriterSupplier implements ProcessorSupplier {
static final long serialVersionUID = 1L;
private final String name;
private final SerializableClientConfig clientConfig;
private final DistributedFunction toKeyFn;
private final DistributedFunction> toEntryProcessorFn;
private final boolean isLocal;
private transient HazelcastInstance client;
private transient HazelcastInstance instance;
private EntryProcessorWriterSupplier(String name, SerializableClientConfig clientConfig,
DistributedFunction toKeyFn,
DistributedFunction> toEntryProcessorFn,
boolean isLocal) {
this.name = name;
this.clientConfig = clientConfig;
this.toKeyFn = toKeyFn;
this.toEntryProcessorFn = toEntryProcessorFn;
this.isLocal = isLocal;
}
@Override
public void init(@Nonnull Context context) {
if (isRemote()) {
instance = client = newHazelcastClient(clientConfig.asClientConfig());
} else {
instance = context.jetInstance().getHazelcastInstance();
}
}
@Override
public void close(Throwable error) {
if (client != null) {
client.shutdown();
}
}
private boolean isRemote() {
return clientConfig != null;
}
@Override @Nonnull
public List get(int count) {
return Stream.generate(() ->
new EntryProcessorWriter<>(instance, name, toKeyFn, toEntryProcessorFn, isLocal))
.limit(count)
.collect(toList());
}
}
private static class HazelcastWriterSupplier implements ProcessorSupplier {
static final long serialVersionUID = 1L;
private final SerializableClientConfig clientConfig;
private final DistributedFunction> instanceToFlushBufferFn;
private final DistributedFunction newBufferFn;
private final DistributedBiConsumer addToBufferFn;
private final DistributedConsumer disposeBufferFn;
private transient DistributedConsumer flushBuffer;
private transient HazelcastInstance client;
HazelcastWriterSupplier(
SerializableClientConfig clientConfig,
DistributedFunction newBufferFn,
DistributedBiConsumer addToBufferFn,
DistributedFunction> instanceToFlushBufferFn,
DistributedConsumer disposeBufferFn
) {
this.clientConfig = clientConfig;
this.instanceToFlushBufferFn = instanceToFlushBufferFn;
this.newBufferFn = newBufferFn;
this.addToBufferFn = addToBufferFn;
this.disposeBufferFn = disposeBufferFn;
}
@Override
public void init(@Nonnull Context context) {
HazelcastInstance instance;
if (isRemote()) {
instance = client = newHazelcastClient(clientConfig.asClientConfig());
} else {
instance = context.jetInstance().getHazelcastInstance();
}
flushBuffer = instanceToFlushBufferFn.apply(instance);
}
@Override
public void close(Throwable error) {
if (client != null) {
client.shutdown();
}
}
private boolean isRemote() {
return clientConfig != null;
}
@Override @Nonnull
public List get(int count) {
return Stream.generate(() -> new WriteBufferedP<>(newBufferFn, addToBufferFn, flushBuffer, disposeBufferFn))
.limit(count).collect(toList());
}
}
public static class ApplyFnEntryProcessor implements EntryProcessor, EntryBackupProcessor,
IdentifiedDataSerializable {
private Map keysToUpdate;
private DistributedBiFunction updateFn;
public ApplyFnEntryProcessor() {
}
public ApplyFnEntryProcessor(Map keysToUpdate, DistributedBiFunction updateFn) {
this.keysToUpdate = keysToUpdate;
this.updateFn = updateFn;
}
@Override
public Object process(Entry entry) {
V oldValue = entry.getValue();
T item = keysToUpdate.get(entry.getKey());
V newValue = updateFn.apply(oldValue, item);
entry.setValue(newValue);
return null;
}
@Override
public EntryBackupProcessor getBackupProcessor() {
return this;
}
@Override
public void writeData(ObjectDataOutput out) throws IOException {
out.writeObject(keysToUpdate);
out.writeObject(updateFn);
}
@Override
public void readData(ObjectDataInput in) throws IOException {
keysToUpdate = in.readObject();
updateFn = in.readObject();
}
@Override
public void processBackup(Entry entry) {
process(entry);
}
@Override
public int getFactoryId() {
return SerializationConstants.FACTORY_ID;
}
@Override
public int getId() {
return SerializationConstants.APPLY_FN_ENTRY_PROCESSOR;
}
}
}
