com.hazelcast.mapreduce.impl.AbstractJob Maven / Gradle / Ivy
/*
* Copyright (c) 2008-2017, Hazelcast, Inc. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.hazelcast.mapreduce.impl;
import com.hazelcast.mapreduce.Collator;
import com.hazelcast.mapreduce.CombinerFactory;
import com.hazelcast.mapreduce.Job;
import com.hazelcast.mapreduce.JobCompletableFuture;
import com.hazelcast.mapreduce.JobTracker;
import com.hazelcast.mapreduce.KeyPredicate;
import com.hazelcast.mapreduce.KeyValueSource;
import com.hazelcast.mapreduce.Mapper;
import com.hazelcast.mapreduce.MappingJob;
import com.hazelcast.mapreduce.ReducerFactory;
import com.hazelcast.mapreduce.ReducingJob;
import com.hazelcast.mapreduce.ReducingSubmittableJob;
import com.hazelcast.mapreduce.TopologyChangedStrategy;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import static com.hazelcast.util.Preconditions.isNotNull;
/**
* Base class for all map reduce job implementations
*
* @param type of the input key
* @param type of the input value
*/
public abstract class AbstractJob
implements Job {
protected final String name;
protected final JobTracker jobTracker;
protected final KeyValueSource keyValueSource;
protected Mapper mapper;
protected CombinerFactory combinerFactory;
protected ReducerFactory reducerFactory;
protected Collection keys;
protected KeyPredicate predicate;
protected int chunkSize = -1;
protected TopologyChangedStrategy topologyChangedStrategy;
public AbstractJob(String name, JobTracker jobTracker, KeyValueSource keyValueSource) {
this.name = name;
this.jobTracker = jobTracker;
this.keyValueSource = keyValueSource;
}
@Override
public MappingJob mapper(
Mapper mapper) {
isNotNull(mapper, "mapper");
if (this.mapper != null) {
throw new IllegalStateException("mapper already set");
}
this.mapper = mapper;
return new MappingJobImpl();
}
@Override
public Job onKeys(Iterable keys) {
addKeys(keys);
return this;
}
@Override
public Job onKeys(KeyIn... keys) {
addKeys(keys);
return this;
}
@Override
public Job keyPredicate(KeyPredicate predicate) {
setKeyPredicate(predicate);
return this;
}
@Override
public Job chunkSize(int chunkSize) {
this.chunkSize = chunkSize;
return this;
}
@Override
public Job topologyChangedStrategy(TopologyChangedStrategy topologyChangedStrategy) {
this.topologyChangedStrategy = topologyChangedStrategy;
return this;
}
protected JobCompletableFuture submit(Collator collator) {
prepareKeyPredicate();
return invoke(collator);
}
protected abstract JobCompletableFuture invoke(Collator collator);
protected void prepareKeyPredicate() {
if (predicate == null) {
return;
}
if (keyValueSource.isAllKeysSupported()) {
Collection allKeys = keyValueSource.getAllKeys();
for (KeyIn key : allKeys) {
if (predicate.evaluate(key)) {
if (this.keys == null) {
this.keys = new HashSet();
}
this.keys.add(key);
}
}
}
}
private void addKeys(Iterable keys) {
if (this.keys == null) {
this.keys = new HashSet();
}
for (KeyIn key : keys) {
this.keys.add(key);
}
}
private void addKeys(KeyIn... keys) {
if (this.keys == null) {
this.keys = new ArrayList();
}
this.keys.addAll(Arrays.asList(keys));
}
private void setKeyPredicate(KeyPredicate predicate) {
isNotNull(predicate, "predicate");
this.predicate = predicate;
}
private JobCompletableFuture submit() {
return submit(null);
}
/**
* This class is just used to comply to the public DSL style API
*
* @param type of the original base key
* @param type of the key at that processing state
* @param type of the value at that processing state
*/
protected class MappingJobImpl
implements MappingJob {
@Override
public MappingJob onKeys(Iterable keys) {
addKeys((Iterable) keys);
return this;
}
@Override
public MappingJob onKeys(EntryKey... keys) {
addKeys((KeyIn[]) keys);
return this;
}
@Override
public MappingJob keyPredicate(KeyPredicate predicate) {
setKeyPredicate((KeyPredicate) predicate);
return this;
}
@Override
public MappingJob chunkSize(int chunkSize) {
AbstractJob.this.chunkSize = chunkSize;
return this;
}
@Override
public MappingJob topologyChangedStrategy(
TopologyChangedStrategy topologyChangedStrategy) {
AbstractJob.this.topologyChangedStrategy = topologyChangedStrategy;
return this;
}
@Override
public ReducingJob combiner(
CombinerFactory combinerFactory) {
isNotNull(combinerFactory, "combinerFactory");
if (AbstractJob.this.combinerFactory != null) {
throw new IllegalStateException("combinerFactory already set");
}
AbstractJob.this.combinerFactory = combinerFactory;
return new ReducingJobImpl();
}
@Override
public ReducingSubmittableJob reducer(
ReducerFactory reducerFactory) {
isNotNull(reducerFactory, "reducerFactory");
if (AbstractJob.this.reducerFactory != null) {
throw new IllegalStateException("reducerFactory already set");
}
AbstractJob.this.reducerFactory = reducerFactory;
return new ReducingSubmittableJobImpl();
}
@Override
public JobCompletableFuture>> submit() {
return AbstractJob.this.submit();
}
@Override
public JobCompletableFuture submit(
Collator>, ValueOut> collator) {
return AbstractJob.this.submit(collator);
}
}
/**
* This class is just used to comply to the public DSL style API
*
* @param type of the original base key
* @param type of the key at that processing state
* @param type of the value at that processing state
*/
protected class ReducingJobImpl
implements ReducingJob {
@Override
public ReducingSubmittableJob reducer(
ReducerFactory reducerFactory) {
isNotNull(reducerFactory, "reducerFactory");
if (AbstractJob.this.reducerFactory != null) {
throw new IllegalStateException("reducerFactory already set");
}
AbstractJob.this.reducerFactory = reducerFactory;
return new ReducingSubmittableJobImpl();
}
@Override
public ReducingJob onKeys(Iterable keys) {
addKeys((Iterable) keys);
return this;
}
@Override
public ReducingJob onKeys(EntryKey... keys) {
addKeys((KeyIn[]) keys);
return this;
}
@Override
public ReducingJob keyPredicate(KeyPredicate predicate) {
setKeyPredicate((KeyPredicate) predicate);
return this;
}
@Override
public ReducingJob chunkSize(int chunkSize) {
AbstractJob.this.chunkSize = chunkSize;
return this;
}
@Override
public ReducingJob topologyChangedStrategy(
TopologyChangedStrategy topologyChangedStrategy) {
AbstractJob.this.topologyChangedStrategy = topologyChangedStrategy;
return this;
}
@Override
public JobCompletableFuture>> submit() {
return AbstractJob.this.submit();
}
@Override
public JobCompletableFuture submit(
Collator>, ValueOut> collator) {
return AbstractJob.this.submit(collator);
}
}
/**
* This class is just used to comply to the public DSL style API
*
* @param type of the original base key
* @param type of the key at that processing state
* @param type of the value at that processing state
*/
protected class ReducingSubmittableJobImpl
implements ReducingSubmittableJob {
@Override
public ReducingSubmittableJob onKeys(Iterable keys) {
addKeys((Iterable) keys);
return this;
}
@Override
public ReducingSubmittableJob onKeys(EntryKey... keys) {
addKeys((KeyIn[]) keys);
return this;
}
@Override
public ReducingSubmittableJob keyPredicate(KeyPredicate predicate) {
setKeyPredicate((KeyPredicate) predicate);
return this;
}
@Override
public ReducingSubmittableJob chunkSize(int chunkSize) {
AbstractJob.this.chunkSize = chunkSize;
return this;
}
@Override
public ReducingSubmittableJob topologyChangedStrategy(
TopologyChangedStrategy topologyChangedStrategy) {
AbstractJob.this.topologyChangedStrategy = topologyChangedStrategy;
return this;
}
@Override
public JobCompletableFuture> submit() {
return AbstractJob.this.submit();
}
@Override
public JobCompletableFuture submit(Collator, ValueOut> collator) {
return AbstractJob.this.submit(collator);
}
}
}