com.hazelcast.mapreduce.impl.task.ReducerTask Maven / Gradle / Ivy

Go to download
/*
 * Copyright (c) 2008-2016, Hazelcast, Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.hazelcast.mapreduce.impl.task;

import com.hazelcast.mapreduce.Reducer;
import com.hazelcast.mapreduce.impl.CombinerResultList;
import com.hazelcast.mapreduce.impl.MapReduceService;
import com.hazelcast.mapreduce.impl.notification.ReducingFinishedNotification;
import com.hazelcast.nio.Address;
import com.hazelcast.util.ExceptionUtil;

import java.util.List;
import java.util.Map;
import java.util.Queue;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.atomic.AtomicBoolean;

import static com.hazelcast.mapreduce.impl.MapReduceUtil.notifyRemoteException;

/**
 * This task implementation executes the reducing phase. It collects all arriving chunks and processes them
 * one by one. There is one ReducerTask per job per node to have a clear idea of the resulting load a job
 * may create.
 *
 * @param  type of the emitted key
 * @param  type of the intermediate chunk data
 */
public class ReducerTask
        implements Runnable {

    // This variable is used for piggybacking the internal state before
    // suspension and continuation of the reducers
    private volatile boolean visibility;

    private final AtomicBoolean cancelled = new AtomicBoolean();

    private final JobSupervisor supervisor;
    private final Queue> reducerQueue;
    private final String name;
    private final String jobId;
    private final ReducerTaskScheduler scheduler;

    public ReducerTask(String name, String jobId, JobSupervisor supervisor) {
        this.name = name;
        this.jobId = jobId;
        this.supervisor = supervisor;
        this.reducerQueue = new ConcurrentLinkedQueue>();
        this.scheduler = new ReducerTaskScheduler(getExecutorService(), this);
    }

    public String getName() {
        return name;
    }

    public String getJobId() {
        return jobId;
    }

    public void cancel() {
        cancelled.set(true);
    }

    public void processChunk(Map chunk) {
        processChunk(-1, null, chunk);
    }

    public void processChunk(int partitionId, Address sender, Map chunk) {
        if (cancelled.get()) {
            return;
        }
        reducerQueue.offer(new ReducerChunk(chunk, partitionId, sender));
        scheduler.requestExecution();
    }

    private ExecutorService getExecutorService() {
        MapReduceService mapReduceService = supervisor.getMapReduceService();
        return mapReduceService.getExecutorService(name);
    }

    @Override
    public void run() {
        boolean visibility = this.visibility;
        try {
            ReducerChunk reducerChunk;
            while ((reducerChunk = reducerQueue.poll()) != null) {
                if (cancelled.get()) {
                    return;
                }

                reduceChunk(reducerChunk.chunk);
                processProcessedState(reducerChunk);
            }
        } catch (Throwable t) {
            notifyRemoteException(supervisor, t);
            if (t instanceof Error) {
                ExceptionUtil.sneakyThrow(t);
            }
        } finally {
            this.visibility = !visibility;
            scheduler.afterExecution();
        }
    }

    private void reduceChunk(Map chunk) {
        for (Map.Entry entry : chunk.entrySet()) {
            Reducer reducer = supervisor.getReducerByKey(entry.getKey());
            if (reducer != null) {
                Chunk chunkValue = entry.getValue();
                if (chunkValue instanceof CombinerResultList) {
                    for (Object value : (List) chunkValue) {
                        reducer.reduce(value);
                    }
                } else {
                    reducer.reduce(chunkValue);
                }
            }
        }
    }

    private void processProcessedState(ReducerChunk reducerChunk) {
        // If partitionId is set this was the last chunk for this partition
        if (reducerChunk.partitionId != -1) {
            MapReduceService mapReduceService = supervisor.getMapReduceService();
            ReducingFinishedNotification notification = new ReducingFinishedNotification(mapReduceService.getLocalAddress(), name,
                    jobId, reducerChunk.partitionId);

            mapReduceService.sendNotification(reducerChunk.sender, notification);
        }
    }

}