eu.stratosphere.nephele.profiling.impl.JobProfilingData Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of stratosphere-runtime Show documentation
There is a newer version: 0.5.2-hadoop2
/***********************************************************************************************************************
 * Copyright (C) 2010-2013 by the Stratosphere project (http://stratosphere.eu)
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
 * the License. You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
 * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
 * specific language governing permissions and limitations under the License.
 **********************************************************************************************************************/

package eu.stratosphere.nephele.profiling.impl;

import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;

import eu.stratosphere.nephele.executiongraph.ExecutionGraph;
import eu.stratosphere.nephele.executiongraph.ExecutionGroupVertex;
import eu.stratosphere.nephele.executiongraph.ExecutionGroupVertexIterator;
import eu.stratosphere.nephele.executiongraph.ExecutionVertex;
import eu.stratosphere.nephele.instance.AbstractInstance;
import eu.stratosphere.nephele.instance.DummyInstance;
import eu.stratosphere.nephele.instance.InstanceConnectionInfo;
import eu.stratosphere.nephele.profiling.impl.types.InternalInstanceProfilingData;
import eu.stratosphere.nephele.profiling.types.InstanceSummaryProfilingEvent;

public class JobProfilingData {

	private final ExecutionGraph executionGraph;

	private final long profilingStart;

	private final Map collectedInstanceProfilingData = new HashMap();

	public JobProfilingData(ExecutionGraph executionGraph) {
		this.executionGraph = executionGraph;

		this.profilingStart = System.currentTimeMillis();
	}

	public long getProfilingStart() {
		return this.profilingStart;
	}

	public ExecutionGraph getExecutionGraph() {

		return this.executionGraph;
	}

	public boolean instanceAllocatedByJob(InternalInstanceProfilingData instanceProfilingData) {

		final ExecutionGroupVertexIterator it = new ExecutionGroupVertexIterator(this.executionGraph, true,
			this.executionGraph.getIndexOfCurrentExecutionStage());
		while (it.hasNext()) {

			final ExecutionGroupVertex groupVertex = it.next();
			for (int i = 0; i < groupVertex.getCurrentNumberOfGroupMembers(); i++) {
				final ExecutionVertex executionVertex = groupVertex.getGroupMember(i);
				if (instanceProfilingData.getInstanceConnectionInfo().equals(
					executionVertex.getAllocatedResource().getInstance().getInstanceConnectionInfo())) {
					this.collectedInstanceProfilingData.put(instanceProfilingData.getInstanceConnectionInfo(),
						instanceProfilingData);
					return true;
				}
			}
		}

		return false;
	}

	public InstanceSummaryProfilingEvent getInstanceSummaryProfilingData(long timestamp) {

		final Set tempSet = new HashSet();
		// First determine the number of allocated instances in the current stage
		final ExecutionGroupVertexIterator it = new ExecutionGroupVertexIterator(this.executionGraph, true,
			this.executionGraph.getIndexOfCurrentExecutionStage());
		while (it.hasNext()) {

			final ExecutionGroupVertex groupVertex = it.next();
			for (int i = 0; i < groupVertex.getCurrentNumberOfGroupMembers(); i++) {
				final ExecutionVertex executionVertex = groupVertex.getGroupMember(i);
				final AbstractInstance instance = executionVertex.getAllocatedResource().getInstance();
				if(!(instance instanceof DummyInstance)) {
					tempSet.add(instance);
				}
			}
		}

		/*
		 * Now compare the size of the collected data set and the allocated instance set.
		 * If their sizes are equal we can issue an instance summary.
		 */
		if (tempSet.size() != this.collectedInstanceProfilingData.size()) {
			return null;
		}

		return constructInstanceSummary(timestamp);
	}

	private InstanceSummaryProfilingEvent constructInstanceSummary(long timestamp) {

		final int numberOfInstances = this.collectedInstanceProfilingData.size();
		final Iterator instanceIterator = this.collectedInstanceProfilingData.keySet()
			.iterator();

		long freeMemorySum = 0;
		long totalMemorySum = 0;
		long bufferedMemorySum = 0;
		long cachedMemorySum = 0;
		long cachedSwapMemorySum = 0;

		int ioWaitCPUSum = 0;
		int idleCPUSum = 0;
		int profilingIntervalSum = 0;
		int systemCPUSum = 0;
		int hardIrqCPUSum = 0;
		int softIrqCPUSum = 0;

		int userCPUSum = 0;
		long receivedBytesSum = 0;
		long transmittedBytesSum = 0;

		// Sum up the individual values
		while (instanceIterator.hasNext()) {

			final InternalInstanceProfilingData profilingData = this.collectedInstanceProfilingData
				.get(instanceIterator.next());

			freeMemorySum += profilingData.getFreeMemory();
			ioWaitCPUSum += profilingData.getIOWaitCPU();
			idleCPUSum += profilingData.getIdleCPU();
			profilingIntervalSum += profilingData.getProfilingInterval();
			systemCPUSum += profilingData.getSystemCPU();
			hardIrqCPUSum += profilingData.getHardIrqCPU();
			softIrqCPUSum += profilingData.getSoftIrqCPU();
			totalMemorySum += profilingData.getTotalMemory();
			userCPUSum += profilingData.getUserCPU();
			receivedBytesSum += profilingData.getReceivedBytes();
			transmittedBytesSum += profilingData.getTransmittedBytes();
			bufferedMemorySum += profilingData.getBufferedMemory();
			cachedMemorySum += profilingData.getCachedMemory();
			cachedSwapMemorySum += profilingData.getCachedSwapMemory();
		}

		final InstanceSummaryProfilingEvent instanceSummary = new InstanceSummaryProfilingEvent(profilingIntervalSum
			/ numberOfInstances, ioWaitCPUSum / numberOfInstances, idleCPUSum / numberOfInstances, userCPUSum
			/ numberOfInstances, systemCPUSum / numberOfInstances, hardIrqCPUSum / numberOfInstances, softIrqCPUSum
			/ numberOfInstances, totalMemorySum / (long) numberOfInstances, freeMemorySum / (long) numberOfInstances,
			bufferedMemorySum / (long) numberOfInstances, cachedMemorySum / (long) numberOfInstances,
			cachedSwapMemorySum / (long) numberOfInstances, receivedBytesSum / (long) numberOfInstances,
			transmittedBytesSum / (long) numberOfInstances, this.executionGraph.getJobID(), timestamp,
			(timestamp - this.profilingStart));

		this.collectedInstanceProfilingData.clear();

		return instanceSummary;
	}
}