Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance. Project price only 1 $
You can buy this project and download/modify it how often you want.
/*
* Copyright (c) 2013 mgm technology partners GmbH
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.mgmtp.perfload.loadprofiles.generation;
import static com.google.common.base.Preconditions.checkState;
import java.io.File;
import java.io.IOException;
import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import org.apache.commons.io.IOUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.mgmtp.perfload.loadprofiles.model.BaseLoadProfileEvent;
import com.mgmtp.perfload.loadprofiles.model.Client;
import com.mgmtp.perfload.loadprofiles.model.LoadCurve;
import com.mgmtp.perfload.loadprofiles.model.LoadCurveAssignment;
import com.mgmtp.perfload.loadprofiles.model.LoadEvent;
import com.mgmtp.perfload.loadprofiles.model.LoadEventComparator;
import com.mgmtp.perfload.loadprofiles.model.LoadTestConfiguration;
import com.mgmtp.perfload.loadprofiles.model.MarkerEvent;
import com.mgmtp.perfload.loadprofiles.model.Operation;
import com.mgmtp.perfload.loadprofiles.model.Target;
/**
* Distribute loadtest events to clients balancing the load on each client according to the relative
* power of the given clients and taking the relative load of each operation (caused on the client
* by this operation) into account.
*
* @author mvarendo
*/
public class EventDistributor {
private static final Logger log = LoggerFactory.getLogger(EventDistributor.class);
private static final String eventSeparator = ";";
/**
* Distributes events from different operations with different client loads to Clients with
* different performance. The events from the given list of events, which must be sorted
* according to time (not verified in this method) are distributed onto processes within daemons
* on clients. All processes of all daemons of one client should get the same load The daemon is
* only introduced, since it exists in the configuration of perfload It is not necessary to
* balance the distribution of events between daemons. The load of all processes of one client
* and the total load per client is balanced by this algorithm. While looping over all events,
* the algorithm derives for each event which client has up to now the lowest total load,
* assigns the client load of the operation of this event to the client and then continues with
* the next event. The algorithm for the distribution on processes is analogous. This algorithn
* takes only the time sequence of the events into account, the time difference between the
* events is not considered. To correct for this, not the total load per client up to the time
* of the event to be distributed has to be taken into account, but the load distributed over
* time, taking into account, that after a certain time, the 'current load' of a client might be
* 0. I have no idea yet how to do this.
*
* @param allEvents
* ArrayList of all load events
* @return Lists of Events. For each event the client, daemon and process is now defined.
*/
private static List distributeEvents(final List allEvents,
final LoadTestConfiguration loadTestConfiguration, final double totalWeightedNevent) {
List clients = loadTestConfiguration.getClients();
int numClients = clients.size();
List operations = loadTestConfiguration.getOperations();
double[] relativeClientPower = getRelativeClientPower(loadTestConfiguration);
// distribute events to processes and daemons of all clients
ArrayList clientEventList = new ArrayList();
double[][] cumulatedProcessLoad = new double[numClients][];
double[][] deficitProcessLoad = new double[numClients][];
int[] processesPerClient = new int[numClients];
for (int iClient = 0; iClient < numClients; iClient++) {
processesPerClient[iClient] = clients.get(iClient).getNumDaemons() * clients.get(iClient).getNumProcesses();
cumulatedProcessLoad[iClient] = new double[processesPerClient[iClient]];
deficitProcessLoad[iClient] = new double[processesPerClient[iClient]];
}
double cumulatedClientLoad[] = new double[numClients];
double desiredClientLoad[] = new double[numClients];
double deficitClientLoad[] = new double[numClients];
// log input parameter
log.info("totalWeightedNevent = " + totalWeightedNevent);
log.info("nClient = " + numClients);
for (Client client : clients) {
log.info("nDaemon = " + client.getNumDaemons()
+ ", nProcess = " + client.getNumProcesses()
+ ", relativeClientPower = " + client.getRelativePower());
}
for (Operation operation : operations) {
log.info("operation " + operation.getName() + " relativeClientLoad = " + operation.getRelativeClientLoad());
}
if (log.isDebugEnabled()) {
log.debug("weight; cumulatedWeightedNevent;");
for (int iClient = 0; iClient < numClients; iClient++) {
log.debug("targetClientLoad[" + iClient + "]; ");
}
for (int iClient = 0; iClient < numClients; iClient++) {
log.debug("deficitClientLoad[" + iClient + "]; ");
}
log.debug("iClientMax; OperationType; StartTime");
}
double cumulatedWeightedNevent = 0.;
for (LoadEvent event : allEvents) {
double weight = event.getOperation().getRelativeClientLoad();
cumulatedWeightedNevent += weight;
for (int iClient = 0; iClient < numClients; iClient++) {
desiredClientLoad[iClient] = cumulatedWeightedNevent * relativeClientPower[iClient];
deficitClientLoad[iClient] = desiredClientLoad[iClient] - cumulatedClientLoad[iClient];
}
int iClientMax = findHighestDeficit(deficitClientLoad);
cumulatedClientLoad[iClientMax] += weight;
double desiredProcessLoad = desiredClientLoad[iClientMax] / processesPerClient[iClientMax];
for (int iProcess = 0; iProcess < processesPerClient[iClientMax]; iProcess++) {
deficitProcessLoad[iClientMax][iProcess] = desiredProcessLoad - cumulatedProcessLoad[iClientMax][iProcess];
}
int iProcessMax = findHighestDeficit(deficitProcessLoad[iClientMax]);
cumulatedProcessLoad[iClientMax][iProcessMax] += weight;
event.setClientId(iClientMax);
event.setProcessId(iProcessMax);
int daemonId = getDaemonIdForProcessId(iProcessMax, clients, iClientMax);
event.setDaemonId(daemonId);
clientEventList.add(event);
if (log.isDebugEnabled()) {
log.debug(weight + "; " + cumulatedWeightedNevent + "; ");
for (int iClient = 0; iClient < numClients; iClient++) {
log.debug(desiredClientLoad[iClient] + "; ");
}
for (int iClient = 0; iClient < numClients; iClient++) {
log.debug(deficitClientLoad[iClient] + "; ");
}
log.debug(iClientMax + "; " + event.getOperation().getName() + "; " + event.getTime());
}
}
// numeric check
double numericRelativeDifference = (cumulatedWeightedNevent - totalWeightedNevent) / totalWeightedNevent;
log.info("Numeric relative difference cumulatedWeightedNevent - totalWeightedNevent " + numericRelativeDifference);
// check residua for clients
for (int iClient = 0; iClient < numClients; iClient++) {
deficitClientLoad[iClient] = desiredClientLoad[iClient] - cumulatedClientLoad[iClient];
log.info("Residuum Client " + iClient + ": " + deficitClientLoad[iClient]);
}
return clientEventList;
}
/*
* the daemonid has to be unique within one load test configuration of perfLoad. To ensure a
* unique id, daemons are numbered starting at the first daemon of the first client. the
* numbering starts with 1. The performance of this method can be improved by deriving and
* storing the number of the first daemon for each client.
*/
private static int getDaemonIdForProcessId(final int processId, final List clients, final int iClient) {
// derive the numbering of the daemons up to the given client iClient, excluding
// this client
int daemonNumber = 1;
for (int i = 0; i < iClient; i++) {
daemonNumber += clients.get(i).getNumDaemons();
}
// continue the numbering for the actual client iClient
Client client = clients.get(iClient);
int nProcessesPerClient = client.getNumProcesses();
int iDaemon = processId / nProcessesPerClient;
daemonNumber += iDaemon;
return daemonNumber;
}
/**
* Create events according to the given load curve assignment and distribute the events onto the
* targets according to the given partition between targets. The distribution algorithm is
* analogous to the distribution of events to clients described in the method distributeEvents()
* in this class. The shift value is used to shift each event within its time interval. The time
* interval is the time for which the integral over the load curve increases from the last
* integer number of events to the next integer number of events. The value indicates the
* relative shift within this interval. A value of 0.5 puts the event at the time, where the
* integral value is equal to the last integer number plus 0.5. The shifting is used to be able
* to smooth out the distribution of events on small time scales. If the load of an operation
* following a load curve is created separately for parts of the load, then it is possible to
* shift the start time of each event to smooth out the distribution. Creating events for half
* the load of a load curve with a shift value of 0. and another list of events with a shift
* value of 0.5 and combining them is equal to creating a list of events with the full load and
* a shift value of 0.
*
* @param loadCurveAssignment
* The assignment of a load curve to targets
* @param shift
* The shift of all events for the given assignment
* @return list of load events
*/
private static ArrayList createEvents(final LoadCurveAssignment loadCurveAssignment, final double shift) {
LoadCurve loadCurve = loadCurveAssignment.getLoadCurve();
List targets = loadCurveAssignment.getTargets();
Operation operation = loadCurveAssignment.getOperation();
String targetList = "";
for (Target target : targets) {
targetList += target.getName() + ", ";
}
log.info("Creating events according to loadCurve " + loadCurve.getName() +
"\n\tof operation " + operation.getName() +
"\n\tfor targets " + targetList);
loadCurve.dump(log);
int nEvents = (int) loadCurve.getNEvents();
ArrayList events = new ArrayList(nEvents);
// calculate events from load curve in given time interval
for (int iEvent = 0; iEvent < nEvents; iEvent++) {
double eventIndex = iEvent + shift;
double Tn = LoadCurveCalculator.deriveStartTime(loadCurve, eventIndex);
LoadEvent event = new LoadEvent(Tn, operation);
events.add(iEvent, event);
}
// assign events to targets according to load part for each targets
double[] desiredTargetLoad = new double[targets.size()];
double[] cumulatedTargetLoad = new double[targets.size()];
double[] deficitTargetLoad = new double[targets.size()];
double cumulatedNevent = 0.;
for (int iEvent = 0; iEvent < events.size(); iEvent++) {
LoadEvent event = events.get(iEvent);
cumulatedNevent += 1.;
for (int iTarget = 0; iTarget < targets.size(); iTarget++) {
desiredTargetLoad[iTarget] = cumulatedNevent * targets.get(iTarget).getLoadPart();
deficitTargetLoad[iTarget] = desiredTargetLoad[iTarget] - cumulatedTargetLoad[iTarget];
}
int iTargetMax = findHighestDeficit(deficitTargetLoad);
cumulatedTargetLoad[iTargetMax] += 1.;
event.setTarget(targets.get(iTargetMax));
}
return events;
}
/**
* Create a list of events containing events for each client according to the given load test
* configuration. The events are first created for each load curve assignment and then merged to
* one event list sorted by time. Then the events are distributed to clients (and daemons and
* processes). The algorithm for the distribution on clients and targets is described in the
* methods performing the task. The resulting event list contains events, which are sorted by
* time and contain defined values for the targets (server), the operation and the client (and
* daemon and process), which executes this operation against the given targets. The serialized
* form (.csv-file) of this list is used as an input file for the perfload clients.
*
* @param loadTestConfiguration
* configuration data of the load test
*/
public static List createClientEventList(final LoadTestConfiguration loadTestConfiguration) {
verifyArguments(loadTestConfiguration);
// convert load curve units to hours if necessary
List loadCurveAssignments = loadTestConfiguration.getLoadCurveAssignments();
for (LoadCurveAssignment loadCurveAssignment : loadCurveAssignments) {
LoadCurve loadCurve = loadCurveAssignment.getLoadCurve();
if (!LoadCurveCalculator.timeUnit_hour.equals(loadCurve.getTimeUnit())) {
LoadCurveCalculator.transformToHours(loadCurve);
}
}
int numAssignments = loadCurveAssignments.size();
// derive relative shift for each assignment. The relative shifts are evenly distributed
// from 0 to 1. The assignment with the highest client load gets the shift nearest to 0.5,
// then the others are distributed around this value going from the highest client load to lowest.
// This avoids that all clients start simultaneously.
double shiftValues[] = new double[numAssignments];
// implement the sorting here!
log.warn("Sorting by relative client load for optimisation of shiftValues not yet implemented!");
for (int iAssignment = 0; iAssignment < numAssignments; iAssignment++) {
shiftValues[iAssignment] = (iAssignment + 0.5) / numAssignments;
}
// derive for all assignments an event list
List allAssignmentsEventList = new ArrayList();
for (int iAssignment = 0; iAssignment < numAssignments; iAssignment++) {
LoadCurveAssignment loadCurveAssignment = loadCurveAssignments.get(iAssignment);
allAssignmentsEventList.addAll(createEvents(loadCurveAssignment, shiftValues[iAssignment]));
}
Collections.sort(allAssignmentsEventList, new LoadEventComparator());
// for numeric check
double totalWeightedNevent = 0.;
for (LoadCurveAssignment loadCurveAssignment : loadCurveAssignments) {
LoadCurve loadCurve = loadCurveAssignment.getLoadCurve();
totalWeightedNevent += loadCurve.getNEvents() * loadCurveAssignment.getOperation().getRelativeClientLoad();
}
// double relativeOperationClientLoads[] = new double[numAssignments];
// for (int iAssignment = 0; iAssignment < numAssignments; iAssignment++) {
// LoadCurveAssignment loadCurveAssignment = loadCurveAssignments.get(iAssignment);
// relativeOperationClientLoads[iAssignment] = loadCurveAssignment.getOperation().getRelativeClientLoad();
// }
List clientEventList = distributeEvents(allAssignmentsEventList, loadTestConfiguration, totalWeightedNevent);
return clientEventList;
}
/**
* Write the List of load test events for a perfLoadClient The list contains following values: -
* start time of the operation in milliseconds since start of the load test - name of the
* operation (i.e. UStVA) - name of the host on which this operation is triggered - port of the
* host over which the operation is triggered - number of daemon executing this test
* (=identifier of operation) - number of the process of the daemon (here always 1) The format
* of the file is one load test event per line, arguments separated by the given eventSeparator.
*
* @param file
* The events file
* @param eventList
* List of load test events
*/
public static void writeEventListForPerfLoadClientsToFile(final File file, final String headerLines,
final List eventList)
throws IOException {
PrintWriter pw = null;
try {
pw = new PrintWriter(file, "UTF-8");
pw.println(headerLines);
for (BaseLoadProfileEvent event : eventList) {
if (event instanceof LoadEvent) {
LoadEvent loadEvent = (LoadEvent) event;
// convert time from hours to milliseconds
long t = Math.round(loadEvent.getTime() * 60. * 60. * 1000.);
String operation = loadEvent.getOperation().getName();
String target = loadEvent.getTarget().getName();
int daemonId = loadEvent.getDaemonId();
int processId = loadEvent.getProcessId() + 1;
pw.println(t + eventSeparator +
operation + eventSeparator +
target + eventSeparator +
daemonId + eventSeparator +
processId);
} else if (event instanceof MarkerEvent) {
MarkerEvent marker = (MarkerEvent) event;
long t = Math.round(marker.getTime() * 60. * 60. * 1000.);
pw.println(t + eventSeparator +
"[[marker]]" + eventSeparator +
marker.getName() + eventSeparator +
marker.getType() + eventSeparator);
}
}
} finally {
IOUtils.closeQuietly(pw);
}
}
// search for the maximum (currently the first found maximum is taken, a different
// distribution algorithm might be useful
private static int findHighestDeficit(final double deficitLoad[]) {
double deficitMax = deficitLoad[0];
int iMax = 0;
for (int i = 1; i < deficitLoad.length; i++) {
if (deficitLoad[i] > deficitMax) {
deficitMax = deficitLoad[i];
iMax = i;
}
}
return iMax;
}
/**
* verifies the validity of the given arguments. Throws an IllegalArgumentException, if an
* argument is invalid. For load curves it is checked, that the array and its elements are not
* null. Additionally it is verified, that the time values of the points in the load curve are
* strictly increasing.
*/
public static void verifyArguments(final LoadTestConfiguration loadTestConfiguration) {
for (LoadCurveAssignment assignment : loadTestConfiguration.getLoadCurveAssignments()) {
LoadCurve loadCurve = assignment.getLoadCurve();
if (loadCurve == null) {
throw new java.lang.IllegalArgumentException("load curve of assignment " + assignment + " is null");
}
double[] timeValues = loadCurve.getTimeValues();
if (timeValues == null) {
throw new java.lang.IllegalArgumentException("time values of load curve with name " + loadCurve.getName()
+ " is null");
}
int nPoint = timeValues.length;
for (int iPoint = 1; iPoint < nPoint; iPoint++) {
if (timeValues[iPoint] <= timeValues[iPoint - 1]) {
throw new java.lang.IllegalArgumentException("In loadCurve of assignment " + assignment +
"], timeValues[" + Integer.toString(iPoint - 1) + "] >= timeValues[" + iPoint +
"], time value of lower point must be smaller than time value of upper point.");
}
}
}
}
/**
* Get the relative client power of all clients in the load test configuration as an array of
* doubles. The sequence of the values in the array is according to their occurence in the
* clients definition of the load test configuration.
*
* @param loadTestConfiguration
* The load test configuration, for which the relative client power are returned
* @return Array of realtive client powers
*/
public static double[] getRelativeClientPower(final LoadTestConfiguration loadTestConfiguration) {
List clients = loadTestConfiguration.getClients();
int size = clients.size();
double[] relativeClientPower = new double[size];
for (int i = 0; i < size; i++) {
relativeClientPower[i] = clients.get(i).getRelativePower();
}
return relativeClientPower;
}
/**
* In assignments of a load test configuration load curves are referenced by name. In this
* method the given load curves are scaled with the given scaling factor of the assignment and
* then added as a direct reference. Since load curves can be used in more than one assignment,
* load curves are cloned before they are scaled.
*
* @param loadTestConfiguration
* The load test configuration to which scaled load curves are be added
* @param loadCurves
* The load curves to be added to the assignments
*/
public static void addScaledLoadCurvesToAssignments(final LoadTestConfiguration loadTestConfiguration,
final List loadCurves) {
for (LoadCurve loadCurve : loadCurves) {
LoadCurveCalculator.transformToHours(loadCurve);
}
List loadCurveAssignments = loadTestConfiguration.getLoadCurveAssignments();
for (LoadCurveAssignment loadCurveAssignment : loadCurveAssignments) {
String loadCurveName = loadCurveAssignment.getLoadCurveName();
boolean found = false;
for (LoadCurve loadCurve : loadCurves) {
if (loadCurve.getName().equalsIgnoreCase(loadCurveName)) {
LoadCurve clonedLoadCurve = loadCurve.clone();
LoadCurve scaledLoadCurve = LoadCurveCalculator.scaleLoadCurve(clonedLoadCurve,
loadCurveAssignment.getLoadCurveScaling());
loadCurveAssignment.setLoadCurve(scaledLoadCurve);
found = true;
break;
}
}
checkState(found, "No matching load curve found for load curve: " + loadCurveName);
}
}
}
