org.apache.uima.ducc.ws.server.DuccHandlerClassic Maven / Gradle / Ivy
The newest version!
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.uima.ducc.ws.server;
import java.io.File;
import java.io.IOException;
import java.text.DecimalFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Properties;
import java.util.TreeMap;
import java.util.concurrent.ConcurrentSkipListMap;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.apache.uima.ducc.common.ConvertSafely;
import org.apache.uima.ducc.common.IDuccEnv;
import org.apache.uima.ducc.common.NodeConfiguration;
import org.apache.uima.ducc.common.SizeBytes;
import org.apache.uima.ducc.common.SizeBytes.Type;
import org.apache.uima.ducc.common.boot.DuccDaemonRuntimeProperties;
import org.apache.uima.ducc.common.boot.DuccDaemonRuntimeProperties.DaemonName;
import org.apache.uima.ducc.common.head.IDuccHead;
import org.apache.uima.ducc.common.internationalization.Messages;
import org.apache.uima.ducc.common.utils.ComponentHelper;
import org.apache.uima.ducc.common.utils.DuccLogger;
import org.apache.uima.ducc.common.utils.DuccProperties;
import org.apache.uima.ducc.common.utils.DuccPropertiesHelper;
import org.apache.uima.ducc.common.utils.DuccPropertiesResolver;
import org.apache.uima.ducc.common.utils.DuccSchedulerClasses;
import org.apache.uima.ducc.common.utils.InetHelper;
import org.apache.uima.ducc.common.utils.TimeStamp;
import org.apache.uima.ducc.common.utils.id.DuccId;
import org.apache.uima.ducc.orchestrator.jd.scheduler.JdReservation;
import org.apache.uima.ducc.transport.event.common.DuccWorkJob;
import org.apache.uima.ducc.transport.event.common.DuccWorkReservation;
import org.apache.uima.ducc.transport.event.common.IDuccPerWorkItemStatistics;
import org.apache.uima.ducc.transport.event.common.IDuccProcess;
import org.apache.uima.ducc.transport.event.common.IDuccReservation;
import org.apache.uima.ducc.transport.event.common.IDuccReservationMap;
import org.apache.uima.ducc.transport.event.common.IDuccSchedulingInfo;
import org.apache.uima.ducc.transport.event.common.IDuccWork;
import org.apache.uima.ducc.transport.event.common.IDuccWorkJob;
import org.apache.uima.ducc.transport.event.common.IRationale;
import org.apache.uima.ducc.transport.event.common.JdReservationBean;
import org.apache.uima.ducc.ws.Distiller;
import org.apache.uima.ducc.ws.DuccDaemonsData;
import org.apache.uima.ducc.ws.DuccData;
import org.apache.uima.ducc.ws.DuccHead;
import org.apache.uima.ducc.ws.DuccMachinesData;
import org.apache.uima.ducc.ws.DuccMachinesDataHelper;
import org.apache.uima.ducc.ws.Info;
import org.apache.uima.ducc.ws.JobInfo;
import org.apache.uima.ducc.ws.MachineInfo;
import org.apache.uima.ducc.ws.helper.BrokerHelper;
import org.apache.uima.ducc.ws.helper.BrokerHelper.FrameworkAttribute;
import org.apache.uima.ducc.ws.helper.BrokerHelper.JmxKeyWord;
import org.apache.uima.ducc.ws.helper.DatabaseHelper;
import org.apache.uima.ducc.ws.registry.ServiceInterpreter.StartState;
import org.apache.uima.ducc.ws.registry.ServicesRegistry;
import org.apache.uima.ducc.ws.registry.sort.IServiceAdapter;
import org.apache.uima.ducc.ws.registry.sort.ServicesHelper;
import org.apache.uima.ducc.ws.registry.sort.ServicesSortCache;
import org.apache.uima.ducc.ws.server.DuccCookies.DisplayStyle;
import org.apache.uima.ducc.ws.server.IWebMonitor.MonitorType;
import org.apache.uima.ducc.ws.state.monitoring.INodeState;
import org.apache.uima.ducc.ws.state.monitoring.NodeState;
import org.apache.uima.ducc.ws.types.NodeId;
import org.apache.uima.ducc.ws.types.UserId;
import org.apache.uima.ducc.ws.utils.FormatHelper.Precision;
import org.apache.uima.ducc.ws.utils.alien.EffectiveUser;
import org.eclipse.jetty.server.Request;
public class DuccHandlerClassic extends DuccAbstractHandler {
private static DuccLogger duccLogger = DuccLogger.getLogger(DuccHandlerClassic.class);
private static Messages messages = Messages.getInstance();
private static DuccId jobid = null;
private static DatabaseHelper databaseHelper = DatabaseHelper.getInstance();
private static INodeState nodeState = NodeState.getInstance();
private static IDuccHead dh = DuccHead.getInstance();
public final String classicJobs = duccContextClassic+"-jobs-data";
public final String classicReservations = duccContextClassic+"-reservations-data";
public final String classicServices = duccContextClassic+"-services-data";
public final String classicSystemClasses = duccContextClassic+"-system-classes-data";
public final String classicSystemDaemons = duccContextClassic+"-system-daemons-data";
public final String classicSystemMachines = duccContextClassic+"-system-machines-data";
public final String classicSystemBroker = duccContextClassic+"-system-broker-data";
public DuccHandlerClassic(DuccWebServer duccWebServer) {
super.init(duccWebServer);
}
public String getFileName() {
return dir_home+File.separator+dir_resources+File.separator+getDuccWebServer().getClassDefinitionFile();
}
private void buildJobsListEntry(HttpServletRequest request, StringBuffer sb, DuccId duccId, IDuccWorkJob job, DuccData duccData, long now, ServicesRegistry servicesRegistry) {
EffectiveUser eu = EffectiveUser.create(request);
String id = normalize(duccId);
// Terminate
sb.append("");
if(terminateEnabled) {
if(!job.isFinished()) {
sb.append("");
}
}
sb.append(" ");
// Id
sb.append("");
sb.append(""+id+"");
sb.append(" ");
// Start
sb.append("");
sb.append("");
sb.append(getTimeStamp(request,job.getDuccId(), job.getStandardInfo().getDateOfSubmission()));
sb.append("");
sb.append(" ");
// Duration
sb.append("");
if(job.isCompleted()) {
String duration = getDuration(request,job, Precision.Whole);
String decoratedDuration = decorateDuration(request,job,duration,Precision.Whole);
sb.append("");
sb.append(decoratedDuration);
sb.append("");
}
else {
String duration = getDuration(request,job,now,Precision.Whole);
String decoratedDuration = decorateDuration(request,job,duration,Precision.Whole);
sb.append("");
sb.append(decoratedDuration);
sb.append("");
String projection = getProjection(request,job,Precision.Whole);
sb.append(projection);
}
sb.append(" ");
// User
String title = "";
String submitter = job.getStandardInfo().getSubmitter();
if(submitter != null) {
title = "title=\"submitter PID@host: "+submitter+"\" ";
}
sb.append("");
sb.append(job.getStandardInfo().getUser());
sb.append(" ");
// Class
sb.append("");
String schedulingClass = stringNormalize(job.getSchedulingInfo().getSchedulingClass(),messages.fetch("default"));
long debugPortDriver = job.getDebugPortDriver();
long debugPortProcess = job.getDebugPortProcess();
title = "debug ports:";
if(debugPortDriver >= 0) {
title = title + " driver="+debugPortDriver;
}
if(debugPortProcess >= 0) {
title = title + " process="+debugPortProcess;
}
switch(DuccCookies.getDisplayStyle(request)) {
case Textual:
default:
sb.append(schedulingClass);
if((debugPortDriver >= 0) || (debugPortProcess >= 0)) {
sb.append("
");
if(job.isCompleted()) {
sb.append("");
}
else {
sb.append("");
}
sb.append("DEBUG");
sb.append("");
}
break;
case Visual:
// Below
String key = "bug";
String bugFile = DuccWebServerHelper.getImageFileName(key);
sb.append(schedulingClass);
if((debugPortDriver >= 0) || (debugPortProcess >= 0)) {
sb.append("
");
if(job.isCompleted()) {
sb.append("");
}
else {
sb.append("");
}
if(bugFile != null) {
sb.append("![](\""+bugFile+"\")
");
}
sb.append("");
}
break;
}
sb.append(" ");
// State
sb.append("");
if(duccData.isLive(duccId)) {
if(job.isOperational()) {
sb.append("");
}
else {
sb.append("");
}
}
else {
sb.append("");
}
sb.append(job.getStateObject().toString());
if(duccData.isLive(duccId)) {
sb.append("");
}
sb.append(" ");
// Reason
String reason = getReason(job, MonitorType.Job).toString();
sb.append("");
sb.append(reason);
sb.append(" ");
// Services
sb.append("");
sb.append(evaluateServices(job,servicesRegistry));
sb.append(" ");
// Processes
sb.append("");
if(duccData.isLive(duccId)) {
sb.append(job.getProcessMap().getAliveProcessCount());
}
else {
sb.append("0");
}
sb.append(" ");
// Initialize Failures
sb.append("");
long initFails = job.getProcessInitFailureCount();
if(initFails > 0) {
if(job.getSchedulingInfo().getLongProcessesMax() < 0) {
DisplayStyle style = DuccCookies.getDisplayStyle(request);
String key = "cap.small";
String capFile = DuccWebServerHelper.getImageFileName(key);
switch(style) {
case Visual:
if(capFile == null) {
style = DisplayStyle.Textual;
}
break;
default:
break;
}
switch(style) {
case Textual:
default:
sb.append(buildInitializeFailuresLink(job));
sb.append("");
sb.append("");
sb.append("");
sb.append("capped");
sb.append("");
sb.append("");
sb.append("");
sb.append("
");
break;
case Visual:
sb.append("");
sb.append("![](\""+capFile+"\")
");
sb.append("");
sb.append("
");
sb.append(buildInitializeFailuresLink(job));
break;
}
}
else {
sb.append(buildInitializeFailuresLink(job));
}
}
else {
sb.append(""+initFails);
}
sb.append(" ");
// Runtime Failures
sb.append("");
sb.append(buildRuntimeFailuresLink(job));
sb.append(" ");
// Pgin
sb.append("");
long faults = 0;
try {
faults = job.getPgInCount();
}
catch(Exception e) {
}
int ifaults = (int)faults;
switch(ifaults) {
case -3: // (some do and some don't have cgroups) but retVal would have been > 0
sb.append("");
sb.append(inc);
break;
case -2: // (some do and some don't have cgroups) but retVal would have been == 0
sb.append("");
sb.append(inc);
break;
case -1: // (none have cgroups)
sb.append("");
sb.append(notAvailable);
break;
default: // (all have cgroups)
double swapping = job.getSwapUsageGbMax();
if((swapping * faults) > 0) {
sb.append("");
}
else {
sb.append("");
}
sb.append(faults);
break;
}
sb.append("");
sb.append(" ");
// Swap
sb.append("");
String swapSizeDisplay = "";
String swapSizeHover = "";
title = "";
double swap = 0;
if(job.isCompleted()) {
swap = job.getSwapUsageGbMax();
}
else {
swap = job.getSwapUsageGb();
}
int iswap = (int)swap;
switch(iswap) {
case -3: // (some do and some don't have cgroups) but retVal would have been > 0
sb.append("");
sb.append(inc);
break;
case -2: // (some do and some don't have cgroups) but retVal would have been == 0
sb.append("");
sb.append(inc);
break;
case -1: // (none have cgroups)
sb.append("");
sb.append(notAvailable);
break;
default: // (all have cgroups)
double swapBytes = swap*DuccHandlerUtils.GB;
swapSizeDisplay = DuccHandlerUtils.getSwapSizeDisplay(swapBytes);
swapSizeHover = DuccHandlerUtils.getSwapSizeHover(swapBytes);
title = "title="+"\""+swapSizeHover+"\"";
if(swapBytes > 0) {
sb.append("");
}
else {
sb.append("");
}
sb.append(swapSizeDisplay);
break;
}
sb.append("");
sb.append(" ");
// Memory
IDuccSchedulingInfo si;
SizeBytes sizeBytes;
String requested;
String actual;
si = job.getSchedulingInfo();
sizeBytes = new SizeBytes(SizeBytes.Type.Bytes, si.getMemorySizeAllocatedInBytes());
actual = getProcessMemorySize(duccId,sizeBytes);
sizeBytes = new SizeBytes(si.getMemoryUnits().name(), Long.parseLong(si.getMemorySizeRequested()));
requested = getProcessMemorySize(duccId,sizeBytes);
sb.append("");
sb.append("");
sb.append(actual);
sb.append("");
sb.append(" ");
// Total
sb.append("");
sb.append(job.getSchedulingInfo().getWorkItemsTotal());
sb.append(" ");
// Done
sb.append("");
IDuccPerWorkItemStatistics perWorkItemStatistics = job.getSchedulingInfo().getPerWorkItemStatistics();
String done = job.getSchedulingInfo().getWorkItemsCompleted();
if (perWorkItemStatistics != null) {
double max = Math.round(perWorkItemStatistics.getMax()/100.0)/10.0;
double min = Math.round(perWorkItemStatistics.getMin()/100.0)/10.0;
double avg = Math.round(perWorkItemStatistics.getMean()/100.0)/10.0;
double dev = Math.round(perWorkItemStatistics.getStandardDeviation()/100.0)/10.0;
done = ""+done+"";
}
sb.append(done);
sb.append(" ");
// Error
sb.append("");
sb.append(buildErrorLink(eu,job));
sb.append(" ");
// Dispatch
sb.append("");
String d0 = "";
String d1 = "0";
String d2 = "";
if(duccData.isLive(duccId)) {
int dispatch = 0;
int unassigned = job.getSchedulingInfo().getCasQueuedMap().size();
try {
dispatch = Integer.parseInt(job.getSchedulingInfo().getWorkItemsDispatched())-unassigned;
}
catch(Exception e) {
}
if(dispatch < 0) {
d0 = "";
//d1 = "0";
}
else {
d1 = ""+dispatch;
}
}
sb.append(d0);
sb.append(d1);
sb.append(d2);
sb.append(" ");
// Retry
sb.append("");
sb.append(job.getSchedulingInfo().getWorkItemsRetry());
sb.append(" ");
// Preempt
sb.append("");
sb.append(job.getSchedulingInfo().getWorkItemsPreempt());
sb.append(" ");
// Description
sb.append("");
String description = stringNormalize(job.getStandardInfo().getDescription(),messages.fetch("none"));
switch(DuccCookies.getDescriptionStyle(request)) {
case Long:
default:
sb.append("");
sb.append(description);
sb.append("");
break;
case Short:
String shortDescription = getShortDescription(description);
if(shortDescription == null) {
sb.append("");
sb.append(description);
sb.append("");
}
else {
sb.append("");
sb.append(shortDescription);
sb.append("");
}
break;
}
sb.append(" ");
sb.append("");
}
private void handleServletClassicJobs(String target,Request baseRequest,HttpServletRequest request,HttpServletResponse response)
throws IOException, ServletException
{
String methodName = "handleServletClassicJobs";
duccLogger.trace(methodName, jobid, messages.fetch("enter"));
StringBuffer sb = new StringBuffer();
if(dh.is_ducc_head_backup()) {
sb.append("");
sb.append("");
sb.append(messages.fetch("no data - not master"));
sb.append(" ");
sb.append(" ");
}
else {
ServicesRegistry servicesRegistry = ServicesRegistry.getInstance();
long now = System.currentTimeMillis();
int maxRecords = getJobsMax(request);
ArrayList users = getJobsUsers(request);
DuccData duccData = DuccData.getInstance();
ConcurrentSkipListMap sortedJobs = duccData.getSortedJobs();
if(sortedJobs.size()> 0) {
Iterator> iterator = sortedJobs.entrySet().iterator();
int counter = 0;
while(iterator.hasNext()) {
JobInfo jobInfo = iterator.next().getValue();
DuccWorkJob job = jobInfo.getJob();
boolean list = DuccWebUtil.isListable(request, users, maxRecords, counter, job);
if(list) {
counter++;
sb.append(trGet(counter));
buildJobsListEntry(request, sb, job.getDuccId(), job, duccData, now, servicesRegistry);
}
}
}
else {
sb.append("");
sb.append("");
if(DuccData.getInstance().isPublished()) {
sb.append(messages.fetch("no jobs"));
}
else {
sb.append(messages.fetch("no data"));
}
sb.append(" ");
sb.append(" ");
}
}
duccLogger.debug(methodName, jobid, sb);
response.getWriter().println(sb);
duccLogger.trace(methodName, jobid, messages.fetch("exit"));
}
private void buildReservationsListEntry(HttpServletRequest request, StringBuffer sb, DuccId duccId, IDuccWork duccwork, DuccData duccData, long now) {
String id = normalize(duccId);
String reservationType = "Unmanaged";
if(duccwork instanceof DuccWorkJob) {
reservationType = "Managed";
}
sb.append("");
if(terminateEnabled) {
if(!duccwork.isCompleted()) {
String disabled = getDisabledWithHover(request,duccwork);
String user = duccwork.getStandardInfo().getUser();
if(user != null) {
DuccPropertiesResolver dpr = DuccPropertiesResolver.getInstance();
String jdHostUser = dpr.getCachedProperty(DuccPropertiesResolver.ducc_jd_host_user);
// We presume that user is sufficient to identify JD allocation
if(user.equals(jdHostUser)) {
disabled = "disabled=\"disabled\"";
}
}
if(duccwork instanceof DuccWorkReservation) {
sb.append("");
}
else if(duccwork instanceof DuccWorkJob) {
sb.append("");
}
else {
//huh?
}
}
}
sb.append(" ");
// Id
if(reservationType.equals("Managed")) {
sb.append("");
sb.append(""+id+"");
sb.append(" ");
}
else {
sb.append("");
sb.append(id);
sb.append(" ");
}
// Start
sb.append("");
sb.append("");
sb.append(getTimeStamp(request,duccwork.getDuccId(),duccwork.getStandardInfo().getDateOfSubmission()));
sb.append("");
sb.append(" ");
// Duration
sb.append("");
if(duccwork instanceof DuccWorkReservation) {
DuccWorkReservation reservation = (DuccWorkReservation) duccwork;
String duration;
String decoratedDuration;
switch(reservation.getReservationState()) {
case Completed:
sb.append("");
duration = getDuration(request,reservation,Precision.Whole);
decoratedDuration = decorateDuration(request,reservation, duration);
sb.append(decoratedDuration);
sb.append("");
break;
default:
sb.append("");
duration = getDuration(request,reservation,now,Precision.Whole);
decoratedDuration = decorateDuration(request,reservation, duration);
sb.append(decoratedDuration);
sb.append("");
break;
}
}
else if(duccwork instanceof DuccWorkJob) {
DuccWorkJob job = (DuccWorkJob) duccwork;
switch(job.getJobState()) {
case Completed:
sb.append("");
String duration = getDuration(request,job,Precision.Whole);
String decoratedDuration = decorateDuration(request,job,duration,Precision.Whole);
sb.append(decoratedDuration);
sb.append("");
break;
default:
sb.append("");
duration = getDuration(request,job,now,Precision.Whole);
decoratedDuration = decorateDuration(request,job,duration,Precision.Whole);
sb.append(decoratedDuration);
sb.append("");
break;
}
}
sb.append(" ");
// User
String title = "";
String submitter = duccwork.getStandardInfo().getSubmitter();
if(submitter != null) {
title = "title=\"submitter PID@host: "+submitter+"\"";
}
sb.append("");
UserId userId = new UserId(duccwork.getStandardInfo().getUser());
sb.append(userId.toString());
sb.append(" ");
// Class
sb.append("");
sb.append(stringNormalize(duccwork.getSchedulingInfo().getSchedulingClass(),messages.fetch("default")));
sb.append(" ");
// Type
sb.append("");
sb.append(reservationType);
sb.append(" ");
// State
sb.append("");
if(duccData.isLive(duccId)) {
if(duccwork.isOperational()) {
sb.append("");
}
else {
sb.append("");
}
}
else {
sb.append("");
}
sb.append(duccwork.getStateObject().toString());
if(duccData.isLive(duccId)) {
sb.append("");
}
sb.append(" ");
// Reason
if(duccwork instanceof DuccWorkReservation) {
DuccWorkReservation reservation = (DuccWorkReservation) duccwork;
sb.append("");
switch(reservation.getReservationState()) {
case WaitingForResources:
String rmReason = reservation.getRmReason();
if(rmReason != null) {
sb.append("");
sb.append(rmReason);
sb.append("");
}
break;
case Assigned:
List list = reservation.getJdReservationBeanList();
long inuse = 0;
long total = 0;
if(list != null) {
for(JdReservationBean jdReservationBean : list) {
JdReservation jdReservation = (JdReservation) jdReservationBean;
inuse += jdReservation.getSlicesInuse();
total += jdReservation.getSlicesTotal();
}
title = "title=\"the number of job driver allocations inuse for this reservation\"";
sb.append("");
sb.append("inuse: "+inuse);
sb.append("");
sb.append(" ");
title = "title=\"the number of job driver allocations maximum capacity for this reservation\"";
sb.append("");
sb.append("limit: "+total);
sb.append("");
}
break;
default:
switch(reservation.getCompletionType()) {
case Undefined:
break;
case CanceledByUser:
case CanceledByAdmin:
try {
String cancelUser = duccwork.getStandardInfo().getCancelUser();
if(cancelUser != null) {
sb.append("");
sb.append(duccwork.getCompletionTypeObject().toString());
sb.append("");
}
else {
IRationale rationale = reservation.getCompletionRationale();
if(rationale != null) {
sb.append("");
sb.append(duccwork.getCompletionTypeObject().toString());
sb.append("");
}
else {
sb.append(duccwork.getCompletionTypeObject().toString());
}
}
}
catch(Exception e) {
IRationale rationale = reservation.getCompletionRationale();
if(rationale != null) {
sb.append("");
sb.append(duccwork.getCompletionTypeObject().toString());
sb.append("");
}
else {
sb.append(duccwork.getCompletionTypeObject().toString());
}
}
break;
default:
IRationale rationale = reservation.getCompletionRationale();
if(rationale != null) {
sb.append("");
sb.append(duccwork.getCompletionTypeObject().toString());
sb.append("");
}
else {
sb.append(duccwork.getCompletionTypeObject().toString());
}
break;
}
break;
}
sb.append(" ");
}
else if(duccwork instanceof DuccWorkJob) {
DuccWorkJob job = (DuccWorkJob) duccwork;
String reason = getReason(job, MonitorType.ManagedReservation).toString();
sb.append("");
sb.append(reason);
sb.append(" ");
}
// Allocation
/*
sb.append("");
sb.append(duccwork.getSchedulingInfo().getInstancesCount());
sb.append(" ");
*/
// User Processes
sb.append("");
TreeMap nodeMap = new TreeMap();
if(duccwork instanceof DuccWorkReservation) {
DuccWorkReservation reservation = (DuccWorkReservation) duccwork;
if(!reservation.getReservationMap().isEmpty()) {
IDuccReservationMap map = reservation.getReservationMap();
for (DuccId key : map.keySet()) {
IDuccReservation value = reservation.getReservationMap().get(key);
String node = value.getNodeIdentity().getCanonicalName();
if(!nodeMap.containsKey(node)) {
nodeMap.put(node,new Integer(0));
}
Integer count = nodeMap.get(node);
count++;
nodeMap.put(node,count);
}
}
boolean qualify = false;
if(!nodeMap.isEmpty()) {
if(nodeMap.keySet().size() > 1) {
qualify = true;
}
}
ArrayList qualifiedPids = new ArrayList();
if(duccwork.isOperational()) {
DuccMachinesData machinesData = DuccMachinesData.getInstance();
for (String node: nodeMap.keySet()) {
NodeId nodeId = new NodeId(node);
List nodePids = machinesData.getPids(nodeId, userId);
for( String pid : nodePids ) {
if(qualify) {
qualifiedPids.add(node+":"+pid);
}
else {
qualifiedPids.add(pid);
}
}
}
}
if(qualifiedPids.size() > 0) {
String list = "";
for( String entry : qualifiedPids ) {
list += entry+" ";
}
sb.append("");
sb.append(""+qualifiedPids.size());
sb.append("");
}
else {
sb.append(""+qualifiedPids.size());
}
}
else {
DuccWorkJob job = (DuccWorkJob) duccwork;
if(job.isOperational()) {
sb.append(duccwork.getSchedulingInfo().getInstancesCount());
}
else {
sb.append("0");
}
Iterator iterator = job.getProcessMap().keySet().iterator();
while(iterator.hasNext()) {
DuccId processId = iterator.next();
IDuccProcess process = job.getProcessMap().get(processId);
String node = process.getNodeIdentity().getCanonicalName();
nodeMap.put(node, 1);
}
}
sb.append(" ");
// PgIn
sb.append("");
if(duccwork instanceof DuccWorkJob) {
DuccWorkJob job = (DuccWorkJob) duccwork;
long faults = 0;
try {
faults = job.getPgInCount();
}
catch(Exception e) {
}
int ifaults = (int)faults;
switch(ifaults) {
case -3: // (some do and some don't have cgroups) but retVal would have been > 0
sb.append("");
sb.append(inc);
break;
case -2: // (some do and some don't have cgroups) but retVal would have been == 0
sb.append("");
sb.append(inc);
break;
case -1: // (none have cgroups)
sb.append("");
sb.append(notAvailable);
break;
default: // (all have cgroups)
double swapping = job.getSwapUsageGbMax();
if((swapping * faults) > 0) {
sb.append("");
}
else {
sb.append("");
}
sb.append(faults);
break;
}
sb.append("");
}
sb.append(" ");
// Swap
sb.append("");
if(duccwork instanceof DuccWorkJob) {
DuccWorkJob job = (DuccWorkJob) duccwork;
String swapSizeDisplay = "";
String swapSizeHover = "";
title = "";
double swap = 0;
if(job.isCompleted()) {
swap = job.getSwapUsageGbMax();
}
else {
swap = job.getSwapUsageGb();
}
int iswap = (int)swap;
switch(iswap) {
case -3: // (some do and some don't have cgroups) but retVal would have been > 0
sb.append("");
sb.append(inc);
break;
case -2: // (some do and some don't have cgroups) but retVal would have been == 0
sb.append("");
sb.append(inc);
break;
case -1: // (none have cgroups)
sb.append("");
sb.append(notAvailable);
break;
default: // (all have cgroups)
double swapBytes = swap*DuccHandlerUtils.GB;
swapSizeDisplay = DuccHandlerUtils.getSwapSizeDisplay(swapBytes);
swapSizeHover = DuccHandlerUtils.getSwapSizeHover(swapBytes);
title = "title="+"\""+swapSizeHover+"\"";
if(swapBytes > 0) {
sb.append("");
}
else {
sb.append("");
}
sb.append(swapSizeDisplay);
break;
}
sb.append("");
}
sb.append(" ");
// Memory
IDuccSchedulingInfo si;
SizeBytes sizeBytes;
String requested;
String actual;
si = duccwork.getSchedulingInfo();
sizeBytes = new SizeBytes(SizeBytes.Type.Bytes, si.getMemorySizeAllocatedInBytes());
actual = getProcessMemorySize(duccId,sizeBytes);
sizeBytes = new SizeBytes(si.getMemoryUnits().name(), Long.parseLong(si.getMemorySizeRequested()));
requested = getProcessMemorySize(duccId,sizeBytes);
sb.append("");
sb.append("");
sb.append(actual);
sb.append("");
sb.append(" ");
// Host Names
sb.append("");
if(!nodeMap.isEmpty()) {
boolean useList = false;
if(nodeMap.size() > 1) {
useList = true;
}
if(useList) {
sb.append("");
}
}
sb.append(" ");
// Description
sb.append("");
String description = stringNormalize(duccwork.getStandardInfo().getDescription(),messages.fetch("none"));
switch(DuccCookies.getDescriptionStyle(request)) {
case Long:
default:
sb.append("");
sb.append(description);
sb.append("");
break;
case Short:
String shortDescription = getShortDescription(description);
if(shortDescription == null) {
sb.append("");
sb.append(description);
sb.append("");
}
else {
sb.append("");
sb.append(shortDescription);
sb.append("");
}
break;
}
sb.append(" ");
sb.append("");
}
private void handleServletClassicReservations(String target,Request baseRequest,HttpServletRequest request,HttpServletResponse response)
throws IOException, ServletException
{
String methodName = "handleServletClassicReservations";
duccLogger.trace(methodName, jobid, messages.fetch("enter"));
StringBuffer sb = new StringBuffer();
if(dh.is_ducc_head_backup()) {
sb.append("");
sb.append("");
sb.append(messages.fetch("no data - not master"));
sb.append(" ");
sb.append(" ");
}
else {
int maxRecords = getReservationsMax(request);
DuccData duccData = DuccData.getInstance();
ConcurrentSkipListMap sortedCombinedReservations = duccData.getSortedCombinedReservations();
ArrayList users = getReservationsUsers(request);
long now = System.currentTimeMillis();
if((sortedCombinedReservations.size() > 0)) {
int counter = 0;
Iterator> iR = sortedCombinedReservations.entrySet().iterator();
while(iR.hasNext()) {
Info info = iR.next().getValue();
IDuccWork dw = info.getDuccWork();
boolean list = DuccWebUtil.isListable(request, users, maxRecords, counter, dw);
if(list) {
counter++;
if(dw instanceof DuccWorkReservation) {
DuccWorkReservation reservation = (DuccWorkReservation) dw;
sb.append(trGet(counter));
buildReservationsListEntry(request, sb, reservation.getDuccId(), reservation, duccData, now);
}
else if(dw instanceof DuccWorkJob) {
DuccWorkJob job = (DuccWorkJob) dw;
sb.append(trGet(counter));
buildReservationsListEntry(request, sb, job.getDuccId(), job, duccData, now);
}
else {
// huh?
}
}
}
}
else {
sb.append("");
sb.append("");
if(DuccData.getInstance().isPublished()) {
sb.append(messages.fetch("no reservations"));
}
else {
sb.append(messages.fetch("no data"));
}
sb.append(" ");
sb.append(" ");
}
}
duccLogger.debug(methodName, jobid, sb);
response.getWriter().println(sb);
duccLogger.trace(methodName, jobid, messages.fetch("exit"));
}
private void handleServletClassicServices(String target,Request baseRequest,HttpServletRequest request,HttpServletResponse response)
throws IOException, ServletException
{
String methodName = "handleServletClassicServices";
duccLogger.trace(methodName, jobid, messages.fetch("enter"));
StringBuffer sb = new StringBuffer();
if(dh.is_ducc_head_backup()) {
sb.append("");
sb.append("");
sb.append(messages.fetch("no data - not master"));
sb.append(" ");
sb.append(" ");
}
else {
ServicesSortCache servicesSortCache = ServicesSortCache.getInstance();
Collection servicesSortedCollection = servicesSortCache.getSortedCollection();
if(!servicesSortedCollection.isEmpty()) {
int maxRecords = getServicesMax(request);
ArrayList users = getServicesUsers(request);
int counter = 0;
for(IServiceAdapter service : servicesSortedCollection) {
boolean list = DuccWebUtil.isListable(request, users, maxRecords, counter, service);
if(!list) {
continue;
}
counter++;
// Row Begin
sb.append("");
int sid = service.getId();
String user = service.getUser();
long deployments = service.getDeployments();
long instances = service.getInstances();
// Enable
sb.append("");
if(service.isRegistered()) {
if(buttonsEnabled) {
if(service.isDisabled()) {
sb.append("");
}
}
}
sb.append(" ");
// Stop
sb.append("");
if(service.isRegistered()) {
if(buttonsEnabled) {
if(service.isPingOnly()) {
if(service.isPingActive()) {
sb.append("");
}
}
else {
if(deployments != 0) {
sb.append("");
}
}
}
}
sb.append(" ");
// Id
String name = service.getName();
sb.append("");
String id = ""+sid+"";
sb.append(""+id);
sb.append(" ");
// Name
sb.append("");
sb.append(name);
sb.append(" ");
// State
sb.append("");
String state = service.getState();
boolean alert = service.isAlert();
boolean available = service.isStateAvailable();
if(alert) {
state += "+Alert";
}
String style = "class=\"health_black\";";
if(alert) {
style = "class=\"health_red\"";
}
else if(available) {
style = "class=\"health_green\"";
}
String stateHover = ServicesHelper.getInstance().getStateHover(service);
if(stateHover.length() > 0) {
stateHover = "title="+"\""+stateHover+"\"";
}
sb.append("");
sb.append(state);
sb.append("");
sb.append(" ");
// Last Use
sb.append("");
long lastUse = service.getLastUse();
if(lastUse > 0) {
sb.append(getTimeStamp(request, jobid, ""+lastUse));
}
sb.append(" ");
// Instances
sb.append("");
sb.append(instances);
sb.append(" ");
// Deployments
sb.append("");
sb.append(deployments);
sb.append(" ");
// Start-State
StartState startState = service.getStartState();
sb.append("");
sb.append("");
sb.append(startState.name());
if(service.isDisabled()) {
sb.append("
");
String health = "class=\"health_red\"";
String reason = "title=\""+service.getDisableReason()+"\"";
sb.append("");
sb.append("Disabled");
sb.append("");
}
sb.append("");
sb.append(" ");
// User
sb.append("");
sb.append(user);
sb.append(" ");
// Share Class (or Type)
sb.append("");
if(service.isPingOnly()) {
String schedulingClass = service.getSchedulingClass();
sb.append("");
String serviceType = "ping-only";
sb.append("");
sb.append(serviceType);
}
else {
String schedulingClass = service.getSchedulingClass();
sb.append(schedulingClass);
}
sb.append(" ");
// PgIn
sb.append("");
long faults = 0;
try {
faults = service.getPgIn();
}
catch(Exception e) {
}
int ifaults = (int)faults;
switch(ifaults) {
case -3: // (some do and some don't have cgroups) but retVal would have been > 0
sb.append("");
sb.append(inc);
break;
case -2: // (some do and some don't have cgroups) but retVal would have been == 0
sb.append("");
sb.append(inc);
break;
case -1: // (none have cgroups)
sb.append("");
sb.append(notAvailable);
break;
default: // (all have cgroups)
double swapping = service.getSwap();
if((swapping * faults) > 0) {
sb.append("");
}
else {
sb.append("");
}
sb.append(faults);
break;
}
sb.append("");
sb.append(" ");
// Swap
sb.append("");
String swapSizeDisplay = "";
String swapSizeHover = "";
String title = "";
double swap = service.getSwap();
int iswap = (int)swap;
switch(iswap) {
case -3: // (some do and some don't have cgroups) but retVal would have been > 0
sb.append("");
sb.append(inc);
break;
case -2: // (some do and some don't have cgroups) but retVal would have been == 0
sb.append("");
sb.append(inc);
break;
case -1: // (none have cgroups)
sb.append("");
sb.append(notAvailable);
break;
default: // (all have cgroups)
double swapBytes = swap;
swapSizeDisplay = DuccHandlerUtils.getSwapSizeDisplay(swapBytes);
swapSizeHover = DuccHandlerUtils.getSwapSizeHover(swapBytes);
title = "title="+"\""+swapSizeHover+"\"";
if(swapBytes > 0) {
sb.append("");
}
else {
sb.append("");
}
sb.append(swapSizeDisplay);
break;
}
sb.append("");
sb.append(" ");
// Size
sb.append("");
long size = service.getSize();
if(size < 0) {
size = 0;
}
sb.append(size);
sb.append(" ");
// Jobs
sb.append("");
ArrayList dependentJobs = service.getDependentJobs();
int countDependentJobs = dependentJobs.size();
String titleJobs = "";
if(countDependentJobs > 0) {
StringBuffer idList = new StringBuffer();
for(String duccId : dependentJobs) {
if(idList.length() > 0) {
idList.append(",");
}
idList.append(duccId);
}
titleJobs = "dependent Job Id list: "+idList;
}
String jobs = ""+countDependentJobs+"";
sb.append(jobs);
sb.append(" ");
// Services
sb.append("");
ArrayList dependentServices = service.getDependentServices();
int countDependentServices = dependentServices.size();
String titleServices = "";
if(countDependentServices > 0) {
StringBuffer idList = new StringBuffer();
for(String duccId : dependentServices) {
if(idList.length() > 0) {
idList.append(",");
}
idList.append(duccId);
}
titleServices = "dependent Service Name list: "+idList;
}
String services = ""+countDependentServices+"";
sb.append(services);
sb.append(" ");
// Reservations
sb.append("");
ArrayList dependentReservations = service.getDependentReservations();
int countDependentReservations = dependentReservations.size();
String titleReservations = "";
if(countDependentReservations > 0) {
StringBuffer idList = new StringBuffer();
for(String duccId : dependentReservations) {
if(idList.length() > 0) {
idList.append(",");
}
idList.append(duccId);
}
titleReservations = "dependent Reservation Id list: "+idList;
}
String reservations = ""+countDependentReservations+"";
sb.append(reservations);
sb.append(" ");
// Description
sb.append("");
String description = service.getDescription();
sb.append(description);
sb.append(" ");
// Row End
sb.append(" ");
}
}
else {
sb.append("");
sb.append("");
if(DuccData.getInstance().isPublished()) {
sb.append(messages.fetch("no services"));
}
else {
sb.append(messages.fetch("no data"));
}
sb.append(" ");
sb.append(" ");
}
}
duccLogger.debug(methodName, jobid, sb);
response.getWriter().println(sb);
duccLogger.trace(methodName, jobid, messages.fetch("exit"));
}
private void handleServletClassicSystemClasses(String target,Request baseRequest,HttpServletRequest request,HttpServletResponse response)
throws Exception
{
String methodName = "handleServletClassicSystemClasses";
duccLogger.trace(methodName, jobid, messages.fetch("enter"));
StringBuffer sb = new StringBuffer();
String val = null;
NodeConfiguration nc = getNodeConfiguration();
DuccSchedulerClasses schedulerClasses = DuccSchedulerClasses.getInstance();
Map clmap = schedulerClasses.getClasses();
if ( clmap != null ) {
DuccProperties[] class_set = clmap.values().toArray(new DuccProperties[clmap.size()]);
Arrays.sort(class_set, new NodeConfiguration.ClassSorter());
int i = 0;
for ( DuccProperties cl : class_set) {
sb.append(trGet(i+1));
// Name
String class_name = cl.getProperty("name");
sb.append("");
sb.append(class_name);
sb.append(" ");
// Nodepool
sb.append("");
val = cl.getProperty("nodepool");
sb.append(val);
sb.append(" ");
// Policy
sb.append("");
String policy = cl.getProperty("policy");
sb.append(policy);
sb.append(" ");
// Quantum
sb.append("");
sb.append(getQuantum(nc,class_name));
sb.append(" ");
// Weight
sb.append("");
sb.append(cl.getStringProperty("weight", "-"));
sb.append(" ");
// Priority
sb.append("");
sb.append(cl.getProperty("priority"));
sb.append(" ");
// Non-preemptable Class
sb.append("");
val = "-";
if(schedulerClasses.isPreemptable(class_name)) {
String v1 = cl.getStringProperty("debug", "");
if(!v1.equals("")) {
val = v1;
}
}
sb.append(val);
sb.append(" ");
sb.append("");
}
}
duccLogger.debug(methodName, jobid, sb);
response.getWriter().println(sb);
duccLogger.trace(methodName, jobid, messages.fetch("exit"));
}
private void handleServletClassicSystemDaemons(String target,Request baseRequest,HttpServletRequest request,HttpServletResponse response)
throws IOException, ServletException
{
String methodName = "handleServletClassicSystemDaemons";
duccLogger.trace(methodName, jobid, messages.fetch("enter"));
StringBuffer sb = new StringBuffer();
String wsHostIP = getWebServerHostIP();
String wsHostName = getWebServerHostName();
DuccDaemonsData duccDaemonsData = DuccDaemonsData.getInstance();
DuccMachinesData duccMachinesData = DuccMachinesData.getInstance();
BrokerHelper brokerHelper = BrokerHelper.getInstance();
int counter = 0;
boolean brokerAlive = brokerHelper.isAlive();
daemons:
for(DaemonName daemonName : DuccDaemonRuntimeProperties.daemonNames) {
switch(daemonName) {
case Database:
if(databaseHelper.isDisabled()) {
continue daemons;
}
default:
break;
}
String status = "";
String bootTime = "";
String hostIP = "";
String hostName = "";
String pid = "";
String pubSizeLast = "";
String pubSizeMax = "";
String heartbeatLast = "";
String heartbeatMax = "";
String heartbeatMaxTOD = "";
String jmxUrl = null;
Properties properties = DuccDaemonRuntimeProperties.getInstance().get(daemonName);
switch(daemonName) {
case Broker:
if(brokerAlive) {
status = DuccHandlerUtils.up();
}
else {
status = DuccHandlerUtils.down();
}
bootTime = getTimeStamp(DuccCookies.getDateStyle(request),brokerHelper.getStartTime());
hostName = useWS(wsHostName, brokerHelper.getHost());
hostIP = useWS(wsHostName, hostName, wsHostIP);
pid = ""+brokerHelper.getPID();
pubSizeLast = "-";
pubSizeMax = "-";
heartbeatLast = "";
heartbeatMax = "";
heartbeatMaxTOD = "";
jmxUrl = brokerHelper.getJmxUrl();
break;
case Database:
if(databaseHelper.isAlive()) {
status = DuccHandlerUtils.up();
}
else {
status = DuccHandlerUtils.down();
}
if(DuccPropertiesHelper.isDatabaseAutomanaged()) {
bootTime = getTimeStamp(DuccCookies.getDateStyle(request),databaseHelper.getStartTime());
hostName = useWS(wsHostName, databaseHelper.getHostListString());
hostIP = useWS(wsHostName, hostName, wsHostIP);
pid = ""+databaseHelper.getPID();
}
else {
bootTime = "-";
hostName = useWS(wsHostName, databaseHelper.getHostListString());
String[] hostList = hostName.split("\\s+");
String[] ipList = InetHelper.getIP(hostList);
StringBuffer sbuf = new StringBuffer();
if(ipList != null) {
for(String ip : ipList) {
sbuf.append(ip);
sbuf.append(" ");
}
}
hostIP = sbuf.toString();
pid = "-";
}
//
hostName = tableize(hostName);
hostIP = tableize(hostIP);
//
pubSizeLast = "-";
pubSizeMax = "-";
heartbeatLast = "";
heartbeatMax = "";
heartbeatMaxTOD = "";
jmxUrl = databaseHelper.getJmxUrl();
break;
case Webserver:
status = DuccHandlerUtils.up();
bootTime = getTimeStamp(DuccCookies.getDateStyle(request),getPropertiesValue(properties,DuccDaemonRuntimeProperties.keyBootTime,""));
hostIP = getPropertiesValue(properties,DuccDaemonRuntimeProperties.keyNodeIpAddress,"");
hostName = getPropertiesValue(properties,DuccDaemonRuntimeProperties.keyNodeName,"");
pid = getPropertiesValue(properties,DuccDaemonRuntimeProperties.keyPid,"");
pubSizeLast = "*";
pubSizeMax = "*";
heartbeatLast = "";
heartbeatMax = "";
heartbeatMaxTOD = "";
jmxUrl = getPropertiesValue(properties,DuccDaemonRuntimeProperties.keyJmxUrl,"");
break;
default:
status = DuccHandlerUtils.unknown();
if(daemonName.equals(DaemonName.Orchestrator)) {
if(ComponentHelper.isLocked(IDuccEnv.DUCC_STATE_DIR,"orchestrator")) {
String filename = ComponentHelper.getLockFileName(IDuccEnv.DUCC_STATE_DIR,"orchestrator");
String hover = "title=\""+ComponentHelper.getLockFileNameWithPath(IDuccEnv.DUCC_STATE_DIR,"orchestrator")+"\"";
String fileNameWithHover = ""+filename+"";
status += ", "+DuccHandlerUtils.warn("warning: ")+fileNameWithHover+" found.";
}
}
bootTime = getTimeStamp(DuccCookies.getDateStyle(request),getPropertiesValue(properties,DuccDaemonRuntimeProperties.keyBootTime,""));
hostIP = getPropertiesValue(properties,DuccDaemonRuntimeProperties.keyNodeIpAddress,"");
hostName = getPropertiesValue(properties,DuccDaemonRuntimeProperties.keyNodeName,"");
pid = getPropertiesValue(properties,DuccDaemonRuntimeProperties.keyPid,"");
pubSizeLast = ""+duccDaemonsData.getEventSize(daemonName);
pubSizeMax = ""+duccDaemonsData.getEventSizeMax(daemonName);
heartbeatLast = DuccDaemonsData.getInstance().getHeartbeat(daemonName);
long timeout = DuccWebProperties.get_ducc_ws_monitored_daemon_down_millis_expiry()/1000;
if(timeout > 0) {
try {
long overtime = timeout - Long.parseLong(heartbeatLast);
if(overtime < 0) {
if(brokerAlive) {
status = DuccHandlerUtils.down();
}
if(daemonName.equals(DaemonName.Orchestrator)) {
if(ComponentHelper.isLocked(IDuccEnv.DUCC_STATE_DIR,"orchestrator")) {
String filename = ComponentHelper.getLockFileName(IDuccEnv.DUCC_STATE_DIR,"orchestrator");
String hover = "title=\""+ComponentHelper.getLockFileNameWithPath(IDuccEnv.DUCC_STATE_DIR,"orchestrator")+"\"";
String fileNameWithHover = ""+filename+"";
status += ", "+DuccHandlerUtils.warn("warning: ")+fileNameWithHover+" found.";
}
}
}
else {
if(brokerAlive) {
status = DuccHandlerUtils.up();
}
if(daemonName.equals(DaemonName.Orchestrator)) {
if(dh.is_ducc_head_virtual_master()) {
boolean reqMet = DuccData.getInstance().getLive().isJobDriverMinimalAllocateRequirementMet();
if(!reqMet) {
status = DuccHandlerUtils.up_provisional(", pending JD allocation");
}
}
}
}
}
catch(Throwable t) {
}
}
heartbeatMax = DuccDaemonsData.getInstance().getMaxHeartbeat(daemonName);
heartbeatMaxTOD = TimeStamp.simpleFormat(DuccDaemonsData.getInstance().getMaxHeartbeatTOD(daemonName));
try {
heartbeatMaxTOD = getTimeStamp(DuccCookies.getDateStyle(request),heartbeatMaxTOD);
}
catch(Exception e) {
}
jmxUrl = getPropertiesValue(properties,DuccDaemonRuntimeProperties.keyJmxUrl,"");
break;
}
// Status
sb.append(trGet(counter));
sb.append("");
sb.append(status);
sb.append(" ");
// Daemon Name
sb.append("");
sb.append(daemonName);
sb.append(" ");
// Boot Time
sb.append("");
sb.append(bootTime);
sb.append(" ");
// Host IP
sb.append("");
sb.append(hostIP);
sb.append(" ");
// Host Name
sb.append("");
sb.append(hostName);
sb.append(" ");
// PID
sb.append("");
sb.append(pid);
sb.append(" ");
// Publication Size (last)
sb.append("");
sb.append(""+pubSizeLast);
sb.append(" ");
// Publication Size (max)
sb.append("");
sb.append(pubSizeMax);
sb.append(" ");
// Heartbeat (last)
sb.append("");
sb.append(heartbeatLast);
sb.append(" ");
// Heartbeat (max)
sb.append("");
sb.append(heartbeatMax);
sb.append(" ");
// Heartbeat (max) TOD
sb.append("");
sb.append(heartbeatMaxTOD);
sb.append(" ");
// JConsole URL
sb.append("");
if(jmxUrl != null) {
sb.append(buildjConsoleLink(jmxUrl));
}
sb.append(" ");
//
sb.append("");
counter++;
}
//
String cookie = DuccCookies.getCookie(request,DuccCookies.cookieAgents);
if(cookie.equals(DuccCookies.valueAgentsShow)) {
duccLogger.trace(methodName, jobid, "== show: "+cookie);
if(dh.is_ducc_head_backup()) {
sb.append("");
sb.append("");
sb.append(messages.fetch("no agents - not master"));
sb.append(" ");
sb.append(" ");
}
else {
Map machines = duccMachinesData.getMachines();
Iterator iterator = machines.keySet().iterator();
while(iterator.hasNext()) {
MachineInfo machineInfo = iterator.next();
DuccDaemonRuntimeProperties drp = DuccDaemonRuntimeProperties.getInstance();
String machineName = machineInfo.getShortName();
if(machineName.startsWith("=")) {
continue;
}
Properties properties = drp.getAgent(machineName);
sb.append(trGet(counter));
// Status
StringBuffer status = new StringBuffer();
if(brokerAlive) {
String machineStatus = machineInfo.getStatus();
if(machineStatus.equals("down")) {
//status.append("");
status.append(DuccHandlerUtils.down());
//status.append("");
}
else if(machineStatus.equals("up")) {
//status.append("");
status.append(DuccHandlerUtils.up());
//status.append("");
}
else {
status.append(DuccHandlerUtils.unknown());
}
}
else {
status.append(DuccHandlerUtils.unknown());
}
sb.append("");
sb.append(status);
sb.append(" ");
sb.append("");
// Daemon Name
String daemonName = "Agent";
sb.append("");
sb.append(daemonName);
sb.append(" ");
// Boot Time
String bootTime = getTimeStamp(DuccCookies.getDateStyle(request),getPropertiesValue(properties,DuccDaemonRuntimeProperties.keyBootTime,""));
sb.append("");
sb.append(bootTime);
sb.append(" ");
// Host IP
String hostIP = getPropertiesValue(properties,DuccDaemonRuntimeProperties.keyNodeIpAddress,"");
sb.append("");
sb.append(hostIP);
sb.append(" ");
// Host Name
String hostName = machineInfo.getName();
sb.append("");
sb.append(hostName);
sb.append(" ");
// PID
String pid = getPropertiesValue(properties,DuccDaemonRuntimeProperties.keyPid,"");
sb.append("");
sb.append(pid);
sb.append(" ");
// Publication Size (last)
String publicationSizeLast = machineInfo.getPublicationSizeLast();
sb.append("");
sb.append(publicationSizeLast);
sb.append(" ");
// Publication Size (max)
String publicationSizeMax = machineInfo.getPublicationSizeMax();
sb.append("");
sb.append(publicationSizeMax);
sb.append(" ");
// Heartbeat (last)
String heartbeatLast = machineInfo.getHeartbeatLast();
sb.append("");
sb.append(heartbeatLast);
sb.append(" ");
// Heartbeat (max)
long heartbeatMax = machineInfo.getHeartbeatMax();
sb.append("");
if(heartbeatMax > 0) {
sb.append(heartbeatMax);
}
sb.append(" ");
// Heartbeat (max) TOD
String fmtHeartbeatMaxTOD = "";
long heartbeatMaxTOD = machineInfo.getHeartbeatMaxTOD();
if(heartbeatMaxTOD > 0) {
fmtHeartbeatMaxTOD = TimeStamp.simpleFormat(""+heartbeatMaxTOD);
try {
fmtHeartbeatMaxTOD = getTimeStamp(DuccCookies.getDateStyle(request),fmtHeartbeatMaxTOD);
}
catch(Exception e) {
}
}
sb.append("");
sb.append(fmtHeartbeatMaxTOD);
sb.append(" ");
// JConsole URL
sb.append("");
String jmxUrl = getPropertiesValue(properties,DuccDaemonRuntimeProperties.keyJmxUrl,"");
if(jmxUrl != null) {
sb.append(buildjConsoleLink(jmxUrl));
}
sb.append(" ");
//
sb.append("");
counter++;
}
}
}
else {
duccLogger.trace(methodName, jobid, "!= show: "+cookie);
}
//
duccLogger.debug(methodName, jobid, sb);
response.getWriter().println(sb);
duccLogger.trace(methodName, jobid, messages.fetch("exit"));
}
// Individual Machine
private void buildRowForIndividualMachine(StringBuffer data, int counter, MachineInfo machineInfo, SizeBytes allocated) {
StringBuffer row = new StringBuffer();
row.append((trGet(counter)));
StringBuffer sb = new StringBuffer();
// Status
String status = machineInfo.getStatus();
if(status == null) {
status = "?";
}
String hover = "title=\""+machineInfo.getMachineStatusReason()+"\"";
if(status.equals("down")) {
sb.append("");
sb.append(status);
sb.append("");
}
else if(status.equals("up")) {
sb.append("");
sb.append(status);
sb.append("");
}
else {
sb.append(status);
}
row.append("");
row.append(sb);
row.append(" ");
// Online
row.append("");
row.append(nodeState.getOnline(machineInfo.getName(), "-"));
row.append(" ");
// Quiesced
row.append("");
row.append(nodeState.getQuiesced(machineInfo.getName(), "-"));
row.append(" ");
// IP
row.append("");
row.append(machineInfo.getIp());
row.append(" ");
// Name
row.append("");
row.append(machineInfo.getName());
row.append(" ");
// Nodepool
row.append("");
String nodepool = DuccSchedulerClasses.getInstance().getNodepool(machineInfo.getName());
row.append(nodepool);
row.append(" ");
// Memory: usable
if(status.equals("up")) {
sb = new StringBuffer();
sb.append("total="+machineInfo.getMemTotal());
Integer quantum = machineInfo.getQuantum();
if(quantum != null) {
sb.append(" ");
sb.append("quantum="+quantum);
}
hover = "title=\""+sb.toString()+"\"";
row.append("");
row.append(machineInfo.getMemReserve());
row.append(" ");
}
else if(status.equals("down")) {
row.append("");
row.append("0");
row.append(" ");
}
else {
row.append("");
row.append(" ");
}
// Memory: free
if(status.equals("up")) {
long memFree = ConvertSafely.String2Long(machineInfo.getMemReserve());
memFree = memFree - allocated.getGBytes();
row.append("");
row.append(memFree);
row.append(" ");
}
else if(status.equals("down")) {
row.append("");
row.append("0");
row.append(" ");
}
else {
row.append("");
row.append(" ");
}
// CPU: load average
row.append("");
if(status.equals("up")) {
String cpu = formatter1.format(machineInfo.getCpu());
row.append(cpu);
}
row.append(" ");
// Swap: inuse
sb = new StringBuffer();
String swapping = machineInfo.getSwapInuse();
if(swapping.equals("0")) {
sb.append(swapping);
}
else {
sb.append("");
sb.append(swapping);
sb.append("");
}
row.append("");
if(!status.equals("defined")) {
row.append(sb);
}
row.append(" ");
// Swap: free
row.append("");
if(!status.equals("defined")) {
row.append(machineInfo.getSwapFree());
}
row.append(" ");
// C-Groups
boolean isCgroupsEnabled = machineInfo.getCgroupsEnabled();
boolean isCgroupsCpuReportingEnabled = machineInfo.getCgroupsCpuReportingEnabled();
sb = new StringBuffer();
if(status.equals("up")) {
if(isCgroupsEnabled) {
if(isCgroupsCpuReportingEnabled) {
sb.append("");
sb.append("on");
sb.append("");
}
else {
sb.append("");
sb.append("noCPU%");
sb.append("");
}
}
else {
sb.append("");
sb.append("off");
sb.append("");
}
}
String cgroups = sb.toString();
row.append("");
row.append(""+cgroups);
row.append(" ");
// Alien PIDs
sb = new StringBuffer();
long aliens = machineInfo.getAliens().size();
if(aliens == 0) {
sb.append(aliens);
}
else {
StringBuffer title = new StringBuffer();
title.append("title=");
title.append("\"");
for(String pid : machineInfo.getAliens()) {
title.append(pid+" ");
}
title.append("\"");
sb.append("");
sb.append(aliens);
sb.append("");
}
row.append("");
if(!status.equals("defined")) {
row.append(sb);
}
row.append(" ");
// Heartbeat: last
row.append("");
if(!status.equals("defined")) {
row.append(machineInfo.getHeartbeatLast());
}
row.append(" ");
row.append("");
data.append(row);
}
private void handleServletClassicSystemMachines(String target,Request baseRequest,HttpServletRequest request,HttpServletResponse response)
throws IOException, ServletException
{
String methodName = "handleServletClassicSystemMachines";
duccLogger.trace(methodName, jobid, messages.fetch("enter"));
StringBuffer data = new StringBuffer();
if(dh.is_ducc_head_backup()) {
data.append("");
data.append("");
data.append(messages.fetch("no data - not master"));
data.append(" ");
data.append(" ");
}
else {
int counter = 0;
long sumMemTotal = 0; // Memory(GB):reported by Agent
long sumMemFree = 0; // Memory(GB):free
long sumMemReserve = 0; // Memory(GB):usable
long sumMemAllocated = 0;
double sumCPU = 0;
long sumMachines = 0;
long sumSwapInuse = 0;
long sumSwapFree = 0;
long sumAliens = 0;
String hover;
StringBuffer row;
StringBuffer individualMachines = new StringBuffer();
DuccMachinesData instance = DuccMachinesData.getInstance();
Map machines = instance.getMachines();
if(!machines.isEmpty()) {
Map allocatedMap = Distiller.getMap();
for(Entry entry : machines.entrySet()) {
MachineInfo machineInfo = entry.getKey();
SizeBytes sb = new SizeBytes(Type.Bytes, 0);
if(DuccMachinesDataHelper.isUp(machineInfo)) {
try {
sumMemTotal += ConvertSafely.String2Long(machineInfo.getMemTotal());
// Calculate total for Memory(GB):usable
sumMemReserve += ConvertSafely.String2Long(machineInfo.getMemReserve());
sumSwapInuse += ConvertSafely.String2Long(machineInfo.getSwapInuse());
sumSwapFree += ConvertSafely.String2Long(machineInfo.getSwapFree());
sumCPU += machineInfo.getCpu();
sumMachines += 1;
sumAliens += machineInfo.getAlienPidsCount();
String machineName = machineInfo.getName();
long bytes = allocatedMap.get(machineName);
sumMemAllocated += bytes;
sb = new SizeBytes(Type.Bytes, bytes);
String text = "allocated "+machineName+"="+sb.getGBytes();
duccLogger.trace(methodName, jobid, text);
}
catch(Exception e) {
duccLogger.trace(methodName, jobid, e);
}
}
buildRowForIndividualMachine(individualMachines, counter, machineInfo, sb);
counter++;
}
SizeBytes sbAllocated = new SizeBytes(Type.Bytes, sumMemAllocated);
sumMemFree = sumMemReserve - sbAllocated.getGBytes();
//
row = new StringBuffer();
row.append("");
// Status
row.append("");
row.append(""+"Total");
row.append(" ");
// Online
row.append("");
row.append("");
row.append(" ");
// Quiesced
row.append("");
row.append("");
row.append(" ");
// IP
row.append("");
row.append("");
row.append(" ");
// Name
row.append("");
row.append("");
row.append(" ");
// Nodepool
row.append("");
row.append("");
row.append(" ");
// Memory: usable
hover = "title=\"total="+sumMemTotal+"\"";
row.append("");
row.append(""+sumMemReserve);
row.append(" ");
// Memory: free
row.append("");
row.append(""+sumMemFree);
row.append(" ");
// CPU: load average
row.append("");
String cpuTotal = formatter1.format(sumCPU/sumMachines);
row.append(""+cpuTotal);
row.append(" ");
// Swap: inuse
row.append("");
row.append(""+sumSwapInuse);
row.append(" ");
// Swap: free
row.append("");
row.append(""+sumSwapFree);
row.append(" ");
// C-Groups
row.append("");
row.append("");
row.append(" ");
// Alien PIDs
row.append("");
row.append(""+sumAliens);
row.append(" ");
// Heartbeat: last
row.append("");
row.append("");
row.append(" ");
row.append(" ");
//
data.append(row);
data.append(individualMachines);
}
else {
row = new StringBuffer();
row.append((trGet(counter)));
// Release
row.append("");
row.append("");
row.append(" ");
// Status
row.append("");
row.append("");
row.append(" ");
// Online
row.append("");
row.append("");
row.append(" ");
// IP
row.append("");
row.append("");
row.append(" ");
// Name
row.append("");
row.append("");
row.append(" ");
// Reserve
row.append("");
row.append("");
row.append(" ");
// Memory: total
row.append("");
row.append("");
row.append(" ");
// Swap: inuse
row.append("");
row.append("");
row.append(" ");
// Alien PIDs
row.append("");
row.append("");
row.append(" ");
// Shares: total
row.append("");
row.append("");
row.append(" ");
// Shares:inuse
row.append("");
row.append("");
row.append(" ");
// Heartbeat: last
row.append("");
row.append("");
row.append(" ");
row.append("");
//
data.append(row);
}
}
duccLogger.debug(methodName, jobid, data);
response.getWriter().println(data);
duccLogger.trace(methodName, jobid, messages.fetch("exit"));
}
private static DecimalFormat formatter1 = new DecimalFormat("##0.0");
private static DecimalFormat formatter3 = new DecimalFormat("##0.000");
private void handleServletClassicSystemBroker(String target,Request baseRequest,HttpServletRequest request,HttpServletResponse response)
throws IOException, ServletException
{
String methodName = "handleServletClassicBroker";
duccLogger.trace(methodName, jobid, messages.fetch("enter"));
StringBuffer sb = new StringBuffer();
BrokerHelper brokerHelper = BrokerHelper.getInstance();
StringBuffer topics = new StringBuffer();
StringBuffer queues = new StringBuffer();
Map> topicAttributes = brokerHelper.getEntityAttributes();
int taSize = 0;
if(topicAttributes != null) {
taSize = topicAttributes.size();
}
if(taSize > 0) {
for(Entry> entry : topicAttributes.entrySet()) {
String topic = entry.getKey();
String attrValue;
Map map = entry.getValue();
StringBuffer row = new StringBuffer();
row.append(messages.fetch(""));
// name
row.append(messages.fetch(""));
row.append(messages.fetch(topic));
row.append(messages.fetch(" "));
// type
String type = map.get(JmxKeyWord.Type.name());
row.append(messages.fetch(""));
row.append(messages.fetch(type));
row.append(messages.fetch(" "));
// ConsumerCount
attrValue = map.get(FrameworkAttribute.ConsumerCount.name());
row.append(messages.fetch(""));
row.append(messages.fetch(attrValue));
row.append(messages.fetch(" "));
// QueueSize
attrValue = map.get(FrameworkAttribute.QueueSize.name());
row.append(messages.fetch(""));
row.append(messages.fetch(attrValue));
row.append(messages.fetch(" "));
// MaxEnqueueTime
attrValue = map.get(FrameworkAttribute.MaxEnqueueTime.name());
row.append(messages.fetch(""));
row.append(messages.fetch(attrValue));
row.append(messages.fetch(" "));
// AverageEnqueueTime
attrValue = map.get(FrameworkAttribute.AverageEnqueueTime.name());
try {
Double d = Double.valueOf(attrValue);
attrValue = formatter3.format(d);
}
catch(Exception e) {
}
row.append(messages.fetch(""));
row.append(messages.fetch(attrValue));
row.append(messages.fetch(" "));
// MemoryPercentUsage
attrValue = map.get(FrameworkAttribute.MemoryPercentUsage.name());
row.append(messages.fetch(""));
row.append(messages.fetch(attrValue));
row.append(messages.fetch(" "));
//
row.append(messages.fetch(" "));
if(type.equals(JmxKeyWord.Type.name())) {
topics.append(row);
}
else {
queues.append(row);
}
}
sb.append(topics);
sb.append(queues);
}
else {
StringBuffer row = new StringBuffer();
row.append(messages.fetch(""));
// name
row.append(messages.fetch(""));
row.append(messages.fetch("no data"));
row.append(messages.fetch(" "));
// ConsumerCount
row.append(messages.fetch(""));
row.append(messages.fetch(""));
row.append(messages.fetch(" "));
// MaxEnqueueTime
row.append(messages.fetch(""));
row.append(messages.fetch(""));
row.append(messages.fetch(" "));
// AverageEnqueueTime
row.append(messages.fetch(""));
row.append(messages.fetch(""));
row.append(messages.fetch(" "));
// MemoryPercentUsage
row.append(messages.fetch(""));
row.append(messages.fetch(""));
row.append(messages.fetch(" "));
//
row.append(messages.fetch(" "));
sb.append(row);
}
duccLogger.debug(methodName, jobid, sb);
response.getWriter().println(sb);
duccLogger.trace(methodName, jobid, messages.fetch("exit"));
}
private void handleServletUnknown(String target,Request baseRequest,HttpServletRequest request,HttpServletResponse response)
throws IOException, ServletException
{
String methodName = "handleServletUnknown";
duccLogger.trace(methodName, jobid, messages.fetch("enter"));
duccLogger.info(methodName, jobid, request.toString());
duccLogger.trace(methodName, jobid, messages.fetch("exit"));
}
private void handleDuccRequest(String target,Request baseRequest,HttpServletRequest request,HttpServletResponse response)
throws Exception
{
String methodName = "handleDuccRequest";
duccLogger.trace(methodName, jobid, messages.fetch("enter"));
duccLogger.debug(methodName, jobid,request.toString());
duccLogger.debug(methodName, jobid,"getRequestURI():"+request.getRequestURI());
String reqURI = request.getRequestURI()+"";
if(reqURI.startsWith(classicJobs)) {
handleServletClassicJobs(target, baseRequest, request, response);
}
else if(reqURI.startsWith(classicReservations)) {
handleServletClassicReservations(target, baseRequest, request, response);
}
else if(reqURI.startsWith(classicServices)) {
handleServletClassicServices(target, baseRequest, request, response);
}
else if(reqURI.startsWith(classicSystemClasses)) {
handleServletClassicSystemClasses(target, baseRequest, request, response);
}
else if(reqURI.startsWith(classicSystemDaemons)) {
handleServletClassicSystemDaemons(target, baseRequest, request, response);
}
else if(reqURI.startsWith(classicSystemMachines)) {
handleServletClassicSystemMachines(target, baseRequest, request, response);
}
else if(reqURI.startsWith(classicSystemBroker)) {
handleServletClassicSystemBroker(target, baseRequest, request, response);
}
else {
handleServletUnknown(target, baseRequest, request, response);
}
duccLogger.trace(methodName, jobid, messages.fetch("exit"));
}
public void handle(String target,Request baseRequest,HttpServletRequest request,HttpServletResponse response)
throws IOException, ServletException {
String methodName = "handle";
try{
duccLogger.debug(methodName, jobid,request.toString());
duccLogger.debug(methodName, jobid,"getRequestURI():"+request.getRequestURI());
String reqURI = request.getRequestURI()+"";
if(reqURI.startsWith(duccContextClassic)) {
response.setContentType("text/html;charset=utf-8");
response.setStatus(HttpServletResponse.SC_OK);
baseRequest.setHandled(true);
handleDuccRequest(target, baseRequest, request, response);
DuccWebUtil.noCache(response);
}
}
catch(Throwable t) {
if(isIgnorable(t)) {
duccLogger.debug(methodName, jobid, t);
}
else {
duccLogger.info(methodName, jobid, "", t.getMessage(), t);
duccLogger.error(methodName, jobid, t);
}
}
}
}