edu.internet2.middleware.grouper.misc.GrouperReport Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of grouper Show documentation
Show all versions of grouper Show documentation
Internet2 Groups Management Toolkit
/**
* Copyright 2014 Internet2
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/*
* @author mchyzer
* $Id: GrouperReport.java,v 1.7 2009-06-09 22:55:40 shilen Exp $
*/
package edu.internet2.middleware.grouper.misc;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.PrintStream;
import java.text.DecimalFormat;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Date;
import java.util.List;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import edu.internet2.middleware.grouper.GrouperSession;
import edu.internet2.middleware.grouper.app.loader.GrouperDaemonUtils;
import edu.internet2.middleware.grouper.app.loader.GrouperLoaderConfig;
import edu.internet2.middleware.grouper.app.loader.GrouperLoaderStatus;
import edu.internet2.middleware.grouper.app.loader.OtherJobBase;
import edu.internet2.middleware.grouper.app.loader.db.Hib3GrouperLoaderLog;
import edu.internet2.middleware.grouper.app.usdu.SubjectResolutionStat;
import edu.internet2.middleware.grouper.app.usdu.UsduService;
import edu.internet2.middleware.grouper.cfg.GrouperConfig;
import edu.internet2.middleware.grouper.exception.GrouperSessionException;
import edu.internet2.middleware.grouper.hibernate.HibernateSession;
import edu.internet2.middleware.grouper.internal.dao.QueryOptions;
import edu.internet2.middleware.grouper.util.GrouperEmail;
import edu.internet2.middleware.grouper.util.GrouperUtil;
/**
*
*/
public class GrouperReport extends OtherJobBase {
/**
* logger
*/
@SuppressWarnings("unused")
private static final Log LOG = GrouperUtil.getLog(GrouperReport.class);
/**
* @param args
*/
public static void main(String[] args) {
System.out.println(report());
}
@Override
public OtherJobOutput run(OtherJobInput otherJobInput) {
String emailTo = GrouperLoaderConfig.retrieveConfig().propertyValueString("daily.report.emailTo");
String reportDirectory = GrouperLoaderConfig.retrieveConfig().propertyValueString("daily.report.saveInDirectory");
if (StringUtils.isBlank(emailTo) && StringUtils.isBlank(reportDirectory)) {
otherJobInput.getHib3GrouperLoaderLog().setJobMessage("Not running report since both daily.report.emailTo and daily.report.saveInDirectory are null");
return null;
}
String report = null;
//keep track if ends up being error
RuntimeException re = null;
try {
report = new GrouperReport().runReport();
} catch (RuntimeException e) {
report = e.toString() + "\n\n" + GrouperUtil.getFullStackTrace(e) + "\n\n";
if (e instanceof GrouperReportException) {
report += ((GrouperReportException)e).getResult();
}
re = e;
}
//if we are emailing
if (!StringUtils.isBlank(emailTo)) {
new GrouperEmail().setBody(report).setSubject("Grouper report").setTo(emailTo).send();
}
//if we are saving to dir on server
if (!StringUtils.isBlank(reportDirectory)) {
reportDirectory = GrouperUtil.stripLastSlashIfExists(reportDirectory) + File.separator;
GrouperUtil.mkdirs(new File(reportDirectory));
File reportFile = new File(reportDirectory + "grouperReport_"
+ new SimpleDateFormat("yyyyMMdd_HH_mm_ss").format(new Date()) + ".txt");
GrouperUtil.saveStringIntoFile(reportFile, report);
}
//end in error if error
if (re != null) {
throw re;
}
otherJobInput.getHib3GrouperLoaderLog().setJobMessage("Ran the grouper report, emailTo=" + emailTo + ", reportDirectory=" + reportDirectory);
return null;
}
/**
* format with commas
* @param theLong
* @return the string
*/
public static String formatCommas(Long theLong) {
if (theLong == null) {
return "";
}
DecimalFormat myFormatter = new DecimalFormat("###,###");
String output = myFormatter.format(theLong);
return output;
}
/**
* @return the report
* @throws GrouperReportException
*/
public static String report() {
return new GrouperReport().runReport();
}
/**
* @return the report
* @throws GrouperReportException
*/
public String runReport() {
GrouperStartup.startup();
return (String)GrouperSession.internal_callbackRootGrouperSession(new GrouperSessionHandler() {
@Override
public Object callback(GrouperSession grouperSession) throws GrouperSessionException {
StringBuilder result = new StringBuilder();
try {
Calendar calendar = Calendar.getInstance();
calendar.add(Calendar.DAY_OF_YEAR, -1);
Date yesterday = calendar.getTime();
result.append("Grouper daily report\n\n");
result.append("----------------\n");
result.append("OVERALL:\n");
result.append("----------------\n");
result.append("environment: ").append(GrouperConfig.retrieveConfig().propertyValueString("grouper.env.name")).append("\n");
Long membershipCountTotal = HibernateSession.byHqlStatic().createQuery(
"select count(*) from MembershipEntry").uniqueResult(Long.class);
result.append("memberships: ").append(formatCommas(membershipCountTotal)).append("\n");
GrouperDaemonUtils.stopProcessingIfJobPaused();
Long groupCountTotal = HibernateSession.byHqlStatic().createQuery(
"select count(*) from Group").uniqueResult(Long.class);
result.append("groups: ").append(formatCommas(groupCountTotal)).append("\n");
GrouperDaemonUtils.stopProcessingIfJobPaused();
Long memberCountTotal = HibernateSession.byHqlStatic().createQuery(
"select count(*) from Member").uniqueResult(Long.class);
result.append("members: ").append(formatCommas(memberCountTotal)).append("\n");
GrouperDaemonUtils.stopProcessingIfJobPaused();
Long folderCountTotal = HibernateSession.byHqlStatic().createQuery(
"select count(*) from Stem").uniqueResult(Long.class);
result.append("folders: ").append(formatCommas(folderCountTotal)).append("\n");
GrouperDaemonUtils.stopProcessingIfJobPaused();
List subjectResolutionStats = UsduService.getSubjectResolutionStats();
int unresolvableResultCount = 0;
for (SubjectResolutionStat subjectResolutionStat : subjectResolutionStats) {
unresolvableResultCount += subjectResolutionStat.getUnresolvedCount();
}
result.append("unresolvable subjects: ").append(formatCommas(Long.valueOf(unresolvableResultCount))).append("\n");
GrouperDaemonUtils.stopProcessingIfJobPaused();
result.append("\n----------------\n");
result.append("WITHIN LAST DAY:\n");
result.append("----------------\n");
Long membershipNewCountDay = HibernateSession.byHqlStatic().createQuery(
"select count(*) from MembershipEntry where createTimeLong > :createTime or groupSetCreateTimeLong > :createTime")
.setLong("createTime", yesterday.getTime()).uniqueResult(Long.class);
result.append("new memberships: ").append(formatCommas(membershipNewCountDay)).append("\n");
GrouperDaemonUtils.stopProcessingIfJobPaused();
Long groupNewCountDay = HibernateSession.byHqlStatic().createQuery(
"select count(*) from Group where createTimeLong > :createTime")
.setLong("createTime", yesterday.getTime()).uniqueResult(Long.class);
result.append("new groups: ").append(formatCommas(groupNewCountDay)).append("\n");
GrouperDaemonUtils.stopProcessingIfJobPaused();
Long groupUpdatedCountDay = HibernateSession.byHqlStatic().createQuery(
"select count(*) from Group where createTimeLong > :createTime and modifyTimeLong < :createTime2")
.setLong("createTime", yesterday.getTime())
.setLong("createTime2", yesterday.getTime()).uniqueResult(Long.class);
result.append("updated groups: ").append(formatCommas(groupUpdatedCountDay)).append("\n");
GrouperDaemonUtils.stopProcessingIfJobPaused();
Long stemNewCountDay = HibernateSession.byHqlStatic().createQuery(
"select count(*) from Stem where createTimeLong > :createTime")
.setLong("createTime", yesterday.getTime()).uniqueResult(Long.class);
result.append("new folders: ").append(formatCommas(stemNewCountDay)).append("\n");
GrouperDaemonUtils.stopProcessingIfJobPaused();
result.append("\n----------------\n");
result.append("LOADER SUMMARY WITHIN LAST DAY\n");
result.append("----------------\n");
Long loaderLogCount = HibernateSession.byHqlStatic().createQuery(
"select count(*) from Hib3GrouperLoaderLog where lastUpdated > :lastUpdated")
.setTimestamp("lastUpdated", yesterday).uniqueResult(Long.class);
result.append("jobs: ").append(formatCommas(loaderLogCount)).append("\n");
GrouperDaemonUtils.stopProcessingIfJobPaused();
long loaderErrorCount = -1;
for (GrouperLoaderStatus grouperLoaderStatus : GrouperLoaderStatus.values()) {
if (GrouperLoaderStatus.SUCCESS.equals(grouperLoaderStatus)) {
continue;
}
Long loaderCount = HibernateSession.byHqlStatic().createQuery(
"select count(*) from Hib3GrouperLoaderLog where status = '" + grouperLoaderStatus.name() + "' and lastUpdated > :lastUpdated")
.setTimestamp("lastUpdated", yesterday).uniqueResult(Long.class);
GrouperDaemonUtils.stopProcessingIfJobPaused();
if (grouperLoaderStatus == GrouperLoaderStatus.ERROR) {
loaderErrorCount = loaderCount;
}
if (grouperLoaderStatus != GrouperLoaderStatus.SUCCESS
&& (grouperLoaderStatus == GrouperLoaderStatus.ERROR || loaderCount > 0)) {
String label = grouperLoaderStatus.getFriendlyString();
result.append(label).append(":").append(StringUtils.repeat(" ", 22-label.length()))
.append(formatCommas(loaderCount)).append("\n");
}
}
Long loaderUsduCount = HibernateSession.byHqlStatic().createQuery(
"select sum(unresolvableSubjectCount) from Hib3GrouperLoaderLog where lastUpdated > :lastUpdated" +
" and jobType like 'SQL%' and parentJobId is null")
.setTimestamp("lastUpdated", yesterday).uniqueResult(Long.class);
result.append("unresolvable subjects: ").append(formatCommas(loaderUsduCount)).append("\n");
GrouperDaemonUtils.stopProcessingIfJobPaused();
Long loaderInsertCount = HibernateSession.byHqlStatic().createQuery(
"select sum(insertCount) from Hib3GrouperLoaderLog where lastUpdated > :lastUpdated" +
" and jobType like 'SQL%' and parentJobId is null")
.setTimestamp("lastUpdated", yesterday).uniqueResult(Long.class);
result.append("inserts: ").append(formatCommas(loaderInsertCount)).append("\n");
GrouperDaemonUtils.stopProcessingIfJobPaused();
Long loaderUpdateCount = HibernateSession.byHqlStatic().createQuery(
"select sum(updateCount) from Hib3GrouperLoaderLog where lastUpdated > :lastUpdated" +
" and jobType like 'SQL%' and parentJobId is null")
.setTimestamp("lastUpdated", yesterday).uniqueResult(Long.class);
result.append("updates: ").append(formatCommas(loaderUpdateCount)).append("\n");
GrouperDaemonUtils.stopProcessingIfJobPaused();
Long loaderDeleteCount = HibernateSession.byHqlStatic().createQuery(
"select sum (deleteCount) from Hib3GrouperLoaderLog where lastUpdated > :lastUpdated" +
" and jobType like 'SQL%' and parentJobId is null")
.setTimestamp("lastUpdated", yesterday).uniqueResult(Long.class);
result.append("deletes: ").append(formatCommas(loaderDeleteCount)).append("\n");
GrouperDaemonUtils.stopProcessingIfJobPaused();
Long loaderTotalCount = HibernateSession.byHqlStatic().createQuery(
"select sum (totalCount) from Hib3GrouperLoaderLog where lastUpdated > :lastUpdated" +
" and jobType like 'SQL%' and parentJobId is null")
.setTimestamp("lastUpdated", yesterday).uniqueResult(Long.class);
result.append("total loader mships: ").append(formatCommas(loaderTotalCount)).append("\n");
GrouperDaemonUtils.stopProcessingIfJobPaused();
Long processingSum = GrouperUtil.defaultIfNull(HibernateSession.byHqlStatic().createQuery(
"select sum(millis) from Hib3GrouperLoaderLog where lastUpdated > :lastUpdated and parentJobId is null")
.setTimestamp("lastUpdated", yesterday).uniqueResult(Long.class), new Long(0));
result.append("processing time: ").append(GrouperUtil.convertMillisToFriendlyString(processingSum)).append("\n");
GrouperDaemonUtils.stopProcessingIfJobPaused();
if (loaderErrorCount > 0) {
List loaderLogs = HibernateSession.byHqlStatic().createQuery(
"from Hib3GrouperLoaderLog where lastUpdated > :lastUpdated and status != 'SUCCESS'")
.setTimestamp("lastUpdated", yesterday).options(new QueryOptions().paging(50, 1, false).sortDesc("lastUpdated")).list(Hib3GrouperLoaderLog.class);
result.append("\n----------------\n");
result.append("LOADER JOBS WITH NON-SUCCESS (max 50 of them)\n");
result.append("----------------\n");
for (Hib3GrouperLoaderLog loaderLog : loaderLogs) {
result.append("\njob: ").append(loaderLog.getJobName())
.append("\n");
result.append("status: ").append(loaderLog.getStatus()).append(", started: ")
.append(loaderLog.getStartedTime()).append(" (")
.append(GrouperUtil.convertMillisToFriendlyString(loaderLog.getMillis())).append(")\n");
result.append("ins/upd/del/tot: ").append(loaderLog.getInsertCount()).append("/")
.append(loaderLog.getUpdateCount()).append("/").append(loaderLog.getDeleteCount()).append("/")
.append(loaderLog.getTotalCount()).append("\n");
if (loaderLog.getUnresolvableSubjectCount() > 0) {
result.append("unresolv subjects: ").append(loaderLog.getUnresolvableSubjectCount()).append("\n");
}
result.append("error: ").append(loaderLog.getJobDescription()).append("\n");
}
}
GrouperDaemonUtils.stopProcessingIfJobPaused();
if (loaderLogCount > 0) {
List loaderLogs = HibernateSession.byHqlStatic().createQuery(
"from Hib3GrouperLoaderLog where lastUpdated > :lastUpdated " +
" and status != 'ERROR' and jobName != 'CHANGE_LOG_changeLogTempToChangeLog'")
.setTimestamp("lastUpdated", yesterday).options(new QueryOptions().paging(50, 1, false).sortDesc("lastUpdated"))
.list(Hib3GrouperLoaderLog.class);
if (loaderLogs.size() > 0) {
result.append("\n----------------\n");
result.append("LOADER JOBS SUCCESS (max 50 of them)\n");
result.append("----------------\n");
for (Hib3GrouperLoaderLog loaderLog : loaderLogs) {
result.append("\njob: ").append(loaderLog.getJobName())
.append("\n");
result.append("status: ").append(loaderLog.getStatus()).append(", started: ")
.append(loaderLog.getStartedTime()).append(" (")
.append(GrouperUtil.convertMillisToFriendlyString(loaderLog.getMillis())).append(")\n");
result.append("ins/upd/del/tot: ").append(loaderLog.getInsertCount()).append("/")
.append(loaderLog.getUpdateCount()).append("/").append(loaderLog.getDeleteCount())
.append("/").append(loaderLog.getTotalCount()).append("\n");
if (loaderLog.getUnresolvableSubjectCount() > 0) {
result.append("unresolv subjects: ").append(loaderLog.getUnresolvableSubjectCount()).append("\n");
}
}
}
}
GrouperDaemonUtils.stopProcessingIfJobPaused();
result.append("\n----------------\n");
result.append("GROUPER INFO\n");
result.append("----------------\n");
ByteArrayOutputStream baos = new ByteArrayOutputStream();
PrintStream printStream = new PrintStream(baos);
GrouperInfo.grouperInfo(printStream, false);
result.append(new String(baos.toByteArray()));
} catch (Exception e) {
GrouperReportException gre = new GrouperReportException("Problem generating daily report", e);
gre.setResult(result.toString());
throw gre;
}
return result.toString();
}
});
}
}