com.liferay.jenkins.results.parser.metrics.BuildHistoryProcessor Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of com.liferay.jenkins.results.parser
Show all versions of com.liferay.jenkins.results.parser
Liferay Jenkins Results Parser
The newest version!
/**
* SPDX-FileCopyrightText: (c) 2023 Liferay, Inc. https://liferay.com
* SPDX-License-Identifier: LGPL-2.1-or-later OR LicenseRef-Liferay-DXP-EULA-2.0.0-2023-06
*/
package com.liferay.jenkins.results.parser.metrics;
import com.liferay.jenkins.results.parser.JenkinsResultsParserUtil;
import com.liferay.jenkins.results.parser.ParallelExecutor;
import java.io.File;
import java.io.IOException;
import java.time.DayOfWeek;
import java.time.LocalDate;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.TimeoutException;
import java.util.function.BiConsumer;
import java.util.function.Function;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.json.JSONArray;
/**
* @author Kenji Heigel
*/
public class BuildHistoryProcessor {
public static ExecutorService getExecutorService() {
return _executorService;
}
public static BuildHistory mergeBuildHistories(
Collection buildHistories, String name) {
return _mergeBuildHistories(new ArrayList<>(buildHistories), name);
}
public static BuildHistory mergeBuildHistories(
String name, BuildHistory... buildHistories) {
return _mergeBuildHistories(Arrays.asList(buildHistories), name);
}
public static Collection newAggregateJobHistories(
long duration, long startTime) {
BiConsumer, Map> biConsumer =
new BiConsumer, Map>() {
@Override
public void accept(
Set buildJSONObjects,
Map buildHistories) {
_addToBuildHistoriesMap(
buildJSONObjects, buildHistories, duration,
new GroupByCategory(), startTime);
}
};
return _getBuildHistories(duration, null, null, biConsumer, startTime);
}
public static Collection newDefaultJobHistories(
long duration, long startTime) {
BiConsumer, Map> biConsumer =
new BiConsumer, Map>() {
@Override
public void accept(
Set buildJSONObjects,
Map buildHistories) {
_addToBuildHistoriesMap(
buildJSONObjects, buildHistories, duration,
new GroupByJobName(), startTime);
}
};
return _getBuildHistories(duration, null, null, biConsumer, startTime);
}
public static Collection newTestSuiteJobHistories(
long duration, Pattern jobNamePattern, long startTime) {
Function groupByTopLevelTestSuite =
new GroupByTopLevelTestSuite();
BiConsumer, Map> biConsumer =
new BiConsumer, Map>() {
@Override
public void accept(
Set buildJSONObjects,
Map buildHistories) {
Set downstreamBuildJSONObjects =
new HashSet<>();
Set topLevelBuildJSONObjects =
new HashSet<>();
for (BuildJSONObject buildJSONObject : buildJSONObjects) {
if (buildJSONObject.isTopLevelBuild()) {
topLevelBuildJSONObjects.add(buildJSONObject);
}
else {
downstreamBuildJSONObjects.add(buildJSONObject);
}
}
_addToBuildHistoriesMap(
topLevelBuildJSONObjects, buildHistories, duration,
groupByTopLevelTestSuite, startTime);
Map>
groupedBuildDataJSONObjectsMap =
_getGroupedBuildDataJSONObjectsMap(
downstreamBuildJSONObjects,
groupByTopLevelTestSuite);
for (Map.Entry> entry :
groupedBuildDataJSONObjectsMap.entrySet()) {
String key = entry.getKey();
if (!buildHistories.containsKey(key)) {
BuildHistory buildHistory = new BuildHistory(
duration, key, startTime);
buildHistories.put(key, buildHistory);
}
BuildHistory buildHistory = buildHistories.get(key);
buildHistory.addBuildJSONObjects(
groupedBuildDataJSONObjectsMap.get(key));
}
}
};
return _getBuildHistories(
duration, null, jobNamePattern, biConsumer, startTime);
}
public static Collection newUtilizationBuildHistories(
long duration, long startTime) {
BiConsumer, Map> biConsumer =
new BiConsumer, Map>() {
@Override
public void accept(
Set buildJSONObjects,
Map buildHistories) {
_addToBuildHistoriesMap(
buildJSONObjects, buildHistories, duration,
new GroupByWeeklyUtilization(), startTime);
}
};
return _getBuildHistories(duration, null, null, biConsumer, startTime);
}
private static void _addToBuildHistoriesMap(
Collection buildJSONObjects,
Map buildHistoriesMap, long duration,
Function groupingFunction, long startTime) {
Map> groupedBuildDataJSONObjectsMap =
_getGroupedBuildDataJSONObjectsMap(
buildJSONObjects, groupingFunction);
for (Map.Entry> entry :
groupedBuildDataJSONObjectsMap.entrySet()) {
if (!buildHistoriesMap.containsKey(entry.getKey())) {
BuildHistory buildHistory = new BuildHistory(
duration, entry.getKey(), startTime);
buildHistoriesMap.put(entry.getKey(), buildHistory);
}
BuildHistory buildHistory = buildHistoriesMap.get(entry.getKey());
buildHistory.addBuildJSONObjects(entry.getValue());
}
}
private static Collection _getBuildHistories(
long duration, Pattern jobNameExcludesPattern,
Pattern jobNameIncludesPattern,
BiConsumer, Map>
buildHistoryBiConsumer,
long startTime) {
Map buildHistoriesMap = new HashMap<>();
for (String dateString :
JenkinsResultsParserUtil.getDateStrings(startTime, duration)) {
Set buildJSONObjects = new HashSet<>();
for (BuildJSONObject buildJSONObject :
_getBuildJSONObjects(dateString)) {
if (jobNameExcludesPattern != null) {
Matcher jobNameExcludesMatcher =
jobNameExcludesPattern.matcher(
buildJSONObject.getJobName());
if (jobNameExcludesMatcher.matches()) {
continue;
}
}
if (jobNameIncludesPattern == null) {
buildJSONObjects.add(buildJSONObject);
continue;
}
Matcher jobNameIncludesMatcher = jobNameIncludesPattern.matcher(
buildJSONObject.getJobName());
if (jobNameIncludesMatcher.matches()) {
buildJSONObjects.add(buildJSONObject);
}
}
buildHistoryBiConsumer.accept(buildJSONObjects, buildHistoriesMap);
}
return _getSortedBuildHistories(buildHistoriesMap.values());
}
private static Set _getBuildJSONObjects(
String dateString) {
File dateDir = new File(_BASE_DIR, dateString);
if (dateDir.listFiles() == null) {
return Collections.emptySet();
}
Set buildJSONObjects = Collections.synchronizedSet(
new HashSet());
System.out.println("Reading files from: " + dateDir.toPath());
List> callables = new ArrayList<>();
for (final File jsonFile : dateDir.listFiles()) {
callables.add(
new Callable() {
@Override
public Void call() throws Exception {
try {
String jsonFileName = jsonFile.getCanonicalPath();
if (jsonFileName.contains("test-1-0") ||
jsonFileName.contains("test-1-41")) {
return null;
}
String content = JenkinsResultsParserUtil.read(
jsonFile);
JSONArray jsonArray = new JSONArray(content.trim());
Set newBuildJSONObjects =
new HashSet<>();
for (int i = 0; i < jsonArray.length(); i++) {
newBuildJSONObjects.add(
new BuildJSONObject(
jsonArray.getJSONObject(i)));
}
buildJSONObjects.addAll(newBuildJSONObjects);
}
catch (IOException ioException) {
System.out.println("Unable to read " + jsonFile);
}
return null;
}
});
}
ParallelExecutor parallelExecutor = new ParallelExecutor<>(
callables, _executorService, "_getBuildJSONObjects");
try {
parallelExecutor.execute();
}
catch (TimeoutException timeoutException) {
throw new RuntimeException(timeoutException);
}
return buildJSONObjects;
}
private static Map>
_getGroupedBuildDataJSONObjectsMap(
Collection buildJSONObjects,
Function groupingFunction) {
Map> groupedBuildDataJSONObjectsMap =
new HashMap<>();
for (BuildJSONObject buildJSONObject : buildJSONObjects) {
String groupName = groupingFunction.apply(buildJSONObject);
if (!groupedBuildDataJSONObjectsMap.containsKey(groupName)) {
groupedBuildDataJSONObjectsMap.put(
groupName, new HashSet());
}
Set groupedBuildJSONObjects =
groupedBuildDataJSONObjectsMap.get(groupName);
groupedBuildJSONObjects.add(buildJSONObject);
}
return groupedBuildDataJSONObjectsMap;
}
private static List _getSortedBuildHistories(
Collection buildHistories) {
List buildHistoryList = new ArrayList<>(buildHistories);
Collections.sort(
buildHistoryList,
new Comparator() {
@Override
public int compare(
BuildHistory buildHistory1, BuildHistory buildHistory2) {
Integer buildCount1 =
(int)buildHistory1.getInvokedBuildCount();
Integer buildCount2 =
(int)buildHistory2.getInvokedBuildCount();
return buildCount2.compareTo(buildCount1);
}
});
return buildHistoryList;
}
private static BuildHistory _mergeBuildHistories(
List buildHistories, String name) {
BuildHistory mergedBuildHistory = new BuildHistory(
0, name, System.currentTimeMillis());
for (BuildHistory buildHistory : buildHistories) {
mergedBuildHistory.merge(buildHistory);
}
return mergedBuildHistory;
}
private static final File _BASE_DIR;
private static final Integer _THREAD_COUNT = 8;
private static final Properties _buildProperties;
private static final ExecutorService _executorService =
JenkinsResultsParserUtil.getNewThreadPoolExecutor(_THREAD_COUNT, true);
static {
_buildProperties = new Properties() {
{
try {
putAll(JenkinsResultsParserUtil.getBuildProperties());
}
catch (IOException ioException) {
throw new RuntimeException(ioException);
}
}
};
_BASE_DIR = new File(
_buildProperties.getProperty("archive.ci.build.data.tmp.dir"),
"builds");
}
private static class GroupByCategory
implements Function {
public String apply(BuildJSONObject buildJSONObject) {
String jobName = buildJSONObject.getJobName();
jobName = jobName.replace("-batch", "");
jobName = jobName.replace("-downstream", "");
jobName = jobName.replace("-validation", "");
if (jobName.contains("maintenance-") ||
jobName.contains("mirrors-") ||
jobName.contains("verification-")) {
return Category.MAINTENANCE.toString();
}
if (jobName.equals("test-portal-acceptance-pullrequest(master)")) {
return Category.PORTAL_MASTER_PULLREQUEST.toString();
}
if (jobName.equals("test-portal-acceptance-upstream(master)") ||
jobName.equals("test-portal-acceptance-upstream-dxp(master)") ||
jobName.equals("test-portal-testsuite-upstream(master)")) {
return Category.PORTAL_MASTER_UPSTREAM.toString();
}
if (jobName.equals("test-portal-fixpack-release") ||
jobName.equals("test-portal-hotfix-release") ||
jobName.equals("test-portal-release")) {
return Category.PORTAL_RELEASE.toString();
}
if (jobName.contains("test-portal-")) {
return Category.PORTAL_OTHER.toString();
}
return Category.OTHER.toString();
}
private enum Category {
MAINTENANCE("CI Maintenance"), OTHER("Other"),
PORTAL_MASTER_PULLREQUEST("liferay-portal/master PR's"),
PORTAL_MASTER_UPSTREAM("liferay-portal/master Upstream"),
PORTAL_OTHER("liferay-portal-ee PR's & Upstream"),
PORTAL_OTHER_RELEASE("Portal Fixpack & Hotfix Release"),
PORTAL_RELEASE("Portal Release");
@Override
public String toString() {
return _string;
}
private Category(String string) {
_string = string;
}
private final String _string;
}
}
private static class GroupByJobName
implements Function {
public String apply(BuildJSONObject buildJSONObject) {
String jobName = buildJSONObject.getJobName();
String name = jobName.replace("-batch", "");
name = name.replace("-downstream", "");
name = name.replace("-validation", "");
return name;
}
}
private static class GroupByTopLevelTestSuite
implements Function {
public String apply(BuildJSONObject buildJSONObject) {
String jobName = buildJSONObject.getJobName();
if (jobName.contains("acceptance-upstream-dxp")) {
return "acceptance-dxp";
}
if (buildJSONObject.isTopLevelBuild()) {
Map parameters =
buildJSONObject.getParameters();
if (parameters.containsKey("CI_TEST_SUITE")) {
_topLevelBuildTestSuiteMap.put(
buildJSONObject.getURL(),
parameters.get("CI_TEST_SUITE"));
return parameters.get("CI_TEST_SUITE");
}
return "[Unknown]";
}
String topLevelBuildURL = buildJSONObject.getTopLevelBuildURL();
if (_topLevelBuildTestSuiteMap.containsKey(topLevelBuildURL)) {
return _topLevelBuildTestSuiteMap.get(topLevelBuildURL);
}
return "[Unknown]";
}
private final Map _topLevelBuildTestSuiteMap =
new HashMap<>();
}
private static class GroupByWeeklyUtilization
implements Function {
public String apply(BuildJSONObject buildJSONObject) {
String jobName = buildJSONObject.getJobName();
LocalDate localDate = LocalDate.parse(
buildJSONObject.getStartDateString(),
DateTimeFormatter.ofPattern("yyyyMMdd"));
DayOfWeek dayOfWeek = localDate.getDayOfWeek();
boolean weekday = false;
if (dayOfWeek.getValue() <= 5) {
weekday = true;
}
if (jobName.contains("test-portal-acceptance-pullrequest")) {
if (weekday) {
return Category.PORTAL_PULLREQUEST_WEEKDAYS.toString();
}
return Category.PORTAL_PULLREQUEST_WEEKENDS.toString();
}
if (jobName.contains("release") || jobName.contains("upstream")) {
if (weekday) {
return Category.PORTAL_RELEASE_AND_UPSTREAM_WEEKDAYS.
toString();
}
return Category.PORTAL_RELEASE_AND_UPSTREAM_WEEKENDS.toString();
}
if (weekday) {
return Category.OTHER_WEEKDAYS.toString();
}
return Category.OTHER_WEEKENDS.toString();
}
private enum Category {
OTHER_WEEKDAYS("Other (Weekdays)"),
OTHER_WEEKENDS("Other (Weekends)"),
PORTAL_PULLREQUEST_WEEKDAYS("Portal Pull Requests (Weekdays)"),
PORTAL_PULLREQUEST_WEEKENDS("Portal Pull Requests (Weekends)"),
PORTAL_RELEASE_AND_UPSTREAM_WEEKDAYS(
"Portal Release & Upstream (Weekdays)"),
PORTAL_RELEASE_AND_UPSTREAM_WEEKENDS(
"Portal Release & Upstream (Weekends)");
@Override
public String toString() {
return _string;
}
private Category(String string) {
_string = string;
}
private final String _string;
}
}
}