com.liferay.jenkins.results.parser.metrics.BuildHistoryProcessor Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of com.liferay.jenkins.results.parser
Show all versions of com.liferay.jenkins.results.parser
Liferay Jenkins Results Parser
/**
* SPDX-FileCopyrightText: (c) 2023 Liferay, Inc. https://liferay.com
* SPDX-License-Identifier: LGPL-2.1-or-later OR LicenseRef-Liferay-DXP-EULA-2.0.0-2023-06
*/
package com.liferay.jenkins.results.parser.metrics;
import com.liferay.jenkins.results.parser.JenkinsResultsParserUtil;
import com.liferay.jenkins.results.parser.ParallelExecutor;
import java.io.File;
import java.io.IOException;
import java.time.DayOfWeek;
import java.time.LocalDate;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.TimeoutException;
import java.util.function.BiConsumer;
import java.util.function.Function;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.json.JSONArray;
/**
* @author Kenji Heigel
*/
public class BuildHistoryProcessor {
public static ExecutorService getExecutorService() {
return _executorService;
}
public static BuildHistory mergeBuildHistories(
Collection buildHistories, String name) {
return _mergeBuildHistories(new ArrayList<>(buildHistories), name);
}
public static BuildHistory mergeBuildHistories(
String name, BuildHistory... buildHistories) {
return _mergeBuildHistories(Arrays.asList(buildHistories), name);
}
public static Collection newAggregateJobHistories(
long duration, long startTime) {
BiConsumer, Map> biConsumer =
new BiConsumer, Map>() {
@Override
public void accept(
Set buildJSONObjects,
Map buildHistories) {
_addToBuildHistoriesMap(
buildJSONObjects, buildHistories, duration,
new GroupByCategory(), startTime);
}
};
return _getBuildHistories(duration, null, null, biConsumer, startTime);
}
public static Collection newDefaultJobHistories(
long duration, long startTime) {
BiConsumer, Map> biConsumer =
new BiConsumer, Map>() {
@Override
public void accept(
Set buildJSONObjects,
Map buildHistories) {
_addToBuildHistoriesMap(
buildJSONObjects, buildHistories, duration,
new GroupByJobName(), startTime);
}
};
return _getBuildHistories(duration, null, null, biConsumer, startTime);
}
public static Collection newTestSuiteJobHistories(
long duration, Pattern jobNamePattern, long startTime) {
Function groupByTopLevelTestSuite =
new GroupByTopLevelTestSuite();
BiConsumer, Map> biConsumer =
new BiConsumer, Map>() {
@Override
public void accept(
Set buildJSONObjects,
Map buildHistories) {
Set downstreamBuildJSONObjects =
new HashSet<>();
Set topLevelBuildJSONObjects =
new HashSet<>();
for (BuildJSONObject buildJSONObject : buildJSONObjects) {
if (buildJSONObject.isTopLevelBuild()) {
topLevelBuildJSONObjects.add(buildJSONObject);
}
else {
downstreamBuildJSONObjects.add(buildJSONObject);
}
}
_addToBuildHistoriesMap(
topLevelBuildJSONObjects, buildHistories, duration,
groupByTopLevelTestSuite, startTime);
Map>
groupedBuildDataJSONObjectsMap =
_getGroupedBuildDataJSONObjectsMap(
downstreamBuildJSONObjects,
groupByTopLevelTestSuite);
for (Map.Entry> entry :
groupedBuildDataJSONObjectsMap.entrySet()) {
String key = entry.getKey();
if (!buildHistories.containsKey(key)) {
BuildHistory buildHistory = new BuildHistory(
duration, key, startTime);
buildHistories.put(key, buildHistory);
}
BuildHistory buildHistory = buildHistories.get(key);
buildHistory.addBuildJSONObjects(
groupedBuildDataJSONObjectsMap.get(key));
}
}
};
return _getBuildHistories(
duration, null, jobNamePattern, biConsumer, startTime);
}
public static Collection newUtilizationBuildHistories(
long duration, long startTime) {
BiConsumer, Map> biConsumer =
new BiConsumer, Map>() {
@Override
public void accept(
Set buildJSONObjects,
Map buildHistories) {
_addToBuildHistoriesMap(
buildJSONObjects, buildHistories, duration,
new GroupByWeeklyUtilization(), startTime);
}
};
return _getBuildHistories(duration, null, null, biConsumer, startTime);
}
private static void _addToBuildHistoriesMap(
Collection buildJSONObjects,
Map buildHistoriesMap, long duration,
Function groupingFunction, long startTime) {
Map> groupedBuildDataJSONObjectsMap =
_getGroupedBuildDataJSONObjectsMap(
buildJSONObjects, groupingFunction);
for (Map.Entry> entry :
groupedBuildDataJSONObjectsMap.entrySet()) {
if (!buildHistoriesMap.containsKey(entry.getKey())) {
BuildHistory buildHistory = new BuildHistory(
duration, entry.getKey(), startTime);
buildHistoriesMap.put(entry.getKey(), buildHistory);
}
BuildHistory buildHistory = buildHistoriesMap.get(entry.getKey());
buildHistory.addBuildJSONObjects(entry.getValue());
}
}
private static Collection _getBuildHistories(
long duration, Pattern jobNameExcludesPattern,
Pattern jobNameIncludesPattern,
BiConsumer, Map>
buildHistoryBiConsumer,
long startTime) {
Map buildHistoriesMap = new HashMap<>();
for (String dateString :
JenkinsResultsParserUtil.getDateStrings(startTime, duration)) {
Set buildJSONObjects = new HashSet<>();
for (BuildJSONObject buildJSONObject :
_getBuildJSONObjects(dateString)) {
if (jobNameExcludesPattern != null) {
Matcher jobNameExcludesMatcher =
jobNameExcludesPattern.matcher(
buildJSONObject.getJobName());
if (jobNameExcludesMatcher.matches()) {
continue;
}
}
if (jobNameIncludesPattern == null) {
buildJSONObjects.add(buildJSONObject);
continue;
}
Matcher jobNameIncludesMatcher = jobNameIncludesPattern.matcher(
buildJSONObject.getJobName());
if (jobNameIncludesMatcher.matches()) {
buildJSONObjects.add(buildJSONObject);
}
}
buildHistoryBiConsumer.accept(buildJSONObjects, buildHistoriesMap);
}
return _getSortedBuildHistories(buildHistoriesMap.values());
}
private static Set _getBuildJSONObjects(
String dateString) {
File dateDir = new File(_BASE_DIR, dateString);
if (dateDir.listFiles() == null) {
return Collections.emptySet();
}
Set buildJSONObjects = Collections.synchronizedSet(
new HashSet());
System.out.println("Reading files from: " + dateDir.toPath());
List> callables = new ArrayList<>();
for (final File jsonFile : dateDir.listFiles()) {
callables.add(
new Callable() {
@Override
public Void call() throws Exception {
try {
String jsonFileName = jsonFile.getCanonicalPath();
if (jsonFileName.contains("test-1-0") ||
jsonFileName.contains("test-1-41")) {
return null;
}
String content = JenkinsResultsParserUtil.read(
jsonFile);
JSONArray jsonArray = new JSONArray(content.trim());
Set newBuildJSONObjects =
new HashSet<>();
for (int i = 0; i < jsonArray.length(); i++) {
newBuildJSONObjects.add(
new BuildJSONObject(
jsonArray.getJSONObject(i)));
}
buildJSONObjects.addAll(newBuildJSONObjects);
}
catch (IOException ioException) {
System.out.println("Unable to read " + jsonFile);
}
return null;
}
});
}
ParallelExecutor parallelExecutor = new ParallelExecutor<>(
callables, _executorService, "_getBuildJSONObjects");
try {
parallelExecutor.execute();
}
catch (TimeoutException timeoutException) {
throw new RuntimeException(timeoutException);
}
return buildJSONObjects;
}
private static Map>
_getGroupedBuildDataJSONObjectsMap(
Collection buildJSONObjects,
Function groupingFunction) {
Map> groupedBuildDataJSONObjectsMap =
new HashMap<>();
for (BuildJSONObject buildJSONObject : buildJSONObjects) {
String groupName = groupingFunction.apply(buildJSONObject);
if (!groupedBuildDataJSONObjectsMap.containsKey(groupName)) {
groupedBuildDataJSONObjectsMap.put(
groupName, new HashSet());
}
Set groupedBuildJSONObjects =
groupedBuildDataJSONObjectsMap.get(groupName);
groupedBuildJSONObjects.add(buildJSONObject);
}
return groupedBuildDataJSONObjectsMap;
}
private static List _getSortedBuildHistories(
Collection buildHistories) {
List buildHistoryList = new ArrayList<>(buildHistories);
Collections.sort(
buildHistoryList,
new Comparator() {
@Override
public int compare(
BuildHistory buildHistory1, BuildHistory buildHistory2) {
Integer buildCount1 =
(int)buildHistory1.getInvokedBuildCount();
Integer buildCount2 =
(int)buildHistory2.getInvokedBuildCount();
return buildCount2.compareTo(buildCount1);
}
});
return buildHistoryList;
}
private static BuildHistory _mergeBuildHistories(
List buildHistories, String name) {
BuildHistory mergedBuildHistory = new BuildHistory(
0, name, System.currentTimeMillis());
for (BuildHistory buildHistory : buildHistories) {
mergedBuildHistory.merge(buildHistory);
}
return mergedBuildHistory;
}
private static final File _BASE_DIR;
private static final Integer _THREAD_COUNT = 8;
private static final Properties _buildProperties;
private static final ExecutorService _executorService =
JenkinsResultsParserUtil.getNewThreadPoolExecutor(_THREAD_COUNT, true);
static {
_buildProperties = new Properties() {
{
try {
putAll(JenkinsResultsParserUtil.getBuildProperties());
}
catch (IOException ioException) {
throw new RuntimeException(ioException);
}
}
};
_BASE_DIR = new File(
_buildProperties.getProperty("archive.ci.build.data.tmp.dir"),
"builds");
}
private static class GroupByCategory
implements Function {
public String apply(BuildJSONObject buildJSONObject) {
String jobName = buildJSONObject.getJobName();
jobName = jobName.replace("-batch", "");
jobName = jobName.replace("-downstream", "");
jobName = jobName.replace("-validation", "");
if (jobName.contains("maintenance-") ||
jobName.contains("mirrors-") ||
jobName.contains("verification-")) {
return Category.MAINTENANCE.toString();
}
if (jobName.equals("test-portal-acceptance-pullrequest(master)")) {
return Category.PORTAL_MASTER_PULLREQUEST.toString();
}
if (jobName.equals("test-portal-acceptance-upstream(master)") ||
jobName.equals("test-portal-acceptance-upstream-dxp(master)") ||
jobName.equals("test-portal-testsuite-upstream(master)")) {
return Category.PORTAL_MASTER_UPSTREAM.toString();
}
if (jobName.equals("test-portal-fixpack-release") ||
jobName.equals("test-portal-hotfix-release") ||
jobName.equals("test-portal-release")) {
return Category.PORTAL_RELEASE.toString();
}
if (jobName.contains("test-portal-")) {
return Category.PORTAL_OTHER.toString();
}
return Category.OTHER.toString();
}
private enum Category {
MAINTENANCE("CI Maintenance"), OTHER("Other"),
PORTAL_MASTER_PULLREQUEST("liferay-portal/master PR's"),
PORTAL_MASTER_UPSTREAM("liferay-portal/master Upstream"),
PORTAL_OTHER("liferay-portal-ee PR's & Upstream"),
PORTAL_OTHER_RELEASE("Portal Fixpack & Hotfix Release"),
PORTAL_RELEASE("Portal Release");
@Override
public String toString() {
return _string;
}
private Category(String string) {
_string = string;
}
private final String _string;
}
}
private static class GroupByJobName
implements Function {
public String apply(BuildJSONObject buildJSONObject) {
String jobName = buildJSONObject.getJobName();
String name = jobName.replace("-batch", "");
name = name.replace("-downstream", "");
name = name.replace("-validation", "");
return name;
}
}
private static class GroupByTopLevelTestSuite
implements Function {
public String apply(BuildJSONObject buildJSONObject) {
String jobName = buildJSONObject.getJobName();
if (jobName.contains("acceptance-upstream-dxp")) {
return "acceptance-dxp";
}
if (buildJSONObject.isTopLevelBuild()) {
Map parameters =
buildJSONObject.getParameters();
if (parameters.containsKey("CI_TEST_SUITE")) {
_topLevelBuildTestSuiteMap.put(
buildJSONObject.getURL(),
parameters.get("CI_TEST_SUITE"));
return parameters.get("CI_TEST_SUITE");
}
return "[Unknown]";
}
String topLevelBuildURL = buildJSONObject.getTopLevelBuildURL();
if (_topLevelBuildTestSuiteMap.containsKey(topLevelBuildURL)) {
return _topLevelBuildTestSuiteMap.get(topLevelBuildURL);
}
return "[Unknown]";
}
private final Map _topLevelBuildTestSuiteMap =
new HashMap<>();
}
private static class GroupByWeeklyUtilization
implements Function {
public String apply(BuildJSONObject buildJSONObject) {
String jobName = buildJSONObject.getJobName();
LocalDate localDate = LocalDate.parse(
buildJSONObject.getStartDateString(),
DateTimeFormatter.ofPattern("yyyyMMdd"));
DayOfWeek dayOfWeek = localDate.getDayOfWeek();
boolean weekday = false;
if (dayOfWeek.getValue() <= 5) {
weekday = true;
}
if (jobName.contains("test-portal-acceptance-pullrequest")) {
if (weekday) {
return Category.PORTAL_PULLREQUEST_WEEKDAYS.toString();
}
return Category.PORTAL_PULLREQUEST_WEEKENDS.toString();
}
if (jobName.contains("release") || jobName.contains("upstream")) {
if (weekday) {
return Category.PORTAL_RELEASE_AND_UPSTREAM_WEEKDAYS.
toString();
}
return Category.PORTAL_RELEASE_AND_UPSTREAM_WEEKENDS.toString();
}
if (weekday) {
return Category.OTHER_WEEKDAYS.toString();
}
return Category.OTHER_WEEKENDS.toString();
}
private enum Category {
OTHER_WEEKDAYS("Other (Weekdays)"),
OTHER_WEEKENDS("Other (Weekends)"),
PORTAL_PULLREQUEST_WEEKDAYS("Portal Pull Requests (Weekdays)"),
PORTAL_PULLREQUEST_WEEKENDS("Portal Pull Requests (Weekends)"),
PORTAL_RELEASE_AND_UPSTREAM_WEEKDAYS(
"Portal Release & Upstream (Weekdays)"),
PORTAL_RELEASE_AND_UPSTREAM_WEEKENDS(
"Portal Release & Upstream (Weekends)");
@Override
public String toString() {
return _string;
}
private Category(String string) {
_string = string;
}
private final String _string;
}
}
}