
com.facebook.presto.verifier.Validator Maven / Gradle / Ivy
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.facebook.presto.verifier;
import com.facebook.presto.jdbc.PrestoConnection;
import com.facebook.presto.jdbc.PrestoStatement;
import com.facebook.presto.jdbc.QueryStats;
import com.facebook.presto.spi.type.SqlVarbinary;
import com.facebook.presto.verifier.Validator.ChangedRow.Changed;
import com.google.common.base.Joiner;
import com.google.common.base.Stopwatch;
import com.google.common.base.Throwables;
import com.google.common.collect.ComparisonChain;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSortedMultiset;
import com.google.common.collect.Iterables;
import com.google.common.collect.Multiset;
import com.google.common.collect.Multisets;
import com.google.common.collect.Ordering;
import com.google.common.collect.SortedMultiset;
import com.google.common.util.concurrent.SimpleTimeLimiter;
import com.google.common.util.concurrent.TimeLimiter;
import com.google.common.util.concurrent.UncheckedTimeoutException;
import io.airlift.units.Duration;
import java.math.BigDecimal;
import java.math.MathContext;
import java.sql.Array;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.SQLClientInfoException;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
import java.util.concurrent.Callable;
import java.util.concurrent.TimeUnit;
import java.util.function.Consumer;
import java.util.function.Function;
import static com.facebook.presto.verifier.QueryResult.State;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkState;
import static com.google.common.base.Strings.nullToEmpty;
import static com.google.common.primitives.Doubles.isFinite;
import static io.airlift.units.Duration.nanosSince;
import static java.lang.String.format;
import static java.util.Collections.unmodifiableList;
import static java.util.Objects.requireNonNull;
public class Validator
{
private final String testUsername;
private final String controlUsername;
private final String testPassword;
private final String controlPassword;
private final String controlGateway;
private final String testGateway;
private final Duration controlTimeout;
private final Duration testTimeout;
private final int maxRowCount;
private final boolean checkCorrectness;
private final boolean checkDeterministic;
private final boolean verboseResultsComparison;
private final QueryPair queryPair;
private final boolean explainOnly;
private final Map sessionProperties;
private final int precision;
private final int controlTeardownRetries;
private final int testTeardownRetries;
private Boolean valid;
private QueryResult controlResult;
private QueryResult testResult;
private final List controlPreQueryResults = new ArrayList<>();
private final List controlPostQueryResults = new ArrayList<>();
private final List testPreQueryResults = new ArrayList<>();
private final List testPostQueryResults = new ArrayList<>();
private boolean deterministic = true;
public Validator(
String controlGateway,
String testGateway,
Duration controlTimeout,
Duration testTimeout,
int maxRowCount,
boolean explainOnly,
int precision,
boolean checkCorrectness,
boolean checkDeterministic,
boolean verboseResultsComparison,
int controlTeardownRetries,
int testTeardownRetries,
QueryPair queryPair)
{
this.testUsername = requireNonNull(queryPair.getTest().getUsername(), "test username is null");
this.controlUsername = requireNonNull(queryPair.getControl().getUsername(), "control username is null");
this.testPassword = queryPair.getTest().getPassword();
this.controlPassword = queryPair.getControl().getPassword();
this.controlGateway = requireNonNull(controlGateway, "controlGateway is null");
this.testGateway = requireNonNull(testGateway, "testGateway is null");
this.controlTimeout = controlTimeout;
this.testTimeout = testTimeout;
this.maxRowCount = maxRowCount;
this.explainOnly = explainOnly;
this.precision = precision;
this.checkCorrectness = checkCorrectness;
this.checkDeterministic = checkDeterministic;
this.verboseResultsComparison = verboseResultsComparison;
this.controlTeardownRetries = controlTeardownRetries;
this.testTeardownRetries = testTeardownRetries;
this.queryPair = requireNonNull(queryPair, "queryPair is null");
// Test and Control always have the same session properties.
this.sessionProperties = queryPair.getTest().getSessionProperties();
}
public boolean isSkipped()
{
if (queryPair.getControl().getQuery().isEmpty() || queryPair.getTest().getQuery().isEmpty()) {
return true;
}
if (getControlResult().getState() != State.SUCCESS) {
return true;
}
if (!isDeterministic()) {
return true;
}
if (getTestResult().getState() == State.TIMEOUT) {
return true;
}
return false;
}
public String getSkippedMessage()
{
StringBuilder sb = new StringBuilder();
if (getControlResult().getState() == State.TOO_MANY_ROWS) {
sb.append("----------\n");
sb.append("Name: " + queryPair.getName() + "\n");
sb.append("Schema (control): " + queryPair.getControl().getSchema() + "\n");
sb.append("Too many rows.\n");
}
else if (!isDeterministic()) {
sb.append("----------\n");
sb.append("Name: " + queryPair.getName() + "\n");
sb.append("Schema (control): " + queryPair.getControl().getSchema() + "\n");
sb.append("NON DETERMINISTIC\n");
}
else if (getControlResult().getState() == State.TIMEOUT || getTestResult().getState() == State.TIMEOUT) {
sb.append("----------\n");
sb.append("Name: " + queryPair.getName() + "\n");
sb.append("Schema (control): " + queryPair.getControl().getSchema() + "\n");
sb.append("TIMEOUT\n");
}
else {
sb.append("SKIPPED: ");
if (getControlResult().getException() != null) {
sb.append(getControlResult().getException().getMessage());
}
}
return sb.toString();
}
public boolean valid()
{
if (valid == null) {
valid = validate();
}
return valid;
}
public boolean isDeterministic()
{
if (valid == null) {
valid = validate();
}
return deterministic;
}
private boolean validate()
{
controlResult = executeQueryControl();
// query has too many rows. Consider blacklisting.
if (controlResult.getState() == State.TOO_MANY_ROWS) {
testResult = new QueryResult(State.INVALID, null, null, null, ImmutableList.>of());
return false;
}
// query failed in the control
if (controlResult.getState() != State.SUCCESS) {
testResult = new QueryResult(State.INVALID, null, null, null, ImmutableList.>of());
return true;
}
testResult = executeQueryTest();
if (controlResult.getState() != State.SUCCESS || testResult.getState() != State.SUCCESS) {
return false;
}
if (!checkCorrectness) {
return true;
}
boolean matches = resultsMatch(controlResult, testResult, precision);
if (!matches && checkDeterministic) {
return checkForDeterministicAndRerunTestQueriesIfNeeded();
}
return matches;
}
private static QueryResult tearDown(Query query, List postQueryResults, Function executor)
{
postQueryResults.clear();
for (String postqueryString : query.getPostQueries()) {
QueryResult queryResult = executor.apply(postqueryString);
postQueryResults.add(queryResult);
if (queryResult.getState() != State.SUCCESS) {
return new QueryResult(State.FAILED_TO_TEARDOWN, queryResult.getException(), queryResult.getWallTime(), queryResult.getCpuTime(), ImmutableList.>of());
}
}
return new QueryResult(State.SUCCESS, null, null, null, ImmutableList.of());
}
private static QueryResult setup(Query query, List preQueryResults, Function executor)
{
preQueryResults.clear();
for (String prequeryString : query.getPreQueries()) {
QueryResult queryResult = executor.apply(prequeryString);
preQueryResults.add(queryResult);
if (queryResult.getState() != State.SUCCESS) {
return new QueryResult(State.FAILED_TO_SETUP, queryResult.getException(), queryResult.getWallTime(), queryResult.getCpuTime(), ImmutableList.>of());
}
}
return new QueryResult(State.SUCCESS, null, null, null, ImmutableList.of());
}
private boolean checkForDeterministicAndRerunTestQueriesIfNeeded()
{
// check if the control query is deterministic
for (int i = 0; i < 3; i++) {
QueryResult results = executeQueryControl();
if (results.getState() != State.SUCCESS) {
return false;
}
if (!resultsMatch(controlResult, results, precision)) {
deterministic = false;
return false;
}
}
// Re-run the test query to confirm that the results don't match, in case there was caching on the test tier,
// but require that it matches 3 times in a row to rule out a non-deterministic correctness bug.
for (int i = 0; i < 3; i++) {
testResult = executeQueryTest();
if (testResult.getState() != State.SUCCESS) {
return false;
}
if (!resultsMatch(controlResult, testResult, precision)) {
return false;
}
}
// test result agrees with control result 3 times in a row although the first test result didn't agree
return true;
}
private QueryResult executeQueryTest()
{
Query query = queryPair.getTest();
QueryResult queryResult = new QueryResult(State.INVALID, null, null, null, ImmutableList.>of());
try {
// startup
queryResult = setup(query, testPreQueryResults, testPrequery -> executeQuery(testGateway, testUsername, testPassword, queryPair.getTest(), testPrequery, testTimeout, sessionProperties));
// if startup is successful -> execute query
if (queryResult.getState() == State.SUCCESS) {
queryResult = executeQuery(testGateway, testUsername, testPassword, queryPair.getTest(), query.getQuery(), testTimeout, sessionProperties);
}
}
finally {
int retry = 0;
QueryResult tearDownResult;
do {
tearDownResult = tearDown(query, testPostQueryResults, testPostquery -> executeQuery(testGateway, testUsername, testPassword, queryPair.getTest(), testPostquery, testTimeout, sessionProperties));
if (tearDownResult.getState() == State.SUCCESS) {
break;
}
try {
TimeUnit.MINUTES.sleep(1);
}
catch (InterruptedException e) {
Thread.currentThread().interrupt();
break;
}
retry++;
}
while (retry < testTeardownRetries);
// if teardown is not successful the query fails
queryResult = tearDownResult.getState() == State.SUCCESS ? queryResult : tearDownResult;
}
return queryResult;
}
private QueryResult executeQueryControl()
{
Query query = queryPair.getControl();
QueryResult queryResult = new QueryResult(State.INVALID, null, null, null, ImmutableList.>of());
try {
// startup
queryResult = setup(query, controlPreQueryResults, controlPrequery -> executeQuery(controlGateway, controlUsername, controlPassword, queryPair.getControl(), controlPrequery, controlTimeout, sessionProperties));
// if startup is successful -> execute query
if (queryResult.getState() == State.SUCCESS) {
queryResult = executeQuery(controlGateway, controlUsername, controlPassword, queryPair.getControl(), query.getQuery(), controlTimeout, sessionProperties);
}
}
finally {
int retry = 0;
QueryResult tearDownResult;
do {
tearDownResult = tearDown(query, controlPostQueryResults, controlPostquery -> executeQuery(controlGateway, controlUsername, controlPassword, queryPair.getControl(), controlPostquery, controlTimeout, sessionProperties));
if (tearDownResult.getState() == State.SUCCESS) {
break;
}
try {
TimeUnit.MINUTES.sleep(1);
}
catch (InterruptedException e) {
Thread.currentThread().interrupt();
break;
}
retry++;
}
while (retry < controlTeardownRetries);
// if teardown is not successful the query fails
queryResult = tearDownResult.getState() == State.SUCCESS ? queryResult : tearDownResult;
}
return queryResult;
}
public QueryPair getQueryPair()
{
return queryPair;
}
public QueryResult getControlResult()
{
return controlResult;
}
public QueryResult getTestResult()
{
return testResult;
}
public List getControlPreQueryResults()
{
return controlPreQueryResults;
}
public List getControlPostQueryResults()
{
return controlPostQueryResults;
}
public List getTestPreQueryResults()
{
return testPreQueryResults;
}
public List getTestPostQueryResults()
{
return testPostQueryResults;
}
private QueryResult executeQuery(String url, String username, String password, Query query, String sql, Duration timeout, Map sessionProperties)
{
try (Connection connection = DriverManager.getConnection(url, username, password)) {
trySetConnectionProperties(query, connection);
for (Map.Entry entry : sessionProperties.entrySet()) {
connection.unwrap(PrestoConnection.class).setSessionProperty(entry.getKey(), entry.getValue());
}
try (Statement statement = connection.createStatement()) {
TimeLimiter limiter = new SimpleTimeLimiter();
Stopwatch stopwatch = Stopwatch.createStarted();
Statement limitedStatement = limiter.newProxy(statement, Statement.class, timeout.toMillis(), TimeUnit.MILLISECONDS);
if (explainOnly) {
sql = "EXPLAIN " + sql;
}
long start = System.nanoTime();
PrestoStatement prestoStatement = limitedStatement.unwrap(PrestoStatement.class);
ProgressMonitor progressMonitor = new ProgressMonitor();
prestoStatement.setProgressMonitor(progressMonitor);
try {
boolean isSelectQuery = limitedStatement.execute(sql);
List> results = null;
if (isSelectQuery) {
results = limiter.callWithTimeout(
getResultSetConverter(limitedStatement.getResultSet()),
timeout.toMillis() - stopwatch.elapsed(TimeUnit.MILLISECONDS),
TimeUnit.MILLISECONDS, true);
}
else {
results = ImmutableList.of(ImmutableList.of(limitedStatement.getLargeUpdateCount()));
}
prestoStatement.clearProgressMonitor();
QueryStats queryStats = progressMonitor.getFinalQueryStats();
if (queryStats == null) {
throw new VerifierException("Cannot fetch query stats");
}
Duration queryCpuTime = new Duration(queryStats.getCpuTimeMillis(), TimeUnit.MILLISECONDS);
return new QueryResult(State.SUCCESS, null, nanosSince(start), queryCpuTime, results);
}
catch (AssertionError e) {
if (e.getMessage().startsWith("unimplemented type:")) {
return new QueryResult(State.INVALID, null, null, null, ImmutableList.>of());
}
throw e;
}
catch (SQLException | VerifierException e) {
throw e;
}
catch (UncheckedTimeoutException e) {
return new QueryResult(State.TIMEOUT, null, null, null, ImmutableList.>of());
}
catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw Throwables.propagate(e);
}
catch (Exception e) {
throw Throwables.propagate(e);
}
}
}
catch (SQLException e) {
Exception exception = e;
if (("Error executing query".equals(e.getMessage()) || "Error fetching results".equals(e.getMessage())) &&
(e.getCause() instanceof Exception)) {
exception = (Exception) e.getCause();
}
State state = isPrestoQueryInvalid(e) ? State.INVALID : State.FAILED;
return new QueryResult(state, exception, null, null, null);
}
catch (VerifierException e) {
return new QueryResult(State.TOO_MANY_ROWS, e, null, null, null);
}
}
private void trySetConnectionProperties(Query query, Connection connection)
throws SQLException
{
// Required for jdbc drivers that do not implement all/some of these functions (eg. impala jdbc driver)
// For these drivers, set the database default values in the query database
try {
connection.setClientInfo("ApplicationName", "verifier-test:" + queryPair.getName());
connection.setCatalog(query.getCatalog());
connection.setSchema(query.getSchema());
}
catch (SQLClientInfoException ignored) {
// Do nothing
}
}
private Callable>> getResultSetConverter(ResultSet resultSet)
{
return () -> convertJdbcResultSet(resultSet);
}
private static boolean isPrestoQueryInvalid(SQLException e)
{
for (Throwable t = e.getCause(); t != null; t = t.getCause()) {
if (t.toString().contains(".SemanticException:")) {
return true;
}
if (t.toString().contains(".ParsingException:")) {
return true;
}
if (nullToEmpty(t.getMessage()).matches("Function .* not registered")) {
return true;
}
}
return false;
}
private List> convertJdbcResultSet(ResultSet resultSet)
throws SQLException, VerifierException
{
int rowCount = 0;
int columnCount = resultSet.getMetaData().getColumnCount();
ImmutableList.Builder> rows = ImmutableList.builder();
while (resultSet.next()) {
List
© 2015 - 2025 Weber Informatics LLC | Privacy Policy