io.trino.verifier.Validator Maven / Gradle / Ivy

Show more of this group Show more artifacts with this name
Show all versions of trino-verifier Show documentation
There is a newer version: 451
/*
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package io.trino.verifier;

import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Joiner;
import com.google.common.base.Stopwatch;
import com.google.common.collect.ComparisonChain;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSortedMultiset;
import com.google.common.collect.Iterables;
import com.google.common.collect.Multiset;
import com.google.common.collect.Multisets;
import com.google.common.collect.Ordering;
import com.google.common.collect.SortedMultiset;
import com.google.common.util.concurrent.SimpleTimeLimiter;
import com.google.common.util.concurrent.TimeLimiter;
import com.google.common.util.concurrent.UncheckedTimeoutException;
import io.airlift.log.Logger;
import io.airlift.units.Duration;
import io.trino.jdbc.QueryStats;
import io.trino.jdbc.TrinoConnection;
import io.trino.jdbc.TrinoStatement;
import io.trino.spi.type.SqlVarbinary;
import io.trino.verifier.Validator.ChangedRow.Changed;

import java.math.BigDecimal;
import java.sql.Array;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
import java.sql.SQLClientInfoException;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.function.Consumer;
import java.util.function.Function;

import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkState;
import static com.google.common.base.Strings.nullToEmpty;
import static io.airlift.units.Duration.nanosSince;
import static io.trino.verifier.QueryResult.State;
import static java.lang.Double.isFinite;
import static java.lang.String.format;
import static java.util.Collections.unmodifiableList;
import static java.util.Objects.requireNonNull;
import static java.util.concurrent.Executors.newSingleThreadExecutor;
import static java.util.concurrent.TimeUnit.MILLISECONDS;

public class Validator
{
    private static final Logger log = Logger.get(Validator.class);

    private final String testUsername;
    private final String controlUsername;
    private final String testPassword;
    private final String controlPassword;
    private final String controlGateway;
    private final String testGateway;
    private final Duration controlTimeout;
    private final Duration testTimeout;
    private final int maxRowCount;
    private final boolean checkCorrectness;
    private final boolean checkDeterministic;
    private final boolean verboseResultsComparison;
    private final QueryPair queryPair;
    private final boolean explainOnly;
    private final Map controlSessionProperties;
    private final Map testSessionProperties;
    private final int precision;
    private final int controlTeardownRetries;
    private final int testTeardownRetries;
    private final boolean runTearDownOnResultMismatch;
    private final boolean skipControl;

    private Boolean valid;

    private QueryResult controlResult;
    private QueryResult testResult;

    private final List controlPreQueryResults = new ArrayList<>();
    private final List controlPostQueryResults = new ArrayList<>();
    private final List testPreQueryResults = new ArrayList<>();
    private final List testPostQueryResults = new ArrayList<>();

    private boolean deterministic = true;

    public Validator(
            String controlGateway,
            String testGateway,
            Duration controlTimeout,
            Duration testTimeout,
            int maxRowCount,
            boolean explainOnly,
            int precision,
            boolean checkCorrectness,
            boolean checkDeterministic,
            boolean verboseResultsComparison,
            int controlTeardownRetries,
            int testTeardownRetries,
            boolean runTearDownOnResultMismatch,
            boolean skipControl,
            QueryPair queryPair)
    {
        this.testUsername = requireNonNull(queryPair.getTest().getUsername(), "test username is null");
        this.controlUsername = requireNonNull(queryPair.getControl().getUsername(), "control username is null");
        this.testPassword = queryPair.getTest().getPassword();
        this.controlPassword = queryPair.getControl().getPassword();
        this.controlGateway = requireNonNull(controlGateway, "controlGateway is null");
        this.testGateway = requireNonNull(testGateway, "testGateway is null");
        this.controlTimeout = controlTimeout;
        this.testTimeout = testTimeout;
        this.maxRowCount = maxRowCount;
        this.explainOnly = explainOnly;
        this.precision = precision;
        this.checkCorrectness = checkCorrectness;
        this.checkDeterministic = checkDeterministic;
        this.verboseResultsComparison = verboseResultsComparison;
        this.controlTeardownRetries = controlTeardownRetries;
        this.testTeardownRetries = testTeardownRetries;
        this.runTearDownOnResultMismatch = runTearDownOnResultMismatch;
        this.skipControl = skipControl;

        this.queryPair = requireNonNull(queryPair, "queryPair is null");
        this.controlSessionProperties = queryPair.getControl().getSessionProperties();
        this.testSessionProperties = queryPair.getTest().getSessionProperties();
    }

    public boolean isSkipped()
    {
        if (queryPair.getControl().getQuery().isEmpty() || queryPair.getTest().getQuery().isEmpty()) {
            return true;
        }

        if (!skipControl && getControlResult().getState() != State.SUCCESS) {
            return true;
        }

        if (!isDeterministic()) {
            return true;
        }

        if (getTestResult().getState() == State.TIMEOUT) {
            return true;
        }

        return false;
    }

    public String getSkippedMessage()
    {
        StringBuilder sb = new StringBuilder();
        if (getControlResult().getState() == State.TOO_MANY_ROWS) {
            sb.append("----------\n");
            sb.append("Name: " + queryPair.getName() + "\n");
            sb.append("Schema (control): " + queryPair.getControl().getSchema() + "\n");
            sb.append("Too many rows.\n");
        }
        else if (!isDeterministic()) {
            sb.append("----------\n");
            sb.append("Name: " + queryPair.getName() + "\n");
            sb.append("Schema (control): " + queryPair.getControl().getSchema() + "\n");
            sb.append("NON DETERMINISTIC\n");
        }
        else if (getControlResult().getState() == State.TIMEOUT || getTestResult().getState() == State.TIMEOUT) {
            sb.append("----------\n");
            sb.append("Name: " + queryPair.getName() + "\n");
            sb.append("Schema (control): " + queryPair.getControl().getSchema() + "\n");
            sb.append("TIMEOUT\n");
        }
        else {
            sb.append("SKIPPED: ");
            if (getControlResult().getException() != null) {
                sb.append(getControlResult().getException().getMessage());
            }
        }
        return sb.toString();
    }

    public boolean valid()
    {
        if (valid == null) {
            valid = validate();
        }
        return valid;
    }

    public boolean isDeterministic()
    {
        if (valid == null) {
            valid = validate();
        }
        return deterministic;
    }

    private boolean validate()
    {
        boolean tearDownControl = true;
        boolean tearDownTest = false;
        try {
            if (skipControl) {
                controlResult = new QueryResult(State.SKIPPED, null, null, null, null, ImmutableList.of(), ImmutableList.of());
            }
            else {
                controlResult = executePreAndMainForControl();
            }

            // query has too many rows. Consider banning it.
            if (controlResult.getState() == State.TOO_MANY_ROWS) {
                testResult = new QueryResult(State.INVALID, null, null, null, null, ImmutableList.of(), ImmutableList.of());
                return false;
            }
            // query failed in the control
            if (!skipControl && controlResult.getState() != State.SUCCESS) {
                testResult = new QueryResult(State.INVALID, null, null, null, null, ImmutableList.of(), ImmutableList.of());
                return true;
            }

            testResult = executePreAndMainForTest();
            tearDownTest = true;

            if ((!skipControl && controlResult.getState() != State.SUCCESS) || testResult.getState() != State.SUCCESS) {
                return false;
            }

            if (skipControl || !checkCorrectness) {
                return true;
            }

            boolean matches = resultsMatch(controlResult, testResult, precision);
            if (!matches && checkDeterministic) {
                matches = checkForDeterministicAndRerunTestQueriesIfNeeded();
            }
            if (!matches && !runTearDownOnResultMismatch) {
                tearDownControl = false;
                tearDownTest = false;
            }
            return matches;
        }
        finally {
            if (tearDownControl) {
                tearDownControl();
            }
            if (tearDownTest) {
                tearDownTest();
            }
        }
    }

    private void tearDownControl()
    {
        QueryResult controlTearDownResult = executeTearDown(
                queryPair.getControl(),
                controlGateway,
                controlUsername,
                controlPassword,
                controlTimeout,
                controlPostQueryResults,
                controlTeardownRetries,
                controlSessionProperties);
        if (controlTearDownResult.getState() != State.SUCCESS) {
            log.warn("Control table teardown failed");
        }
    }

    private void tearDownTest()
    {
        QueryResult testTearDownResult = executeTearDown(
                queryPair.getTest(),
                testGateway,
                testUsername,
                testPassword,
                testTimeout,
                testPostQueryResults,
                testTeardownRetries,
                testSessionProperties);
        if (testTearDownResult.getState() != State.SUCCESS) {
            log.warn("Test table teardown failed");
        }
    }

    private QueryResult tearDown(Query query, List postQueryResults, Function executor)
    {
        postQueryResults.clear();
        for (String postqueryString : query.getPostQueries()) {
            QueryResult queryResult = executor.apply(postqueryString);
            postQueryResults.add(queryResult);
            if (queryResult.getState() != State.SUCCESS) {
                return new QueryResult(State.FAILED_TO_TEARDOWN, queryResult.getException(), queryResult.getWallTime(), queryResult.getCpuTime(), queryResult.getQueryId(), ImmutableList.of(), ImmutableList.of());
            }
        }

        return new QueryResult(State.SUCCESS, null, null, null, null, ImmutableList.of(), ImmutableList.of());
    }

    private static QueryResult setup(Query query, List preQueryResults, Function executor)
    {
        preQueryResults.clear();
        for (String prequeryString : query.getPreQueries()) {
            QueryResult queryResult = executor.apply(prequeryString);
            preQueryResults.add(queryResult);
            if (queryResult.getState() == State.TIMEOUT) {
                return queryResult;
            }
            if (queryResult.getState() != State.SUCCESS) {
                return new QueryResult(State.FAILED_TO_SETUP, queryResult.getException(), queryResult.getWallTime(), queryResult.getCpuTime(), queryResult.getQueryId(), ImmutableList.of(), ImmutableList.of());
            }
        }

        return new QueryResult(State.SUCCESS, null, null, null, null, ImmutableList.of(), ImmutableList.of());
    }

    private boolean checkForDeterministicAndRerunTestQueriesIfNeeded()
    {
        // check if the control query is deterministic
        for (int i = 0; i < 3; i++) {
            QueryResult results = executePreAndMainForControl();
            if (results.getState() != State.SUCCESS) {
                return false;
            }

            if (!resultsMatch(controlResult, results, precision)) {
                deterministic = false;
                return false;
            }
        }

        // Re-run the test query to confirm that the results don't match, in case there was caching on the test tier,
        // but require that it matches 3 times in a row to rule out a non-deterministic correctness bug.
        for (int i = 0; i < 3; i++) {
            testResult = executePreAndMainForTest();
            if (testResult.getState() != State.SUCCESS) {
                return false;
            }
            if (!resultsMatch(controlResult, testResult, precision)) {
                return false;
            }
        }

        // test result agrees with control result 3 times in a row although the first test result didn't agree
        return true;
    }

    private QueryResult executePreAndMainForTest()
    {
        return executePreAndMain(
                queryPair.getTest(),
                testPreQueryResults,
                testGateway,
                testUsername,
                testPassword,
                testTimeout,
                testPostQueryResults,
                testTeardownRetries,
                testSessionProperties);
    }

    private QueryResult executePreAndMainForControl()
    {
        return executePreAndMain(
                queryPair.getControl(),
                controlPreQueryResults,
                controlGateway,
                controlUsername,
                controlPassword,
                controlTimeout,
                controlPostQueryResults,
                controlTeardownRetries,
                controlSessionProperties);
    }

    private QueryResult executePreAndMain(
            Query query,
            List preQueryResults,
            String gateway,
            String username,
            String password,
            Duration timeout,
            List postQueryResults,
            int teardownRetries,
            Map sessionProperties)
    {
        try {
            // startup
            QueryResult queryResult = setup(query, preQueryResults, preQuery ->
                    executeQuery(gateway, username, password, query, preQuery, timeout, sessionProperties));

            // if startup is successful -> execute query
            if (queryResult.getState() == State.SUCCESS) {
                queryResult = executeQuery(gateway, username, password, query, query.getQuery(), timeout, sessionProperties);
            }

            return queryResult;
        }
        catch (Exception e) {
            executeTearDown(query, gateway, username, password, timeout, postQueryResults, teardownRetries, sessionProperties);
            throw e;
        }
    }

    private QueryResult executeTearDown(
            Query query,
            String gateway,
            String username,
            String password,
            Duration timeout,
            List postQueryResults,
            int teardownRetries,
            Map sessionProperties)
    {
        int attempt = 0;
        QueryResult tearDownResult;
        do {
            tearDownResult = tearDown(query, postQueryResults, postQuery ->
                    executeQuery(gateway, username, password, query, postQuery, timeout, sessionProperties));
            if (tearDownResult.getState() == State.SUCCESS) {
                break;
            }
            try {
                TimeUnit.MINUTES.sleep(1);
                log.info("Query teardown failed on attempt #%s, will sleep and retry", attempt);
            }
            catch (InterruptedException e) {
                Thread.currentThread().interrupt();
                break;
            }
            attempt++;
        }
        while (attempt < teardownRetries);
        return tearDown(query, postQueryResults, postQuery ->
                executeQuery(gateway, username, password, query, postQuery, timeout, sessionProperties));
    }

    public QueryPair getQueryPair()
    {
        return queryPair;
    }

    public QueryResult getControlResult()
    {
        return controlResult;
    }

    public QueryResult getTestResult()
    {
        return testResult;
    }

    public List getControlPreQueryResults()
    {
        return controlPreQueryResults;
    }

    public List getControlPostQueryResults()
    {
        return controlPostQueryResults;
    }

    public List getTestPreQueryResults()
    {
        return testPreQueryResults;
    }

    public List getTestPostQueryResults()
    {
        return testPostQueryResults;
    }

    private QueryResult executeQuery(String url, String username, String password, Query query, String sql, Duration timeout, Map sessionProperties)
    {
        ExecutorService executor = newSingleThreadExecutor();
        TimeLimiter limiter = SimpleTimeLimiter.create(executor);

        long start = System.nanoTime();
        String queryId = null;
        Duration queryCpuTime = null;
        try (Connection connection = DriverManager.getConnection(url, username, password)) {
            trySetConnectionProperties(query, connection);
            for (Map.Entry entry : sessionProperties.entrySet()) {
                connection.unwrap(TrinoConnection.class).setSessionProperty(entry.getKey(), entry.getValue());
            }

            ProgressMonitor progressMonitor = new ProgressMonitor();
            List> results;
            List columnTypes;
            try (Statement statement = connection.createStatement()) {
                Stopwatch stopwatch = Stopwatch.createStarted();
                Statement limitedStatement = limiter.newProxy(statement, Statement.class, timeout.toMillis(), MILLISECONDS);
                if (explainOnly) {
                    sql = "EXPLAIN " + sql;
                }

                TrinoStatement trinoStatement = limitedStatement.unwrap(TrinoStatement.class);
                trinoStatement.setProgressMonitor(progressMonitor);
                boolean isSelectQuery = limitedStatement.execute(sql);

                if (isSelectQuery) {
                    ResultSetConverter converter = limiter.newProxy(
                            this::convertJdbcResultSet,
                            ResultSetConverter.class,
                            timeout.toMillis() - stopwatch.elapsed(MILLISECONDS),
                            MILLISECONDS);
                    ResultSet resultSet = limitedStatement.getResultSet();
                    results = converter.convert(resultSet);
                    columnTypes = getColumnTypes(resultSet);
                }
                else {
                    results = ImmutableList.of(ImmutableList.of(limitedStatement.getLargeUpdateCount()));
                    columnTypes = ImmutableList.of("BIGINT");
                }

                trinoStatement.clearProgressMonitor();
                if (progressMonitor.getFinalQueryStats() == null) {
                    throw new VerifierException("Cannot fetch query stats");
                }
            }
            finally {
                QueryStats queryStats = progressMonitor.getFinalQueryStats();
                if (queryStats != null) {
                    queryCpuTime = new Duration(queryStats.getCpuTimeMillis(), MILLISECONDS);
                    queryId = queryStats.getQueryId();
                }
            }

            return new QueryResult(State.SUCCESS, null, nanosSince(start), queryCpuTime, queryId, columnTypes, results);
        }
        catch (SQLException e) {
            Exception exception = e;
            if ((e.getMessage().startsWith("Error executing query") || "Error fetching results".equals(e.getMessage())) &&
                    (e.getCause() instanceof Exception)) {
                exception = (Exception) e.getCause();
            }
            State state = isTrinoQueryInvalid(e) ? State.INVALID : State.FAILED;
            return new QueryResult(state, exception, nanosSince(start), queryCpuTime, queryId, ImmutableList.of(), ImmutableList.of());
        }
        catch (VerifierException e) {
            return new QueryResult(State.TOO_MANY_ROWS, e, nanosSince(start), queryCpuTime, queryId, ImmutableList.of(), ImmutableList.of());
        }
        catch (UncheckedTimeoutException e) {
            return new QueryResult(State.TIMEOUT, e, nanosSince(start), queryCpuTime, queryId, ImmutableList.of(), ImmutableList.of());
        }
        finally {
            executor.shutdownNow();
        }
    }

    private void trySetConnectionProperties(Query query, Connection connection)
            throws SQLException
    {
        // Required for jdbc drivers that do not implement all/some of these functions (eg. impala jdbc driver)
        // For these drivers, set the database default values in the query database
        try {
            connection.setClientInfo("ApplicationName", "verifier-test:" + queryPair.getName());
            connection.setCatalog(query.getCatalog());
            connection.setSchema(query.getSchema());
        }
        catch (SQLClientInfoException ignored) {
            // Do nothing
        }
    }

    private static boolean isTrinoQueryInvalid(SQLException e)
    {
        for (Throwable t = e.getCause(); t != null; t = t.getCause()) {
            if (t.toString().contains(".SemanticException:")) {
                return true;
            }
            if (t.toString().contains(".ParsingException:")) {
                return true;
            }
            if (nullToEmpty(t.getMessage()).matches("Function .* not registered")) {
                return true;
            }
        }
        return false;
    }

    private List> convertJdbcResultSet(ResultSet resultSet)
            throws SQLException, VerifierException
    {
        int rowCount = 0;
        int columnCount = resultSet.getMetaData().getColumnCount();

        ImmutableList.Builder> rows = ImmutableList.builder();
        while (resultSet.next()) {
            List