io.trino.plugin.hive.GenericHiveRecordCursorProvider Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of trino-hive Show documentation
Trino - Hive connector
There is a newer version: 468
/*
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package io.trino.plugin.hive;

import io.airlift.units.DataSize;
import io.trino.plugin.hive.util.HiveUtil;
import io.trino.spi.TrinoException;
import io.trino.spi.connector.ConnectorSession;
import io.trino.spi.connector.RecordCursor;
import io.trino.spi.predicate.TupleDomain;
import io.trino.spi.type.TypeManager;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapred.RecordReader;
import org.apache.hadoop.mapreduce.lib.input.LineRecordReader;

import javax.inject.Inject;

import java.io.IOException;
import java.util.List;
import java.util.Optional;
import java.util.Properties;

import static com.google.common.base.Preconditions.checkArgument;
import static io.trino.plugin.hive.HiveErrorCode.HIVE_FILESYSTEM_ERROR;
import static io.trino.plugin.hive.HivePageSourceProvider.projectBaseColumns;
import static java.lang.Math.toIntExact;
import static java.util.Objects.requireNonNull;
import static java.util.stream.Collectors.toUnmodifiableList;

public class GenericHiveRecordCursorProvider
        implements HiveRecordCursorProvider
{
    private final HdfsEnvironment hdfsEnvironment;
    private final int textMaxLineLengthBytes;

    @Inject
    public GenericHiveRecordCursorProvider(HdfsEnvironment hdfsEnvironment, HiveConfig config)
    {
        this(hdfsEnvironment, config.getTextMaxLineLength());
    }

    public GenericHiveRecordCursorProvider(HdfsEnvironment hdfsEnvironment, DataSize textMaxLineLength)
    {
        this.hdfsEnvironment = requireNonNull(hdfsEnvironment, "hdfsEnvironment is null");
        this.textMaxLineLengthBytes = toIntExact(textMaxLineLength.toBytes());
        checkArgument(textMaxLineLengthBytes >= 1, "textMaxLineLength must be at least 1 byte");
    }

    @Override
    public Optional createRecordCursor(
            Configuration configuration,
            ConnectorSession session,
            Path path,
            long start,
            long length,
            long fileSize,
            Properties schema,
            List columns,
            TupleDomain effectivePredicate,
            TypeManager typeManager,
            boolean s3SelectPushdownEnabled)
    {
        configuration.setInt(LineRecordReader.MAX_LINE_LENGTH, textMaxLineLengthBytes);

        // make sure the FileSystem is created with the proper Configuration object
        try {
            this.hdfsEnvironment.getFileSystem(session.getIdentity(), path, configuration);
        }
        catch (IOException e) {
            throw new TrinoException(HIVE_FILESYSTEM_ERROR, "Failed getting FileSystem: " + path, e);
        }

        Optional projections = projectBaseColumns(columns);
        List readerColumns = projections
                .map(ReaderColumns::get)
                .map(columnHandles -> columnHandles.stream()
                        .map(HiveColumnHandle.class::cast)
                        .collect(toUnmodifiableList()))
                .orElse(columns);

        RecordCursor cursor = hdfsEnvironment.doAs(session.getIdentity(), () -> {
            RecordReader recordReader = HiveUtil.createRecordReader(
                    configuration,
                    path,
                    start,
                    length,
                    schema,
                    readerColumns);

            try {
                return new GenericHiveRecordCursor<>(
                        configuration,
                        path,
                        genericRecordReader(recordReader),
                        length,
                        schema,
                        readerColumns);
            }
            catch (Exception e) {
                try {
                    recordReader.close();
                }
                catch (IOException closeException) {
                    if (e != closeException) {
                        e.addSuppressed(closeException);
                    }
                }
                throw e;
            }
        });

        return Optional.of(new ReaderRecordCursorWithProjections(cursor, projections));
    }

    @SuppressWarnings("unchecked")
    private static RecordReader genericRecordReader(RecordReader recordReader)
    {
        return (RecordReader) recordReader;
    }
}