org.apache.atlas.hive.hook.events.CreateTable Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of hive-bridge
Apache Atlas Hive Bridge Module
There is a newer version: 2.3.0
/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 * 
 * http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.atlas.hive.hook.events;

import org.apache.atlas.hive.hook.AtlasHiveHookContext;
import org.apache.atlas.hive.hook.HiveHook;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasEntity.AtlasEntitiesWithExtInfo;
import org.apache.atlas.model.notification.HookNotification;
import org.apache.atlas.model.notification.HookNotification.EntityCreateRequestV2;
import org.apache.commons.collections.CollectionUtils;
import org.apache.hadoop.hive.metastore.TableType;
import org.apache.hadoop.hive.metastore.events.AlterTableEvent;
import org.apache.hadoop.hive.metastore.events.CreateTableEvent;
import org.apache.hadoop.hive.metastore.events.ListenerEvent;
import org.apache.hadoop.hive.ql.hooks.Entity;
import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.plan.HiveOperation;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.Collections;
import java.util.List;

import static org.apache.hadoop.hive.metastore.TableType.EXTERNAL_TABLE;
import static org.apache.hadoop.hive.ql.plan.HiveOperation.*;

public class CreateTable extends BaseHiveEvent {
    private static final Logger LOG = LoggerFactory.getLogger(CreateTable.class);

    public CreateTable(AtlasHiveHookContext context) {
        super(context);
    }

    @Override
    public List getNotificationMessages() throws Exception {
        List ret = null;
        AtlasEntitiesWithExtInfo entities = context.isMetastoreHook() ? getHiveMetastoreEntities() : getHiveEntities();

        if (entities != null && CollectionUtils.isNotEmpty(entities.getEntities())) {
            ret = Collections.singletonList(new EntityCreateRequestV2(getUserName(), entities));
        }

        return ret;
    }

    public AtlasEntitiesWithExtInfo getHiveMetastoreEntities() throws Exception {
        AtlasEntitiesWithExtInfo ret   = new AtlasEntitiesWithExtInfo();
        ListenerEvent            event = context.getMetastoreEvent();
        HiveOperation            oper  = context.getHiveOperation();
        Table                    table;

        if (isAlterTable(oper)) {
            table = toTable(((AlterTableEvent) event).getNewTable());
        } else {
            table = toTable(((CreateTableEvent) event).getTable());
        }

        if (skipTemporaryTable(table)) {
            table = null;
        }

        processTable(table, ret);

        addProcessedEntities(ret);

        return ret;
    }

    public AtlasEntitiesWithExtInfo getHiveEntities() throws Exception {
        AtlasEntitiesWithExtInfo ret   = new AtlasEntitiesWithExtInfo();
        Table                    table = null;

        if (CollectionUtils.isNotEmpty(getOutputs())) {
            for (Entity entity : getOutputs()) {
                if (entity.getType() == Entity.Type.TABLE) {
                    table = entity.getTable();

                    if (table != null) {
                        table = getHive().getTable(table.getDbName(), table.getTableName());

                        if (table != null) {
                            if (skipTemporaryTable(table)) {
                                table = null;
                            } else {
                                break;
                            }
                        }
                    }
                }
            }
        }

        processTable(table, ret);

        addProcessedEntities(ret);

        return ret;
    }

    // create process entities for lineages from HBase/HDFS to hive table
    private void processTable(Table table, AtlasEntitiesWithExtInfo ret) throws Exception {
        if (table != null) {
            AtlasEntity tblEntity = toTableEntity(table, ret);

            if (tblEntity != null) {
                if (isHBaseStore(table)) {
                    if (context.isMetastoreHook()) {
                        //do nothing
                    } else {
                        // This create lineage to HBase table in case of Hive on HBase
                        AtlasEntity hbaseTableEntity = toReferencedHBaseTable(table, ret);

                        //not a hive metastore hook
                        //it is running in the context of Hbase.
                        if (hbaseTableEntity != null) {
                            final AtlasEntity processEntity;

                            if (EXTERNAL_TABLE.equals(table.getTableType())) {
                                processEntity = getHiveProcessEntity(Collections.singletonList(hbaseTableEntity), Collections.singletonList(tblEntity));
                            } else {
                                processEntity = getHiveProcessEntity(Collections.singletonList(tblEntity), Collections.singletonList(hbaseTableEntity));
                            }
                            ret.addEntity(processEntity);

                            AtlasEntity processExecution = getHiveProcessExecutionEntity(processEntity);
                            ret.addEntity(processExecution);
                        }
                    }

                } else {
                    if (context.isMetastoreHook()) {
                        //it is running in the context of HiveMetastore
                        //not a hive metastore hook
                        if (isCreateExtTableOperation(table)) {
                            if (LOG.isDebugEnabled()) {
                                LOG.debug("Creating a dummy process with lineage from hdfs path to hive table");
                            }
                            AtlasEntity hdfsPathEntity = getPathEntity(table.getDataLocation(), ret);
                            AtlasEntity processEntity  = getHiveProcessEntity(Collections.singletonList(hdfsPathEntity), Collections.singletonList(tblEntity));

                            ret.addEntity(processEntity);
                            ret.addReferredEntity(hdfsPathEntity);
                        }
                    } else {
                        //not a hive metastore hook
                        //it is running in the context of HiveServer2
                        if (EXTERNAL_TABLE.equals(table.getTableType())) {
                            AtlasEntity hdfsPathEntity = getPathEntity(table.getDataLocation(), ret);
                            AtlasEntity processEntity  = getHiveProcessEntity(Collections.singletonList(hdfsPathEntity), Collections.singletonList(tblEntity));

                            ret.addEntity(processEntity);
                            ret.addReferredEntity(hdfsPathEntity);

                            AtlasEntity processExecution = getHiveProcessExecutionEntity(processEntity);
                            ret.addEntity(processExecution);
                        }
                    }
                }

                if (!context.isMetastoreHook()) {
                    AtlasEntity tableDDLEntity = createHiveDDLEntity(tblEntity);

                    if (tableDDLEntity != null) {
                        ret.addEntity(tableDDLEntity);
                    }
                }
            }
        }
    }

    private static boolean isAlterTable(HiveOperation oper) {
        return (oper == ALTERTABLE_PROPERTIES || oper == ALTERTABLE_RENAME || oper == ALTERTABLE_RENAMECOL);
    }

    private boolean skipTemporaryTable(Table table) {
        /**
         * If its an external table, even though the temp table skip flag(skip.temp.tables) is on, we create the table since we need the HDFS path to temp table lineage.
         * We skip external temp table only on enabling flag for skip all temp tables including external tables(skip.all.temp.tables)
         **/
        return table != null && skipTempTables && table.isTemporary() && (!EXTERNAL_TABLE.equals(table.getTableType()) || HiveHook.isSkipAllTempTablesIncludingExternal());
    }

    private boolean isCreateExtTableOperation(Table table) {
        HiveOperation oper      = context.getHiveOperation();
        TableType     tableType = table.getTableType();

        return EXTERNAL_TABLE.equals(tableType) && (oper == CREATETABLE || oper == CREATETABLE_AS_SELECT);
    }
}