com.netflix.metacat.testdata.provider.PigDataDtoProvider.groovy Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of metacat-testdata-provider Show documentation
metacat-testdata-provider
There is a newer version: 1.3.1
Show newest version
/*
 *  Copyright 2017 Netflix, Inc.
 *
 *     Licensed under the Apache License, Version 2.0 (the "License");
 *     you may not use this file except in compliance with the License.
 *     You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 *     Unless required by applicable law or agreed to in writing, software
 *     distributed under the License is distributed on an "AS IS" BASIS,
 *     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *     See the License for the specific language governing permissions and
 *     limitations under the License.
 */
package com.netflix.metacat.testdata.provider

import com.netflix.metacat.common.QualifiedName
import com.netflix.metacat.common.dto.*
import com.netflix.metacat.common.json.MetacatJson
import com.netflix.metacat.common.json.MetacatJsonLocator

/**
 * Created by amajumdar on 5/15/15.
 */
class PigDataDtoProvider {
    private static final MetacatJson metacatJson = MetacatJsonLocator.INSTANCE
    /**
     * Returns a tableDto with the provided information.
     */
    def static TableDto getTable(String sourceName, String databaseName, String tableName, String owner, String uri){
        if( uri == null){
            uri = String.format("s3://wh/%s.db/%s", databaseName, tableName);
        }
        return new TableDto(
                name: QualifiedName.ofTable(sourceName, databaseName, tableName),
                serde: new StorageDto(
                        owner: owner,
                        inputFormat: 'org.apache.hadoop.mapred.TextInputFormat',
                        outputFormat: 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',
                        serializationLib: 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe',
                        serdeInfoParameters: [
                                'serialization.format': '1'
                        ],
                        uri: uri
                ),
                audit: new AuditDto(
                        createdBy: owner,
                        createdDate: new Date(),
                        lastModifiedBy: owner,
                        lastModifiedDate: new Date()
                ),
                fields: [
                        new FieldDto(
                                comment: 'added 1st - partition key',
                                name: 'field1',
                                pos: 0,
                                type: 'chararray',
                                partition_key: true
                        ),
                        new FieldDto(
                                comment: 'added 2st',
                                name: 'field2',
                                pos: 1,
                                type: 'chararray',
                                partition_key: false
                        ),
                        new FieldDto(
                                comment: 'added 3st - partition key',
                                name: 'field3',
                                pos: 2,
                                type: 'chararray',
                                partition_key: true
                        ),
                        new FieldDto(
                                comment: 'added 4st',
                                name: 'field4',
                                pos: 3,
                                type: 'chararray',
                                partition_key: false
                        )
                ],
                definitionMetadata: getDefinitionMetadata(owner)
        )
    }

    /**
     * Returns a list of tableDtos with the provided information. Table names and uris will be appended by the index.
     */
    def static List getTables(String sourceName, String databaseName, String tableName, String owner, String uri, int noOfTables){
        def result = [] as List
        for(int i=0;i getPartitions(String sourceName, String databaseName, String tableName, String name, String uri, int noOfPartitions){
        def result = [] as List
        for(int i=0;i