test.resources.csvToDocDB.pig Maven / Gradle / Ivy

Go to download

Show more of this group Show more artifacts with this name
Show all versions of azure-documentdb-hadoop Show documentation

Hadoop Connector for Microsoft Azure DocumentDB

The newest version!

-- You can  run this pig script as : 
-- pig -x local /home/cloudera/sdk/hadoop/DocumentDBHadoopConnector/src/test/resources/load_docdb_dump.pig

-- Register our Jar package.
REGISTER /home/cloudera/sdk/Java/DocumentDB/target/DocumentDB-1.1.0-jar-with-dependencies.jar
REGISTER /home/cloudera/sdk/hadoop/DocumentDBHadoopConnector/target/DocumentDBHadoopConnector-1.1.0.jar

%declare  RoutesCSV '/home/cloudera/sdk/hadoop/DocumentDBHadoopConnector/src/test/resources/routes.csv'
%declare  StopsCSV '/home/cloudera/sdk/hadoop/DocumentDBHadoopConnector/src/test/resources/stops.csv'

-- here id is route_id
routes = LOAD '$RoutesCSV'  
         USING PigStorage(',') 
         AS (agency_id:chararray, id:chararray,
             route_short_name:chararray,route_long_name:chararray,
             route_type:chararray, route_desc:chararray,
             route_url:chararray,route_color:chararray, route_text_color:chararray); 

STORE routes INTO 'https://ming2.documents.azure.com:443/'
               USING com.microsoft.azure.documentdb.hadoop.pig.DocumentDBStorage(
             'R17FHH6bn0Knz/yhb4jMBRkcQMe1tVeiQ6e0SlQnDjtXRJG9OufFEt7sgaC+PaCMeAmcYY9r+gphaJuc1jdKTg==', 
             'DemoDatabase', 'Routes');

-- here id is trip_id
stops = LOAD '$StopsCSV'  
         USING PigStorage(',') 
         AS (route_id:chararray,
             id:chararray,service_id:chararray,
             trip_headsign, direction_id:chararray,
             shape_id:chararray);
             
STORE stops INTO 'https://ming2.documents.azure.com:443/'
               USING com.microsoft.azure.documentdb.hadoop.pig.DocumentDBStorage(
             'R17FHH6bn0Knz/yhb4jMBRkcQMe1tVeiQ6e0SlQnDjtXRJG9OufFEt7sgaC+PaCMeAmcYY9r+gphaJuc1jdKTg==', 
             'DemoDatabase', 'Stops');