org.datavec.camel.component.DataVecProducer Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of datavec-camel Show documentation
The newest version!
/*******************************************************************************
 * Copyright (c) 2015-2018 Skymind, Inc.
 *
 * This program and the accompanying materials are made available under the
 * terms of the Apache License, Version 2.0 which is available at
 * https://www.apache.org/licenses/LICENSE-2.0.
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 * License for the specific language governing permissions and limitations
 * under the License.
 *
 * SPDX-License-Identifier: Apache-2.0
 ******************************************************************************/

package org.datavec.camel.component;

import org.apache.camel.Exchange;
import org.apache.camel.impl.DefaultProducer;
import org.datavec.api.conf.Configuration;
import org.datavec.api.formats.input.InputFormat;
import org.datavec.api.io.WritableConverter;
import org.datavec.api.io.converters.SelfWritableConverter;
import org.datavec.api.records.reader.RecordReader;
import org.datavec.api.split.InputSplit;
import org.datavec.api.writable.Writable;

import java.util.ArrayList;
import java.util.Collection;


/**
 * The DataVec producer.
 * Converts input records in to their final form
 * based on the input split generated from
 * the given exchange.
 *
 * @author Adam Gibson
 */
public class DataVecProducer extends DefaultProducer {
    private Class inputFormatClazz;
    private Class marshallerClazz;
    private InputFormat inputFormat;
    private Configuration configuration;
    private WritableConverter writableConverter;
    private DataVecMarshaller marshaller;


    public DataVecProducer(DataVecEndpoint endpoint) {
        super(endpoint);
        if (endpoint.getInputFormat() != null) {
            try {
                inputFormatClazz = (Class) Class.forName(endpoint.getInputFormat());
                inputFormat = inputFormatClazz.newInstance();
                marshallerClazz = (Class) Class.forName(endpoint.getInputMarshaller());
                Class converterClazz =
                                (Class) Class.forName(endpoint.getWritableConverter());
                writableConverter = converterClazz.newInstance();
                marshaller = marshallerClazz.newInstance();
                configuration = new Configuration();
                for (String prop : endpoint.getConsumerProperties().keySet())
                    configuration.set(prop, endpoint.getConsumerProperties().get(prop).toString());

            } catch (Exception e) {
                throw new RuntimeException(e);
            }
        }

    }


    //stub, still need to fill out more of the end point yet..endpoint will likely be initialized with a split
    protected InputSplit inputFromExchange(Exchange exchange) {
        return marshaller.getSplit(exchange);
    }


    @Override
    public void process(Exchange exchange) throws Exception {
        InputSplit split = inputFromExchange(exchange);
        RecordReader reader = inputFormat.createReader(split, configuration);
        Collection> newRecord = new ArrayList<>();
        if (!(writableConverter instanceof SelfWritableConverter)) {
            newRecord = new ArrayList<>();
            while (reader.hasNext()) {
                Collection newRecordAdd = new ArrayList<>();
                // create a message body
                Collection next = reader.next();
                for (Writable writable : next) {
                    newRecordAdd.add(writableConverter.convert(writable));
                }


                newRecord.add(newRecordAdd);
            }
        } else {
            while (reader.hasNext()) {
                // create a message body
                Collection next = reader.next();
                newRecord.add(next);
            }
        }


        exchange.getIn().setBody(newRecord);
        exchange.getOut().setBody(newRecord);
    }
}