All Downloads are FREE. Search and download functionalities are using the official Maven repository.

com.yahoo.bullet.storm.DSLSpout Maven / Gradle / Ivy

Go to download

This is the implementation of Bullet - a real-time query engine - in Apache Storm.

The newest version!
/*
 *  Copyright 2018, Yahoo Inc.
 *  Licensed under the terms of the Apache License, Version 2.0.
 *  See the LICENSE file associated with the project for terms.
 */
package com.yahoo.bullet.storm;

import com.yahoo.bullet.dsl.BulletDSLConfig;
import com.yahoo.bullet.dsl.BulletDSLException;
import com.yahoo.bullet.dsl.connector.BulletConnector;
import com.yahoo.bullet.dsl.converter.BulletRecordConverter;
import com.yahoo.bullet.dsl.deserializer.BulletDeserializer;
import com.yahoo.bullet.dsl.deserializer.IdentityDeserializer;
import com.yahoo.bullet.record.BulletRecord;
import lombok.extern.slf4j.Slf4j;
import org.apache.storm.spout.SpoutOutputCollector;
import org.apache.storm.task.TopologyContext;
import org.apache.storm.topology.IRichSpout;
import org.apache.storm.topology.OutputFieldsDeclarer;
import org.apache.storm.tuple.Fields;
import org.apache.storm.tuple.Values;

import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.stream.Collectors;

@Slf4j
public class DSLSpout extends ConfigComponent implements IRichSpout {
    private static final long serialVersionUID = 9218045272408135524L;
    private static final Long DUMMY_ID = 42L;

    protected transient SpoutOutputCollector collector;
    protected T connector;
    private BulletRecordConverter converter;
    private BulletDeserializer deserializer;
    private boolean dslBoltEnable;

    /**
     * Creates a DSLSpout with a given {@link BulletStormConfig}.
     *
     * @param bulletStormConfig The non-null BulletStormConfig to use. It should contain the settings to initialize a BulletConnector and a BulletRecordConverter.
     */
    public DSLSpout(BulletStormConfig bulletStormConfig) {
        super(bulletStormConfig);
        BulletDSLConfig config = new BulletDSLConfig(bulletStormConfig);
        connector = getConnector(config);
        dslBoltEnable = config.getAs(BulletStormConfig.DSL_BOLT_ENABLE, Boolean.class);
        if (!dslBoltEnable) {
            boolean dslDeserializerEnable = config.getAs(BulletStormConfig.DSL_DESERIALIZER_ENABLE, Boolean.class);
            converter = BulletRecordConverter.from(config);
            deserializer = dslDeserializerEnable ? BulletDeserializer.from(config) : new IdentityDeserializer(config);
        }
    }

    @Override
    public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
        this.collector = collector;
        try {
            connector.initialize();
        } catch (BulletDSLException e) {
            throw new RuntimeException("Could not open DSLSpout.", e);
        }
    }

    @Override
    public void activate() {
        log.info("DSLSpout activated");
    }

    @Override
    public void deactivate() {
        log.info("DSLSpout deactivated");
    }

    @Override
    public void nextTuple() {
        List objects = readObjects();
        if (objects.isEmpty()) {
            return;
        }
        if (dslBoltEnable) {
            collector.emit(new Values(objects, System.currentTimeMillis()), DUMMY_ID);
            return;
        }
        objects.forEach(this::convertAndEmit);
    }

    private List readObjects() {
        try {
            return connector.read().stream().filter(Objects::nonNull).collect(Collectors.toList());
        } catch (BulletDSLException e) {
            log.error("Could not read from BulletConnector.", e);
        }
        return Collections.emptyList();
    }

    private void convertAndEmit(Object object) {
        BulletRecord record;
        try {
            record = converter.convert(deserializer.deserialize(object));
        } catch (BulletDSLException e) {
            log.error("Could not convert object.", e);
            return;
        }
        collector.emit(new Values(record, System.currentTimeMillis()), DUMMY_ID);
    }

    @Override
    public void declareOutputFields(OutputFieldsDeclarer declarer) {
        declarer.declare(new Fields(TopologyConstants.RECORD_FIELD, TopologyConstants.RECORD_TIMESTAMP_FIELD));
    }

    @Override
    public void ack(Object id) {
    }

    @Override
    public void fail(Object id) {
    }

    @Override
    public void close() {
        try {
            connector.close();
        } catch (Exception e) {
            log.error("Could not close BulletConnector.", e);
        }
        log.info("DSLSpout closed");
    }

    /**
     * Creates the {@link BulletConnector} of the given type from the config.
     *
     * @param config The {@link BulletDSLConfig} with the relevant settings.
     * @return An instance of the relevant {@link BulletConnector}.
     */
    @SuppressWarnings("unchecked")
    protected T getConnector(BulletDSLConfig config) {
        return (T) BulletConnector.from(config);
    }
}
    

    

    

            

    

            



    
    






    
© 2015 - 2024 Weber Informatics LLC | Privacy Policy