org.apache.jena.fuseki.kafka.lib.FKLib Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of jena-fuseki-kafka-module Show documentation
Apache Jena Fuseki module for Kafka connector
The newest version!
/*
 *  Copyright (c) Telicent Ltd.
 *
 *  Licensed under the Apache License, Version 2.0 (the "License");
 *  you may not use this file except in compliance with the License.
 *  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing, software
 *  distributed under the License is distributed on an "AS IS" BASIS,
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  See the License for the specific language governing permissions and
 *  limitations under the License.
 */

package org.apache.jena.fuseki.kafka.lib;

import java.nio.charset.StandardCharsets;
import java.time.Duration;
import java.util.List;
import java.util.Properties;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.function.BiConsumer;

import org.apache.jena.atlas.io.IO;
import org.apache.jena.atlas.lib.Lib;
import org.apache.jena.atlas.logging.FmtLog;
import org.apache.jena.kafka.FusekiKafka;
import org.apache.jena.kafka.common.DataState;
import org.apache.jena.kafka.common.DeserializerDump;
import org.apache.jena.riot.Lang;
import org.apache.jena.riot.RDFLanguages;
import org.apache.jena.riot.WebContent;
import org.apache.jena.riot.web.HttpNames;
import org.apache.jena.util.FileUtils;
import org.apache.kafka.clients.consumer.Consumer;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.Producer;
import org.apache.kafka.clients.producer.ProducerRecord;
import org.apache.kafka.clients.producer.RecordMetadata;
import org.apache.kafka.common.TopicPartition;
import org.apache.kafka.common.header.Header;
import org.apache.kafka.common.header.internals.RecordHeader;
import org.apache.kafka.common.serialization.StringDeserializer;
import org.apache.kafka.common.serialization.StringSerializer;
import org.slf4j.Logger;

public class FKLib {

    private static Logger LOG = FusekiKafka.LOG;

    // -- send file
    public static String  ctForFile(String fn) {
        String ct = null;
        if ( ct == null ) {
            String ext = FileUtils.getFilenameExt(fn);
            if ( Lib.equals("ru", ext) )
                ct = WebContent.contentTypeSPARQLUpdate;
            else if ( Lib.equals("rdfp", ext) )
                ct = WebContent.contentTypePatch;
            else {
                Lang lang = RDFLanguages.filenameToLang(fn);
                if ( lang != null )
                    ct = lang.getContentType().getContentTypeStr();
            }
        }
        return ct;
    }

    /** Send files */
    public static Producer producer(Properties props) {
        StringSerializer serString1 = new StringSerializer();
        StringSerializer serString2 = new StringSerializer();
        Producer producer = new KafkaProducer<>(props, serString1, serString2);
        return producer;
    }

    public static void sendFiles(Properties props, String topic, List files) {
        try ( StringSerializer serString1 = new StringSerializer();
              StringSerializer serString2 = new StringSerializer();
              Producer producer = producer(props) ) {
            sendFiles(producer, null, topic, files);
        }
    }

    public static void sendString(Properties props, String topic, String contentType, String content) {
        try ( StringSerializer serString1 = new StringSerializer();
              StringSerializer serString2 = new StringSerializer();
              Producer producer = producer(props) ) {
            sendString(producer, null, topic, contentType, content);
        }
    }

    private static Header header(String key, String value) {
        return new RecordHeader(key, value.getBytes(StandardCharsets.UTF_8));
    }

    private static void sendFiles(Producer producer, Integer partition, String topic, List files) {
        for ( String fn : files ) {
            RecordMetadata res = sendFile(producer, partition, topic, fn);
            if ( res == null )
                FmtLog.error(LOG, "[%s] Error: sendFile %s", topic, fn);
            else if ( ! res.hasOffset() )
                FmtLog.info(LOG, "[%s] No offset", topic);
            else
                FmtLog.info(LOG, "[%s] Send: %s: Offset = %s", topic, fn, res.offset());
        }
    }

    private static void sendString(Producer producer, Integer partition, String topic, String contentType, String content) {
        List headers = ( contentType != null ) ? List.of(header(HttpNames.hContentType, contentType)) : List.of();
        RecordMetadata res = sendBody(producer, partition, topic, headers, content);
        if ( res == null )
            FmtLog.error(LOG, "[%s] Error: sendString", topic);
        else if ( ! res.hasOffset() )
            FmtLog.info(LOG, "[%s] sendString: No offset", topic);
        else
            FmtLog.info(LOG, "[%s] sendString: Offset = %s", topic, res.offset());
    }

    private static RecordMetadata sendFile(Producer producer, Integer partition, String topic, String fn) {
        String ct = ctForFile(fn);
        String body = IO.readWholeFileAsUTF8(fn);
        List headers = ( ct != null ) ? List.of(header(HttpNames.hContentType, ct)) : List.of();
        RecordMetadata res = sendBody(producer, partition, topic, headers, body);
        return res;
    }

    private static RecordMetadata sendBody(Producer producer, Integer partition, String topic, List headers, String body) {
        try {
            ProducerRecord pRec = new ProducerRecord<>(topic, partition, null, null, body, headers);
            Future f = producer.send(pRec);
            RecordMetadata res = f.get();
            return res;
        } catch (InterruptedException | ExecutionException e) {
            e.printStackTrace();
        }
        return null;
    }

    public static void receive(DataState dataState, Properties cProps, String topic, BiConsumer, Long> handler) {
        try ( StringDeserializer strDeser = new StringDeserializer();
              DeserializerDump deSer = new DeserializerDump();
              Consumer consumer = new KafkaConsumer<>(cProps, strDeser, deSer)){
            TopicPartition topicPartition = new TopicPartition(topic, 0);
            consumer.assign(List.of(topicPartition));
            long initialOffset = dataState.getLastOffset();
            if ( initialOffset < 0 )
                consumer.seekToBeginning(List.of(topicPartition));
            receiverLoop(consumer, dataState, handler);
        }
    }

    // Receive until empty.
    private static void receiverLoop(Consumer consumer,
                                DataState dataState, BiConsumer, Long> handler) {
        for ( ;; ) {
            boolean somethingReceived = receiver(consumer, dataState, handler);
            if ( ! somethingReceived )
                break;
        }
    }

    // Once round the polling loop.

    private static boolean receiver(Consumer consumer, DataState dataState,
                                    BiConsumer, Long> handler) {
        final long lastOffsetState = dataState.getLastOffset();
        long newOffset = receiverStep(dataState.getLastOffset(), consumer, handler);
        //System.out.println("Batch end");
        if ( newOffset == lastOffsetState )
            return false;
        //FmtLog.info(LOG, "Offset: %d -> %d", lastOffsetState, newOffset);
        dataState.setLastOffset(newOffset);
        return true;
    }

    // Do the Kafka-poll/wait.
    private static long receiverStep(final long lastOffsetState, Consumer consumer,
                                     BiConsumer, Long> handler) {
        ConsumerRecords cRec = consumer.poll(Duration.ofMillis(1000));
        long lastOffset = lastOffsetState;
        int count = cRec.count();
        for ( ConsumerRecord rec : cRec ) {
            handler.accept(rec, rec.offset());
            long offset = rec.offset();
            if ( offset != lastOffset+1 )
                FmtLog.warn(FusekiKafka.LOG, "WARNING: Inconsistent offsets: offset=%d, lastOffset = %d\n", offset, lastOffset);
            lastOffset = offset;
        }
        return lastOffset;
    }
}