org.apache.jena.fuseki.kafka.lib.FKLib Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of jena-fmod-kafka Show documentation
Show all versions of jena-fmod-kafka Show documentation
Apache Jena Fuseki server Kafka connector
The newest version!
/*
* Copyright (c) Telicent Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.jena.fuseki.kafka.lib;
import java.nio.charset.StandardCharsets;
import java.time.Duration;
import java.util.List;
import java.util.Properties;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.function.BiConsumer;
import org.apache.jena.atlas.io.IO;
import org.apache.jena.atlas.lib.Lib;
import org.apache.jena.atlas.logging.FmtLog;
import org.apache.jena.kafka.FusekiKafka;
import org.apache.jena.kafka.common.DataState;
import org.apache.jena.kafka.common.DeserializerDump;
import org.apache.jena.riot.Lang;
import org.apache.jena.riot.RDFLanguages;
import org.apache.jena.riot.WebContent;
import org.apache.jena.riot.web.HttpNames;
import org.apache.jena.util.FileUtils;
import org.apache.kafka.clients.consumer.Consumer;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.Producer;
import org.apache.kafka.clients.producer.ProducerRecord;
import org.apache.kafka.clients.producer.RecordMetadata;
import org.apache.kafka.common.TopicPartition;
import org.apache.kafka.common.header.Header;
import org.apache.kafka.common.header.internals.RecordHeader;
import org.apache.kafka.common.serialization.StringDeserializer;
import org.apache.kafka.common.serialization.StringSerializer;
import org.slf4j.Logger;
public class FKLib {
private static Logger LOG = FusekiKafka.LOG;
// -- send file
public static String ctForFile(String fn) {
String ct = null;
if ( ct == null ) {
String ext = FileUtils.getFilenameExt(fn);
if ( Lib.equals("ru", ext) )
ct = WebContent.contentTypeSPARQLUpdate;
else if ( Lib.equals("rdfp", ext) )
ct = WebContent.contentTypePatch;
else {
Lang lang = RDFLanguages.filenameToLang(fn);
if ( lang != null )
ct = lang.getContentType().getContentTypeStr();
}
}
return ct;
}
/** Send files */
public static Producer producer(Properties props) {
StringSerializer serString1 = new StringSerializer();
StringSerializer serString2 = new StringSerializer();
Producer producer = new KafkaProducer<>(props, serString1, serString2);
return producer;
}
public static void sendFiles(Properties props, String topic, List files) {
try ( StringSerializer serString1 = new StringSerializer();
StringSerializer serString2 = new StringSerializer();
Producer producer = producer(props) ) {
sendFiles(producer, null, topic, files);
}
}
public static void sendString(Properties props, String topic, String contentType, String content) {
try ( StringSerializer serString1 = new StringSerializer();
StringSerializer serString2 = new StringSerializer();
Producer producer = producer(props) ) {
sendString(producer, null, topic, contentType, content);
}
}
private static Header header(String key, String value) {
return new RecordHeader(key, value.getBytes(StandardCharsets.UTF_8));
}
private static void sendFiles(Producer producer, Integer partition, String topic, List files) {
for ( String fn : files ) {
RecordMetadata res = sendFile(producer, partition, topic, fn);
if ( res == null )
FmtLog.error(LOG, "[%s] Error: sendFile %s", topic, fn);
else if ( ! res.hasOffset() )
FmtLog.info(LOG, "[%s] No offset", topic);
else
FmtLog.info(LOG, "[%s] Send: %s: Offset = %s", topic, fn, res.offset());
}
}
private static void sendString(Producer producer, Integer partition, String topic, String contentType, String content) {
List headers = ( contentType != null ) ? List.of(header(HttpNames.hContentType, contentType)) : List.of();
RecordMetadata res = sendBody(producer, partition, topic, headers, content);
if ( res == null )
FmtLog.error(LOG, "[%s] Error: sendString", topic);
else if ( ! res.hasOffset() )
FmtLog.info(LOG, "[%s] sendString: No offset", topic);
else
FmtLog.info(LOG, "[%s] sendString: Offset = %s", topic, res.offset());
}
private static RecordMetadata sendFile(Producer producer, Integer partition, String topic, String fn) {
String ct = ctForFile(fn);
String body = IO.readWholeFileAsUTF8(fn);
List headers = ( ct != null ) ? List.of(header(HttpNames.hContentType, ct)) : List.of();
RecordMetadata res = sendBody(producer, partition, topic, headers, body);
return res;
}
private static RecordMetadata sendBody(Producer producer, Integer partition, String topic, List headers, String body) {
try {
ProducerRecord pRec = new ProducerRecord<>(topic, partition, null, null, body, headers);
Future f = producer.send(pRec);
RecordMetadata res = f.get();
return res;
} catch (InterruptedException | ExecutionException e) {
e.printStackTrace();
}
return null;
}
public static void receive(DataState dataState, Properties cProps, String topic, BiConsumer, Long> handler) {
try ( StringDeserializer strDeser = new StringDeserializer();
DeserializerDump deSer = new DeserializerDump();
Consumer consumer = new KafkaConsumer<>(cProps, strDeser, deSer)){
TopicPartition topicPartition = new TopicPartition(topic, 0);
consumer.assign(List.of(topicPartition));
long initialOffset = dataState.getLastOffset();
if ( initialOffset < 0 )
consumer.seekToBeginning(List.of(topicPartition));
receiverLoop(consumer, dataState, handler);
}
}
// Receive until empty.
private static void receiverLoop(Consumer consumer,
DataState dataState, BiConsumer, Long> handler) {
for ( ;; ) {
boolean somethingReceived = receiver(consumer, dataState, handler);
if ( ! somethingReceived )
break;
}
}
// Once round the polling loop.
private static boolean receiver(Consumer consumer, DataState dataState,
BiConsumer, Long> handler) {
final long lastOffsetState = dataState.getLastOffset();
long newOffset = receiverStep(dataState.getLastOffset(), consumer, handler);
//System.out.println("Batch end");
if ( newOffset == lastOffsetState )
return false;
//FmtLog.info(LOG, "Offset: %d -> %d", lastOffsetState, newOffset);
dataState.setLastOffset(newOffset);
return true;
}
// Do the Kafka-poll/wait.
private static long receiverStep(final long lastOffsetState, Consumer consumer,
BiConsumer, Long> handler) {
ConsumerRecords cRec = consumer.poll(Duration.ofMillis(1000));
long lastOffset = lastOffsetState;
int count = cRec.count();
for ( ConsumerRecord rec : cRec ) {
handler.accept(rec, rec.offset());
long offset = rec.offset();
if ( offset != lastOffset+1 )
FmtLog.warn(FusekiKafka.LOG, "WARNING: Inconsistent offsets: offset=%d, lastOffset = %d\n", offset, lastOffset);
lastOffset = offset;
}
return lastOffset;
}
}