com.github.castorm.kafka.connect.http.HttpSourceTask Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of kafka-connect-http-plugin Show documentation
Set of Kafka Connect connectors that enable Kafka integration with external systems via HTTP
There is a newer version: 0.5.0
Show newest version
package com.github.castorm.kafka.connect.http;

/*-
 * #%L
 * kafka-connect-http-plugin
 * %%
 * Copyright (C) 2020 CastorM
 * %%
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 * 
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 * 
 * You should have received a copy of the GNU General Public
 * License along with this program.  If not, see
 * .
 * #L%
 */

import com.github.castorm.kafka.connect.http.client.spi.HttpClient;
import com.github.castorm.kafka.connect.http.model.HttpRequest;
import com.github.castorm.kafka.connect.http.model.HttpResponse;
import com.github.castorm.kafka.connect.http.poll.spi.PollInterceptor;
import com.github.castorm.kafka.connect.http.record.spi.SourceRecordMapper;
import com.github.castorm.kafka.connect.http.request.spi.HttpRequestFactory;
import com.github.castorm.kafka.connect.http.response.spi.HttpResponseParser;
import org.apache.kafka.connect.errors.RetriableException;
import org.apache.kafka.connect.source.SourceRecord;
import org.apache.kafka.connect.source.SourceTask;

import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.function.Function;

import static com.github.castorm.kafka.connect.common.VersionUtils.getVersion;
import static java.util.Collections.emptyMap;
import static java.util.stream.Collectors.toList;

public class HttpSourceTask extends SourceTask {

    private final Function, HttpSourceConnectorConfig> configFactory;

    private PollInterceptor pollInterceptor;

    private HttpRequestFactory requestFactory;

    private HttpClient requestExecutor;

    private HttpResponseParser responseParser;

    private SourceRecordMapper recordMapper;

    HttpSourceTask(Function, HttpSourceConnectorConfig> configFactory) {
        this.configFactory = configFactory;
    }

    public HttpSourceTask() {
        this(HttpSourceConnectorConfig::new);
    }

    @Override
    public void start(Map settings) {

        HttpSourceConnectorConfig config = configFactory.apply(settings);

        pollInterceptor = config.getPollInterceptor();
        requestFactory = config.getRequestFactory();
        requestFactory.setOffset(resolveStartingOffset(config.getInitialOffset()));
        requestExecutor = config.getClient();
        responseParser = config.getResponseParser();
        recordMapper = config.getRecordMapper();
    }

    private Map resolveStartingOffset(Map initialOffset) {

        Map latestOffset = context.offsetStorageReader().offset(emptyMap());

        return !latestOffset.isEmpty() ? latestOffset : initialOffset;
    }

    @Override
    public List poll() throws InterruptedException {

        pollInterceptor.beforePoll();

        HttpRequest request = requestFactory.createRequest();

        HttpResponse response = execute(request);

        List records = responseParser.parse(response).stream()
                .map(recordMapper::map)
                .collect(toList());

        return pollInterceptor.afterPoll(records);
    }

    private HttpResponse execute(HttpRequest request) {
        try {
            return requestExecutor.execute(request);
        } catch (IOException e) {
            throw new RetriableException(e);
        }
    }

    @Override
    public void commitRecord(SourceRecord record) {

        requestFactory.setOffset(record.sourceOffset());
    }

    @Override
    public void stop() {
    }

    @Override
    public String version() {
        return getVersion(getClass());
    }
}