fi.evolver.ai.spring.provider.anthropic.AnthropicService Maven / Gradle / Ivy

Go to download
package fi.evolver.ai.spring.provider.anthropic;

import java.io.IOException;
import java.io.UncheckedIOException;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpRequest.BodyPublishers;
import java.net.http.HttpRequest.Builder;
import java.net.http.HttpResponse;
import java.net.http.HttpResponse.BodyHandlers;
import java.time.Duration;
import java.util.Set;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.http.HttpHeaders;
import org.springframework.http.MediaType;
import org.springframework.stereotype.Component;

import com.fasterxml.jackson.core.JsonProcessingException;

import fi.evolver.ai.spring.ApiResponseException;
import fi.evolver.ai.spring.Model;
import fi.evolver.ai.spring.Tokenizer;
import fi.evolver.ai.spring.chat.ChatApi;
import fi.evolver.ai.spring.chat.ChatResponse;
import fi.evolver.ai.spring.chat.prompt.ChatPrompt;
import fi.evolver.ai.spring.config.ApiConfigurationService;
import fi.evolver.ai.spring.config.ApiEndpointParameters;
import fi.evolver.ai.spring.provider.ConditionalOnProviderConfigured;
import fi.evolver.ai.spring.provider.anthropic.response.AChatStreamingResponse;
import fi.evolver.ai.spring.provider.anthropic.response.AMessage;
import fi.evolver.ai.spring.provider.anthropic.response.ARateLimitHeaders;
import fi.evolver.ai.spring.provider.openai.OpenAiRequestParameters;
import fi.evolver.ai.spring.util.BodyHandlerWrapper;
import fi.evolver.ai.spring.util.Json;
import fi.evolver.ai.spring.util.ResponseInfoCallBack;
import fi.evolver.ai.spring.util.SseUtils;
import fi.evolver.basics.spring.http.LoggingHttpClient;
import fi.evolver.basics.spring.http.SseSubscriber;
import fi.evolver.basics.spring.http.SseSubscriber.SseEvent;
import fi.evolver.basics.spring.http.SseSubscriber.SseEventConsumer;
import fi.evolver.basics.spring.log.MessageLogService;

@Component
@ConditionalOnProviderConfigured(AnthropicService.class)
public class AnthropicService implements ChatApi {
	private static final Logger LOG = LoggerFactory.getLogger(AnthropicService.class);

	 // Note: using wrong tokenizer as the actual algorithm has not been released
	public static final Model CLAUDE_3_HAIKU = new Model<>("claude-3-haiku", 200_000, Tokenizer.CL100K_BASE);
	public static final Model CLAUDE_3_SONNET = new Model<>("claude-3-sonnet", 200_000, Tokenizer.CL100K_BASE);
	public static final Model CLAUDE_3_OPUS = new Model<>("claude-3-opus", 200_000, Tokenizer.CL100K_BASE);

	static final Set FINISH_REASONS_OK = Set.of("end_turn", "stop_sequence", "tool_use");


	private final ApiConfigurationService apiConfigurationService;
	private final LoggingHttpClient httpClient;


	public AnthropicService(
			ApiConfigurationService apiConfigurationService,
			MessageLogService messageLogService,
			@Value("${evolver.anthropic-service.connection.timeout.seconds:5}") int connectionTimeoutSeconds) {

		this.apiConfigurationService = apiConfigurationService;
		this.httpClient = new LoggingHttpClient(
				messageLogService,
				HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(connectionTimeoutSeconds)).build());
	}


	@Override
	public ChatResponse send(ChatPrompt prompt) {
		String body = AnthropicRequestGenerator.generate(prompt);

		ApiEndpointParameters endpointParameters = apiConfigurationService.getEndpointParameters(
				AnthropicService.class,
				prompt.getStringProperty(ChatApi.PROVIDER),
				ChatApi.class,
				prompt.model());

		Builder builder = HttpRequest.newBuilder(endpointParameters.prepareUri())
				.header(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_VALUE)
				.timeout(prompt.timeout().orElse(ChatApi.DEFAULT_TIMEOUT))
				.POST(BodyPublishers.ofString(body));
		endpointParameters.headers().forEach(builder::header);
		HttpRequest request = builder.build();

		// Anthropic does not support streaming function calls for now.
		if (prompt.getBooleanProperty(OpenAiRequestParameters.STREAM).orElse(false))
			return makeStreamingRequest(request, prompt);
		else
			return makeNonStreamingRequest(request, prompt);
	}


	private ChatResponse makeStreamingRequest(HttpRequest request, ChatPrompt prompt) {
		AnthropicStreamingChatResponse response = new AnthropicStreamingChatResponse(prompt);

		HttpResponse.BodyHandler bodyHandler = SseSubscriber.createBodyHandler(new StreamingCompletionsEventConsumer(response));
		ResponseInfoCallBack headerParser = responseInfo -> {
			java.net.http.HttpHeaders headers = responseInfo.headers();
			ARateLimitHeaders rateLimitHeaders = ARateLimitHeaders.fromHttpHeaders(headers);
			response.addRateLimitHeaders(rateLimitHeaders);
		};
		HttpResponse.BodyHandler wrappedBodyHandler = BodyHandlerWrapper.wrapBodyHandler(bodyHandler, headerParser);


		httpClient.sendAsync(
				request,
				wrappedBodyHandler,
				createLogParameters("ChatRequest")
		).exceptionally(e -> {
			response.handleError(e);
			return null;
		});

		return response;
	}


	private ChatResponse makeNonStreamingRequest(HttpRequest request, ChatPrompt prompt) {
		try {
			HttpResponse httpResponse = httpClient.send(request, BodyHandlers.ofString(), createLogParameters("ChatRequest"));
			if (httpResponse.statusCode() != 200)
				throw new ApiResponseException("Failed non-streaming chat request: HTTP %s", httpResponse.statusCode());

			AMessage chatResponse = Json.OBJECT_MAPPER.readValue(httpResponse.body(), AMessage.class);
			ARateLimitHeaders rateLimitHeaders = ARateLimitHeaders.fromHttpHeaders(httpResponse.headers());

			return new AnthropicChatResponse(prompt, chatResponse, rateLimitHeaders);
		}
		catch (IOException | InterruptedException e) {
			throw new ApiResponseException(e, "Failed non-streaming chat request");
		}
	}


	private static class StreamingCompletionsEventConsumer implements SseEventConsumer {
		private final AnthropicStreamingChatResponse response;

		public StreamingCompletionsEventConsumer(AnthropicStreamingChatResponse response) {
			this.response = response;
		}

		@Override
		public void onEvent(SseEvent event) {
			if (!event.data().startsWith("{")) {
				LOG.warn("Unknown chunk: {}", event.data());
				return;
			}

			try {
				AChatStreamingResponse result = Json.OBJECT_MAPPER.readValue(event.data(), AChatStreamingResponse.class);
				response.addResult(result);
			}
			catch (JsonProcessingException e) {
				LOG.warn("Bad SSE event", e);
			}
		}


		@Override
		public void onError(Throwable throwable) {
			response.handleError(throwable);
		}

		@Override
		public void onComplete() {
			response.handleStreamEnd();
		}

	}


	@Override
	public ChatResponse parseChatResponse(String rawResponse) {
		try {
			ChatPrompt prompt = ChatPrompt.builder(CLAUDE_3_HAIKU).build();
			if (SseUtils.isStreamResponse(rawResponse)) {
				AnthropicStreamingChatResponse chatResponse = new AnthropicStreamingChatResponse(prompt);

				SseUtils.handleStreamContent(rawResponse, res -> chatResponse.addResult(res), AChatStreamingResponse.class);
				chatResponse.handleStreamEnd();

				return chatResponse;
			}
			else {
				return new AnthropicChatResponse(
						prompt,
						Json.OBJECT_MAPPER.readValue(rawResponse, AMessage.class),
						null);
			}
		}
		catch (JsonProcessingException e) {
			throw new UncheckedIOException(e);
		}
	}

}