dev.langchain4j.model.Tokenizer Maven / Gradle / Ivy

Go to download
package dev.langchain4j.model;

import dev.langchain4j.agent.tool.ToolExecutionRequest;
import dev.langchain4j.agent.tool.ToolSpecification;
import dev.langchain4j.data.message.ChatMessage;

import java.util.ArrayList;
import java.util.List;

import static dev.langchain4j.agent.tool.ToolSpecifications.toolSpecificationsFrom;
import static java.util.Collections.singletonList;

/**
 * Represents an interface for estimating the count of tokens in various text types such as a text, prompt, text segment, etc.
 * This can be useful when it's necessary to know in advance the cost of processing a specified text by the LLM.
 */
public interface Tokenizer {

    /**
     * Estimates the count of tokens in the given text.
     * @param text the text.
     * @return the estimated count of tokens.
     */
    int estimateTokenCountInText(String text);

    /**
     * Estimates the count of tokens in the given message.
     * @param message the message.
     * @return the estimated count of tokens.
     */
    int estimateTokenCountInMessage(ChatMessage message);

    /**
     * Estimates the count of tokens in the given messages.
     * @param messages the messages.
     * @return the estimated count of tokens.
     */
    int estimateTokenCountInMessages(Iterable messages);

    /**
     * Estimates the count of tokens in {@code Tool} annotations of the given object.
     * @param objectWithTools the object.
     * @return the estimated count of tokens.
     */
    default int estimateTokenCountInTools(Object objectWithTools) {
        return estimateTokenCountInTools(singletonList(objectWithTools));
    }

    /**
     * Estimates the count of tokens in {@code Tool} annotations of the given objects.
     * @param objectsWithTools the objects.
     * @return the estimated count of tokens.
     */
    default int estimateTokenCountInTools(Iterable