org.bytedeco.tensorrt.nvinfer.ICudaEngine Maven / Gradle / Ivy
// Targeted by JavaCPP version 1.5.7: DO NOT EDIT THIS FILE
package org.bytedeco.tensorrt.nvinfer;
import java.nio.*;
import org.bytedeco.javacpp.*;
import org.bytedeco.javacpp.annotation.*;
import static org.bytedeco.javacpp.presets.javacpp.*;
import org.bytedeco.cuda.cudart.*;
import static org.bytedeco.cuda.global.cudart.*;
import org.bytedeco.cuda.cublas.*;
import static org.bytedeco.cuda.global.cublas.*;
import org.bytedeco.cuda.cudnn.*;
import static org.bytedeco.cuda.global.cudnn.*;
import org.bytedeco.cuda.nvrtc.*;
import static org.bytedeco.cuda.global.nvrtc.*;
import static org.bytedeco.tensorrt.global.nvinfer.*;
/**
* \class ICudaEngine
*
* \brief An engine for executing inference on a built network, with functionally unsafe features.
*
* \warning Do not inherit from this class, as doing so will break forward-compatibility of the API and ABI.
* */
@Namespace("nvinfer1") @NoOffset @Properties(inherit = org.bytedeco.tensorrt.presets.nvinfer.class)
public class ICudaEngine extends INoCopy {
static { Loader.load(); }
/** Default native constructor. */
public ICudaEngine() { super((Pointer)null); allocate(); }
/** Native array allocator. Access with {@link Pointer#position(long)}. */
public ICudaEngine(long size) { super((Pointer)null); allocateArray(size); }
/** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */
public ICudaEngine(Pointer p) { super(p); }
private native void allocate();
private native void allocateArray(long size);
@Override public ICudaEngine position(long position) {
return (ICudaEngine)super.position(position);
}
@Override public ICudaEngine getPointer(long i) {
return new ICudaEngine((Pointer)this).offsetAddress(i);
}
/**
* \brief Get the number of binding indices.
*
* There are separate binding indices for each optimization profile.
* This method returns the total over all profiles.
* If the engine has been built for K profiles, the first getNbBindings() / K bindings are used by profile
* number 0, the following getNbBindings() / K bindings are used by profile number 1 etc.
*
* @see getBindingIndex();
* */
//!
//!
//!
//!
//!
//!
//!
public native @NoException(true) int getNbBindings();
/**
* \brief Retrieve the binding index for a named tensor.
*
* IExecutionContext::enqueue() and IExecutionContext::execute() require an array of buffers.
*
* Engine bindings map from tensor names to indices in this array.
* Binding indices are assigned at engine build time, and take values in the range [0 ... n-1] where n is the total
* number of inputs and outputs.
*
* To get the binding index of the name in an optimization profile with index k > 0,
* mangle the name by appending " [profile k]", as described for method getBindingName().
*
* @param name The tensor name.
* @return The binding index for the named tensor, or -1 if the name is not found.
*
* @see getNbBindings() getBindingName()
* */
//!
//!
//!
//!
//!
//!
public native @NoException(true) int getBindingIndex(String name);
public native @NoException(true) int getBindingIndex(@Cast("const char*") BytePointer name);
/**
* \brief Retrieve the name corresponding to a binding index.
*
* This is the reverse mapping to that provided by getBindingIndex().
*
* For optimization profiles with an index k > 0, the name is mangled by appending
* " [profile k]", with k written in decimal. For example, if the tensor in the
* INetworkDefinition had the name "foo", and bindingIndex refers to that tensor in the
* optimization profile with index 3, getBindingName returns "foo [profile 3]".
*
* @param bindingIndex The binding index.
* @return The name corresponding to the index, or nullptr if the index is out of range.
*
* @see getBindingIndex()
* */
//!
//!
//!
//!
public native @NoException(true) String getBindingName(int bindingIndex);
/**
* \brief Determine whether a binding is an input binding.
*
* @param bindingIndex The binding index.
* @return True if the index corresponds to an input binding and the index is in range.
*
* @see getBindingIndex()
* */
//!
//!
//!
//!
//!
//!
public native @Cast("bool") @NoException(true) boolean bindingIsInput(int bindingIndex);
/**
* \brief Get the dimensions of a binding.
*
* @param bindingIndex The binding index.
* @return The dimensions of the binding if the index is in range, otherwise Dims().
* Has -1 for any dimension that varies within the optimization profile.
*
* For example, suppose an INetworkDefinition has an input with shape [-1,-1]
* that becomes a binding b in the engine. If the associated optimization profile
* specifies that b has minimum dimensions as [6,9] and maximum dimensions [7,9],
* getBindingDimensions(b) returns [-1,9], despite the second dimension being
* dynamic in the INetworkDefinition.
*
* Because each optimization profile has separate bindings, the returned value can
* differ across profiles. Consider another binding b' for the same network input,
* but for another optimization profile. If that other profile specifies minimum
* dimensions [5,8] and maximum dimensions [5,9], getBindingDimensions(b') returns [5,-1].
*
* @see getBindingIndex()
* */
//!
//!
//!
//!
public native @ByVal @Cast("nvinfer1::Dims*") @NoException(true) Dims32 getBindingDimensions(int bindingIndex);
/**
* \brief Determine the required data type for a buffer from its binding index.
*
* @param bindingIndex The binding index.
* @return The type of the data in the buffer.
*
* @see getBindingIndex()
* */
//!
//!
//!
//!
public native @NoException(true) DataType getBindingDataType(int bindingIndex);
/**
* \brief Get the maximum batch size which can be used for inference.
*
* For an engine built from an INetworkDefinition without an implicit batch dimension, this will always return 1.
*
* @return The maximum batch size for this engine.
* */
//!
//!
//!
//!
public native @NoException(true) int getMaxBatchSize();
/**
* \brief Get the number of layers in the network.
*
* The number of layers in the network is not necessarily the number in the original network definition, as layers
* may be combined or eliminated as the engine is optimized. This value can be useful when building per-layer
* tables, such as when aggregating profiling data over a number of executions.
*
* @return The number of layers in the network.
* */
//!
//!
//!
//!
//!
public native @NoException(true) int getNbLayers();
/**
* \brief Serialize the network to a stream.
*
* @return A IHostMemory object that contains the serialized engine.
*
* The network may be deserialized with IRuntime::deserializeCudaEngine().
*
* @see IRuntime::deserializeCudaEngine()
* */
//!
//!
//!
//!
public native @NoException(true) IHostMemory serialize();
/**
* \brief Create an execution context.
*
* If the engine supports dynamic shapes, each execution context in concurrent use must use a separate optimization
* profile. The first execution context created will call setOptimizationProfile(0) implicitly. For other execution
* contexts, setOptimizationProfile() must be called with unique profile index before calling execute or enqueue.
* If an error recorder has been set for the engine, it will also be passed to the execution context.
*
* @see IExecutionContext.
* @see IExecutionContext::setOptimizationProfile()
* */
//!
//!
//!
//!
public native @NoException(true) IExecutionContext createExecutionContext();
/**
* \brief Destroy this object;
*
* @deprecated Deprecated interface will be removed in TensorRT 10.0.
*
* \warning Calling destroy on a managed pointer will result in a double-free error.
* */
//!
//!
//!
//!
//!
public native @Deprecated @NoException(true) void destroy();
/**
* \brief Get location of binding
*
* This lets you know whether the binding should be a pointer to device or host memory.
*
* @see ITensor::setLocation() ITensor::getLocation()
*
* @param bindingIndex The binding index.
* @return The location of the bound tensor with given index.
* */
//!
//!
public native @NoException(true) TensorLocation getLocation(int bindingIndex);
/** \brief create an execution context without any device memory allocated
*
* The memory for execution of this device context must be supplied by the application.
* */
//!
//!
//!
public native @NoException(true) IExecutionContext createExecutionContextWithoutDeviceMemory();
/**
* \brief Return the amount of device memory required by an execution context.
*
* @see IExecutionContext::setDeviceMemory()
* */
//!
//!
//!
public native @Cast("size_t") @NoException(true) long getDeviceMemorySize();
/**
* \brief Return true if an engine can be refit.
*
* @see nvinfer1::createInferRefitter()
* */
//!
//!
//!
//!
//!
public native @Cast("bool") @NoException(true) boolean isRefittable();
/**
* \brief Return the number of bytes per component of an element.
*
* The vector component size is returned if getBindingVectorizedDim() != -1.
*
* @param bindingIndex The binding Index.
*
* @see ICudaEngine::getBindingVectorizedDim()
* */
//!
//!
//!
//!
//!
public native @NoException(true) int getBindingBytesPerComponent(int bindingIndex);
/**
* \brief Return the number of components included in one element.
*
* The number of elements in the vectors is returned if getBindingVectorizedDim() != -1.
*
* @param bindingIndex The binding Index.
*
* @see ICudaEngine::getBindingVectorizedDim()
* */
//!
//!
//!
public native @NoException(true) int getBindingComponentsPerElement(int bindingIndex);
/**
* \brief Return the binding format.
*
* @param bindingIndex The binding Index.
* */
//!
//!
//!
//!
public native @NoException(true) TensorFormat getBindingFormat(int bindingIndex);
/**
* \brief Return the human readable description of the tensor format.
*
* The description includes the order, vectorization, data type, strides,
* and etc. Examples are shown as follows:
* Example 1: kCHW + FP32
* "Row major linear FP32 format"
* Example 2: kCHW2 + FP16
* "Two wide channel vectorized row major FP16 format"
* Example 3: kHWC8 + FP16 + Line Stride = 32
* "Channel major FP16 format where C % 8 == 0 and H Stride % 32 == 0"
*
* @param bindingIndex The binding Index.
* */
//!
//!
//!
//!
public native @NoException(true) String getBindingFormatDesc(int bindingIndex);
/**
* \brief Return the dimension index that the buffer is vectorized.
*
* Specifically -1 is returned if scalars per vector is 1.
*
* @param bindingIndex The binding Index.
* */
//!
//!
//!
//!
//!
public native @NoException(true) int getBindingVectorizedDim(int bindingIndex);
/**
* \brief Returns the name of the network associated with the engine.
*
* The name is set during network creation and is retrieved after
* building or deserialization.
*
* @see INetworkDefinition::setName(), INetworkDefinition::getName()
*
* @return A null-terminated C-style string representing the name of the network.
* */
//!
//!
//!
public native @NoException(true) String getName();
/**
* \brief Get the number of optimization profiles defined for this engine.
*
* @return Number of optimization profiles. It is always at least 1.
*
* @see IExecutionContext::setOptimizationProfile() */
//!
//!
//!
//!
//!
//!
//!
//!
//!
public native @NoException(true) int getNbOptimizationProfiles();
/**
* \brief Get the minimum / optimum / maximum dimensions for a particular binding under an optimization profile.
*
* @param bindingIndex The binding index, which must belong to the given profile,
* or be between 0 and bindingsPerProfile-1 as described below.
*
* @param profileIndex The profile index, which must be between 0 and getNbOptimizationProfiles()-1.
*
* @param select Whether to query the minimum, optimum, or maximum dimensions for this binding.
*
* @return The minimum / optimum / maximum dimensions for this binding in this profile.
* If the profileIndex or bindingIndex are invalid, return Dims with nbDims=-1.
*
* For backwards compatibility with earlier versions of TensorRT, if the bindingIndex
* does not belong to the current optimization profile, but is between 0 and bindingsPerProfile-1,
* where bindingsPerProfile = getNbBindings()/getNbOptimizationProfiles,
* then a corrected bindingIndex is used instead, computed by:
*
* profileIndex * bindingsPerProfile + bindingIndex % bindingsPerProfile
*
* Otherwise the bindingIndex is considered invalid.
* */
//!
//!
//!
//!
//!
//!
//!
//!
public native @ByVal @Cast("nvinfer1::Dims*") @NoException(true) Dims32 getProfileDimensions(int bindingIndex, int profileIndex, OptProfileSelector select);
public native @ByVal @Cast("nvinfer1::Dims*") @NoException(true) Dims32 getProfileDimensions(int bindingIndex, int profileIndex, @Cast("nvinfer1::OptProfileSelector") int select);
/**
* \brief Get minimum / optimum / maximum values for an input shape binding under an optimization profile.
*
* @param profileIndex The profile index (must be between 0 and getNbOptimizationProfiles()-1)
*
* @param inputIndex The input index (must be between 0 and getNbBindings() - 1)
*
* @param select Whether to query the minimum, optimum, or maximum shape values for this binding.
*
* @return If the binding is an input shape binding, return a pointer to an array that has
* the same number of elements as the corresponding tensor, i.e. 1 if dims.nbDims == 0, or dims.d[0]
* if dims.nbDims == 1, where dims = getBindingDimensions(inputIndex). The array contains
* the elementwise minimum / optimum / maximum values for this shape binding under the profile.
* If either of the indices is out of range, or if the binding is not an input shape binding, return
* nullptr.
*
* For backwards compatibility with earlier versions of TensorRT, a bindingIndex that does not belong
* to the profile is corrected as described for getProfileDimensions.
*
* @see ICudaEngine::getProfileDimensions
* */
//!
//!
//!
//!
//!
//!
//!
//!
//!
//!
//!
public native @Const @NoException(true) IntPointer getProfileShapeValues(int profileIndex, int inputIndex, OptProfileSelector select);
public native @Const @NoException(true) IntBuffer getProfileShapeValues(int profileIndex, int inputIndex, @Cast("nvinfer1::OptProfileSelector") int select);
/**
* \brief True if tensor is required as input for shape calculations or output from them.
*
* TensorRT evaluates a network in two phases:
*
* 1. Compute shape information required to determine memory allocation requirements
* and validate that runtime sizes make sense.
*
* 2. Process tensors on the device.
*
* Some tensors are required in phase 1. These tensors are called "shape tensors", and always
* have type Int32 and no more than one dimension. These tensors are not always shapes
* themselves, but might be used to calculate tensor shapes for phase 2.
*
* isShapeBinding(i) returns true if the tensor is a required input or an output computed in phase 1.
* isExecutionBinding(i) returns true if the tensor is a required input or an output computed in phase 2.
*
* For example, if a network uses an input tensor with binding i as an addend
* to an IElementWiseLayer that computes the "reshape dimensions" for IShuffleLayer,
* then isShapeBinding(i) == true.
*
* It's possible to have a tensor be required by both phases. For instance, a tensor
* can be used for the "reshape dimensions" and as the indices for an IGatherLayer
* collecting floating-point data.
*
* It's also possible to have a tensor be required by neither phase, but nonetheless
* shows up in the engine's inputs. For example, if an input tensor is used only
* as an input to IShapeLayer, only its shape matters and its values are irrelevant.
*
* @see isExecutionBinding()
* */
//!
//!
//!
//!
public native @Cast("bool") @NoException(true) boolean isShapeBinding(int bindingIndex);
/**
* \brief True if pointer to tensor data is required for execution phase, false if nullptr can be supplied.
*
* For example, if a network uses an input tensor with binding i ONLY as the "reshape dimensions"
* input of IShuffleLayer, then isExecutionBinding(i) is false, and a nullptr can be
* supplied for it when calling IExecutionContext::execute or IExecutionContext::enqueue.
*
* @see isShapeBinding()
* */
//!
//!
//!
//!
public native @Cast("bool") @NoException(true) boolean isExecutionBinding(int bindingIndex);
/**
* \brief Determine what execution capability this engine has.
*
* If the engine has EngineCapability::kSTANDARD, then all engine functionality is valid.
* If the engine has EngineCapability::kSAFETY, then only the functionality in safe engine is valid.
* If the engine has EngineCapability::kDLA_STANDALONE, then only serialize, destroy, and const-accessor functions are
* valid.
*
* @return The EngineCapability flag that the engine was built for.
* */
//!
//!
//!
//!
public native @NoException(true) EngineCapability getEngineCapability();
/** \brief Set the ErrorRecorder for this interface
*
* Assigns the ErrorRecorder to this interface. The ErrorRecorder will track all errors during execution.
* This function will call incRefCount of the registered ErrorRecorder at least once. Setting
* recorder to nullptr unregisters the recorder with the interface, resulting in a call to decRefCount if
* a recorder has been registered.
*
* If an error recorder is not set, messages will be sent to the global log stream.
*
* @param recorder The error recorder to register with this interface. */
//
/** @see getErrorRecorder()
/** */
//!
//!
//!
//!
//!
public native @NoException(true) void setErrorRecorder(IErrorRecorder recorder);
/**
* \brief Get the ErrorRecorder assigned to this interface.
*
* Retrieves the assigned error recorder object for the given class. A nullptr will be returned if
* an error handler has not been set.
*
* @return A pointer to the IErrorRecorder object that has been registered.
*
* @see setErrorRecorder()
* */
//!
//!
//!
//!
//!
//!
public native @NoException(true) IErrorRecorder getErrorRecorder();
/**
* \brief Query whether the engine was built with an implicit batch dimension.
*
* @return True if tensors have implicit batch dimension, false otherwise.
*
* This is an engine-wide property. Either all tensors in the engine
* have an implicit batch dimension or none of them do.
*
* hasImplicitBatchDimension() is true if and only if the INetworkDefinition
* from which this engine was built was created with createNetwork() or
* createNetworkV2() without NetworkDefinitionCreationFlag::kEXPLICIT_BATCH flag.
*
* @see createNetworkV2
* */
//!
//!
public native @Cast("bool") @NoException(true) boolean hasImplicitBatchDimension();
/** \brief return the tactic sources required by this engine
*
* @see IBuilderConfig::setTacticSources()
* */
//!
//!
//!
public native @Cast("nvinfer1::TacticSources") @NoException(true) int getTacticSources();
/** \brief Return the \ref ProfilingVerbosity the builder config was set to when the engine was built.
*
* @return the profiling verbosity the builder config was set to when the engine was built.
*
* @see IBuilderConfig::setProfilingVerbosity()
* */
//!
//!
//!
public native @NoException(true) ProfilingVerbosity getProfilingVerbosity();
/**
* \brief Create a new engine inspector which prints the layer information in an engine or an execution context.
*
* @see IEngineInspector.
* */
public native @NoException(true) IEngineInspector createEngineInspector();
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy