#include <onnx_multi_batch_infer.h>

类 apollo::perception::inference::MultiBatchInference 继承关系图:

apollo::perception::inference::MultiBatchInference 的协作图:

Public 成员函数
	MultiBatchInference ()

virtual	~MultiBatchInference ()

bool	OnnxToTRTModel (const std::string &model_file)
	Convert ONNX to TensorRT model

void	SetOnnxOptimizationProfile (nvinfer1::IOptimizationProfile *profile, const BlobMap &input_shapes)

void	inference ()

bool	LoadCache (const std::string &path)

bool	Init (const std::map< std::string, std::vector< int > > &shapes) override

void	Infer () override

void	init_blob (std::vector< std::string > *names)

base::BlobPtr< float >	get_blob (const std::string &name) override

void	FileInit ()

void	SetStream (cudaStream_t stream)

Public 成员函数继承自 apollo::perception::inference::Inference
	Inference ()=default

virtual	~Inference ()=default

void	set_max_batch_size (const int &batch_size)

void	set_gpu_id (const int &gpu_id)

void	set_model_info (const std::string &proto_file, const std::vector< std::string > &net_input_names, const std::vector< std::string > &net_output_names)

额外继承的成员函数
Protected 属性继承自 apollo::perception::inference::Inference
int	max_batch_size_ = 1

int	gpu_id_ = 0

std::string	proto_file_ = ""

std::vector< std::string >	net_output_names_

std::vector< std::string >	net_input_names_

详细描述

在文件 onnx_multi_batch_infer.h 第 41 行定义.

构造及析构函数说明

◆ MultiBatchInference()

apollo::perception::inference::MultiBatchInference::MultiBatchInference ( )

inline

在文件 onnx_multi_batch_infer.h 第 43 行定义.

43{}

◆ ~MultiBatchInference()

apollo::perception::inference::MultiBatchInference::~MultiBatchInference ( )

virtual

在文件 onnx_multi_batch_infer.cc 第 92 行定义.

                                          {
    if (network_) {
        network_->destroy();
    }
    if (builder_) {
        builder_->destroy();
    }
    if (context_) {
        context_->destroy();
    }
    if (engine_) {
        engine_->destroy();
    }
}

成员函数说明

◆ FileInit()

void apollo::perception::inference::MultiBatchInference::FileInit ( )

在文件 onnx_multi_batch_infer.cc 第 116 行定义.

                                   {
    model_file_ = proto_file_;
    output_names_ = net_output_names_;
    input_names_ = net_input_names_;
}

◆ get_blob()

base::BlobPtr< float > apollo::perception::inference::MultiBatchInference::get_blob ( const std::string & name )

overridevirtual

实现了 apollo::perception::inference::Inference.

在文件 onnx_multi_batch_infer.cc 第 84 行定义.

                                                                      {
    auto iter = blobs_.find(name);
    if (iter == blobs_.end()) {
        return nullptr;
    }
    return iter->second;
}

◆ Infer()

void apollo::perception::inference::MultiBatchInference::Infer ( )

overridevirtual

实现了 apollo::perception::inference::Inference.

在文件 onnx_multi_batch_infer.cc 第 295 行定义.

                                {
    BASE_GPU_CHECK(cudaSetDevice(gpu_id_));
    BASE_GPU_CHECK(cudaStreamSynchronize(stream_));
    CHECK(input_names_.size() > 0 && blobs_.size() > 0);
    auto input_shape = blobs_[input_names_[0]]->shape();
    kBatchSize = input_shape[0];
    if (kBatchSize == 0) {
        AINFO << "batch size is zero.";
        return;
    }
 
    int batch_size = max_batch_size_;
 
    // binding input blob
    for (auto name : input_names_) {
        auto blob = get_blob(name);
        if (blob != nullptr) {
            int32_t index = engine_->getBindingIndex(name.c_str());
            // careful about this line
            // batch_size = std::min<int>(batch_size, blob->shape(0));
            batch_size = blob->shape(0);
            // set buffer
            buffers_[index] = const_cast<float*>(blob->gpu_data());
 
            // set Dim
            nvinfer1::Dims dims = network_->getInput(index)->getDimensions();
            dims.d[0] = batch_size;
            context_->setOptimizationProfile(0);
            context_->setBindingDimensions(index, dims);
        }
    }
 
    // bingding output blob
    for (auto name : output_names_) {
        auto blob = get_blob(name);
        if (blob != nullptr) {
            blob->mutable_gpu_data();
            int32_t index = engine_->getBindingIndex(name.c_str());
            buffers_[index] = blob->mutable_gpu_data();
        }
    }
 
    {
        std::lock_guard<std::mutex> lock(mutex_);
        context_->enqueueV2(&buffers_[0], stream_, nullptr);
    }
 
    BASE_GPU_CHECK(cudaStreamSynchronize(stream_));
    for (auto name : output_names_) {
        auto blob = get_blob(name);
        if (blob != nullptr) {
            blob->mutable_gpu_data();
        }
    }
}

◆ inference()

void apollo::perception::inference::MultiBatchInference::inference ( )

在文件 onnx_multi_batch_infer.cc 第 278 行定义.

                                    {
    AINFO << "Do Inference";
}

◆ Init()

bool apollo::perception::inference::MultiBatchInference::Init ( const std::map< std::string, std::vector< int > > & shapes )

overridevirtual

实现了 apollo::perception::inference::Inference.

在文件 onnx_multi_batch_infer.cc 第 122 行定义.

                                                                              {
    BASE_GPU_CHECK(cudaSetDevice(gpu_id_));
    BASE_GPU_CHECK(cudaStreamCreate(&stream_));
 
    FileInit();
 
    // create a TensorRT model from the onnx model and load it into an engine
    OnnxToTRTModel(model_file_);
 
    if (engine_ == nullptr) {
        AERROR << "Fail to load obstacle ONNX model";
        return false;
    }
 
    // create execution context from the engine
    context_ = engine_->createExecutionContext();
    if (context_ == nullptr) {
        AERROR << "Fail to create Exceution Context";
        return false;
    }
    for (const auto& name : output_names_) {
        auto iter = shapes.find(name);
        if (iter != shapes.end()) {
            auto blob = std::make_shared<Blob<float>>(iter->second);
            blobs_.emplace(name, blob);
        }
    }
 
    for (const auto& name : input_names_) {
        auto iter = shapes.find(name);
        if (iter != shapes.end()) {
            auto blob = std::make_shared<Blob<float>>(iter->second);
            blobs_.emplace(name, blob);
        }
    }
    buffers_.resize(input_names_.size() + output_names_.size());
 
    // init_blob(&input_names_);
    // init_blob(&output_names_);
 
    return true;
}

◆ init_blob()

void apollo::perception::inference::MultiBatchInference::init_blob ( std::vector< std::string > * names )

在文件 onnx_multi_batch_infer.cc 第 282 行定义.

                                                               {
    for (auto name : *names) {
        auto blob = get_blob(name);
 
        std::vector<int> shape(blob->shape().begin(), blob->shape().end());
        blob.reset(new apollo::perception::base::Blob<float>(shape));
 
        buffers_[bindingIndex_] = blob->mutable_gpu_data();
 
        bindingIndex_ += 1;
    }
}

◆ LoadCache()

bool apollo::perception::inference::MultiBatchInference::LoadCache ( const std::string & path )

在文件 onnx_multi_batch_infer.cc 第 107 行定义.

                                                         {
    struct stat buffer;
    if (stat(path.c_str(), &buffer) != 0) {
        AERROR << "[INFO] cannot find model cache : " << path << ", it will take minutes to generate...";
        return false;
    }
    return true;
}

◆ OnnxToTRTModel()

bool apollo::perception::inference::MultiBatchInference::OnnxToTRTModel ( const std::string & model_file )

Convert ONNX to TensorRT model

参数

[in] model_file ONNX model file path

Load ONNX model, and convert it to TensorRT model

在文件 onnx_multi_batch_infer.cc 第 207 行定义.

                                                                    {
    if (gpu_id_ < 0) {
        AERROR << "must use gpu mode";
        return false;
    }
    // create the builder
    builder_ = nvinfer1::createInferBuilder(onnx_gLogger);
    network_ = builder_->createNetworkV2(
            1U << static_cast<int>(nvinfer1::NetworkDefinitionCreationFlag::kEXPLICIT_BATCH));
 
    // parse onnx model
    nvonnxparser::IParser* parser = nvonnxparser::createParser(*network_, onnx_gLogger.getTRTLogger());
 
    const char* onnx_filename = model_file.c_str();
    int verbosity = static_cast<int>(onnx_gLogger.getReportableSeverity());
    if (!parser->parseFromFile(onnx_filename, verbosity)) {
        AERROR << "Failure while parsing ONNX file: " << onnx_filename;
        return false;
    }
 
    // Build the engine
    builder_->setMaxBatchSize(max_batch_size_);
    nvinfer1::IBuilderConfig* config = builder_->createBuilderConfig();
    config->setMaxWorkspaceSize(1 << 30);
    bool use_fp16 = builder_->platformHasFastFp16();
    if (use_fp16) {
        config->setFlag(nvinfer1::BuilderFlag::kFP16);
    }
    config->setFlag(nvinfer1::BuilderFlag::kDEBUG);
 
    auto trt_cache_path = model_file + ".trt.engine";
 
    if (!LoadCache(trt_cache_path)) {
        nvinfer1::IOptimizationProfile* profile = builder_->createOptimizationProfile();
        SetOnnxOptimizationProfile(profile, blobs_);
        config->addOptimizationProfile(profile);
 
        engine_ = builder_->buildEngineWithConfig(*network_, *config);
        nvinfer1::IHostMemory* ser_mem = engine_->serialize();
 
        if (ser_mem) {
            nvinfer1::IHostMemory* serialized_model(ser_mem);
            std::ofstream f(trt_cache_path, std::ios::binary);
 
            f.write(reinterpret_cast<char*>(serialized_model->data()), serialized_model->size());
            serialized_model->destroy();
        }
    } else {
        AINFO << "Loading TensorRT engine from serialized model file...";
        std::ifstream planFile(trt_cache_path);
 
        if (!planFile.is_open()) {
            AERROR << "Could not open serialized model";
            return false;
        }
        initLibNvInferPlugins(&onnx_gLogger, "");
 
        AINFO << "success open serialized model";
        std::stringstream planBuffer;
        planBuffer << planFile.rdbuf();
        std::string plan = planBuffer.str();
        nvinfer1::IRuntime* runtime = nvinfer1::createInferRuntime(onnx_gLogger);
 
        engine_ = runtime->deserializeCudaEngine(static_cast<const void*>(plan.data()), plan.size(), nullptr);
        runtime->destroy();
    }
    config->destroy();
    parser->destroy();
    return true;
}

◆ SetOnnxOptimizationProfile()

void apollo::perception::inference::MultiBatchInference::SetOnnxOptimizationProfile	(	nvinfer1::IOptimizationProfile *	profile,
		const BlobMap &	input_shapes
	)

在文件 onnx_multi_batch_infer.cc 第 165 行定义.

                                     {
    // Optimizat every input
    // TODO(huanglei14) Distinguish inputs that do not require optimization
    for (int i = 0; i < network_->getNbInputs(); ++i) {
        nvinfer1::ITensor* nv_input = network_->getInput(i);
        const char* nv_input_name = nv_input->getName();
        nvinfer1::Dims nv_input_dims = nv_input->getDimensions();
        auto it = input_shapes.find(nv_input_name);
        if (it != input_shapes.end()) {
            CHECK_EQ(nv_input_dims.nbDims, input_shapes.at(nv_input_name)->num_axes());
        }
        AINFO << "nv_input_name: " << nv_input_name;
        AINFO << "nv_input_dims: ";
        AINFO << "\t" << nv_input_dims.d[0];
        if (nv_input_dims.d[0] > 0) {
            continue;
        }
 
        for (int j = 1; j < nv_input_dims.nbDims; ++j) {
            AINFO << "\t" << nv_input_dims.d[j];
            if (it != input_shapes.end()) {
                // input dims should be dynamic, or equal to those specified by init
                // options
                CHECK(nv_input_dims.d[j] < 0 || nv_input_dims.d[j] == it->second->shape(j));
                nv_input_dims.d[j] = it->second->shape(j);
            } else {
                // input should have fixed dims if not specfied by init options
                CHECK(nv_input_dims.d[j] > 0);
            }
        }
 
        nv_input_dims.d[0] = 1;
        profile->setDimensions(nv_input_name, nvinfer1::OptProfileSelector::kMIN, nv_input_dims);
        nv_input_dims.d[0] = std::max(1, static_cast<int>(max_batch_size_ / 2));
        profile->setDimensions(nv_input_name, nvinfer1::OptProfileSelector::kOPT, nv_input_dims);
        nv_input_dims.d[0] = max_batch_size_;
        profile->setDimensions(nv_input_name, nvinfer1::OptProfileSelector::kMAX, nv_input_dims);
    }
}

◆ SetStream()

void apollo::perception::inference::MultiBatchInference::SetStream ( cudaStream_t stream )

inlinevirtual

重载 apollo::perception::inference::Inference .

在文件 onnx_multi_batch_infer.h 第 65 行定义.

                                        {
        stream_ = stream;
    }

该类的文档由以下文件生成:

modules/perception/perception_plugin/inference/onnx_multi_batch/onnx_multi_batch_infer.h
modules/perception/perception_plugin/inference/onnx_multi_batch/onnx_multi_batch_infer.cc

Public 成员函数

额外继承的成员函数

详细描述

构造及析构函数说明

◆ MultiBatchInference()

◆ ~MultiBatchInference()

成员函数说明

◆ FileInit()

◆ get_blob()

◆ Infer()

◆ inference()

◆ Init()

◆ init_blob()

◆ LoadCache()

◆ OnnxToTRTModel()

◆ SetOnnxOptimizationProfile()

◆ SetStream()