前言

深度学习的推理预测大多是 Python 脚本,一些数据预处理的代码可能是由 C++ 的 OpenCV 和 PCL 来编写,为了在 Python 脚本中调用 C++ 编写的代码,可以使用 Pybind11 。

在使用线段检测分割算法 ELSED 的官方代码时,官方提供了一个 Pybind11 的使用示例,具有很好的参考性。

Open in Colab arXivProject Page

环境配置

# 新建项目目录
mkdir XXX
# ELSED 克隆到本地
git clone https://github.com/iago-suarez/ELSED
# Pybind11 克隆到本地
git clone https://github.com/pybind/pybind11.git -b v2.8.1
# 新建 build 文件夹
mkdir build
# 新建 script 文件夹
mkdir script
# 新建 src 文件夹
mkdir src

代码实现

文件目录结构

.
├── build # 存放编译生成的文件
├── CMakeLists.txt
├── ELSED
├── pybind11
├── script # 存放 Python 脚本
└── src # 存放 C++ 脚本

CMakeLists.txt

代码关键部分如下:

cmake_minimum_required(VERSION 3.0)
project(xxx)

set(CMAKE_POSITION_INDEPENDENT_CODE ON)
set(CMAKE_CXX_STANDARD 14)

# Import pybind11
add_subdirectory(pybind11)

# Create the library to use efficient-descriptors
## PCP PyAPI
include_directories(efficient-descriptors/)

pybind11_add_module(pymain src/PyMainAPI.cpp)
target_link_libraries(pymain PRIVATE ${XXX_LIBRARY_DIRS} main)
target_compile_definitions(pymain PRIVATE VERSION_INFO=${EXAMPLE_VERSION_INFO})

链接一些常用库(如 OpenCV、PCL、Eigen3、Boost)和自己编写的库(这里为 ELSED ),示例代码如下:

cmake_minimum_required(VERSION 3.0)
project(XXX)

set(CMAKE_POSITION_INDEPENDENT_CODE ON)
set(CMAKE_CXX_STANDARD 14)

# Import Eigen3
find_package(Eigen3 REQUIRED)
if(Eigen3_FOUND)
    message( STATUS "Eigen3_FOUND: " ${Eigen3_FOUND})
    message( STATUS "Eigen3_INCLUDE_DIRS: " ${Eigen3_INCLUDE_DIRS})
    message( STATUS "Eigen3_LIBRARY_DIRS: " ${Eigen3_LIBRARY_DIRS})
    include_directories(${Eigen3_INCLUDE_DIRS})
else()
    message(err: Eigen3 not found)
endif()

# Import Boost
find_package(Boost REQUIRED COMPONENTS serialization system filesystem program_options thread)
if(Boost_FOUND)
    message( STATUS "Boost_FOUND: " ${Boost_FOUND})
    message( STATUS "Boost_INCLUDE_DIRS: " ${Boost_INCLUDE_DIRS})
    message( STATUS "Boost_LIBRARY_DIRS: " ${Boost_LIBRARY_DIRS})
    include_directories(${Boost_INCLUDE_DIRS})
    add_definitions(-DBOOST_ALL_DYN_LINK)
else()
    message(err: Boost not found)
endif()

# Import OpenCV
find_package(OpenCV REQUIRED)
if(OpenCV_FOUND)
    message( STATUS "OpenCV_FOUND: " ${OpenCV_FOUND})
    message( STATUS "OpenCV_INCLUDE_DIRS: " ${OpenCV_INCLUDE_DIRS})
    message( STATUS "OpenCV_LIBRARY_DIRS: " ${OpenCV_LIBRARY_DIRS})
    include_directories(${OpenCV_INCLUDE_DIRS})
else()
    message(err: OpenCV not found)
endif()

# Import PCL
find_package(PCL 1.12 REQUIRED)
if(PCL_FOUND)
    message( STATUS "PCL_FOUND: " ${PCL_FOUND})
    message( STATUS "PCL_INCLUDE_DIRS: " ${PCLINCLUDE_DIRS})
    message( STATUS "PCL_LIBRARY_DIRS: " ${PCL_LIBRARY_DIRS})
    include_directories(${PCL_INCLUDE_DIRS})
else()
    message(err: PCL not found)
endif()

# Import pybind11
add_subdirectory(pybind11)

# import ELSED
include_directories(${CMAKE_SOURCE_DIR}/ELSED/include)
add_subdirectory(ELSED)
include_directories(ELSED/src)

# Create the library to use efficient-descriptors
include_directories(efficient-descriptors/)

pybind11_add_module(pyelsed src/PyELSEDAPI.cpp)
target_link_libraries(pyelsed PRIVATE
    ${OpenCV_LIBRARY_DIRS}
    ${EIGEN3_LIBRARIES}
    ${OpenCV_LIBRARIES}
    ${PCL_LIBRARIES}
    ${Boost_LIBRARIES}
    elsed
)
target_compile_definitions(pyelsed PRIVATE VERSION_INFO=${EXAMPLE_VERSION_INFO})

PYAPI.cpp

这个 cpp 文件是联系 C++ 和 Python 的关键,需要调用 pybind11 库,定义 python 函数入口和传入的一些参数,两种语言之间的变量类型也需要进行转换。

以 ELSED 的 C++ 脚本为例,实现代码如下:

#include <pybind11/pybind11.h>
#include <pybind11/numpy.h>
#include <iostream>
#include <opencv2/opencv.hpp>
#include <ELSED.h>

namespace py = pybind11;
using namespace upm;

// Converts C++ descriptors to Numpy
inline py::tuple salient_segments_to_py(const upm::SalientSegments &ssegs) {
    py::array_t<float> scores(ssegs.size());
    py::array_t<float> segments({int(ssegs.size()), 4});
    float *p_scores = scores.mutable_data();
    float *p_segments = segments.mutable_data();
    for (int i = 0; i < ssegs.size(); i++) {
        p_scores[i] = ssegs[i].salience;
        p_segments[i * 4] = ssegs[i].segment[0];
        p_segments[i * 4 + 1] = ssegs[i].segment[1];
        p_segments[i * 4 + 2] = ssegs[i].segment[2];
        p_segments[i * 4 + 3] = ssegs[i].segment[3];
      }
    return pybind11::make_tuple(segments, scores);
}

py::tuple compute_elsed(const py::array &py_img,
                        float sigma = 1,
                        float gradientThreshold = 30,
                        int minLineLen = 15,
                        double lineFitErrThreshold = 0.2,
                        double pxToSegmentDistTh = 1.5,
                        double validationTh = 0.15,
                        bool validate = true,
                        bool treatJunctions = true
) {

    py::buffer_info info = py_img.request();
    cv::Mat img(info.shape[0], info.shape[1], CV_8UC1, (uint8_t *) info.ptr);
    ELSEDParams params;

    params.sigma = sigma;
    params.ksize = cvRound(sigma * 3 * 2 + 1) | 1; // Automatic kernel size detection
    params.gradientThreshold = gradientThreshold;
    params.minLineLen = minLineLen;
    params.lineFitErrThreshold = lineFitErrThreshold;
    params.pxToSegmentDistTh = pxToSegmentDistTh;
    params.validationTh = validationTh;
    params.validate = validate;
    params.treatJunctions = treatJunctions;

    ELSED elsed(params);
    upm::SalientSegments salient_segs = elsed.detectSalient(img);

    return salient_segments_to_py(salient_segs);
}

PYBIND11_MODULE(pyelsed, m) {
    m.def("detect", &compute_elsed, R"pbdoc(
        Computes ELSED: Enhanced Line SEgment Drawing in the input image.
        )pbdoc",
        py::arg("img"),
        py::arg("sigma") = 1,
        py::arg("gradientThreshold") = 30,
        py::arg("minLineLen") = 15,
        py::arg("lineFitErrThreshold") = 0.2,
        py::arg("pxToSegmentDistTh") = 1.5,
        py::arg("validationTh") = 0.15,
        py::arg("validate") = true,
        py::arg("treatJunctions") = true
    );
}

编译项目

执行以下命令:

cd build
cmake ..
make

test_PyELSEDAPI.py

这个 python 文件中需要导入编译生成的包,为了能顺利导入,路径名需要正确设置,如果相对路径存在问题就使用绝对路径。

以 ELSED 的 Python 脚本为例,实现代码如下:

import sys
sys.path.append("绝对路径/build") # 放置自己的 build 文件夹的绝对路径
import pyelsed

import numpy as np
import cv2

img = cv2.imread("xxx.png", cv2.IMREAD_GRAYSCALE) # 以灰度图形式读取

segments = pyelsed.detect(img) # 对灰度图进行线段检测

dbg = cv2.cvtColor(img, cv2.COLOR_GRAY2RGB)
for s in segments[0].astype(np.int32):
    cv2.line(dbg, (s[0], s[1]), (s[2], s[3]), (0, 255, 0), 1, cv2.LINE_AA)

cv2.imshow("test", dbg)
cv2.waitKey(0)
cv2.destroyAllWindows()

Mat 转 nparray 的转换

C++ 的 OpenCV 中使用 cv::Mat 来存储图像数据,而在 Python 的 OpenCV 中使用 nparray 来存储。上面示例已经展示了从 nparraycv::Mat 的转换,关于从 cv::Matnparray 的转换可以参考以下链接:

How to send a cv::Mat to python over shared memory

一个网友整理根据上面的回答整理在 Github 中的:

GitHub - pthom/cvnp: cvnp: pybind11 casts between numpy and OpenCV, possibly with shared memory

另一个 Github 的链接:

GitHub - Algomorph/pyboostcvconverter: Minimalist code necessary for using porting C++ functions/classes using OpenCV’s “Mat” type in functions argument lists directly (w/o explicit conversions) to python.