pybind11とcppimportを使ってC++ラッパーを作成する

pybind11は、numpyアレイとC++ Eigen線形代数ライブラリを含んだC++コードをpython用にラッピングするためのスマートな方法を提供している。cppimportと一緒に使うことで、C++とPythonを融合するための非常に便利なワークフローを提供してくれる。

先ずは今回のチュートリアルに必要なpython moduleをインストールする。

! pip3 install pybind11
Collecting pybind11
  Downloading https://files.pythonhosted.org/packages/12/90/0f92a575dc60c8fba6d0c91d6b45abdb1058da9ebed40400cbcfad2ac0a7/pybind11-2.2.3-py2.py3-none-any.whl (144kB)
    100% |################################| 153kB 5.7MB/s ta 0:00:01
Installing collected packages: pybind11
Successfully installed pybind11-2.2.3
! pip3 install cppimport
Collecting cppimport
  Downloading https://files.pythonhosted.org/packages/4d/cf/690066fd4bcf36599497e2a0b22fcfefa35b0bd1c8924b74a42f3f41e12e/cppimport-18.1.10.tar.gz (4.2MB)
    100% |################################| 4.2MB 12.5MB/s ta 0:00:01
Requirement already satisfied: mako in /root/.pyenv/versions/3.6.5/envs/py365/lib/python3.6/site-packages (from cppimport) (1.0.7)
Requirement already satisfied: pybind11 in /root/.pyenv/versions/3.6.5/envs/py365/lib/python3.6/site-packages (from cppimport) (2.2.3)
Requirement already satisfied: MarkupSafe>=0.9.2 in /root/.pyenv/versions/3.6.5/envs/py365/lib/python3.6/site-packages (from mako->cppimport) (1.0)
Building wheels for collected packages: cppimport
  Running setup.py bdist_wheel for cppimport ... done
  Stored in directory: /root/.cache/pip/wheels/8c/4d/63/d6d3fdff3c5e0b5f78a888d59a81c9f46d517246494f240da8
Successfully built cppimport
Installing collected packages: cppimport
Successfully installed cppimport-18.1.10

今回のチュートリアル用のディレクトリと各種ファイルを作成する。

%mkdir example1
cd example1
/home/workspace/example1
%%file funcs.hpp

int add(int i, int j);
Writing funcs.hpp
%%file funcs.cpp

int add(int i, int j) {
    return i + j;
};
Writing funcs.cpp

次に、wrap.cppにpybind11を使ってC++ラッパーコードを書く。エクスポートされた関数定義の引数”i”_a=1, “j”_a=2は、pybind11に対しadd関数用に初期値1の変数iと初期値2の変数jを生成するように指示する。

%%file wrap1.cpp
#include <pybind11/pybind11.h>
#include "funcs.hpp"

namespace py = pybind11;

using namespace pybind11::literals;

PYBIND11_PLUGIN(wrap1) {
    py::module m("wrap1", "pybind11 example plugin");
    m.def("add", &add, "A function which adds two numbers",
          "i"_a=1, "j"_a=2);
    return m.ptr();
}
Writing wrap1.cpp

最後に、ほぼ定型コードである拡張モジュールをコンパイルするためのsetup.pyを書く。

%%file setup.py
import os, sys

from distutils.core import setup, Extension
from distutils import sysconfig

cpp_args = ['-std=c++11']

ext_modules = [
    Extension(
    'wrap1',
        ['funcs.cpp', 'wrap1.cpp'],
        include_dirs=['pybind11/include'],
    language='c++',
    extra_compile_args = cpp_args,
    ),
]

setup(
    name='wrap1',
    version='0.0.1',
    author='Cliburn Chan',
    author_email='cliburn.chan@duke.edu',
    description='Example',
    ext_modules=ext_modules,
)
Writing setup.py

上で作成したファイルを用いて拡張モジュールをサブディレクトリにビルドする。

!python setup.py build_ext -i
running build_ext
building 'wrap1' extension
creating build
creating build/temp.linux-x86_64-3.6
gcc -pthread -Wno-unused-result -Wsign-compare -DNDEBUG -g -fwrapv -O3 -Wall -Wstrict-prototypes -O2 -O2 -fPIC -Ipybind11/include -I/root/.pyenv/versions/py365/include -I/root/.pyenv/versions/3.6.5/include/python3.6m -c funcs.cpp -o build/temp.linux-x86_64-3.6/funcs.o -std=c++11
cc1plus: warning: command line option '-Wstrict-prototypes' is valid for C/ObjC but not for C++
gcc -pthread -Wno-unused-result -Wsign-compare -DNDEBUG -g -fwrapv -O3 -Wall -Wstrict-prototypes -O2 -O2 -fPIC -Ipybind11/include -I/root/.pyenv/versions/py365/include -I/root/.pyenv/versions/3.6.5/include/python3.6m -c wrap1.cpp -o build/temp.linux-x86_64-3.6/wrap1.o -std=c++11
cc1plus: warning: command line option '-Wstrict-prototypes' is valid for C/ObjC but not for C++
wrap1.cpp:1:31: fatal error: pybind11/pybind11.h: No such file or directory
compilation terminated.
error: command 'gcc' failed with exit status 1

pybind11.hの場所が分からないようなので教えてやる。

!find / -name pybind11.h
/root/.pyenv/versions/3.6.5/envs/py365/lib/python3.6/site-packages/torch/lib/include/pybind11/pybind11.h
/root/.pyenv/versions/3.6.5/envs/py365/include/site/python3.6/pybind11/pybind11.h
/root/.pyenv/versions/miniconda3-4.3.30/envs/caffe2/lib/python3.6/site-packages/torch/lib/include/pybind11/pybind11.h
/root/.pyenv/versions/miniconda3-4.3.30/envs/caffe2/include/python3.6m/pybind11/pybind11.h
/root/.pyenv/versions/miniconda3-4.3.30/envs/caffe2/pkgs/pybind11-2.2.3-py36_0/include/python3.6m/pybind11/pybind11.h
/root/.pyenv/versions/miniconda3-4.3.30/envs/caffe2/pkgs/pytorch-0.4.0-py36_cuda9.0.176_cudnn7.1.2_1/lib/python3.6/site-packages/torch/lib/include/pybind11/pybind11.h
/root/.pyenv/versions/miniconda3-4.3.30/envs/caffe2/pkgs/pytorch-0.4.0-py36hdf912b8_0/lib/python3.6/site-packages/torch/lib/include/pybind11/pybind11.h
/root/.pyenv/versions/3.5.5/envs/py35/lib/python3.5/site-packages/torch/lib/include/pybind11/pybind11.h
/root/pytorch/third_party/onnx/third_party/pybind11/include/pybind11/pybind11.h
/root/pytorch/third_party/pybind11/include/pybind11/pybind11.h
/root/pytorch/third_party/onnx-tensorrt/third_party/onnx/third_party/pybind11/include/pybind11/pybind11.h
!python setup.py build_ext -i -I /root/pytorch/third_party/pybind11/include
running build_ext
building 'wrap1' extension
gcc -pthread -Wno-unused-result -Wsign-compare -DNDEBUG -g -fwrapv -O3 -Wall -Wstrict-prototypes -O2 -O2 -fPIC -Ipybind11/include -I/root/pytorch/third_party/pybind11/include -I/root/.pyenv/versions/py365/include -I/root/.pyenv/versions/3.6.5/include/python3.6m -c funcs.cpp -o build/temp.linux-x86_64-3.6/funcs.o -std=c++11
cc1plus: warning: command line option '-Wstrict-prototypes' is valid for C/ObjC but not for C++
gcc -pthread -Wno-unused-result -Wsign-compare -DNDEBUG -g -fwrapv -O3 -Wall -Wstrict-prototypes -O2 -O2 -fPIC -Ipybind11/include -I/root/pytorch/third_party/pybind11/include -I/root/.pyenv/versions/py365/include -I/root/.pyenv/versions/3.6.5/include/python3.6m -c wrap1.cpp -o build/temp.linux-x86_64-3.6/wrap1.o -std=c++11
cc1plus: warning: command line option '-Wstrict-prototypes' is valid for C/ObjC but not for C++
In file included from /root/pytorch/third_party/pybind11/include/pybind11/pytypes.h:12:0,
                 from /root/pytorch/third_party/pybind11/include/pybind11/cast.h:13,
                 from /root/pytorch/third_party/pybind11/include/pybind11/attr.h:13,
                 from /root/pytorch/third_party/pybind11/include/pybind11/pybind11.h:43,
                 from wrap1.cpp:1:
wrap1.cpp: In function 'PyObject* PyInit_wrap1()':
/root/pytorch/third_party/pybind11/include/pybind11/detail/common.h:246:20: warning: 'PyObject* pybind11_init()' is deprecated: PYBIND11_PLUGIN is deprecated, use PYBIND11_MODULE [-Wdeprecated-declarations]
             return pybind11_init();                                            
                    ^
wrap1.cpp:8:1: note: in expansion of macro 'PYBIND11_PLUGIN'
 PYBIND11_PLUGIN(wrap1) {
 ^
/root/pytorch/third_party/pybind11/include/pybind11/detail/common.h:231:22: note: declared here
     static PyObject *pybind11_init();                                          
                      ^
wrap1.cpp:8:1: note: in expansion of macro 'PYBIND11_PLUGIN'
 PYBIND11_PLUGIN(wrap1) {
 ^
/root/pytorch/third_party/pybind11/include/pybind11/detail/common.h:246:20: warning: 'PyObject* pybind11_init()' is deprecated: PYBIND11_PLUGIN is deprecated, use PYBIND11_MODULE [-Wdeprecated-declarations]
             return pybind11_init();                                            
                    ^
wrap1.cpp:8:1: note: in expansion of macro 'PYBIND11_PLUGIN'
 PYBIND11_PLUGIN(wrap1) {
 ^
/root/pytorch/third_party/pybind11/include/pybind11/detail/common.h:231:22: note: declared here
     static PyObject *pybind11_init();                                          
                      ^
wrap1.cpp:8:1: note: in expansion of macro 'PYBIND11_PLUGIN'
 PYBIND11_PLUGIN(wrap1) {
 ^
/root/pytorch/third_party/pybind11/include/pybind11/detail/common.h:246:34: warning: 'PyObject* pybind11_init()' is deprecated: PYBIND11_PLUGIN is deprecated, use PYBIND11_MODULE [-Wdeprecated-declarations]
             return pybind11_init();                                            
                                  ^
wrap1.cpp:8:1: note: in expansion of macro 'PYBIND11_PLUGIN'
 PYBIND11_PLUGIN(wrap1) {
 ^
/root/pytorch/third_party/pybind11/include/pybind11/detail/common.h:231:22: note: declared here
     static PyObject *pybind11_init();                                          
                      ^
wrap1.cpp:8:1: note: in expansion of macro 'PYBIND11_PLUGIN'
 PYBIND11_PLUGIN(wrap1) {
 ^
g++ -pthread -shared -L/root/.pyenv/versions/3.6.5/lib -Wl,-rpath=/root/.pyenv/versions/3.6.5/lib -L/root/.pyenv/versions/3.6.5/lib -Wl,-rpath=/root/.pyenv/versions/3.6.5/lib build/temp.linux-x86_64-3.6/funcs.o build/temp.linux-x86_64-3.6/wrap1.o -L/root/.pyenv/versions/3.6.5/lib -lpython3.6m -o /home/workspace/example1/wrap1.cpython-36m-x86_64-linux-gnu.so

エラーがなければfuncs.so拡張モジュールができたはずなので、その新しいモジュールをテストするためにtest_funcs.pyを書く。

%%file test_funcs.py

import wrap1

def test_add():
    print(wrap1.add(3, 4))
    assert(wrap1.add(3, 4) == 7)

if __name__ == '__main__':
    test_add()
Writing test_funcs.py

上で書いたテストを実行する。エラーメッセージは出ないはず。

!python test_funcs.py
7

Using cppimport

開発時に、拡張モジュールを再ビルドする度にpython setup.py clean &&
python setup.py build_ext -i
とやるのは面倒なので、cppimportにこの作業をやってもらう。

新しいサブディレクトリexaample2を作成して、example1ディレクトリからfunc.hpp, funcs.cpp, wrap.cppファイルをコピーする。前回の例に対しては、wrap.cppファイルのトップ(<%と %>の間)にいくつかの注釈を付け加えるだけで済む。

cd ..
/home/workspace
%mkdir example2
%cp example1/funcs.* example2/
cd example2
/home/workspace/example2
ls
funcs.cpp  funcs.hpp
%%file wrap2.cpp
<%
cfg['compiler_args'] = ['-std=c++11']
cfg['sources'] = ['funcs.cpp']
setup_pybind11(cfg)
%>

#include "funcs.hpp"
#include <pybind11/pybind11.h>

namespace py = pybind11;

PYBIND11_PLUGIN(wrap2) {
    py::module m("wrap2", "pybind11 example plugin");
    m.def("add", &add, "A function which adds two numbers");
    return m.ptr();
}
Writing wrap2.cpp
%%file test_funcs.py

import cppimport
funcs = cppimport.imp("wrap2")

def test_add():
    assert(funcs.add(3, 4) == 7)

if __name__ == '__main__':
    print(funcs.add(3,4))
    test_add()
Writing test_funcs.py
!python test_funcs.py
cc1plus: warning: command line option '-Wstrict-prototypes' is valid for C/ObjC but not for C++
cc1plus: warning: command line option '-Wstrict-prototypes' is valid for C/ObjC but not for C++
In file included from /root/.pyenv/versions/py365/include/site/python3.6/pybind11/pytypes.h:12:0,
                 from /root/.pyenv/versions/py365/include/site/python3.6/pybind11/cast.h:13,
                 from /root/.pyenv/versions/py365/include/site/python3.6/pybind11/attr.h:13,
                 from /root/.pyenv/versions/py365/include/site/python3.6/pybind11/pybind11.h:43,
                 from /home/workspace/example2/.rendered.wrap2.cpp:4:
/home/workspace/example2/.rendered.wrap2.cpp: In function 'PyObject* PyInit_wrap2()':
/root/.pyenv/versions/py365/include/site/python3.6/pybind11/detail/common.h:242:20: warning: 'PyObject* pybind11_init()' is deprecated: PYBIND11_PLUGIN is deprecated, use PYBIND11_MODULE [-Wdeprecated-declarations]
             return pybind11_init();                                            
                    ^
/home/workspace/example2/.rendered.wrap2.cpp:8:1: note: in expansion of macro 'PYBIND11_PLUGIN'
 PYBIND11_PLUGIN(wrap2) {
 ^
/root/.pyenv/versions/py365/include/site/python3.6/pybind11/detail/common.h:227:22: note: declared here
     static PyObject *pybind11_init();                                          
                      ^
/home/workspace/example2/.rendered.wrap2.cpp:8:1: note: in expansion of macro 'PYBIND11_PLUGIN'
 PYBIND11_PLUGIN(wrap2) {
 ^
/root/.pyenv/versions/py365/include/site/python3.6/pybind11/detail/common.h:242:20: warning: 'PyObject* pybind11_init()' is deprecated: PYBIND11_PLUGIN is deprecated, use PYBIND11_MODULE [-Wdeprecated-declarations]
             return pybind11_init();                                            
                    ^
/home/workspace/example2/.rendered.wrap2.cpp:8:1: note: in expansion of macro 'PYBIND11_PLUGIN'
 PYBIND11_PLUGIN(wrap2) {
 ^
/root/.pyenv/versions/py365/include/site/python3.6/pybind11/detail/common.h:227:22: note: declared here
     static PyObject *pybind11_init();                                          
                      ^
/home/workspace/example2/.rendered.wrap2.cpp:8:1: note: in expansion of macro 'PYBIND11_PLUGIN'
 PYBIND11_PLUGIN(wrap2) {
 ^
/root/.pyenv/versions/py365/include/site/python3.6/pybind11/detail/common.h:242:34: warning: 'PyObject* pybind11_init()' is deprecated: PYBIND11_PLUGIN is deprecated, use PYBIND11_MODULE [-Wdeprecated-declarations]
             return pybind11_init();                                            
                                  ^
/home/workspace/example2/.rendered.wrap2.cpp:8:1: note: in expansion of macro 'PYBIND11_PLUGIN'
 PYBIND11_PLUGIN(wrap2) {
 ^
/root/.pyenv/versions/py365/include/site/python3.6/pybind11/detail/common.h:227:22: note: declared here
     static PyObject *pybind11_init();                                          
                      ^
/home/workspace/example2/.rendered.wrap2.cpp:8:1: note: in expansion of macro 'PYBIND11_PLUGIN'
 PYBIND11_PLUGIN(wrap2) {
 ^
7

または、ノートプックからwrap2関数を直接コールしてもいい。

import cppimport
funcs = cppimport.imp("wrap2")

funcs.add(3, 4)
7

手動で拡張モジュールをビルド必要なしに、全てのアップデートはcppimportが検出して、自動的に再ビルドをトリガーしてくれる。

Vectorizing functions for use with numpy arrays

以下の例でsquare関数のベクトル化方法を示す。ここから、コードスニペット用のヘッダーと実装ファイルを個別に使う面倒は止めて、単にそれらをcode.cppファイルにラッピングコードと一緒に書いていることに留意する。つまり、cppimportを使えば、実際にコードするファイルは、C++用のcode.cppとPython用test fileの2ファイルだけで済むということ。

cd ..
/home/workspace
%mkdir example3
%cd example3
/home/workspace/example3
%%file wrap3.cpp
<%
cfg['compiler_args'] = ['-std=c++11']
setup_pybind11(cfg)
%>

#include <pybind11/pybind11.h>
#include <pybind11/numpy.h>

namespace py = pybind11;
double square(double x) {
    return x * x;
}

PYBIND11_PLUGIN(wrap3) {
    py::module m("wrap3", "pybind11 example plugin");
    m.def("square", py::vectorize(square), "A vectroized square function.");
    return m.ptr();
}
Writing wrap3.cpp
import cppimport

wrap3 = cppimport.imp("wrap3")
wrap3.square([1,2,3])
array([1., 4., 9.])

一旦共用ライブラリがビルドされれば、標準Pythonモジュールとしてそれを使える。

! ls
wrap3.cpp  wrap3.cpython-36m-x86_64-linux-gnu.so
import wrap3

wrap3.square([2,4,6])
array([ 4., 16., 36.])

Using numpy arrays as function arguments and return values

以下の例で関数内外へのnumpyアレイの渡し方を示す。これらのnumpyアレイ引数は、無印py:arrayか型付py:array_tになれる。numpyアレイのプロパティは、要求メソッドを呼び出すことで得られ、以下のフォームのstructを返す。

struct buffer_info {
    void *ptr;
    size_t itemsize;
    std::string format;
    int ndim;
    std::vector<size_t> shape;
    std::vector<size_t> strides;
};

以下に2つの関数のC++コードを記す。関数twiceは、ポインターを使って渡されたnumpyアレイのin-placeでの変換方法を示し、関数sumは、numpyアレイの要素の足し方を示す。buffer_infoの情報を利用して、コードは任意のn-dアレイに対して機能する。

cd ..
/home/workspace
%mkdir example4
cd example4
/home/workspace/example4
%%file wrap4.cpp
<%
cfg['compiler_args'] = ['-std=c++11']
setup_pybind11(cfg)
%>

#include <pybind11/pybind11.h>
#include <pybind11/numpy.h>

namespace py = pybind11;

// Passing in an array of doubles
void twice(py::array_t<double> xs) {
    py::buffer_info info = xs.request();
    auto ptr = static_cast<double *>(info.ptr);

    int n = 1;
    for (auto r: info.shape) {
      n *= r;
    }

    for (int i = 0; i <n; i++) {
        *ptr++ *= 2;
    }
}

// Passing in a generic array
double sum(py::array xs) {
    py::buffer_info info = xs.request();
    auto ptr = static_cast<double *>(info.ptr);

    int n = 1;
    for (auto r: info.shape) {
      n *= r;
    }

    double s = 0.0;
    for (int i = 0; i <n; i++) {
        s += *ptr++;
    }

    return s;
}

PYBIND11_PLUGIN(wrap4) {
    pybind11::module m("wrap4", "auto-compiled c++ extension");
    m.def("sum", &sum);
    m.def("twice", &twice);
    return m.ptr();
}
Writing wrap4.cpp
%%file test_code.py
import cppimport
import numpy as np

code = cppimport.imp("wrap4")

if __name__ == '__main__':
    xs = np.arange(12).reshape(3,4).astype('float')
    print(xs)
    print("np :", xs.sum())
    print("cpp:", code.sum(xs))

    print()
    code.twice(xs)
    print(xs)
Writing test_code.py
!python test_code.py
cc1plus: warning: command line option '-Wstrict-prototypes' is valid for C/ObjC but not for C++
In file included from /root/.pyenv/versions/py365/include/site/python3.6/pybind11/pytypes.h:12:0,
                 from /root/.pyenv/versions/py365/include/site/python3.6/pybind11/cast.h:13,
                 from /root/.pyenv/versions/py365/include/site/python3.6/pybind11/attr.h:13,
                 from /root/.pyenv/versions/py365/include/site/python3.6/pybind11/pybind11.h:43,
                 from /home/workspace/example4/.rendered.wrap4.cpp:3:
/home/workspace/example4/.rendered.wrap4.cpp: In function 'PyObject* PyInit_wrap4()':
/root/.pyenv/versions/py365/include/site/python3.6/pybind11/detail/common.h:242:20: warning: 'PyObject* pybind11_init()' is deprecated: PYBIND11_PLUGIN is deprecated, use PYBIND11_MODULE [-Wdeprecated-declarations]
             return pybind11_init();                                            
                    ^
/home/workspace/example4/.rendered.wrap4.cpp:41:1: note: in expansion of macro 'PYBIND11_PLUGIN'
 PYBIND11_PLUGIN(wrap4) {
 ^
/root/.pyenv/versions/py365/include/site/python3.6/pybind11/detail/common.h:227:22: note: declared here
     static PyObject *pybind11_init();                                          
                      ^
/home/workspace/example4/.rendered.wrap4.cpp:41:1: note: in expansion of macro 'PYBIND11_PLUGIN'
 PYBIND11_PLUGIN(wrap4) {
 ^
/root/.pyenv/versions/py365/include/site/python3.6/pybind11/detail/common.h:242:20: warning: 'PyObject* pybind11_init()' is deprecated: PYBIND11_PLUGIN is deprecated, use PYBIND11_MODULE [-Wdeprecated-declarations]
             return pybind11_init();                                            
                    ^
/home/workspace/example4/.rendered.wrap4.cpp:41:1: note: in expansion of macro 'PYBIND11_PLUGIN'
 PYBIND11_PLUGIN(wrap4) {
 ^
/root/.pyenv/versions/py365/include/site/python3.6/pybind11/detail/common.h:227:22: note: declared here
     static PyObject *pybind11_init();                                          
                      ^
/home/workspace/example4/.rendered.wrap4.cpp:41:1: note: in expansion of macro 'PYBIND11_PLUGIN'
 PYBIND11_PLUGIN(wrap4) {
 ^
/root/.pyenv/versions/py365/include/site/python3.6/pybind11/detail/common.h:242:34: warning: 'PyObject* pybind11_init()' is deprecated: PYBIND11_PLUGIN is deprecated, use PYBIND11_MODULE [-Wdeprecated-declarations]
             return pybind11_init();                                            
                                  ^
/home/workspace/example4/.rendered.wrap4.cpp:41:1: note: in expansion of macro 'PYBIND11_PLUGIN'
 PYBIND11_PLUGIN(wrap4) {
 ^
/root/.pyenv/versions/py365/include/site/python3.6/pybind11/detail/common.h:227:22: note: declared here
     static PyObject *pybind11_init();                                          
                      ^
/home/workspace/example4/.rendered.wrap4.cpp:41:1: note: in expansion of macro 'PYBIND11_PLUGIN'
 PYBIND11_PLUGIN(wrap4) {
 ^
[[ 0.  1.  2.  3.]
 [ 4.  5.  6.  7.]
 [ 8.  9. 10. 11.]]
np : 66.0
cpp: 66.0

[[ 0.  2.  4.  6.]
 [ 8. 10. 12. 14.]
 [16. 18. 20. 22.]]
ls
test_code.py  wrap4.cpp  wrap4.cpython-36m-x86_64-linux-gnu.so*
import wrap4 as wp
import numpy as np

xs = np.arange(12).reshape(3,4).astype('float')
print(xs)
print("np :", wp.sum(xs))

print()
wp.twice(xs)
print(xs)
[[ 0.  1.  2.  3.]
 [ 4.  5.  6.  7.]
 [ 8.  9. 10. 11.]]
np : 66.0

[[ 0.  2.  4.  6.]
 [ 8. 10. 12. 14.]
 [16. 18. 20. 22.]]
参考サイトUsing pybind11