Jupyter NotebookがLD_LIBRARY_PATHを認識しない

このサイトからダウンロードしたコードを実行したところエラーが出た。このエラーを解消するのに結構手間取ったので、また同じようなエラーが出ても詰まらないように記録しておくことにした。

スポンサーリンク

jupyter notebookとLD_LIBRARY_PATH

カフェ2を新たにビルドして下記のコードを実行した所エラーが吐き出された。

# %load demo.py
import torch
import torchvision.utils as vutils
import numpy as np
import torchvision.models as models
from torchvision import datasets
from tensorboardX import SummaryWriter
#import skimage
#from skimage import data, io

resnet18 = models.resnet18(False)
writer = SummaryWriter()
sample_rate = 44100
freqs = [262, 294, 330, 349, 392, 440, 440, 440, 440, 440, 440]
true_positive_counts=[75, 64, 21, 5, 0]
false_positive_counts=[150, 105, 18, 0, 0]
true_negative_counts=[0, 45, 132, 150, 150]
false_negative_counts=[0, 11, 54, 70, 75]
precision=[0.3333333, 0.3786982, 0.5384616, 1.0, 0.0]
recall=[1.0, 0.8533334, 0.28, 0.0666667, 0.0]

for n_iter in range(100):
    s1 = torch.rand(1) # value to keep
    s2 = torch.rand(1)
    writer.add_scalar('data/scalar1', s1[0], n_iter) # data grouping by `slash`
    writer.add_scalars('data/scalar_group', {"xsinx":n_iter*np.sin(n_iter),
                                             "xcosx":n_iter*np.cos(n_iter),
                                             "arctanx": np.arctan(n_iter)}, n_iter)
    x = torch.rand(32, 3, 64, 64) # output from network
    if n_iter%10==0:
        x = vutils.make_grid(x, normalize=True, scale_each=True)
        writer.add_image('Image', x, n_iter) # Tensor
        #writer.add_image('astronaut', skimage.data.astronaut(), n_iter) # numpy
        #writer.add_image('imread', skimage.io.imread('screenshots/audio.png'), n_iter) # numpy
        x = torch.zeros(sample_rate*2)
        for i in range(x.size(0)):
            x[i] = np.cos(freqs[n_iter//10]*np.pi*float(i)/float(sample_rate)) # sound amplitude should in [-1, 1]
        writer.add_audio('myAudio', x, n_iter)
        writer.add_text('Text', 'text logged at step:'+str(n_iter), n_iter)
        writer.add_text('markdown Text', '''a|b\n-|-\nc|d''', n_iter)
        for name, param in resnet18.named_parameters():
            if 'bn' not in name:            
                writer.add_histogram(name, param, n_iter)
        writer.add_pr_curve('xoxo', np.random.randint(2, size=100), np.random.rand(100), n_iter) #needs tensorboard 0.4RC or later
        writer.add_pr_curve_raw('prcurve with raw data', true_positive_counts,
            false_positive_counts,
            true_negative_counts,
            false_negative_counts,
            precision,
            recall, n_iter)
# export scalar data to JSON for external processing
writer.export_scalars_to_json("./all_scalars.json")
dataset = datasets.MNIST('mnist', train=False, download=True)
images = dataset.test_data[:100].float()
label = dataset.test_labels[:100]
features = images.view(100, 784)
writer.add_embedding(features, metadata=label, label_img=images.unsqueeze(1))
writer.add_embedding(features, global_step=1, tag='noMetadata')
dataset = datasets.MNIST('mnist', train=True, download=True)
images_train = dataset.train_data[:100].float()
labels_train = dataset.train_labels[:100]
features_train = images_train.view(100, 784)
all_features = torch.cat((features, features_train))
all_labels = torch.cat((label, labels_train))
all_images = torch.cat((images, images_train))
dataset_label = ['test']*100 + ['train']*100
all_labels = list(zip(all_labels, dataset_label))
writer.add_embedding(all_features, metadata=all_labels, label_img=all_images.unsqueeze(1),
                     metadata_header=['digit', 'dataset'], global_step=2)

# VIDEO
vid_images = dataset.train_data[:16*48]
vid = vid_images.view(16, 1, 48, 28, 28)  # BxCxTxHxW
writer.add_video('video', vid_tensor=vid)
writer.add_video('video_1_fps', vid_tensor=vid, fps=1)
writer.close()
WARNING:root:This caffe2 python run does not have GPU support. Will run in CPU only mode.
WARNING:root:Debug message: No module named 'caffe2.python.caffe2_pybind11_state_hip'
CRITICAL:root:Cannot load caffe2.python. Error: libopencv_imgcodecs.so.3.4: cannot open shared object file: No such file or directory
An exception has occurred, use %tb to see the full traceback.

SystemExit: 1
/root/.pyenv/versions/3.6.5/envs/py365/lib/python3.6/site-packages/IPython/core/interactiveshell.py:2971: UserWarning: To exit: use 'exit', 'quit', or Ctrl-D.
  warn("To exit: use 'exit', 'quit', or Ctrl-D.", stacklevel=1)

先ず、問題のライブラリが存在するのかどうかを確認する。

!find / -name libopencv_imgcodecs.so.3.4
/root/.pyenv/versions/miniconda3-4.3.30/pkgs/libopencv-3.4.1-h1a3b859_1/lib/libopencv_imgcodecs.so.3.4
/root/opencv-3.4.1/build/lib/libopencv_imgcodecs.so.3.4
/root/usr/local/opencv/lib/libopencv_imgcodecs.so.3.4

bashrcでライブラリパスは通してあることを確認する。

&load ~/.bashrc
# ~/.bashrc: executed by bash(1) for non-login shells.
# see /usr/share/doc/bash/examples/startup-files (in the package bash-doc)
# for examples

export LD_LIBRARY_PATH=/root/usr/local/opencv/lib/:$LD_LIBRARY_PATH
Overwriting /root/.bashrc

ライブラリパスが本当にjupyterに通っているのかを確認する。

!echo $LD_LIBRARY_PATH
/usr/lib/x86_64-linux-gnu:/usr/lib/x86_64-linux-gnu/libcuda.so

何とライブラリパスが通っていない!端末上で同じコマンドを叩くと、
/usr/lib/x86_64-linux-gnu/:/root/usr/local/opencv/lib/:/root/pool/lib/:/usr/local/lib/:/usr/local/cuda-9.0/lib64:/home/workspace/TensorRT-4.0.1.6/lib
/usr/local/opencv/lib/にちゃんとパスが通っている。

最終的な解決方法

c.Spawner.env_keep.append([‘LD_LIBRARY_PATH’])は過去に試して成功したので今回も試してみたが結果は駄目だった。最終的な解決方法は以下の通り。

cd /etc/ld.so.conf.d
/etc/ld.so.conf.d
ls
cuda-9-0.conf                   x86_64-linux-gnu.conf
fakeroot-x86_64-linux-gnu.conf  x86_64-linux-gnu_EGL.conf@
libc.conf                       x86_64-linux-gnu_GL.conf@
nvidia.conf                     zz_i386-biarch-compat.conf
opencv.conf
cat opencv.conf
/root/usr/local/opencv/lib

このopencv.confを上記のように書き換えて、ldconfigすることで問題は解決された。

参考サイトhttps://stackoverflow.com/