About running DynaSLAM source code (OpenCV3.x version)

OK, here comes another source code operation record ←←

1, Basic environment

According to the author's source code Readme file, python 2.7 must be available. Later, I installed tensorflow in a virtual environment established by Anaconda. This record is for the version of OpenCV 3.x! For the revision of version 3.x, the fourth point is the general part.

2, ORB-SLAM2 compiled successfully

Installation record details of ORB-SLAM2 click here .
There are some libraries to be installed in advance, such as C++11 or C++0x Compiler, Pangolin, OpenCV and Eigen3. If you don't know how to install them, you can search them manually. DynaSLAM can only support OpenCV2.4.11 when it first came out. However, some enthusiasts submitted code that can support Opencv3.x in 19 years. We will talk about it in detail later. I installed OpenCV 3.4.5 myself

3, Additional libraries to install before compiling DynaSLAM

Readme file according to open source code:

1. Install the boost library

sudo apt-get install libboost-all-dev

2. Download the DynaSLAM source code and put it into the h5 file

git clone https://github.com/BertaBescos/DynaSLAM.git
Then from this page https://github.com/matterport/Mask_RCNN/releases Download h5 the file and save it to DynaSLAM/src/python /

3.Python related environment

Here, first create a new virtual environment in Anaconda and activate it, and then install tensorflow and keras in the virtual environment in turn.

conda create -n MaskRCNN python=2.7
conda activate MaskRCNN
pip install tensorflow==1.14.0  #Or pip install tensorflow GPU = = 1.14.0
pip install keras==2.0.9

After completing the above steps, the python environment is almost ready. You can test it below

cd DynaSLAM
python src/python/Check.py

If the output is Mask R-CNN is correctly working, you can go to the next step. However, things are difficult to go so smoothly. Hahaha, then solve them one by one. I have two problems here:

3.1 scikit image is not installed

sudo pip install scikit-image

3.2 error reporting about pycocotools

be careful! Here, you must install Python 2.7 (cocoapi only supports Python 2)! Otherwise, an error will be reported when running Check.py and cannot be found_ mask, because when Python 3 runs, it will not be generated_ mask.so this file.

git clone https://github.com/waleedka/coco 
python PythonAPI/setup.py build_ext install

After running the above instructions, copy the entire pycotools folder to src/python /

4, Modify some DynaSLAM source code

Thank you very much for this little sister (?), whose original address is here: Pushyami_dev , if you want to see what has been added or deleted in the code, you can click in.
The submitted code mainly makes some modifications to the use of Opencv3. Then, on the basis of this code, remove - march=native (Segment Default error will appear) in CMakeLists.txt, and remember to go to / Thirdparty/DBoW2. The code is mainly modified in the following parts. Just copy the whole code directly to the folder. Pay attention to modifying your OpenCV3.x version.

  • CMakeLists.txt
  • Thirdparty/DBoW2/CMakeLists.txt
  • include/Conversion.h
  • src/Conversion.cc

1. CMakeLists.txt

cmake_minimum_required(VERSION 2.8)
project(DynaSLAM)

IF(NOT CMAKE_BUILD_TYPE)
  SET(CMAKE_BUILD_TYPE Release)
  # SET(CMAKE_BUILD_TYPE Debug)
ENDIF()

MESSAGE("Build type: " ${CMAKE_BUILD_TYPE})

#set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS}  -Wall  -O3 -march=native ")
#set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall   -O3 -march=native")
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS}  -Wall  -O3  ")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall   -O3 ")
# This is required if opencv is built from source locally
#SET(OpenCV_DIR "~/opencv/build")

# set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS}  -Wall  -O0 -march=native ")
# set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall   -O0 -march=native")

# Check C++11 or C++0x support
include(CheckCXXCompilerFlag)
CHECK_CXX_COMPILER_FLAG("-std=c++11" COMPILER_SUPPORTS_CXX11)
CHECK_CXX_COMPILER_FLAG("-std=c++0x" COMPILER_SUPPORTS_CXX0X)
if(COMPILER_SUPPORTS_CXX11)
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
   add_definitions(-DCOMPILEDWITHC11)
   message(STATUS "Using flag -std=c++11.")
elseif(COMPILER_SUPPORTS_CXX0X)
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++0x")
   add_definitions(-DCOMPILEDWITHC0X)
   message(STATUS "Using flag -std=c++0x.")
else()
   message(FATAL_ERROR "The compiler ${CMAKE_CXX_COMPILER} has no C++11 support. Please use a different C++ compiler.")
endif()

LIST(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake_modules)

set(Python_ADDITIONAL_VERSIONS "2.7")
#This is to avoid detecting python 3
find_package(PythonLibs 2.7 EXACT REQUIRED)
if (NOT PythonLibs_FOUND)
    message(FATAL_ERROR "PYTHON LIBS not found.")
else()
    message("PYTHON LIBS were found!")
    message("PYTHON LIBS DIRECTORY: " ${PYTHON_LIBRARY} ${PYTHON_INCLUDE_DIRS})
endif()

message("PROJECT_SOURCE_DIR: " ${OpenCV_DIR})
find_package(OpenCV 3.4 QUIET)
if(NOT OpenCV_FOUND)
    find_package(OpenCV 2.4 QUIET)
    if(NOT OpenCV_FOUND)
        message(FATAL_ERROR "OpenCV > 2.4.x not found.")
    endif()
endif()

find_package(Qt5Widgets REQUIRED)
find_package(Qt5Concurrent REQUIRED)
find_package(Qt5OpenGL REQUIRED)
find_package(Qt5Test REQUIRED)

find_package(Boost REQUIRED COMPONENTS thread)
if(Boost_FOUND)
    message("Boost was found!")
    message("Boost Headers DIRECTORY: " ${Boost_INCLUDE_DIRS})
    message("Boost LIBS DIRECTORY: " ${Boost_LIBRARY_DIRS})
    message("Found Libraries: " ${Boost_LIBRARIES})
endif()

find_package(Eigen3 3.1.0 REQUIRED)
find_package(Pangolin REQUIRED)

set(PYTHON_INCLUDE_DIRS ${PYTHON_INCLUDE_DIRS} /usr/local/lib/python2.7/dist-packages/numpy/core/include/numpy)

include_directories(
${PROJECT_SOURCE_DIR}
${PROJECT_SOURCE_DIR}/include
${EIGEN3_INCLUDE_DIR}
${Pangolin_INCLUDE_DIRS}
${PYTHON_INCLUDE_DIRS}
/usr/include/python2.7/
#/usr/lib/python2.7/dist-packages/numpy/core/include/numpy/
${Boost_INCLUDE_DIRS}
)
message("PROJECT_SOURCE_DIR: " ${PROJECT_SOURCE_DIR})
set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${PROJECT_SOURCE_DIR}/lib)

add_library(${PROJECT_NAME} SHARED
src/System.cc
src/Tracking.cc
src/LocalMapping.cc
src/LoopClosing.cc
src/ORBextractor.cc
src/ORBmatcher.cc
src/FrameDrawer.cc
src/Converter.cc
src/MapPoint.cc
src/KeyFrame.cc
src/Map.cc
src/MapDrawer.cc
src/Optimizer.cc
src/PnPsolver.cc
src/Frame.cc
src/KeyFrameDatabase.cc
src/Sim3Solver.cc
src/Initializer.cc
src/Viewer.cc
src/Conversion.cc
src/MaskNet.cc
src/Geometry.cc
)

target_link_libraries(${PROJECT_NAME}
${OpenCV_LIBS}
${EIGEN3_LIBS}
${Pangolin_LIBRARIES}
${PROJECT_SOURCE_DIR}/Thirdparty/DBoW2/lib/libDBoW2.so
${PROJECT_SOURCE_DIR}/Thirdparty/g2o/lib/libg2o.so
/usr/lib/x86_64-linux-gnu/libpython2.7.so
${Boost_LIBRARIES}
)

# Build examples

set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${PROJECT_SOURCE_DIR}/Examples/RGB-D)

add_executable(rgbd_tum
Examples/RGB-D/rgbd_tum.cc)
target_link_libraries(rgbd_tum ${PROJECT_NAME})

set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${PROJECT_SOURCE_DIR}/Examples/Stereo)

add_executable(stereo_kitti
Examples/Stereo/stereo_kitti.cc)
target_link_libraries(stereo_kitti ${PROJECT_NAME})

set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${PROJECT_SOURCE_DIR}/Examples/Monocular)

add_executable(mono_tum
Examples/Monocular/mono_tum.cc)
target_link_libraries(mono_tum ${PROJECT_NAME})

2.Thirdparty/DBoW2/CMakeLists.txt

cmake_minimum_required(VERSION 2.8)
project(DBoW2)

#set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS}  -Wall  -O3 -march=native ")
#set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall  -O3 -march=native")
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS}  -Wall  -O3 ")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall  -O3 ")

set(HDRS_DBOW2
  DBoW2/BowVector.h
  DBoW2/FORB.h 
  DBoW2/FClass.h       
  DBoW2/FeatureVector.h
  DBoW2/ScoringObject.h   
  DBoW2/TemplatedVocabulary.h)
set(SRCS_DBOW2
  DBoW2/BowVector.cpp
  DBoW2/FORB.cpp      
  DBoW2/FeatureVector.cpp
  DBoW2/ScoringObject.cpp)

set(HDRS_DUTILS
  DUtils/Random.h
  DUtils/Timestamp.h)
set(SRCS_DUTILS
  DUtils/Random.cpp
  DUtils/Timestamp.cpp)

find_package(OpenCV 3.4 QUIET)
if(NOT OpenCV_FOUND)
   find_package(OpenCV 2.4.3 QUIET)
   if(NOT OpenCV_FOUND)
      message(FATAL_ERROR "OpenCV > 2.4.3 not found.")
   endif()
endif()

set(LIBRARY_OUTPUT_PATH ${PROJECT_SOURCE_DIR}/lib)

include_directories(${OpenCV_INCLUDE_DIRS})
add_library(DBoW2 SHARED ${SRCS_DBOW2} ${SRCS_DUTILS})
target_link_libraries(DBoW2 ${OpenCV_LIBS})

3.include/Conversion.h

/**
* This file is part of DynaSLAM.
* Copyright (C) 2018 Berta Bescos <bbescos at unizar dot es> (University of Zaragoza)
* For more information see <https://github.com/bertabescos/DynaSLAM>.
*
*/


#ifndef CONVERSION_H_

#define CONVERSION_H_

#include <Python.h>
#include <opencv2/imgproc/imgproc.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <opencv2/core/core.hpp>
#include "numpy/ndarrayobject.h"
// #include "__multiarray_api.h"

#define NUMPY_IMPORT_ARRAY_RETVAL

namespace DynaSLAM
{

static PyObject* opencv_error = 0;

static int failmsg(const char *fmt, ...);

class PyAllowThreads;

class PyEnsureGIL;

#define ERRWRAP2(expr) \
try \
{ \
    PyAllowThreads allowThreads; \
    expr; \
} \
catch (const cv::Exception &e) \
{ \
    PyErr_SetString(opencv_error, e.what()); \
    return 0; \
}

static PyObject* failmsgp(const char *fmt, ...);

static size_t REFCOUNT_OFFSET = (size_t)&(((PyObject*)0)->ob_refcnt) +
    (0x12345678 != *(const size_t*)"\x78\x56\x34\x12\0\0\0\0\0")*sizeof(int);

static inline PyObject* pyObjectFromRefcount(const int* refcount)
{
    return (PyObject*)((size_t)refcount - REFCOUNT_OFFSET);
}

static inline int* refcountFromPyObject(const PyObject* obj)
{
    return (int*)((size_t)obj + REFCOUNT_OFFSET);
}

class NumpyAllocator;

enum { ARG_NONE = 0, ARG_MAT = 1, ARG_SCALAR = 2 };

class NDArrayConverter
{
private:
    void init();
public:
    NDArrayConverter();
    //cv::Mat toMat(const PyObject* o);   //issue bug
    cv::Mat toMat(PyObject* o);
    PyObject* toNDArray(const cv::Mat& mat);
};

}

#endif /* CONVERSION_H_ */

4. src/Conversion.cc

/**
* This file is part of DynaSLAM.
* Copyright (C) 2018 Berta Bescos <bbescos at unizar dot es> (University of Zaragoza)
* For more information see <https://github.com/bertabescos/DynaSLAM>.
*
*/


#include "Conversion.h"
#include <iostream>


namespace DynaSLAM
{

static void init()
{
    import_array();
}

static int failmsg(const char *fmt, ...)
{
    char str[1000];

    va_list ap;
    va_start(ap, fmt);
    vsnprintf(str, sizeof(str), fmt, ap);
    va_end(ap);

    PyErr_SetString(PyExc_TypeError, str);
    return 0;
}

class PyAllowThreads
{
public:
    PyAllowThreads() : _state(PyEval_SaveThread()) {}
    ~PyAllowThreads()
    {
        PyEval_RestoreThread(_state);
    }
private:
    PyThreadState* _state;
};

class PyEnsureGIL
{
public:
    PyEnsureGIL() : _state(PyGILState_Ensure()) {}
    ~PyEnsureGIL()
    {
        //std::cout << "releasing"<< std::endl;
        PyGILState_Release(_state);
    }
private:
    PyGILState_STATE _state;
};

using namespace cv;

static PyObject* failmsgp(const char *fmt, ...)
{
    char str[1000];

    va_list ap;
    va_start(ap, fmt);
    vsnprintf(str, sizeof(str), fmt, ap);
    va_end(ap);

    PyErr_SetString(PyExc_TypeError, str);
    return 0;
}


class NumpyAllocator : public MatAllocator
{
public:
#if ( CV_MAJOR_VERSION < 3)
    NumpyAllocator() {}
    ~NumpyAllocator() {}

    void allocate(int dims, const int* sizes, int type, int*& refcount,
                  uchar*& datastart, uchar*& data, size_t* step)
    {

        //PyEnsureGIL gil;

        int depth = CV_MAT_DEPTH(type);
        int cn = CV_MAT_CN(type);

        const int f = (int)(sizeof(size_t)/8);
        int typenum = depth == CV_8U ? NPY_UBYTE : depth == CV_8S ? NPY_BYTE :
                                                                    depth == CV_16U ? NPY_USHORT : depth == CV_16S ? NPY_SHORT :
                                                                                                                     depth == CV_32S ? NPY_INT : depth == CV_32F ? NPY_FLOAT :
                                                                                                                                                                   depth == CV_64F ? NPY_DOUBLE : f*NPY_ULONGLONG + (f^1)*NPY_UINT;
        int i;

        npy_intp _sizes[CV_MAX_DIM+1];
        for( i = 0; i < dims; i++ )
        {
            _sizes[i] = sizes[i];
        }

        if( cn > 1 )
        {
            _sizes[dims++] = cn;
        }
        PyObject* o = PyArray_SimpleNew(dims, _sizes, typenum);
        if(!o)
        {

            CV_Error_(CV_StsError, ("The numpy array of typenum=%d, ndims=%d can not be created", typenum, dims));
        }
        refcount = refcountFromPyObject(o);

        npy_intp* _strides = PyArray_STRIDES(o);
        for( i = 0; i < dims - (cn > 1); i++ )
            step[i] = (size_t)_strides[i];

        datastart = data = (uchar*)PyArray_DATA(o);

    }

    void deallocate(int* refcount, uchar*, uchar*)
    {
        //PyEnsureGIL gil;
        if( !refcount )
            return;
        PyObject* o = pyObjectFromRefcount(refcount);
        Py_INCREF(o);
        Py_DECREF(o);
    }
#else

    NumpyAllocator() {
        stdAllocator = Mat::getStdAllocator();
    }
    ~NumpyAllocator() {
        }

        UMatData* allocate(PyObject* o, int dims, const int* sizes, int type,
                           size_t* step) const {
            UMatData* u = new UMatData(this);
            u->data = u->origdata = (uchar*) PyArray_DATA((PyArrayObject*) o);
            npy_intp* _strides = PyArray_STRIDES((PyArrayObject*) o);
            for (int i = 0; i < dims - 1; i++)
                step[i] = (size_t) _strides[i];
            step[dims - 1] = CV_ELEM_SIZE(type);
            u->size = sizes[0] * step[0];
            u->userdata = o;
            return u;
        }

        UMatData* allocate(int dims0, const int* sizes, int type, void* data,
                           size_t* step, int flags, UMatUsageFlags usageFlags) const {
            if (data != 0) {
                CV_Error(Error::StsAssert, "The data should normally be NULL!");
                // probably this is safe to do in such extreme case
                return stdAllocator->allocate(dims0, sizes, type, data, step, flags,
                                              usageFlags);
            }
            PyEnsureGIL gil;

            int depth = CV_MAT_DEPTH(type);
            int cn = CV_MAT_CN(type);
            const int f = (int) (sizeof(size_t) / 8);
            int typenum =
                    depth == CV_8U ? NPY_UBYTE :
                    depth == CV_8S ? NPY_BYTE :
                    depth == CV_16U ? NPY_USHORT :
                    depth == CV_16S ? NPY_SHORT :
                    depth == CV_32S ? NPY_INT :
                    depth == CV_32F ? NPY_FLOAT :
                    depth == CV_64F ?
                    NPY_DOUBLE :
                    f * NPY_ULONGLONG + (f ^ 1) * NPY_UINT;
            int i, dims = dims0;
            cv::AutoBuffer<npy_intp> _sizes(dims + 1);
            for (i = 0; i < dims; i++)
                _sizes[i] = sizes[i];
            if (cn > 1)
                _sizes[dims++] = cn;
            PyObject* o = PyArray_SimpleNew(dims, _sizes, typenum);
            if (!o)
                CV_Error_(Error::StsError,
                          ("The numpy array of typenum=%d, ndims=%d can not be created", typenum, dims));
            return allocate(o, dims0, sizes, type, step);
        }

        bool allocate(UMatData* u, int accessFlags,
                      UMatUsageFlags usageFlags) const {
            return stdAllocator->allocate(u, accessFlags, usageFlags);
        }

        void deallocate(UMatData* u) const {
            if (u) {
                PyEnsureGIL gil;
                PyObject* o = (PyObject*) u->userdata;
                Py_XDECREF(o);
                delete u;
            }
        }

        const MatAllocator* stdAllocator;
#endif
};

NumpyAllocator g_numpyAllocator;

NDArrayConverter::NDArrayConverter() { init(); }

void NDArrayConverter::init()
{
    import_array();
}


cv::Mat NDArrayConverter::toMat( PyObject *o)
{
    cv::Mat m;

    if(!o || o == Py_None)
    {
        if( !m.data )
            m.allocator = &g_numpyAllocator;
    }

    if( !PyArray_Check(o) )
    {
        failmsg("toMat: Object is not a numpy array");
    }

    int typenum = PyArray_TYPE(o);
    int type = typenum == NPY_UBYTE ? CV_8U : typenum == NPY_BYTE ? CV_8S :
                                                                    typenum == NPY_USHORT ? CV_16U : typenum == NPY_SHORT ? CV_16S :
                                                                                                                            typenum == NPY_INT || typenum == NPY_LONG ? CV_32S :
                                                                                                                                                                        typenum == NPY_FLOAT ? CV_32F :
                                                                                                                                                                                               typenum == NPY_DOUBLE ? CV_64F : -1;

    if( type < 0 )
    {
        failmsg("toMat: Data type = %d is not supported", typenum);
    }

    int ndims = PyArray_NDIM(o);

    if(ndims >= CV_MAX_DIM)
    {
        failmsg("toMat: Dimensionality (=%d) is too high", ndims);
    }

    int size[CV_MAX_DIM+1];
    size_t step[CV_MAX_DIM+1], elemsize = CV_ELEM_SIZE1(type);
    const npy_intp* _sizes = PyArray_DIMS(o);
    const npy_intp* _strides = PyArray_STRIDES(o);
    bool transposed = false;

    for(int i = 0; i < ndims; i++)
    {
        size[i] = (int)_sizes[i];
        step[i] = (size_t)_strides[i];
    }

    if( ndims == 0 || step[ndims-1] > elemsize ) {
        size[ndims] = 1;
        step[ndims] = elemsize;
        ndims++;
    }

    if( ndims >= 2 && step[0] < step[1] )
    {
        std::swap(size[0], size[1]);
        std::swap(step[0], step[1]);
        transposed = true;
    }

    if( ndims == 3 && size[2] <= CV_CN_MAX && step[1] == elemsize*size[2] )
    {
        ndims--;
        type |= CV_MAKETYPE(0, size[2]);
    }

    if( ndims > 2)
    {
        failmsg("toMat: Object has more than 2 dimensions");
    }

    m = Mat(ndims, size, type, PyArray_DATA(o), step);

    if( m.data )
    {
#if ( CV_MAJOR_VERSION < 3)
        m.refcount = refcountFromPyObject(o);
        m.addref(); // protect the original numpy array from deallocation
        // (since Mat destructor will decrement the reference counter)
#else
        m.u = g_numpyAllocator.allocate(o, ndims, size, type, step);
        m.addref();
        Py_INCREF(o);
        //m.u->refcount = *refcountFromPyObject(o);
#endif

    };
    m.allocator = &g_numpyAllocator;

    if( transposed )
    {
        Mat tmp;
        tmp.allocator = &g_numpyAllocator;
        transpose(m, tmp);
        m = tmp;
    }
    return m;
}

PyObject* NDArrayConverter::toNDArray(const cv::Mat& m)
{
    if( !m.data )
        Py_RETURN_NONE;
    Mat temp;
    Mat *p = (Mat*)&m;
#if ( CV_MAJOR_VERSION < 3)
    if(!p->refcount || p->allocator != &g_numpyAllocator)
    {
        temp.allocator = &g_numpyAllocator;
        m.copyTo(temp);
        p = &temp;
    }
    p->addref();
    return pyObjectFromRefcount(p->refcount);
#else
    if(!p->u || p->allocator != &g_numpyAllocator)
    {
        temp.allocator = &g_numpyAllocator;
        m.copyTo(temp);
        p = &temp;
    }
    //p->addref();
    //return pyObjectFromRefcount(&p->u->refcount);
    PyObject* o = (PyObject*) p->u->userdata;
    Py_INCREF(o);
    return o;
#endif

}

}

5, Compiling source code and running

Compile DynaSLAM source code

cd DynaSLAM
chmod +x build.sh
./build.sh

If the latter two parameters are not given at runtime, it is equivalent to running ORB-SLAM2
If you only want to use the function of MaskRCNN but do not want to save the mask, then in PATH_MASK is written as no_save, otherwise give a folder address to save mask

./Examples/RGB-D/rgbd_tum Vocabulary/ORBvoc.txt Examples/RGB-D/TUM3.yaml /XXX/tum_dataset/ /XXX/tum_dataset/associations.txt masks/ output/

If it is found that the Light Track has been unsuccessful and cannot be initialized, increase the number of feature points in the ORB parameter setting, and generally change it to 3000 on github.

End, sprinkle flowers~

Tags: OpenCV TensorFlow Computer Vision slam

Posted on Mon, 01 Nov 2021 12:40:42 -0400 by activomate