From dbbb290043fe83a3c8fe189654448501bd769b04 Mon Sep 17 00:00:00 2001 From: Skalicky Date: Thu, 12 Aug 2021 16:22:51 -0700 Subject: [PATCH 1/3] stopped closing opened libs --- include/mxnet/c_api.h | 5 +++-- python/mxnet/library.py | 7 +++++-- src/c_api/c_api.cc | 23 +++++++++++++---------- src/initialize.cc | 4 ---- src/initialize.h | 2 +- 5 files changed, 22 insertions(+), 19 deletions(-) diff --git a/include/mxnet/c_api.h b/include/mxnet/c_api.h index 67751a497a05..b2f474627025 100644 --- a/include/mxnet/c_api.h +++ b/include/mxnet/c_api.h @@ -233,10 +233,11 @@ MXNET_DLL const char *MXGetLastError(); /*! * \brief Load library dynamically * \param path to the library .so file - * \param 0 for quiet, 1 for verbose + * \param verbose 0 for quiet, 1 for verbose + * \param lib handle to opened library * \return 0 when success, -1 when failure happens. */ -MXNET_DLL int MXLoadLib(const char *path, unsigned verbose); +MXNET_DLL int MXLoadLib(const char *path, unsigned verbose, void** lib); /*! * \brief Get list of features supported on the runtime diff --git a/python/mxnet/library.py b/python/mxnet/library.py index e0c60d4588f9..02e393af774a 100644 --- a/python/mxnet/library.py +++ b/python/mxnet/library.py @@ -37,7 +37,7 @@ def load(path, verbose=True): Returns --------- - void + ctypes.c_void_p : handle to opened library """ #check if path exists if not os.path.exists(path): @@ -53,7 +53,8 @@ def load(path, verbose=True): verbose_val = 1 if verbose else 0 byt_obj = path.encode('utf-8') chararr = ctypes.c_char_p(byt_obj) - check_call(_LIB.MXLoadLib(chararr, mx_uint(verbose_val))) + lib_ptr = ctypes.c_void_p(0) + check_call(_LIB.MXLoadLib(chararr, mx_uint(verbose_val), ctypes.byref(lib_ptr))) #regenerate operators _init_op_module('mxnet', 'ndarray', _make_ndarray_function) @@ -72,3 +73,5 @@ def load(path, verbose=True): for op in dir(mx_sym_op): func = getattr(mx_sym_op, op) setattr(mx_sym, op, func) + + return lib_ptr diff --git a/src/c_api/c_api.cc b/src/c_api/c_api.cc index 569564a934c0..1c8fb6693c3c 100644 --- a/src/c_api/c_api.cc +++ b/src/c_api/c_api.cc @@ -1514,16 +1514,19 @@ void registerPasses(void *lib, int verbose, mxnet::ext::msgSize_t msgSize, /*! * \brief Loads dynamic custom library and initializes it * \param path library path + * \param verbose 0 for quiet, 1 for verbose + * \param lib handle to opened library + * \return 0 when success, -1 when failure happens. */ -int MXLoadLib(const char *path, unsigned verbose) { +int MXLoadLib(const char *path, unsigned verbose, void** lib) { API_BEGIN(); - void *lib = LibraryInitializer::Get()->lib_load(path); - if (!lib) + *lib = LibraryInitializer::Get()->lib_load(path); + if (!*lib) LOG(FATAL) << "Unable to load library"; // check that library and MXNet use same version of library API mxnet::ext::opVersion_t opVersion = - get_func(lib, const_cast(MXLIB_OPVERSION_STR)); + get_func(*lib, const_cast(MXLIB_OPVERSION_STR)); int libVersion = opVersion(); if (MX_LIBRARY_VERSION != libVersion) LOG(FATAL) << "Library version (" << libVersion << ") does not match MXNet version (" @@ -1531,22 +1534,22 @@ int MXLoadLib(const char *path, unsigned verbose) { // get error messaging APIs mxnet::ext::msgSize_t msgSize = - get_func(lib, const_cast(MXLIB_MSGSIZE_STR)); + get_func(*lib, const_cast(MXLIB_MSGSIZE_STR)); mxnet::ext::msgGet_t msgGet = - get_func(lib, const_cast(MXLIB_MSGGET_STR)); + get_func(*lib, const_cast(MXLIB_MSGGET_STR)); // initialize library by passing MXNet version mxnet::ext::initialize_t initialize = - get_func(lib, const_cast(MXLIB_INITIALIZE_STR)); + get_func(*lib, const_cast(MXLIB_INITIALIZE_STR)); if (!initialize(static_cast(MXNET_VERSION))) { std::string msgs = getExtensionMsgs(msgSize, msgGet); LOG(FATAL) << "Library failed to initialize" << msgs; } // find ops, partitioners, and passes in library - registerOperators(lib, verbose, msgSize, msgGet); - registerPartitioners(lib, verbose, msgSize, msgGet); - registerPasses(lib, verbose, msgSize, msgGet); + registerOperators(*lib, verbose, msgSize, msgGet); + registerPartitioners(*lib, verbose, msgSize, msgGet); + registerPasses(*lib, verbose, msgSize, msgGet); API_END(); } diff --git a/src/initialize.cc b/src/initialize.cc index 84195f961f80..e06f90504a7a 100644 --- a/src/initialize.cc +++ b/src/initialize.cc @@ -96,10 +96,6 @@ LibraryInitializer::LibraryInitializer() install_pthread_atfork_handlers(); } -LibraryInitializer::~LibraryInitializer() { - close_open_libs(); -} - bool LibraryInitializer::lib_is_loaded(const std::string& path) const { return loaded_libs.count(path) > 0; } diff --git a/src/initialize.h b/src/initialize.h index f12b2f6e28c2..04e2322961a7 100644 --- a/src/initialize.h +++ b/src/initialize.h @@ -56,7 +56,7 @@ class LibraryInitializer { */ LibraryInitializer(); - ~LibraryInitializer(); + ~LibraryInitializer() = default; /** * @return true if the current pid doesn't match the one that initialized the library From 03ce69d5a563e608821b0b229b40ba1840989475 Mon Sep 17 00:00:00 2001 From: Skalicky Date: Wed, 18 Aug 2021 10:45:26 -0700 Subject: [PATCH 2/3] removed return handling from load --- include/mxnet/c_api.h | 3 +-- python/mxnet/library.py | 7 ++----- src/c_api/c_api.cc | 19 +++++++++---------- 3 files changed, 12 insertions(+), 17 deletions(-) diff --git a/include/mxnet/c_api.h b/include/mxnet/c_api.h index b2f474627025..22afd4fda592 100644 --- a/include/mxnet/c_api.h +++ b/include/mxnet/c_api.h @@ -234,10 +234,9 @@ MXNET_DLL const char *MXGetLastError(); * \brief Load library dynamically * \param path to the library .so file * \param verbose 0 for quiet, 1 for verbose - * \param lib handle to opened library * \return 0 when success, -1 when failure happens. */ -MXNET_DLL int MXLoadLib(const char *path, unsigned verbose, void** lib); +MXNET_DLL int MXLoadLib(const char *path, unsigned verbose); /*! * \brief Get list of features supported on the runtime diff --git a/python/mxnet/library.py b/python/mxnet/library.py index 02e393af774a..da16af257957 100644 --- a/python/mxnet/library.py +++ b/python/mxnet/library.py @@ -37,7 +37,7 @@ def load(path, verbose=True): Returns --------- - ctypes.c_void_p : handle to opened library + None """ #check if path exists if not os.path.exists(path): @@ -53,8 +53,7 @@ def load(path, verbose=True): verbose_val = 1 if verbose else 0 byt_obj = path.encode('utf-8') chararr = ctypes.c_char_p(byt_obj) - lib_ptr = ctypes.c_void_p(0) - check_call(_LIB.MXLoadLib(chararr, mx_uint(verbose_val), ctypes.byref(lib_ptr))) + check_call(_LIB.MXLoadLib(chararr, mx_uint(verbose_val))) #regenerate operators _init_op_module('mxnet', 'ndarray', _make_ndarray_function) @@ -73,5 +72,3 @@ def load(path, verbose=True): for op in dir(mx_sym_op): func = getattr(mx_sym_op, op) setattr(mx_sym, op, func) - - return lib_ptr diff --git a/src/c_api/c_api.cc b/src/c_api/c_api.cc index 1c8fb6693c3c..9c462b8d0dff 100644 --- a/src/c_api/c_api.cc +++ b/src/c_api/c_api.cc @@ -1515,18 +1515,17 @@ void registerPasses(void *lib, int verbose, mxnet::ext::msgSize_t msgSize, * \brief Loads dynamic custom library and initializes it * \param path library path * \param verbose 0 for quiet, 1 for verbose - * \param lib handle to opened library * \return 0 when success, -1 when failure happens. */ -int MXLoadLib(const char *path, unsigned verbose, void** lib) { +int MXLoadLib(const char *path, unsigned verbose) { API_BEGIN(); - *lib = LibraryInitializer::Get()->lib_load(path); + void *lib = LibraryInitializer::Get()->lib_load(path); if (!*lib) LOG(FATAL) << "Unable to load library"; // check that library and MXNet use same version of library API mxnet::ext::opVersion_t opVersion = - get_func(*lib, const_cast(MXLIB_OPVERSION_STR)); + get_func(lib, const_cast(MXLIB_OPVERSION_STR)); int libVersion = opVersion(); if (MX_LIBRARY_VERSION != libVersion) LOG(FATAL) << "Library version (" << libVersion << ") does not match MXNet version (" @@ -1534,22 +1533,22 @@ int MXLoadLib(const char *path, unsigned verbose, void** lib) { // get error messaging APIs mxnet::ext::msgSize_t msgSize = - get_func(*lib, const_cast(MXLIB_MSGSIZE_STR)); + get_func(lib, const_cast(MXLIB_MSGSIZE_STR)); mxnet::ext::msgGet_t msgGet = - get_func(*lib, const_cast(MXLIB_MSGGET_STR)); + get_func(lib, const_cast(MXLIB_MSGGET_STR)); // initialize library by passing MXNet version mxnet::ext::initialize_t initialize = - get_func(*lib, const_cast(MXLIB_INITIALIZE_STR)); + get_func(lib, const_cast(MXLIB_INITIALIZE_STR)); if (!initialize(static_cast(MXNET_VERSION))) { std::string msgs = getExtensionMsgs(msgSize, msgGet); LOG(FATAL) << "Library failed to initialize" << msgs; } // find ops, partitioners, and passes in library - registerOperators(*lib, verbose, msgSize, msgGet); - registerPartitioners(*lib, verbose, msgSize, msgGet); - registerPasses(*lib, verbose, msgSize, msgGet); + registerOperators(lib, verbose, msgSize, msgGet); + registerPartitioners(lib, verbose, msgSize, msgGet); + registerPasses(lib, verbose, msgSize, msgGet); API_END(); } From fdffbe830267d4699c96c00a7b1f0a2cd7a578ae Mon Sep 17 00:00:00 2001 From: Skalicky Date: Wed, 18 Aug 2021 10:46:40 -0700 Subject: [PATCH 3/3] missed one *lib --- src/c_api/c_api.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/c_api/c_api.cc b/src/c_api/c_api.cc index 9c462b8d0dff..78c7b29a83b0 100644 --- a/src/c_api/c_api.cc +++ b/src/c_api/c_api.cc @@ -1520,7 +1520,7 @@ void registerPasses(void *lib, int verbose, mxnet::ext::msgSize_t msgSize, int MXLoadLib(const char *path, unsigned verbose) { API_BEGIN(); void *lib = LibraryInitializer::Get()->lib_load(path); - if (!*lib) + if (!lib) LOG(FATAL) << "Unable to load library"; // check that library and MXNet use same version of library API