[OpenMP] Start organizing PluginManager, PluginAdaptors #73875
Merged
Conversation
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
`RTLInfoTy` is now `PluginAdaptorTy`, as it provides the offload runtime access to the plugins. `RTLsTy` is now `PluginAdaptorManagerTy`, as it manages all `PluginAdaptorTy` objects. We should reorganize this and at the very least merge the latter with the `PluginManager`.
Member
| @llvm/pr-subscribers-openmp Author: Johannes Doerfert (jdoerfert) ChangesPatch is 27.55 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/73875.diff 11 Files Affected:
diff --git a/openmp/libomptarget/include/PluginManager.h b/openmp/libomptarget/include/PluginManager.h new file mode 100644 index 000000000000000..6a91e245453c868 --- /dev/null +++ b/openmp/libomptarget/include/PluginManager.h @@ -0,0 +1,226 @@ +//===-- PluginManager.h - Plugin loading and communication API --*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Declarations for managing devices that are handled by RTL plugins. +// +//===----------------------------------------------------------------------===// + +#ifndef OMPTARGET_PLUGIN_MANAGER_H +#define OMPTARGET_PLUGIN_MANAGER_H + +#include "Shared/APITypes.h" + +#include "device.h" + +#include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/Support/DynamicLibrary.h" + +#include <list> +#include <mutex> + +struct PluginAdaptorTy { + typedef int32_t(init_plugin_ty)(); + typedef int32_t(is_valid_binary_ty)(void *); + typedef int32_t(is_valid_binary_info_ty)(void *, void *); + typedef int32_t(is_data_exchangable_ty)(int32_t, int32_t); + typedef int32_t(number_of_devices_ty)(); + typedef int32_t(init_device_ty)(int32_t); + typedef __tgt_target_table *(load_binary_ty)(int32_t, void *); + typedef void *(data_alloc_ty)(int32_t, int64_t, void *, int32_t); + typedef int32_t(data_submit_ty)(int32_t, void *, void *, int64_t); + typedef int32_t(data_submit_async_ty)(int32_t, void *, void *, int64_t, + __tgt_async_info *); + typedef int32_t(data_retrieve_ty)(int32_t, void *, void *, int64_t); + typedef int32_t(data_retrieve_async_ty)(int32_t, void *, void *, int64_t, + __tgt_async_info *); + typedef int32_t(data_exchange_ty)(int32_t, void *, int32_t, void *, int64_t); + typedef int32_t(data_exchange_async_ty)(int32_t, void *, int32_t, void *, + int64_t, __tgt_async_info *); + typedef int32_t(data_delete_ty)(int32_t, void *, int32_t); + typedef int32_t(launch_kernel_ty)(int32_t, void *, void **, ptrdiff_t *, + const KernelArgsTy *, __tgt_async_info *); + typedef int64_t(init_requires_ty)(int64_t); + typedef int32_t(synchronize_ty)(int32_t, __tgt_async_info *); + typedef int32_t(query_async_ty)(int32_t, __tgt_async_info *); + typedef int32_t(supports_empty_images_ty)(); + typedef void(print_device_info_ty)(int32_t); + typedef void(set_info_flag_ty)(uint32_t); + typedef int32_t(create_event_ty)(int32_t, void **); + typedef int32_t(record_event_ty)(int32_t, void *, __tgt_async_info *); + typedef int32_t(wait_event_ty)(int32_t, void *, __tgt_async_info *); + typedef int32_t(sync_event_ty)(int32_t, void *); + typedef int32_t(destroy_event_ty)(int32_t, void *); + typedef int32_t(release_async_info_ty)(int32_t, __tgt_async_info *); + typedef int32_t(init_async_info_ty)(int32_t, __tgt_async_info **); + typedef int64_t(init_device_into_ty)(int64_t, __tgt_device_info *, + const char **); + typedef int32_t(data_lock_ty)(int32_t, void *, int64_t, void **); + typedef int32_t(data_unlock_ty)(int32_t, void *); + typedef int32_t(data_notify_mapped_ty)(int32_t, void *, int64_t); + typedef int32_t(data_notify_unmapped_ty)(int32_t, void *); + typedef int32_t(set_device_offset_ty)(int32_t); + typedef int32_t(activate_record_replay_ty)(int32_t, uint64_t, void *, bool, + bool, uint64_t &); + + int32_t Idx = -1; // RTL index, index is the number of devices + // of other RTLs that were registered before, + // i.e. the OpenMP index of the first device + // to be registered with this RTL. + int32_t NumberOfDevices = -1; // Number of devices this RTL deals with. + + std::unique_ptr<llvm::sys::DynamicLibrary> LibraryHandler; + +#ifdef OMPTARGET_DEBUG + std::string RTLName; +#endif + + // Functions implemented in the RTL. + init_plugin_ty *init_plugin = nullptr; + is_valid_binary_ty *is_valid_binary = nullptr; + is_valid_binary_info_ty *is_valid_binary_info = nullptr; + is_data_exchangable_ty *is_data_exchangable = nullptr; + number_of_devices_ty *number_of_devices = nullptr; + init_device_ty *init_device = nullptr; + load_binary_ty *load_binary = nullptr; + data_alloc_ty *data_alloc = nullptr; + data_submit_ty *data_submit = nullptr; + data_submit_async_ty *data_submit_async = nullptr; + data_retrieve_ty *data_retrieve = nullptr; + data_retrieve_async_ty *data_retrieve_async = nullptr; + data_exchange_ty *data_exchange = nullptr; + data_exchange_async_ty *data_exchange_async = nullptr; + data_delete_ty *data_delete = nullptr; + launch_kernel_ty *launch_kernel = nullptr; + init_requires_ty *init_requires = nullptr; + synchronize_ty *synchronize = nullptr; + query_async_ty *query_async = nullptr; + supports_empty_images_ty *supports_empty_images = nullptr; + set_info_flag_ty *set_info_flag = nullptr; + print_device_info_ty *print_device_info = nullptr; + create_event_ty *create_event = nullptr; + record_event_ty *record_event = nullptr; + wait_event_ty *wait_event = nullptr; + sync_event_ty *sync_event = nullptr; + destroy_event_ty *destroy_event = nullptr; + init_async_info_ty *init_async_info = nullptr; + init_device_into_ty *init_device_info = nullptr; + release_async_info_ty *release_async_info = nullptr; + data_lock_ty *data_lock = nullptr; + data_unlock_ty *data_unlock = nullptr; + data_notify_mapped_ty *data_notify_mapped = nullptr; + data_notify_unmapped_ty *data_notify_unmapped = nullptr; + set_device_offset_ty *set_device_offset = nullptr; + activate_record_replay_ty *activate_record_replay = nullptr; + + // Are there images associated with this RTL. + bool IsUsed = false; + + llvm::DenseSet<const __tgt_device_image *> UsedImages; + + // Mutex for thread-safety when calling RTL interface functions. + // It is easier to enforce thread-safety at the libomptarget level, + // so that developers of new RTLs do not have to worry about it. + std::mutex Mtx; +}; + +/// RTLs identified in the system. +struct PluginAdaptorManagerTy { + // List of the detected runtime libraries. + std::list<PluginAdaptorTy> AllRTLs; + + // Array of pointers to the detected runtime libraries that have compatible + // binaries. + llvm::SmallVector<PluginAdaptorTy *> UsedRTLs; + + int64_t RequiresFlags = OMP_REQ_UNDEFINED; + + explicit PluginAdaptorManagerTy() = default; + + // Register the clauses of the requires directive. + void registerRequires(int64_t Flags); + + // Initialize RTL if it has not been initialized + void initRTLonce(PluginAdaptorTy &RTL); + + // Initialize all RTLs + void initAllRTLs(); + + // Register a shared library with all (compatible) RTLs. + void registerLib(__tgt_bin_desc *Desc); + + // Unregister a shared library from all RTLs. + void unregisterLib(__tgt_bin_desc *Desc); + + // not thread-safe, called from global constructor (i.e. once) + void loadRTLs(); + +private: + static bool attemptLoadRTL(const std::string &RTLName, PluginAdaptorTy &RTL); +}; + +/// Struct for the data required to handle plugins +struct PluginManager { + PluginManager(bool UseEventsForAtomicTransfers) + : UseEventsForAtomicTransfers(UseEventsForAtomicTransfers) {} + + /// RTLs identified on the host + PluginAdaptorManagerTy RTLs; + + /// Executable images and information extracted from the input images passed + /// to the runtime. + std::list<std::pair<__tgt_device_image, __tgt_image_info>> Images; + + /// Devices associated with RTLs + llvm::SmallVector<std::unique_ptr<DeviceTy>> Devices; + std::mutex RTLsMtx; ///< For RTLs and Devices + + /// Translation table retreived from the binary + HostEntriesBeginToTransTableTy HostEntriesBeginToTransTable; + std::mutex TrlTblMtx; ///< For Translation Table + /// Host offload entries in order of image registration + llvm::SmallVector<__tgt_offload_entry *> HostEntriesBeginRegistrationOrder; + + /// Map from ptrs on the host to an entry in the Translation Table + HostPtrToTableMapTy HostPtrToTableMap; + std::mutex TblMapMtx; ///< For HostPtrToTableMap + + // Store target policy (disabled, mandatory, default) + kmp_target_offload_kind_t TargetOffloadPolicy = tgt_default; + std::mutex TargetOffloadMtx; ///< For TargetOffloadPolicy + + /// Flag to indicate if we use events to ensure the atomicity of + /// map clauses or not. Can be modified with an environment variable. + const bool UseEventsForAtomicTransfers; + + // Work around for plugins that call dlopen on shared libraries that call + // tgt_register_lib during their initialisation. Stash the pointers in a + // vector until the plugins are all initialised and then register them. + bool delayRegisterLib(__tgt_bin_desc *Desc) { + if (RTLsLoaded) + return false; + DelayedBinDesc.push_back(Desc); + return true; + } + + void registerDelayedLibraries() { + // Only called by libomptarget constructor + RTLsLoaded = true; + for (auto *Desc : DelayedBinDesc) + __tgt_register_lib(Desc); + DelayedBinDesc.clear(); + } + +private: + bool RTLsLoaded = false; + llvm::SmallVector<__tgt_bin_desc *> DelayedBinDesc; +}; + +extern PluginManager *PM; + +#endif // OMPTARGET_PLUGIN_MANAGER_H diff --git a/openmp/libomptarget/include/device.h b/openmp/libomptarget/include/device.h index 9cea6e6c9393012..5e58879bd738bd1 100644 --- a/openmp/libomptarget/include/device.h +++ b/openmp/libomptarget/include/device.h @@ -28,10 +28,8 @@ #include "OpenMP/Mapping.h" -#include "llvm/ADT/SmallVector.h" - // Forward declarations. -struct RTLInfoTy; +struct PluginAdaptorTy; struct __tgt_bin_desc; struct __tgt_target_table; @@ -53,7 +51,7 @@ typedef std::map<__tgt_bin_desc *, PendingCtorDtorListsTy> struct DeviceTy { int32_t DeviceID; - RTLInfoTy *RTL; + PluginAdaptorTy *RTL; int32_t RTLDeviceID; bool IsInit; @@ -77,7 +75,7 @@ struct DeviceTy { std::mutex PendingGlobalsMtx; - DeviceTy(RTLInfoTy *RTL); + DeviceTy(PluginAdaptorTy *RTL); // DeviceTy is not copyable DeviceTy(const DeviceTy &D) = delete; DeviceTy &operator=(const DeviceTy &D) = delete; @@ -243,63 +241,4 @@ struct DeviceTy { extern bool deviceIsReady(int DeviceNum); -/// Struct for the data required to handle plugins -struct PluginManager { - PluginManager(bool UseEventsForAtomicTransfers) - : UseEventsForAtomicTransfers(UseEventsForAtomicTransfers) {} - - /// RTLs identified on the host - RTLsTy RTLs; - - /// Executable images and information extracted from the input images passed - /// to the runtime. - std::list<std::pair<__tgt_device_image, __tgt_image_info>> Images; - - /// Devices associated with RTLs - llvm::SmallVector<std::unique_ptr<DeviceTy>> Devices; - std::mutex RTLsMtx; ///< For RTLs and Devices - - /// Translation table retreived from the binary - HostEntriesBeginToTransTableTy HostEntriesBeginToTransTable; - std::mutex TrlTblMtx; ///< For Translation Table - /// Host offload entries in order of image registration - llvm::SmallVector<__tgt_offload_entry *> HostEntriesBeginRegistrationOrder; - - /// Map from ptrs on the host to an entry in the Translation Table - HostPtrToTableMapTy HostPtrToTableMap; - std::mutex TblMapMtx; ///< For HostPtrToTableMap - - // Store target policy (disabled, mandatory, default) - kmp_target_offload_kind_t TargetOffloadPolicy = tgt_default; - std::mutex TargetOffloadMtx; ///< For TargetOffloadPolicy - - /// Flag to indicate if we use events to ensure the atomicity of - /// map clauses or not. Can be modified with an environment variable. - const bool UseEventsForAtomicTransfers; - - // Work around for plugins that call dlopen on shared libraries that call - // tgt_register_lib during their initialisation. Stash the pointers in a - // vector until the plugins are all initialised and then register them. - bool delayRegisterLib(__tgt_bin_desc *Desc) { - if (RTLsLoaded) - return false; - DelayedBinDesc.push_back(Desc); - return true; - } - - void registerDelayedLibraries() { - // Only called by libomptarget constructor - RTLsLoaded = true; - for (auto *Desc : DelayedBinDesc) - __tgt_register_lib(Desc); - DelayedBinDesc.clear(); - } - -private: - bool RTLsLoaded = false; - llvm::SmallVector<__tgt_bin_desc *> DelayedBinDesc; -}; - -extern PluginManager *PM; - #endif diff --git a/openmp/libomptarget/include/rtl.h b/openmp/libomptarget/include/rtl.h index 49a62685dcdbfa7..d110e89de5f14ea 100644 --- a/openmp/libomptarget/include/rtl.h +++ b/openmp/libomptarget/include/rtl.h @@ -13,162 +13,12 @@ #ifndef _OMPTARGET_RTL_H #define _OMPTARGET_RTL_H -#include "omptarget.h" -#include "llvm/ADT/DenseSet.h" #include "llvm/ADT/SmallVector.h" -#include "llvm/Support/DynamicLibrary.h" #include "omptarget.h" #include <cstdint> -#include <list> #include <map> -#include <mutex> -#include <string> - -// Forward declarations. -struct DeviceTy; -struct __tgt_bin_desc; - -struct RTLInfoTy { - typedef int32_t(init_plugin_ty)(); - typedef int32_t(is_valid_binary_ty)(void *); - typedef int32_t(is_valid_binary_info_ty)(void *, void *); - typedef int32_t(is_data_exchangable_ty)(int32_t, int32_t); - typedef int32_t(number_of_devices_ty)(); - typedef int32_t(init_device_ty)(int32_t); - typedef __tgt_target_table *(load_binary_ty)(int32_t, void *); - typedef void *(data_alloc_ty)(int32_t, int64_t, void *, int32_t); - typedef int32_t(data_submit_ty)(int32_t, void *, void *, int64_t); - typedef int32_t(data_submit_async_ty)(int32_t, void *, void *, int64_t, - __tgt_async_info *); - typedef int32_t(data_retrieve_ty)(int32_t, void *, void *, int64_t); - typedef int32_t(data_retrieve_async_ty)(int32_t, void *, void *, int64_t, - __tgt_async_info *); - typedef int32_t(data_exchange_ty)(int32_t, void *, int32_t, void *, int64_t); - typedef int32_t(data_exchange_async_ty)(int32_t, void *, int32_t, void *, - int64_t, __tgt_async_info *); - typedef int32_t(data_delete_ty)(int32_t, void *, int32_t); - typedef int32_t(launch_kernel_ty)(int32_t, void *, void **, ptrdiff_t *, - const KernelArgsTy *, __tgt_async_info *); - typedef int64_t(init_requires_ty)(int64_t); - typedef int32_t(synchronize_ty)(int32_t, __tgt_async_info *); - typedef int32_t(query_async_ty)(int32_t, __tgt_async_info *); - typedef int32_t(supports_empty_images_ty)(); - typedef void(print_device_info_ty)(int32_t); - typedef void(set_info_flag_ty)(uint32_t); - typedef int32_t(create_event_ty)(int32_t, void **); - typedef int32_t(record_event_ty)(int32_t, void *, __tgt_async_info *); - typedef int32_t(wait_event_ty)(int32_t, void *, __tgt_async_info *); - typedef int32_t(sync_event_ty)(int32_t, void *); - typedef int32_t(destroy_event_ty)(int32_t, void *); - typedef int32_t(release_async_info_ty)(int32_t, __tgt_async_info *); - typedef int32_t(init_async_info_ty)(int32_t, __tgt_async_info **); - typedef int64_t(init_device_into_ty)(int64_t, __tgt_device_info *, - const char **); - typedef int32_t(data_lock_ty)(int32_t, void *, int64_t, void **); - typedef int32_t(data_unlock_ty)(int32_t, void *); - typedef int32_t(data_notify_mapped_ty)(int32_t, void *, int64_t); - typedef int32_t(data_notify_unmapped_ty)(int32_t, void *); - typedef int32_t(set_device_offset_ty)(int32_t); - typedef int32_t(activate_record_replay_ty)(int32_t, uint64_t, void *, bool, - bool, uint64_t &); - - int32_t Idx = -1; // RTL index, index is the number of devices - // of other RTLs that were registered before, - // i.e. the OpenMP index of the first device - // to be registered with this RTL. - int32_t NumberOfDevices = -1; // Number of devices this RTL deals with. - - std::unique_ptr<llvm::sys::DynamicLibrary> LibraryHandler; - -#ifdef OMPTARGET_DEBUG - std::string RTLName; -#endif - - // Functions implemented in the RTL. - init_plugin_ty *init_plugin = nullptr; - is_valid_binary_ty *is_valid_binary = nullptr; - is_valid_binary_info_ty *is_valid_binary_info = nullptr; - is_data_exchangable_ty *is_data_exchangable = nullptr; - number_of_devices_ty *number_of_devices = nullptr; - init_device_ty *init_device = nullptr; - load_binary_ty *load_binary = nullptr; - data_alloc_ty *data_alloc = nullptr; - data_submit_ty *data_submit = nullptr; - data_submit_async_ty *data_submit_async = nullptr; - data_retrieve_ty *data_retrieve = nullptr; - data_retrieve_async_ty *data_retrieve_async = nullptr; - data_exchange_ty *data_exchange = nullptr; - data_exchange_async_ty *data_exchange_async = nullptr; - data_delete_ty *data_delete = nullptr; - launch_kernel_ty *launch_kernel = nullptr; - init_requires_ty *init_requires = nullptr; - synchronize_ty *synchronize = nullptr; - query_async_ty *query_async = nullptr; - supports_empty_images_ty *supports_empty_images = nullptr; - set_info_flag_ty *set_info_flag = nullptr; - print_device_info_ty *print_device_info = nullptr; - create_event_ty *create_event = nullptr; - record_event_ty *record_event = nullptr; - wait_event_ty *wait_event = nullptr; - sync_event_ty *sync_event = nullptr; - destroy_event_ty *destroy_event = nullptr; - init_async_info_ty *init_async_info = nullptr; - init_device_into_ty *init_device_info = nullptr; - release_async_info_ty *release_async_info = nullptr; - data_lock_ty *data_lock = nullptr; - data_unlock_ty *data_unlock = nullptr; - data_notify_mapped_ty *data_notify_mapped = nullptr; - data_notify_unmapped_ty *data_notify_unmapped = nullptr; - set_device_offset_ty *set_device_offset = nullptr; - activate_record_replay_ty *activate_record_replay = nullptr; - - // Are there images associated with this RTL. - bool IsUsed = false; - - llvm::DenseSet<const __tgt_device_image *> UsedImages; - - // Mutex for thread-safety when calling RTL interface functions. - // It is easier to enforce thread-safety at the libomptarget level, - // so that developers of new RTLs do not have to worry about it. - std::mutex Mtx; -}; - -/// RTLs identified in the system. -struct RTLsTy { - // List of the detected runtime libraries. - std::list<RTLInfoTy> AllRTLs; - - // Array of pointers to the detected runtime libraries that have compatible - // binaries. - llvm::SmallVector<RTLInfoTy *> UsedRTLs; - - int64_t RequiresFlags = OMP_REQ_UNDEFINED; - - explicit RTLsTy() = default; - - // Register the clauses of the requires directive. - void registerRequires(int64_t Flags); - - // Initialize RTL if it has not been initialized - void initRTLonce(RTLInfoTy &RTL); - - // Initialize all RTLs - void initAllRTLs(); - - // Register a shared library with all (compatible) RTLs. - void registerLib(__tgt_bin_desc *Desc); - - // Unregister a shared library from all RTLs. - void unregisterLib(__tgt_bin_desc *Desc); - - // not thread-safe, called from global constructor (i.e. once) - void loadRTLs(); - -private: - static bool attemptLoadRTL(const std::string &RTLName, RTLInfoTy &RTL); -}; /// Map between the host entry begin and the translation table. Each /// registered library gets one TranslationTable. Use the map from diff --git a/openmp/libomptarget/src/CMakeLists.txt b/openmp/libomptarget/src/CMakeLists.txt index 8daa448a57e9042..ca40ace456458b4 100644 --- a/openmp/libomptarget/src/CMakeLists.txt +++ b/openmp/libomptarget/src/CMakeLists.txt @@ -21,6 +21,7 @@ add_llvm_library(omptarget omptarget.cpp rtl.cpp LegacyAPI.cpp + PluginManager.cpp OpenMP/Mapping.cpp OpenMP/InteropAPI.cpp diff --git a/openmp/libomptarget/src/OpenMP/InteropAPI.cpp b/openmp/libomptarget/src/OpenMP/InteropAPI.cpp index ace0fecd0a43624..6a40dbca87afd42 100644 --- a/openmp/libomptarget/src/OpenMP/InteropAPI.cpp +++ b/openmp/libomptarget/src/OpenMP/InteropAPI.cpp @@ -10,6 +10,7 @@ ... [truncated] |
You can test this locally with the following command:git-clang-format --diff 2e7f47d4a8a5206fbc565d3eb8c1595c2cb875f8 625a461801d921817e333785e3ce9930a5277973 -- openmp/libomptarget/include/PluginManager.h openmp/libomptarget/src/PluginManager.cpp openmp/libomptarget/include/device.h openmp/libomptarget/include/rtl.h openmp/libomptarget/src/OpenMP/InteropAPI.cpp openmp/libomptarget/src/api.cpp openmp/libomptarget/src/device.cpp openmp/libomptarget/src/interface.cpp openmp/libomptarget/src/omptarget.cpp openmp/libomptarget/src/rtl.cppView the diff from clang-format here.diff --git a/openmp/libomptarget/src/rtl.cpp b/openmp/libomptarget/src/rtl.cpp index 6248acff68..3687208670 100644 --- a/openmp/libomptarget/src/rtl.cpp +++ b/openmp/libomptarget/src/rtl.cpp @@ -115,7 +115,8 @@ void PluginAdaptorManagerTy::loadRTLs() { DP("RTLs loaded!\n"); } -bool PluginAdaptorManagerTy::attemptLoadRTL(const std::string &RTLName, PluginAdaptorTy &RTL) { +bool PluginAdaptorManagerTy::attemptLoadRTL(const std::string &RTLName, + PluginAdaptorTy &RTL) { const char *Name = RTLName.c_str(); DP("Loading library '%s'...\n", Name); |
jhuber6 approved these changes Nov 30, 2023
Contributor
jhuber6 left a comment
jhuber6 left a comment There was a problem hiding this comment.
I don't like a large portion of the handling here, but it's just renaming what we already have.
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
3 participants
Add this suggestion to a batch that can be applied as a single commit. This suggestion is invalid because no changes were made to the code. Suggestions cannot be applied while the pull request is closed. Suggestions cannot be applied while viewing a subset of changes. Only one suggestion per line can be applied in a batch. Add this suggestion to a batch that can be applied as a single commit. Applying suggestions on deleted lines is not supported. You must change the existing code in this line in order to create a valid suggestion. Outdated suggestions cannot be applied. This suggestion has been applied or marked resolved. Suggestions cannot be applied from pending reviews. Suggestions cannot be applied on multi-line comments. Suggestions cannot be applied while the pull request is queued to merge. Suggestion cannot be applied right now. Please check back later.
No description provided.