MDEV-23926: Fix warnings generated during compilation of plugin/auth_pam/mapper/pam_user_map.c on MacOS

Compiler warnings like one listed below are generated during server build on MacOS: [88%] Building C object plugin/auth_pam/CMakeFiles/pam_user_map.dir/mapper/pam_user_map.c.o mariadb/server-10.2/plugin/auth_pam/mapper/pam_user_map.c:87:41: error: passing 'gid_t *' (aka 'unsigned int *') to parameter of type 'int *' converts between pointers to integer types with different sign [-Werror,-Wpointer-sign] if (getgrouplist(user, user_group_id, loc_groups, &ng) < 0) ^~~~~~~~~~ /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX10.15.sdk/usr/include/unistd.h:650:43: note: passing argument to parameter here int getgrouplist(const char *, int, int *, int *); ^ In case MariaDB server is build with -DCMAKE_BUILD_TYPE=Debug it results in build error. The reason of compiler warnings is that declaration of the Posix C API function getgrouplist() on MacOS differs from declaration of getgrouplist() proposed by Posix. To suppress this compiler warning cmake configure was adapted to detect what kind of getgrouplist() function is declared on the build platform and set the macros HAVE_POSIX_GETGROUPLIST in case the building platform supports Posix compatible interface for the getgrouplist() function. Depending on whether this macros is set the compatible type of arguments is used to pass parameter values to the function.

Author: dmitryshulga

plugin/auth_pam/CMakeLists.txt

@@ -5,6 +5,30 @@ CHECK_INCLUDE_FILES (security/pam_ext.h HAVE_PAM_EXT_H)
 CHECK_INCLUDE_FILES (security/pam_appl.h HAVE_PAM_APPL_H)
 CHECK_FUNCTION_EXISTS (strndup HAVE_STRNDUP)
 
+INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR})
+
+# Check whether getgrouplist uses git_t for second and third arguments.
+SET(CMAKE_REQUIRED_FLAGS -Werror)
+CHECK_C_SOURCE_COMPILES(
+"
+#ifdef HAVE_GRP_H
+#include <grp.h>
+#endif
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+int main() {
+ char *arg_1;
+ gid_t arg_2, arg_3;
+ int arg_4;
+ (void)getgrouplist(arg_1,arg_2,&arg_3,arg_4);
+ return 0;
+}
+"
+HAVE_POSIX_GETGROUPLIST
+)
+SET(CMAKE_REQUIRED_FLAGS)
+
 SET(CMAKE_REQUIRED_LIBRARIES pam)
 CHECK_FUNCTION_EXISTS(pam_syslog HAVE_PAM_SYSLOG)
 SET(CMAKE_REQUIRED_LIBRARIES)
@@ -36,3 +60,5 @@ IF(HAVE_PAM_APPL_H)
  ENDIF()
 ENDIF(HAVE_PAM_APPL_H)
 
+CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/config.h.cmake
+ ${CMAKE_CURRENT_BINARY_DIR}/config_auth_pam.h)

plugin/auth_pam/mapper/pam_user_map.c

@@ -31,6 +31,7 @@ These comments are written to the syslog as 'authpriv.debug'
 and usually end up in /var/log/secure file.
 */
 
+#include <config_auth_pam.h>
 #include <stdlib.h>
 #include <stdio.h>
 #include <ctype.h>
@@ -70,10 +71,16 @@ pam_syslog (const pam_handle_t *pamh, int priority,
 #define GROUP_BUFFER_SIZE 100
 static const char debug_keyword[]= "debug";
 
-static int populate_user_groups(const char *user, gid_t **groups)
+#ifdef HAVE_POSIX_GETGROUPLIST
+typedef gid_t my_gid_t;
+#else
+typedef int my_gid_t;
+#endif
+
+static int populate_user_groups(const char *user, my_gid_t **groups)
 {
- gid_t user_group_id;
- gid_t *loc_groups= *groups;
+ my_gid_t user_group_id;
+ my_gid_t *loc_groups= *groups;
  int ng;
 
  {
@@ -88,22 +95,23 @@ static int populate_user_groups(const char *user, gid_t **groups)
  {
  /* The rare case when the user is present in more than */
  /* GROUP_BUFFER_SIZE groups. */
- loc_groups= (gid_t *) malloc(ng * sizeof (gid_t));
+ loc_groups= (my_gid_t *) malloc(ng * sizeof (my_gid_t));
+
  if (!loc_groups)
  return 0;
 
  (void) getgrouplist(user, user_group_id, loc_groups, &ng);
- *groups= loc_groups;
+ *groups= (my_gid_t*)loc_groups;
  }
 
  return ng;
 }
 
 
-static int user_in_group(const gid_t *user_groups, int ng,const char *group)
+static int user_in_group(const my_gid_t *user_groups, int ng,const char *group)
 {
- gid_t group_id;
- const gid_t *groups_end = user_groups + ng;
+ my_gid_t group_id;
+ const my_gid_t *groups_end = user_groups + ng;
 
  {
  struct group *g= getgrnam(group);
@@ -122,7 +130,7 @@ static int user_in_group(const gid_t *user_groups, int ng,const char *group)
 }
 
 
-static void print_groups(pam_handle_t *pamh, const gid_t *user_groups, int ng)
+static void print_groups(pam_handle_t *pamh, const my_gid_t *user_groups, int ng)
 {
  char buf[256];
  char *c_buf= buf, *buf_end= buf+sizeof(buf)-2;
@@ -158,8 +166,8 @@ int pam_sm_authenticate(pam_handle_t *pamh, int flags,
  const char *username;
  char buf[256];
  FILE *f;
- gid_t group_buffer[GROUP_BUFFER_SIZE];
- gid_t *groups= group_buffer;
+ my_gid_t group_buffer[GROUP_BUFFER_SIZE];
+ my_gid_t *groups= group_buffer;
  int n_groups= -1;
 
  for (; argc > 0; argc--)