reallocarray: check for ptrdiff_t overflow

* doc/glibc-functions/reallocarray.texi (reallocarray):
Mention ptrdiff_t overflow.
* lib/reallocarray.c (reallocarray): Reindent as per usual GNU.
* lib/stdlib.in.h (reallocarray): Allow reallocarray to be replaced.
* m4/reallocarray.m4 (gl_FUNC_REALLOCARRAY):
Check for ptrdiff_t overflow.
* m4/stdlib_h.m4 (gl_STDLIB_H_DEFAULTS): Set up REPLACE_REALLOCARRAY.
* modules/reallocarray (Files): Add malloc.m4.
(configure.ac): Also test REPLACE_REALLOCARRAY.
* modules/reallocarray-tests (Depends-on): Add stdint.
* modules/stdlib (stdlib.h): Substitute REPLACE_REALLOCARRAY.
* tests/test-reallocarray.c: Include stdint.h.
(main): Check for ptrdiff_t overflow.

10 files changed, 73 insertions, 29 deletions

diff --git a/ChangeLog b/ChangeLog
index baa74c3c62..c198830bf5 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,20 @@
 2021-04-24  Paul Eggert  <eggert@cs.ucla.edu>
 
+	reallocarray: check for ptrdiff_t overflow
+	* doc/glibc-functions/reallocarray.texi (reallocarray):
+	Mention ptrdiff_t overflow.
+	* lib/reallocarray.c (reallocarray): Reindent as per usual GNU.
+	* lib/stdlib.in.h (reallocarray): Allow reallocarray to be replaced.
+	* m4/reallocarray.m4 (gl_FUNC_REALLOCARRAY):
+	Check for ptrdiff_t overflow.
+	* m4/stdlib_h.m4 (gl_STDLIB_H_DEFAULTS): Set up REPLACE_REALLOCARRAY.
+	* modules/reallocarray (Files): Add malloc.m4.
+	(configure.ac): Also test REPLACE_REALLOCARRAY.
+	* modules/reallocarray-tests (Depends-on): Add stdint.
+	* modules/stdlib (stdlib.h): Substitute REPLACE_REALLOCARRAY.
+	* tests/test-reallocarray.c: Include stdint.h.
+	(main): Check for ptrdiff_t overflow.
+
 	calloc-gnu-tests: add overflow tests
 	* tests/test-calloc-gnu.c (identity): New function, replacing ‘eight’.
 	(main): Do 2 * log2(SIZE_MAX) tests instead of just two tests.
diff --git a/doc/glibc-functions/reallocarray.texi b/doc/glibc-functions/reallocarray.texi
index 903233df19..b1337eff21 100644
--- a/doc/glibc-functions/reallocarray.texi
+++ b/doc/glibc-functions/reallocarray.texi
@@ -26,6 +26,12 @@ Portability problems fixed by Gnulib:
 @item
 This function is missing on many platforms:
 glibc 2.25, macOS 11.1, FreeBSD 10, NetBSD 7, OpenBSD 5.5, Minix 3.3.0, AIX 7.2, HP-UX 11, IRIX, Solaris 11.4, Cygwin 1.7.x, mingw, MSVC 14, Android 9.0.
+
+@item
+On some platforms, @code{reallocarray (n, s)} can succeed even if
+multiplying @code{n} by @code{s} would exceed @code{PTRDIFF_MAX},
+which can lead to undefined behavior later:
+FreeBSD 13, NetBSD 9, OpenBSD 6, musl 1.2.
 @end itemize
 
 Portability problems not fixed by Gnulib:
diff --git a/lib/reallocarray.c b/lib/reallocarray.c
index d10d7d5f57..03197c601b 100644
--- a/lib/reallocarray.c
+++ b/lib/reallocarray.c
@@ -27,12 +27,12 @@
 void *
 reallocarray (void *ptr, size_t nmemb, size_t size)
 {
-    if (xalloc_oversized (nmemb, size))
-      {
-        errno = ENOMEM;
-        return NULL;
-      }
-
-    /* Rely on the semantics of GNU realloc.  */
-    return realloc (ptr, nmemb * size);
+  if (xalloc_oversized (nmemb, size))
+    {
+      errno = ENOMEM;
+      return NULL;
+    }
+
+  /* Rely on the semantics of GNU realloc.  */
+  return realloc (ptr, nmemb * size);
 }
diff --git a/lib/stdlib.in.h b/lib/stdlib.in.h
index c07fd1f5d7..91079b1cf1 100644
--- a/lib/stdlib.in.h
+++ b/lib/stdlib.in.h
@@ -1032,12 +1032,23 @@ _GL_WARN_ON_USE (realloc, "realloc is not POSIX compliant everywhere - "
 
 
 #if @GNULIB_REALLOCARRAY@
-# if ! @HAVE_REALLOCARRAY@
+# if @REPLACE_REALLOCARRAY@
+#  if !(defined __cplusplus && defined GNULIB_NAMESPACE)
+#   undef reallocarray
+#   define reallocarray rpl_reallocarray
+#  endif
+_GL_FUNCDECL_RPL (reallocarray, void *,
+                  (void *ptr, size_t nmemb, size_t size));
+_GL_CXXALIAS_RPL (reallocarray, void *,
+                  (void *ptr, size_t nmemb, size_t size));
+# else
+#  if ! @HAVE_REALLOCARRAY@
 _GL_FUNCDECL_SYS (reallocarray, void *,
                   (void *ptr, size_t nmemb, size_t size));
-# endif
+#  endif
 _GL_CXXALIAS_SYS (reallocarray, void *,
                   (void *ptr, size_t nmemb, size_t size));
+# endif
 _GL_CXXALIASWARN (reallocarray);
 #elif defined GNULIB_POSIXCHECK
 # undef reallocarray
diff --git a/m4/reallocarray.m4 b/m4/reallocarray.m4
index 1d81451862..9d8a626622 100644
--- a/m4/reallocarray.m4
+++ b/m4/reallocarray.m4
@@ -1,4 +1,4 @@
-# reallocarray.m4 serial 2
+# reallocarray.m4 serial 3
 dnl Copyright (C) 2017-2021 Free Software Foundation, Inc.
 dnl This file is free software; the Free Software Foundation
 dnl gives unlimited permission to copy and/or distribute it,
@@ -10,9 +10,12 @@ AC_DEFUN([gl_FUNC_REALLOCARRAY],
   AC_REQUIRE([gl_USE_SYSTEM_EXTENSIONS])
 
   AC_REQUIRE([gl_STDLIB_H_DEFAULTS])
+  AC_REQUIRE([gl_CHECK_MALLOC_PTRDIFF])
   AC_CHECK_FUNCS([reallocarray])
   if test "$ac_cv_func_reallocarray" = no; then
     HAVE_REALLOCARRAY=0
+  elif test "$gl_cv_malloc_ptrdiff" = no; then
+    REPLACE_REALLOCARRAY=1
   fi
 ])
 
diff --git a/m4/stdlib_h.m4 b/m4/stdlib_h.m4
index 2de57f78c4..9c1d1c76c1 100644
--- a/m4/stdlib_h.m4
+++ b/m4/stdlib_h.m4
@@ -1,4 +1,4 @@
-# stdlib_h.m4 serial 62
+# stdlib_h.m4 serial 63
 dnl Copyright (C) 2007-2021 Free Software Foundation, Inc.
 dnl This file is free software; the Free Software Foundation
 dnl gives unlimited permission to copy and/or distribute it,
@@ -179,6 +179,7 @@ AC_DEFUN([gl_STDLIB_H_DEFAULTS],
   REPLACE_RANDOM=0;          AC_SUBST([REPLACE_RANDOM])
   REPLACE_RANDOM_R=0;        AC_SUBST([REPLACE_RANDOM_R])
   REPLACE_REALLOC=0;         AC_SUBST([REPLACE_REALLOC])
+  REPLACE_REALLOCARRAY=0;    AC_SUBST([REPLACE_REALLOCARRAY])
   REPLACE_REALPATH=0;        AC_SUBST([REPLACE_REALPATH])
   REPLACE_SETENV=0;          AC_SUBST([REPLACE_SETENV])
   REPLACE_SETSTATE=0;        AC_SUBST([REPLACE_SETSTATE])
diff --git a/modules/reallocarray b/modules/reallocarray
index a64c6fd2d0..6d762c157a 100644
--- a/modules/reallocarray
+++ b/modules/reallocarray
@@ -3,6 +3,7 @@ reallocarray function that is glibc compatible.
 
 Files:
 lib/reallocarray.c
+m4/malloc.m4
 m4/reallocarray.m4
 
 Depends-on:
@@ -13,7 +14,7 @@ stdlib
 
 configure.ac:
 gl_FUNC_REALLOCARRAY
-if test $HAVE_REALLOCARRAY = 0; then
+if test $HAVE_REALLOCARRAY = 0 || test $REPLACE_REALLOCARRAY = 1; then
   AC_LIBOBJ([reallocarray])
   gl_PREREQ_REALLOCARRAY
 fi
diff --git a/modules/reallocarray-tests b/modules/reallocarray-tests
index 3943d7295b..4b61da1435 100644
--- a/modules/reallocarray-tests
+++ b/modules/reallocarray-tests
@@ -3,6 +3,7 @@ tests/test-reallocarray.c
 tests/signature.h
 
 Depends-on:
+stdint
 
 configure.ac:
 
diff --git a/modules/stdlib b/modules/stdlib
index 47f3af752a..da5629dc9a 100644
--- a/modules/stdlib
+++ b/modules/stdlib
@@ -137,6 +137,7 @@ stdlib.h: stdlib.in.h $(top_builddir)/config.status $(CXXDEFS_H) \
 	      -e 's|@''REPLACE_RANDOM''@|$(REPLACE_RANDOM)|g' \
 	      -e 's|@''REPLACE_RANDOM_R''@|$(REPLACE_RANDOM_R)|g' \
 	      -e 's|@''REPLACE_REALLOC''@|$(REPLACE_REALLOC)|g' \
+	      -e 's|@''REPLACE_REALLOCARRAY''@|$(REPLACE_REALLOCARRAY)|g' \
 	      -e 's|@''REPLACE_REALPATH''@|$(REPLACE_REALPATH)|g' \
 	      -e 's|@''REPLACE_SETENV''@|$(REPLACE_SETENV)|g' \
 	      -e 's|@''REPLACE_SETSTATE''@|$(REPLACE_SETSTATE)|g' \
diff --git a/tests/test-reallocarray.c b/tests/test-reallocarray.c
index 9c4a189efc..973f22ba66 100644
--- a/tests/test-reallocarray.c
+++ b/tests/test-reallocarray.c
@@ -18,6 +18,7 @@
 
 #include <stdlib.h>
 #include <errno.h>
+#include <stdint.h>
 
 #include "signature.h"
 SIGNATURE_CHECK (reallocarray, void *, (void *, size_t, size_t));
@@ -25,20 +26,24 @@ SIGNATURE_CHECK (reallocarray, void *, (void *, size_t, size_t));
 int
 main ()
 {
-   size_t n;
-
-   /* Check that reallocarray fails when requested to allocate a block
-      of memory larger than SIZE_MAX bytes.  */
-   for (n = 2; n != 0; n <<= 1)
-     {
-       void *volatile p = reallocarray (NULL, (size_t) -1 / n + 1, n);
-       if (p)
-         return 1;
-
-       /* Ensure that errno is correctly set.  */
-       if (errno != ENOMEM)
-         return 1;
-     }
-
-   return 0;
+  void *volatile p = NULL;
+
+  /* Check that reallocarray fails when requested to allocate a block
+     of memory larger than PTRDIFF_MAX or SIZE_MAX bytes.  */
+  for (size_t n = 2; n != 0; n <<= 1)
+    {
+      p = reallocarray (p, PTRDIFF_MAX / n + 1, n);
+      if (p)
+        return 1;
+      if (errno != ENOMEM)
+        return 2;
+
+      p = reallocarray (p, SIZE_MAX / n + 1, n);
+      if (p)
+        return 3;
+      if (errno != ENOMEM)
+        return 4;
+    }
+
+  return 0;
 }