As Espie found:

Turns out diff is expecting scandir to return NULL-terminated arrays.
But scandir doesn't ! most specifically, when we fill our buffer exactly,
then we're out of luck. This doesn't happen often, but usually enough to
kill a diff -ur on a large tree with MALLOC_OPTIONS=S

Here's a diff, mostly from Espie, that seems to fix it for me.

Thoughts? Improvements? OK?

.... Ken

Index: diffdir.c
===================================================================
RCS file: /cvs/src/usr.bin/diff/diffdir.c,v
retrieving revision 1.39
diff -u -p -r1.39 diffdir.c
--- diffdir.c	8 Nov 2010 15:49:13 -0000	1.39
+++ diffdir.c	14 Nov 2010 14:21:43 -0000
@@ -38,7 +38,7 @@
 #include "xmalloc.h"
 
 static int selectfile(struct dirent *);
-static struct dirent **slurpdir(char *, int);
+static struct dirent **slurpdir(char *, int, struct dirent ***);
 static void diffit(struct dirent *, char *, size_t, char *, size_t, int);
 
 #define d_status	d_type		/* we need to store status for -l */
@@ -51,6 +51,7 @@ diffdir(char *p1, char *p2, int flags)
 {
 	struct dirent *dent1, **dp1, **dirp1 = NULL;
 	struct dirent *dent2, **dp2, **dirp2 = NULL;
+	struct dirent **end1, **end2;
 	size_t dirlen1, dirlen2;
 	char path1[MAXPATHLEN], path2[MAXPATHLEN];
 	int pos;
@@ -77,8 +78,8 @@ diffdir(char *p1, char *p2, int flags)
 	}
 
 	/* get a list of the entries in each directory */
-	dp1 = dirp1 = slurpdir(path1, Nflag + Pflag);
-	dp2 = dirp2 = slurpdir(path2, Nflag);
+	dp1 = dirp1 = slurpdir(path1, Nflag + Pflag, &end1);
+	dp2 = dirp2 = slurpdir(path2, Nflag, &end2);
 	if (dirp1 == NULL || dirp2 == NULL)
 		goto closem;
 
@@ -86,16 +87,16 @@ diffdir(char *p1, char *p2, int flags)
 	 * If we were given a starting point, find it.
 	 */
 	if (start != NULL) {
-		while (*dp1 != NULL && strcmp((*dp1)->d_name, start) < 0)
+		while (dp1 != end1 && strcmp((*dp1)->d_name, start) < 0)
 			dp1++;
-		while (*dp2 != NULL && strcmp((*dp2)->d_name, start) < 0)
+		while (dp2 != end2 && strcmp((*dp2)->d_name, start) < 0)
 			dp2++;
 	}
 
 	/*
 	 * Iterate through the two directory lists, diffing as we go.
 	 */
-	while (*dp1 != NULL || *dp2 != NULL) {
+	while (dp1 != end1 || dp2 != end2) {
 		dent1 = *dp1;
 		dent2 = *dp2;
 
@@ -131,11 +132,13 @@ diffdir(char *p1, char *p2, int flags)
 	if (lflag) {
 		path1[dirlen1] = '\0';
 		path2[dirlen2] = '\0';
-		for (dp1 = dirp1; (dent1 = *dp1) != NULL; dp1++) {
+		for (dp1 = dirp1; dp1 != end1; dp1++) {
+			dent1 = *dp1;
 			print_status(dent1->d_status, path1, path2,
 			    dent1->d_name);
 		}
-		for (dp2 = dirp2; (dent2 = *dp2) != NULL; dp2++) {
+		for (dp2 = dirp2; dp2 != end2; dp2++) {
+			dent2 = *dp2;
 			if (dent2->d_status == D_ONLY)
 				print_status(dent2->d_status, path2, NULL,
 				    dent2->d_name);
@@ -144,13 +147,13 @@ diffdir(char *p1, char *p2, int flags)
 
 closem:
 	if (dirp1 != NULL) {
-		for (dp1 = dirp1; (dent1 = *dp1) != NULL; dp1++)
-			xfree(dent1);
+		for (dp1 = dirp1; dp1 != end1; dp1++)
+			xfree(*dp1);
 		xfree(dirp1);
 	}
 	if (dirp2 != NULL) {
-		for (dp2 = dirp2; (dent2 = *dp2) != NULL; dp2++)
-			xfree(dent2);
+		for (dp2 = dirp2; dp2 != end2; dp2++)
+			xfree(*dp2);
 		xfree(dirp2);
 	}
 }
@@ -161,7 +164,7 @@ closem:
  * Caller is responsible for free()ing each array element and the array itself.
  */
 static struct dirent **
-slurpdir(char *dirname, int enoentok)
+slurpdir(char *dirname, int enoentok, struct dirent ***endp)
 {
 	struct dirent **namelist = NULL;
 	int rval;
@@ -174,8 +177,9 @@ slurpdir(char *dirname, int enoentok)
 		} else {
 			warn("%s", dirname);
 		}
+		rval = 0;
 	}
-
+	*endp = namelist + rval;
 	return (namelist);
 }