open files with O_CLOEXEC when possible

This helps prevent leaked file descriptors when the library is
used in a process which exec's.

opendir() already uses O_CLOEXEC on platforms that support O_CLOEXEC
so we don't need to make changes there.

In addition read config files using p11_mmap_open() so that we get
the simple benefits of O_CLOEXEC with the open() call there.

https://bugzilla.redhat.com/show_bug.cgi?id=984986

2 files changed, 19 insertions, 3 deletions

diff --git a/common/compat.c b/common/compat.c
index 400e10b..5efc932 100644
--- a/common/compat.c
+++ b/common/compat.c
@@ -192,7 +192,7 @@ p11_mmap_open (const char *path,
 	if (map == NULL)
 		return NULL;
 
-	map->fd = open (path, O_RDONLY);
+	map->fd = open (path, O_RDONLY | O_CLOEXEC);
 	if (map->fd == -1) {
 		free (map);
 		return NULL;
@@ -298,14 +298,20 @@ p11_mmap_open (const char *path,
 	p11_mmap *map;
 
 	map = calloc (1, sizeof (p11_mmap));
-	if (map == NULL)
+	if (map == NULL) {
+		errno = ENOMEM;
 		return NULL;
+	}
 
 	map->file  = CreateFile (path, GENERIC_READ, 0, NULL, OPEN_EXISTING, FILE_FLAG_RANDOM_ACCESS, NULL);
 	if (map->file == INVALID_HANDLE_VALUE) {
 		errn = GetLastError ();
 		free (map);
 		SetLastError (errn);
+		if (errn == ERROR_PATH_NOT_FOUND || errn == ERROR_FILE_NOT_FOUND)
+			errno = ENOENT;
+		else if (errn == ERROR_ACCESS_DENIED)
+			errno = EPERM;
 		return NULL;
 	}
 
@@ -314,6 +320,8 @@ p11_mmap_open (const char *path,
 		CloseHandle (map->file);
 		free (map);
 		SetLastError (errn);
+		if (errn == ERROR_ACCESS_DENIED)
+			errno = EPERM;
 		return NULL;
 	}
 
@@ -323,6 +331,8 @@ p11_mmap_open (const char *path,
 		CloseHandle (map->file);
 		free (map);
 		SetLastError (errn);
+		if (errn == ERROR_ACCESS_DENIED)
+			errno = EPERM;
 		return NULL;
 	}
 
@@ -334,6 +344,8 @@ p11_mmap_open (const char *path,
 		CloseHandle (map->file);
 		free (map);
 		SetLastError (errn);
+		if (errn == ERROR_ACCESS_DENIED)
+			errno = EPERM;
 		return NULL;
 	}
 
@@ -676,7 +688,7 @@ _gettemp (char *path,
 
 	for (;;) {
 		if (doopen) {
-			if ((*doopen = open (path, O_BINARY | O_CREAT | O_EXCL | O_RDWR, 0600)) >= 0)
+			if ((*doopen = open (path, O_BINARY | O_CREAT | O_EXCL | O_RDWR | O_CLOEXEC, 0600)) >= 0)
 				return (1);
 			if (errno != EEXIST)
 				return (0);
diff --git a/common/compat.h b/common/compat.h
index 9127f95..20f9a81 100644
--- a/common/compat.h
+++ b/common/compat.h
@@ -68,6 +68,10 @@
 #define O_BINARY 0
 #endif
 
+#ifndef O_CLOEXEC
+#define O_CLOEXEC 0
+#endif
+
 #ifndef HAVE_GETPROGNAME
 const char * getprogname (void);
 #endif