Don’t trust RLIMIT_NOFILE in src/process.c

Problem discovered on Fedora 44 x86-64 when using GCC 16.1.1 with
-fsanitize=address, with test/src/process-tests.el tests that use
process-tests--with-raised-rlimit.  This function overrides the
default of 1024 for the maximum number of open files, which causes
undefined behavior (subscript errors) in src/process.c.
* src/process.c (inrange_fd, inrange_pipe): New functions.
(allocate_pty, create_process, create_pty, Fmake_pipe_process)
(Fmake_serial_process, connect_network_socket)
(network_interface_info, server_accept_connection)
(Fprocess_send_eof, child_signal_init):
Check that all newly allocated file descriptors are less than
FD_SETSIZE; close them and fail otherwise.
(create_pty, Fmake_pipe_process, Fmake_serial_process)
(connect_network_socket, server_accept_connection)
(child_signal_init): Remove no-longer-needed comparisons
to FD_SETSIZE, now that inrange_fd and inrange_pipe
do the checking for us.
This commit is contained in:
Paul Eggert 2026-05-18 22:59:45 -07:00
parent 71336e837a
commit efb83df331

View file

@ -476,6 +476,44 @@ clear_fd_callback_data (struct fd_callback_data* elem)
elem->waiting_thread = NULL;
}
/* If FD is out of range, close it and return -1, setting errno to
EMFILE. Otherwise, return FD. This module routinely does this for
file descriptors so that fd_set-based primitives work even on
platforms lacking setrlimit (RLIMIT_NOFILE, ...) or if some Emacs
module or even some other process raises Emacs's RLIMIT_NOFILE limit. */
static int
inrange_fd (int fd)
{
if (fd < FD_SETSIZE)
return fd;
emacs_close (fd);
errno = EMFILE;
return -1;
}
/* Create a pipe into FD[0] and fd[1], refusing to create file
descriptors out of range. This is like inrange_fd, that it
only. */
static int
inrange_pipe (int fd[2])
{
int pipefd[2];
int result = emacs_pipe (pipefd);
if (result < 0)
return result;
else if (pipefd[0] < FD_SETSIZE && pipefd[1] < FD_SETSIZE)
{
fd[0] = pipefd[0];
fd[1] = pipefd[1];
return result;
}
else
{
inrange_fd (pipefd[0]);
inrange_fd (pipefd[1]);
return -1;
}
}
/* Add a file descriptor FD to be monitored for when read is possible.
When read is possible, call FUNC with argument DATA. */
@ -493,7 +531,7 @@ add_read_fd (int fd, fd_callback func, void *data)
void
add_non_keyboard_read_fd (int fd, fd_callback func, void *data)
{
add_read_fd(fd, func, data);
add_read_fd (fd, func, data);
fd_callback_info[fd].flags &= ~KEYBOARD_FD;
}
@ -863,6 +901,8 @@ allocate_pty (char pty_name[PTY_NAME_SIZE])
fd = emacs_open (pty_name, O_RDWR | O_NONBLOCK, 0);
#endif /* no PTY_OPEN */
fd = inrange_fd (fd);
if (fd >= 0)
{
#ifdef PTY_TTY_NAME_SPRINTF
@ -892,6 +932,8 @@ allocate_pty (char pty_name[PTY_NAME_SIZE])
setup_pty (fd);
return fd;
}
else if (errno == EMFILE)
return fd;
}
#endif /* HAVE_PTYS */
return -1;
@ -2184,7 +2226,7 @@ create_process (Lisp_Object process, char **new_argv, Lisp_Object current_dir)
then close it and reopen it in the child. */
/* Don't let this terminal become our controlling terminal
(in case we don't have one). */
pty_tty = emacs_open (pty_name, O_RDWR | O_NOCTTY, 0);
pty_tty = inrange_fd (emacs_open (pty_name, O_RDWR | O_NOCTTY, 0));
if (pty_tty < 0)
report_file_error ("Opening pty", Qnil);
#endif /* not USG, or USG_SUBTTY_WORKS */
@ -2201,7 +2243,7 @@ create_process (Lisp_Object process, char **new_argv, Lisp_Object current_dir)
}
else
{
if (emacs_pipe (p->open_fd + SUBPROCESS_STDIN) != 0)
if (inrange_pipe (p->open_fd + SUBPROCESS_STDIN) < 0)
report_file_error ("Creating pipe", Qnil);
forkin = p->open_fd[SUBPROCESS_STDIN];
outchannel = p->open_fd[WRITE_TO_SUBPROCESS];
@ -2215,7 +2257,7 @@ create_process (Lisp_Object process, char **new_argv, Lisp_Object current_dir)
}
else
{
if (emacs_pipe (p->open_fd + READ_FROM_SUBPROCESS) != 0)
if (inrange_pipe (p->open_fd + READ_FROM_SUBPROCESS) < 0)
report_file_error ("Creating pipe", Qnil);
inchannel = p->open_fd[READ_FROM_SUBPROCESS];
forkout = p->open_fd[SUBPROCESS_STDOUT];
@ -2239,11 +2281,8 @@ create_process (Lisp_Object process, char **new_argv, Lisp_Object current_dir)
close_process_fd (&pp->open_fd[SUBPROCESS_STDIN]);
}
if (FD_SETSIZE <= inchannel || FD_SETSIZE <= outchannel)
report_file_errno ("Creating pipe", Qnil, EMFILE);
#ifndef WINDOWSNT
if (emacs_pipe (p->open_fd + READ_FROM_EXEC_MONITOR) != 0)
if (inrange_pipe (p->open_fd + READ_FROM_EXEC_MONITOR) < 0)
report_file_error ("Creating pipe", Qnil);
#endif
@ -2351,14 +2390,12 @@ create_pty (Lisp_Object process)
if (pty_fd >= 0)
{
p->open_fd[SUBPROCESS_STDIN] = pty_fd;
if (FD_SETSIZE <= pty_fd)
report_file_errno ("Opening pty", Qnil, EMFILE);
#if ! defined (USG) || defined (USG_SUBTTY_WORKS)
/* On most USG systems it does not work to open the pty's tty here,
then close it and reopen it in the child. */
/* Don't let this terminal become our controlling terminal
(in case we don't have one). */
int forkout = emacs_open (pty_name, O_RDWR | O_NOCTTY, 0);
int forkout = inrange_fd (emacs_open (pty_name, O_RDWR | O_NOCTTY, 0));
if (forkout < 0)
report_file_error ("Opening pty", Qnil);
p->open_fd[WRITE_TO_SUBPROCESS] = forkout;
@ -2455,15 +2492,11 @@ usage: (make-pipe-process &rest ARGS) */)
record_unwind_protect (remove_process, proc);
p = XPROCESS (proc);
if (emacs_pipe (p->open_fd + SUBPROCESS_STDIN) != 0
|| emacs_pipe (p->open_fd + READ_FROM_SUBPROCESS) != 0)
if (inrange_pipe (p->open_fd + SUBPROCESS_STDIN) < 0
|| inrange_pipe (p->open_fd + READ_FROM_SUBPROCESS) < 0)
report_file_error ("Creating pipe", Qnil);
outchannel = p->open_fd[WRITE_TO_SUBPROCESS];
inchannel = p->open_fd[READ_FROM_SUBPROCESS];
if (FD_SETSIZE <= inchannel || FD_SETSIZE <= outchannel)
report_file_errno ("Creating pipe", Qnil, EMFILE);
fcntl (inchannel, F_SETFL, O_NONBLOCK);
fcntl (outchannel, F_SETFL, O_NONBLOCK);
@ -3209,10 +3242,10 @@ usage: (make-serial-process &rest ARGS) */)
record_unwind_protect (remove_process, proc);
p = XPROCESS (proc);
fd = serial_open (port);
fd = inrange_fd (serial_open (port));
if (fd < 0)
report_file_error ("Opening serial port", port);
p->open_fd[SUBPROCESS_STDIN] = fd;
if (FD_SETSIZE <= fd)
report_file_errno ("Opening serial port", port, EMFILE);
p->infd = fd;
p->outfd = fd;
if (fd > max_desc)
@ -3471,20 +3504,12 @@ connect_network_socket (Lisp_Object proc, Lisp_Object addrinfos,
int socktype = p->socktype | SOCK_CLOEXEC;
if (p->is_non_blocking_client)
socktype |= SOCK_NONBLOCK;
s = socket (family, socktype, protocol);
s = inrange_fd (socket (family, socktype, protocol));
if (s < 0)
{
xerrno = errno;
continue;
}
/* Reject file descriptors that would be too large. */
if (FD_SETSIZE <= s)
{
emacs_close (s);
s = -1;
xerrno = EMFILE;
continue;
}
}
if (p->is_non_blocking_client && ! (SOCK_NONBLOCK && socket_to_use < 0))
@ -4500,7 +4525,7 @@ network_interface_info (Lisp_Object ifname)
error ("Interface name too long");
lispstpcpy (rq.ifr_name, ifname);
s = socket (AF_INET, SOCK_STREAM | SOCK_CLOEXEC, 0);
s = inrange_fd (socket (AF_INET, SOCK_STREAM | SOCK_CLOEXEC, 0));
if (s < 0)
return Qnil;
specpdl_ref count = SPECPDL_INDEX ();
@ -4979,14 +5004,7 @@ server_accept_connection (Lisp_Object server, int channel)
union u_sockaddr saddr;
socklen_t len = sizeof saddr;
s = accept4 (channel, &saddr.sa, &len, SOCK_CLOEXEC);
if (FD_SETSIZE <= s)
{
emacs_close (s);
s = -1;
errno = EMFILE;
}
s = inrange_fd (accept4 (channel, &saddr.sa, &len, SOCK_CLOEXEC));
if (s < 0)
{
@ -7484,7 +7502,7 @@ process has been transmitted to the serial port. */)
shutdown (old_outfd, 1);
#endif
close_process_fd (&p->open_fd[WRITE_TO_SUBPROCESS]);
new_outfd = emacs_open (NULL_DEVICE, O_WRONLY, 0);
new_outfd = inrange_fd (emacs_open (NULL_DEVICE, O_WRONLY, 0));
if (new_outfd < 0)
report_file_error ("Opening null device", Qnil);
p->open_fd[WRITE_TO_SUBPROCESS] = new_outfd;
@ -7567,17 +7585,8 @@ child_signal_init (void)
return; /* already done */
int fds[2];
if (emacs_pipe (fds) < 0)
if (inrange_pipe (fds) < 0)
report_file_error ("Creating pipe for child signal", Qnil);
if (FD_SETSIZE <= fds[0])
{
/* Since we need to `pselect' on the read end, it has to fit
into an `fd_set'. */
emacs_close (fds[0]);
emacs_close (fds[1]);
report_file_errno ("Creating pipe for child signal", Qnil,
EMFILE);
}
/* We leave the file descriptors open until the Emacs process
exits. */
@ -8726,7 +8735,13 @@ init_process_emacs (int sockfd)
#endif
#ifdef HAVE_SETRLIMIT
/* Don't allocate more than FD_SETSIZE file descriptors for Emacs itself. */
/* Don't allocate more than FD_SETSIZE file descriptors for Emacs itself.
This is for performance, so that we needn't open file descriptors
only to immediately close them and fail. The rest of this module
does not rely on emacs_open, accept4, socket, emacs_pipe, etc.
to always return values less than FD_SETSIZE, since not every
platform has setrlimit, and even for those that do, an Emacs
module or even some other process can raise Emacs's limit. */
if (getrlimit (RLIMIT_NOFILE, &nofile_limit) != 0)
nofile_limit.rlim_cur = 0;
else if (FD_SETSIZE < nofile_limit.rlim_cur)