From efb83df331425ae66e9d031a1ac35c2215612b52 Mon Sep 17 00:00:00 2001 From: Paul Eggert Date: Mon, 18 May 2026 22:59:45 -0700 Subject: [PATCH] =?UTF-8?q?Don=E2=80=99t=20trust=20RLIMIT=5FNOFILE=20in=20?= =?UTF-8?q?src/process.c?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Problem discovered on Fedora 44 x86-64 when using GCC 16.1.1 with -fsanitize=address, with test/src/process-tests.el tests that use process-tests--with-raised-rlimit. This function overrides the default of 1024 for the maximum number of open files, which causes undefined behavior (subscript errors) in src/process.c. * src/process.c (inrange_fd, inrange_pipe): New functions. (allocate_pty, create_process, create_pty, Fmake_pipe_process) (Fmake_serial_process, connect_network_socket) (network_interface_info, server_accept_connection) (Fprocess_send_eof, child_signal_init): Check that all newly allocated file descriptors are less than FD_SETSIZE; close them and fail otherwise. (create_pty, Fmake_pipe_process, Fmake_serial_process) (connect_network_socket, server_accept_connection) (child_signal_init): Remove no-longer-needed comparisons to FD_SETSIZE, now that inrange_fd and inrange_pipe do the checking for us. --- src/process.c | 115 ++++++++++++++++++++++++++++---------------------- 1 file changed, 65 insertions(+), 50 deletions(-) diff --git a/src/process.c b/src/process.c index 9ea2b66533b..9e807bef44e 100644 --- a/src/process.c +++ b/src/process.c @@ -476,6 +476,44 @@ clear_fd_callback_data (struct fd_callback_data* elem) elem->waiting_thread = NULL; } +/* If FD is out of range, close it and return -1, setting errno to + EMFILE. Otherwise, return FD. This module routinely does this for + file descriptors so that fd_set-based primitives work even on + platforms lacking setrlimit (RLIMIT_NOFILE, ...) or if some Emacs + module or even some other process raises Emacs's RLIMIT_NOFILE limit. */ +static int +inrange_fd (int fd) +{ + if (fd < FD_SETSIZE) + return fd; + emacs_close (fd); + errno = EMFILE; + return -1; +} + +/* Create a pipe into FD[0] and fd[1], refusing to create file + descriptors out of range. This is like inrange_fd, that it + only. */ +static int +inrange_pipe (int fd[2]) +{ + int pipefd[2]; + int result = emacs_pipe (pipefd); + if (result < 0) + return result; + else if (pipefd[0] < FD_SETSIZE && pipefd[1] < FD_SETSIZE) + { + fd[0] = pipefd[0]; + fd[1] = pipefd[1]; + return result; + } + else + { + inrange_fd (pipefd[0]); + inrange_fd (pipefd[1]); + return -1; + } +} /* Add a file descriptor FD to be monitored for when read is possible. When read is possible, call FUNC with argument DATA. */ @@ -493,7 +531,7 @@ add_read_fd (int fd, fd_callback func, void *data) void add_non_keyboard_read_fd (int fd, fd_callback func, void *data) { - add_read_fd(fd, func, data); + add_read_fd (fd, func, data); fd_callback_info[fd].flags &= ~KEYBOARD_FD; } @@ -863,6 +901,8 @@ allocate_pty (char pty_name[PTY_NAME_SIZE]) fd = emacs_open (pty_name, O_RDWR | O_NONBLOCK, 0); #endif /* no PTY_OPEN */ + fd = inrange_fd (fd); + if (fd >= 0) { #ifdef PTY_TTY_NAME_SPRINTF @@ -892,6 +932,8 @@ allocate_pty (char pty_name[PTY_NAME_SIZE]) setup_pty (fd); return fd; } + else if (errno == EMFILE) + return fd; } #endif /* HAVE_PTYS */ return -1; @@ -2184,7 +2226,7 @@ create_process (Lisp_Object process, char **new_argv, Lisp_Object current_dir) then close it and reopen it in the child. */ /* Don't let this terminal become our controlling terminal (in case we don't have one). */ - pty_tty = emacs_open (pty_name, O_RDWR | O_NOCTTY, 0); + pty_tty = inrange_fd (emacs_open (pty_name, O_RDWR | O_NOCTTY, 0)); if (pty_tty < 0) report_file_error ("Opening pty", Qnil); #endif /* not USG, or USG_SUBTTY_WORKS */ @@ -2201,7 +2243,7 @@ create_process (Lisp_Object process, char **new_argv, Lisp_Object current_dir) } else { - if (emacs_pipe (p->open_fd + SUBPROCESS_STDIN) != 0) + if (inrange_pipe (p->open_fd + SUBPROCESS_STDIN) < 0) report_file_error ("Creating pipe", Qnil); forkin = p->open_fd[SUBPROCESS_STDIN]; outchannel = p->open_fd[WRITE_TO_SUBPROCESS]; @@ -2215,7 +2257,7 @@ create_process (Lisp_Object process, char **new_argv, Lisp_Object current_dir) } else { - if (emacs_pipe (p->open_fd + READ_FROM_SUBPROCESS) != 0) + if (inrange_pipe (p->open_fd + READ_FROM_SUBPROCESS) < 0) report_file_error ("Creating pipe", Qnil); inchannel = p->open_fd[READ_FROM_SUBPROCESS]; forkout = p->open_fd[SUBPROCESS_STDOUT]; @@ -2239,11 +2281,8 @@ create_process (Lisp_Object process, char **new_argv, Lisp_Object current_dir) close_process_fd (&pp->open_fd[SUBPROCESS_STDIN]); } - if (FD_SETSIZE <= inchannel || FD_SETSIZE <= outchannel) - report_file_errno ("Creating pipe", Qnil, EMFILE); - #ifndef WINDOWSNT - if (emacs_pipe (p->open_fd + READ_FROM_EXEC_MONITOR) != 0) + if (inrange_pipe (p->open_fd + READ_FROM_EXEC_MONITOR) < 0) report_file_error ("Creating pipe", Qnil); #endif @@ -2351,14 +2390,12 @@ create_pty (Lisp_Object process) if (pty_fd >= 0) { p->open_fd[SUBPROCESS_STDIN] = pty_fd; - if (FD_SETSIZE <= pty_fd) - report_file_errno ("Opening pty", Qnil, EMFILE); #if ! defined (USG) || defined (USG_SUBTTY_WORKS) /* On most USG systems it does not work to open the pty's tty here, then close it and reopen it in the child. */ /* Don't let this terminal become our controlling terminal (in case we don't have one). */ - int forkout = emacs_open (pty_name, O_RDWR | O_NOCTTY, 0); + int forkout = inrange_fd (emacs_open (pty_name, O_RDWR | O_NOCTTY, 0)); if (forkout < 0) report_file_error ("Opening pty", Qnil); p->open_fd[WRITE_TO_SUBPROCESS] = forkout; @@ -2455,15 +2492,11 @@ usage: (make-pipe-process &rest ARGS) */) record_unwind_protect (remove_process, proc); p = XPROCESS (proc); - if (emacs_pipe (p->open_fd + SUBPROCESS_STDIN) != 0 - || emacs_pipe (p->open_fd + READ_FROM_SUBPROCESS) != 0) + if (inrange_pipe (p->open_fd + SUBPROCESS_STDIN) < 0 + || inrange_pipe (p->open_fd + READ_FROM_SUBPROCESS) < 0) report_file_error ("Creating pipe", Qnil); outchannel = p->open_fd[WRITE_TO_SUBPROCESS]; inchannel = p->open_fd[READ_FROM_SUBPROCESS]; - - if (FD_SETSIZE <= inchannel || FD_SETSIZE <= outchannel) - report_file_errno ("Creating pipe", Qnil, EMFILE); - fcntl (inchannel, F_SETFL, O_NONBLOCK); fcntl (outchannel, F_SETFL, O_NONBLOCK); @@ -3209,10 +3242,10 @@ usage: (make-serial-process &rest ARGS) */) record_unwind_protect (remove_process, proc); p = XPROCESS (proc); - fd = serial_open (port); + fd = inrange_fd (serial_open (port)); + if (fd < 0) + report_file_error ("Opening serial port", port); p->open_fd[SUBPROCESS_STDIN] = fd; - if (FD_SETSIZE <= fd) - report_file_errno ("Opening serial port", port, EMFILE); p->infd = fd; p->outfd = fd; if (fd > max_desc) @@ -3471,20 +3504,12 @@ connect_network_socket (Lisp_Object proc, Lisp_Object addrinfos, int socktype = p->socktype | SOCK_CLOEXEC; if (p->is_non_blocking_client) socktype |= SOCK_NONBLOCK; - s = socket (family, socktype, protocol); + s = inrange_fd (socket (family, socktype, protocol)); if (s < 0) { xerrno = errno; continue; } - /* Reject file descriptors that would be too large. */ - if (FD_SETSIZE <= s) - { - emacs_close (s); - s = -1; - xerrno = EMFILE; - continue; - } } if (p->is_non_blocking_client && ! (SOCK_NONBLOCK && socket_to_use < 0)) @@ -4500,7 +4525,7 @@ network_interface_info (Lisp_Object ifname) error ("Interface name too long"); lispstpcpy (rq.ifr_name, ifname); - s = socket (AF_INET, SOCK_STREAM | SOCK_CLOEXEC, 0); + s = inrange_fd (socket (AF_INET, SOCK_STREAM | SOCK_CLOEXEC, 0)); if (s < 0) return Qnil; specpdl_ref count = SPECPDL_INDEX (); @@ -4979,14 +5004,7 @@ server_accept_connection (Lisp_Object server, int channel) union u_sockaddr saddr; socklen_t len = sizeof saddr; - s = accept4 (channel, &saddr.sa, &len, SOCK_CLOEXEC); - - if (FD_SETSIZE <= s) - { - emacs_close (s); - s = -1; - errno = EMFILE; - } + s = inrange_fd (accept4 (channel, &saddr.sa, &len, SOCK_CLOEXEC)); if (s < 0) { @@ -7484,7 +7502,7 @@ process has been transmitted to the serial port. */) shutdown (old_outfd, 1); #endif close_process_fd (&p->open_fd[WRITE_TO_SUBPROCESS]); - new_outfd = emacs_open (NULL_DEVICE, O_WRONLY, 0); + new_outfd = inrange_fd (emacs_open (NULL_DEVICE, O_WRONLY, 0)); if (new_outfd < 0) report_file_error ("Opening null device", Qnil); p->open_fd[WRITE_TO_SUBPROCESS] = new_outfd; @@ -7567,17 +7585,8 @@ child_signal_init (void) return; /* already done */ int fds[2]; - if (emacs_pipe (fds) < 0) + if (inrange_pipe (fds) < 0) report_file_error ("Creating pipe for child signal", Qnil); - if (FD_SETSIZE <= fds[0]) - { - /* Since we need to `pselect' on the read end, it has to fit - into an `fd_set'. */ - emacs_close (fds[0]); - emacs_close (fds[1]); - report_file_errno ("Creating pipe for child signal", Qnil, - EMFILE); - } /* We leave the file descriptors open until the Emacs process exits. */ @@ -8726,7 +8735,13 @@ init_process_emacs (int sockfd) #endif #ifdef HAVE_SETRLIMIT - /* Don't allocate more than FD_SETSIZE file descriptors for Emacs itself. */ + /* Don't allocate more than FD_SETSIZE file descriptors for Emacs itself. + This is for performance, so that we needn't open file descriptors + only to immediately close them and fail. The rest of this module + does not rely on emacs_open, accept4, socket, emacs_pipe, etc. + to always return values less than FD_SETSIZE, since not every + platform has setrlimit, and even for those that do, an Emacs + module or even some other process can raise Emacs's limit. */ if (getrlimit (RLIMIT_NOFILE, &nofile_limit) != 0) nofile_limit.rlim_cur = 0; else if (FD_SETSIZE < nofile_limit.rlim_cur)