Prepare and load user scripts at startup

* doc/emacs/custom.texi (Early Init File): Document feature and
related user options.
* etc/NEWS: Mention new feature.
* lisp/startup.el (user-lisp-auto-scrape, user-lisp-directory)
(user-lisp-ignored-directories): New user options.
(prepare-user-lisp): New command.
(command-line): Invoke 'prepare-user-lisp' during startup if a
user-lisp directory exists and if not disabled per
'user-lisp-auto-scrape'.

doc/emacs/custom.texi

@@ -2558,6 +2558,7 @@ Manual}.
 * Find Init::           How Emacs finds the init file.
 * Init Non-ASCII::      Using non-@acronym{ASCII} characters in an init file.
 * Early Init File::     Another init file, which is read early on.
+* User Lisp Directory:: Directory of Lisp files to prepare on startup.
 @end menu
 
 @node Init Syntax
@@ -3088,6 +3089,59 @@ provided by the Emacs startup, such as @code{window-setup-hook} or
   For more information on the early init file, @pxref{Init File,,,
 elisp, The Emacs Lisp Reference Manual}.
 
+@node User Lisp Directory
+@subsection User Lisp Directory
+@cindex user-lisp
+@cindex @file{user-lisp/} directory
+
+@vindex user-lisp-directory
+  If the directory specified by @code{user-lisp-directory}, defaulting
+to @file{~/.config/emacs/user-lisp/} or @file{~/.emacs.d/user-lisp/},
+exists, then at startup Emacs will prepare Lisp files within that
+directory for use in the session.  Emacs does the following things:
+@itemize
+@item
+Gather and activate autoload cookies.  This means that you can use
+autoloaded commands and other entry points for the files in your
+@code{user-lisp-directory} without explicitly loading any of the
+files in your initialization file. (@pxref{Autoload,,, elisp, The
+Emacs Lisp Reference Manual})
+@item
+Bytecompile all files, and if supported on your system, natively
+compile them too.  This speeds up the execution of the code in the
+files when they are loaded.  (@pxref{Byte Compilation,,, elisp, The Emacs
+Lisp Reference Manual}, )
+@item
+Adjust @code{load-path} such that all the files can be loaded and
+autoloaded in the usual ways.  (@pxref{Library Search,,, elisp, The
+Emacs Lisp Reference Manual})
+@end itemize
+
+  Keep in mind that as the User Lisp directory is processed at startup,
+before loading the @ref{Init File} file, adjustment have to be made in
+your early-init file (@pxref{Early Init File}).
+
+@vindex user-lisp-ignored-directories
+@vindex user-lisp-auto-scrape
+@findex prepare-user-lisp
+  By default, Emacs considers all Lisp files within
+@code{user-lisp-directory}, even in subdirectories.  You prevent Emacs
+from descending into specific subdirectories by customizing
+@code{user-lisp-ignored-directories}.  You can disable scraping at
+startup by setting @code{user-lisp-auto-scrape} to @code{nil}.  All
+preparations automatically occur at startup if necessary, but can be
+manually invoked during a session using the @code{prepare-user-lisp}
+command.  It is recommended to run the command with a prefix argument
+after upgrading Emacs, to force all generated files to be updated.
+
+  The User Lisp directory is a simpler mechanism than package
+installations (@pxref{Packages}).  In particular, there is no automatic
+dependency resolution or upgrading for files in the User Lisp directory,
+or any facility for building and registering package documentation.
+This means that usually the User Lisp directory is best for code you
+have written only for yourself.  However, it is possible to use the User
+Lisp directory for third party packages or packages you maintain.
+
 @node Authentication
 @section Keeping Persistent Authentication Information
 

etc/NEWS

@@ -68,6 +68,17 @@ user's regular init file, but now site-start.el comes first.  This
 allows site administrators to customize things that can normally only be
 done from early-init.el, such as adding to 'package-directory-list'.
 
++++
+** Emacs prepares a User Lisp directory by default.
+If you have a directory named "user-lisp" in your Emacs configuration
+directory, then the recursive contents will now be byte-compiled,
+scraped for autoload cookies and ensured to be in 'load-path' by
+default.  You can disable the feature by setting 'user-lisp-auto-scrape'
+to nil, or set the 'user-lisp-directory' user option to process any
+other directory on your system.  You can also invoke the
+'prepare-user-lisp' command manually at any time.  See the Info node
+"(emacs) User Lisp Directory" for more details.
+
 
 * Changes in Emacs 31.1
 

lisp/startup.el

@@ -1190,6 +1190,92 @@ This function is called from `load' via `load-path-filter-function'."
                   (directory-files dir nil rx t)))))
            path))))))
 
+(defcustom user-lisp-auto-scrape t
+  "Enable auto-scraping of `user-lisp-directory' at startup.
+If you customize this to nil, you can still invoke the auto-scraping
+with `prepare-user-lisp'.
+
+Note that this variable must be set in your early-init file, as the
+variable's value is used before loading the regular init file.
+Therefore, if you customize it via Customize, you should save your
+customized setting into your `early-init-file'."
+  :type 'boolean
+  :version "31.1")
+
+(defcustom user-lisp-directory
+  (locate-user-emacs-file "user-lisp/")
+  "Activate all Lisp files in this directory, if it exists.
+All regular files below directories are byte-compiled, scraped for
+autoload cookies and ensured to be in `load-path' at startup.  To
+restrict what subdirectories to process, see
+`user-lisp-ignored-directories'.  Note that byte-compilation and
+autoload scraping is lazy, occurring only if the file timestamps
+indicate that it is necessary.  For details on how to override this
+behavior, consult `prepare-user-lisp'.
+
+If you need Emacs to pick up on updates to this directory that occur
+after startup, you can also invoke the `prepare-user-lisp' manually.  To
+disable auto-scraping, see `user-lisp-auto-scrape'.
+
+Note that this variable must be set in your early-init file, as the
+variable's value is used before loading the regular init file.
+Therefore, if you customize it via Customize, you should save your
+customized setting into your `early-init-file'."
+  :initialize #'custom-initialize-delay
+  :type 'directory
+  :version "31.1")
+
+(defcustom user-lisp-ignored-directories
+  '(".git" ".hg" "RCS" "CVS" ".svn" "_svn" ".bzr")
+  "List of directory names for `prepare-user-lisp' to not descend into.
+Each entry of the list is a string that denotes the file name without a
+directory component.  If during recursion any single entry matches the
+file name of any directory, `prepare-user-lisp' will ignore the contents
+of the directory.  This option is most useful to exclude administrative
+directories that do not contain Lisp files."
+  :type '(choice (repeat (string :tag "Directory name")))
+  :version "31.1")
+
+(defun prepare-user-lisp (&optional just-activate autoload-file force)
+  "Byte-compile, scrape autoloads and prepare files in `user-lisp-directory'.
+Write the autoload file to AUTOLOAD-FILE.  If JUST-ACTIVATE is non-nil,
+then the more expensive operations (byte-compilation and autoload
+scraping) are skipped, in effect only processing any previous autoloads.
+If AUTOLOAD-FILE is nil, store the autoload data in a file next to DIR.
+If FORCE is non-nil, or if invoked interactively with a prefix argument,
+re-create the entire autoload file and byte-compile everything
+unconditionally."
+  (interactive (list nil current-prefix-arg))
+  (unless (file-directory-p user-lisp-directory)
+    (error "No such directory: %S" user-lisp-directory))
+  (unless autoload-file
+    (setq autoload-file (expand-file-name ".user-lisp-autoloads.el"
+                                          user-lisp-directory)))
+  (let* ((pred
+          (let ((ignored
+                 (concat "\\`" (regexp-opt user-lisp-ignored-directories) "\\'")))
+            (lambda (dir)
+              (not (string-match-p ignored (file-name-nondirectory dir))))))
+         (dir (expand-file-name user-lisp-directory))
+         (backup-inhibited t)
+         (dirs (list dir)))
+    (add-to-list 'load-path dir)
+    (dolist (file (directory-files-recursively dir "" t pred))
+      (cond
+       ((and (file-regular-p file) (string-suffix-p ".el" file))
+        (unless just-activate
+          (with-demoted-errors "Error while compiling: %S"
+            (byte-recompile-file file force 0)
+            (when (native-comp-available-p)
+              (native-compile-async file)))))
+       ((file-directory-p file)
+        (add-to-list 'load-path file)
+        (push file dirs))))
+    (unless just-activate
+      (loaddefs-generate dirs autoload-file nil nil nil force))
+    (when (file-exists-p autoload-file)
+      (load autoload-file nil t))))
+
 (defun command-line ()
   "A subroutine of `normal-top-level'.
 Amongst another things, it parses the command-line arguments."
@@ -1472,6 +1558,12 @@ please check its value")
 		   (throw 'package-dir-found t)))))))
        (package-activate-all))
 
+  ;; If it enabled and the directory exists, process the contents of the
+  ;; user-lisp/ directory.
+  (when (and init-file-user
+             (file-directory-p user-lisp-directory))
+    (prepare-user-lisp (not user-lisp-auto-scrape)))
+
   ;; Make sure window system's init file was loaded in loadup.el if
   ;; using a window system.
   ;; Initialize the window-system only after processing the command-line