Separate tree-sitter and non-tree-sitter variant of sh-mode

Now there are three modes, sh-base-mode, sh-mode, bash-ts-mode.

The change I made: change sh-mode to sh-base-mode, remove docstring.
Below the new sh-base-mode, create a new definition for sh-mode, paste
the dostring, add setup for font-lock-defaults.  Below sh-mode, add
bash-ts-mode.

* lisp/progmodes/sh-script.el (sh-mode): Moves all setup into
sh-base-mode, except for the setup for font-lock-defaults and the
docstring.
(sh-base-mode): New mode.
(bash-ts-mode): New mode.

lisp/progmodes/c-ts-mode.el

@@ -269,7 +269,7 @@ MODE is either `c' or `cpp'."
    :feature 'definition
    ;; Highlights identifiers in declarations.
    `((declaration
-      declarator: (_) @font-lock-variable-name-face)
+      declarator: (_) @c-ts-mode--fontify-declarator)
 
      (field_declaration
       declarator: (_) @c-ts-mode--fontify-declarator)
@@ -295,21 +295,21 @@ MODE is either `c' or `cpp'."
              (identifier) @font-lock-variable-name-face))
      (assignment_expression
       left: (subscript_expression
-             (identifier) @font-lock-variable-name-face)))
+             (identifier) @font-lock-variable-name-face))
+     (init_declarator declarator: (_) @c-ts-mode--fontify-declarator))
 
    :language mode
-   :feature 'expression
+   :feature 'function
    '((call_expression
-      function: (identifier) @font-lock-function-name-face)
-     (field_expression
-      argument: (identifier) @font-lock-variable-name-face)
-     (pointer_expression
-      (identifier) @font-lock-variable-name-face))
+      function: (identifier) @font-lock-function-name-face))
+
+   :language mode
+   :feature 'variable
+   '((identifier) @c-ts-mode--fontify-variable)
 
    :language mode
    :feature 'label
-   '((expression_statement (identifier) @font-lock-variable-name-face)
-     (labeled_statement
+   '((labeled_statement
       label: (statement_identifier) @font-lock-constant-face))
 
    :language mode
@@ -367,6 +367,19 @@ For NODE, OVERRIDE, START, END, and ARGS, see
         (_ 'font-lock-variable-name-face))
       override))))
 
+(defun c-ts-mode--fontify-variable (node override start end &rest _)
+  "Fontify an identifier node.
+Fontify it if NODE is not a function identifier.  For NODE,
+OVERRIDE, START, END, and ARGS, see `treesit-font-lock-rules'."
+  (when (not (equal (treesit-node-type
+                     (treesit-node-parent node))
+                    "call_expression"))
+    (treesit-fontify-with-override
+     (max (treesit-node-start node) start)
+     (min (treesit-node-end node) end)
+     'font-lock-variable-name-face
+     override)))
+
 (defun c-ts-mode--fontify-defun (node override start end &rest _)
   "Correctly fontify the DEFUN macro.
 For NODE, OVERRIDE, START, and END, see
@@ -497,8 +510,8 @@ the subtrees."
   (setq-local treesit-font-lock-feature-list
               '(( comment constant keyword literal preprocessor string)
                 ( assignment definition label property type)
-                ( bracket delimiter error escape-sequence expression
-                  operator))))
+                ( delimiter error escape-sequence function
+                  operator variable bracket))))
 
 ;;;###autoload
 (define-derived-mode c-ts-mode c-ts-mode--base-mode "C"

lisp/progmodes/java-ts-mode.el

@@ -116,18 +116,14 @@
   "C keywords for tree-sitter font-locking.")
 
 (defvar java-ts-mode--operators
-  '("@" "+" ":" "++" "-" "--" "&" "&&" "|" "||"
-    "!=" "==" "*" "/" "%" "<" "<=" ">" ">=" "="
-    "-=" "+=" "*=" "/=" "%=" "->" "^" "^=" "&="
-    "|=" "~" ">>" ">>>" "<<" "::" "?")
+  '("+" ":" "++" "-" "--" "&" "&&" "|" "||" "="
+    "!=" "==" "*" "/" "%" "<" "<=" ">" ">="
+    "-=" "+=" "*=" "/=" "%=" "->" "^" "^="
+    "|=" "~" ">>" ">>>" "<<" "::" "?" "&=")
   "C operators for tree-sitter font-locking.")
 
 (defvar java-ts-mode--font-lock-settings
   (treesit-font-lock-rules
-   :language 'java
-   :override t
-   :feature 'basic
-   '((identifier) @font-lock-variable-name-face)
    :language 'java
    :override t
    :feature 'comment
@@ -149,7 +145,8 @@
    :language 'java
    :override t
    :feature 'operator
-   `([,@java-ts-mode--operators] @font-lock-builtin-face)
+   `([,@java-ts-mode--operators] @font-lock-operator-face
+     "@" @font-lock-constant-face)
    :language 'java
    :override t
    :feature 'annotation
@@ -191,6 +188,8 @@
 
      (method_reference (identifier) @font-lock-type-face)
 
+     (scoped_identifier (identifier) @font-lock-variable-name-face)
+
      ((scoped_identifier name: (identifier) @font-lock-type-face)
       (:match "^[A-Z]" @font-lock-type-face))
 
@@ -206,6 +205,12 @@
    `((method_declaration
       name: (identifier) @font-lock-function-name-face)
 
+     (variable_declarator
+      name: (identifier) @font-lock-variable-name-face)
+
+     (element_value_pair
+      key: (identifier) @font-lock-property-face)
+
      (formal_parameter
       name: (identifier) @font-lock-variable-name-face)
 
@@ -220,7 +225,15 @@
      (method_invocation
       name: (identifier) @font-lock-function-name-face)
 
-     (argument_list (identifier) @font-lock-variable-name-face)))
+     (argument_list (identifier) @font-lock-variable-name-face))
+
+   :language 'java
+   :feature 'bracket
+   '((["(" ")" "[" "]" "{" "}"]) @font-lock-bracket-face)
+
+   :language 'java
+   :feature 'delimiter
+   '((["," ":" ";"]) @font-lock-delimiter-face))
   "Tree-sitter font-lock settings.")
 
 (defun java-ts-mode--imenu-1 (node)
@@ -248,33 +261,27 @@ the subtrees."
 (defun java-ts-mode--imenu ()
   "Return Imenu alist for the current buffer."
   (let* ((node (treesit-buffer-root-node))
-         (class-tree
-          `("Class" . ,(java-ts-mode--imenu-1
-                        (treesit-induce-sparse-tree
-                         node "^class_declaration$" nil 1000))))
-         (interface-tree
-          `("Interface" . ,(java-ts-mode--imenu-1
-                            (treesit-induce-sparse-tree
-                             node "^interface_declaration$"  nil 1000))))
-         (enum-tree
-          `("Enum" . ,(java-ts-mode--imenu-1
-                       (treesit-induce-sparse-tree
-                        node "^enum_declaration$"  nil 1000))))
-         (record-tree
-          `("Record" . ,(java-ts-mode--imenu-1
-                         (treesit-induce-sparse-tree
-                          node "^record_declaration$"  nil 1000))))
-         (method-tree
-          `("Method" . ,(java-ts-mode--imenu-1
-                         (treesit-induce-sparse-tree
-                          node "^method_declaration$"  nil 1000)))))
-    (cl-remove-if
-     #'null
-     `(,(when (cdr class-tree) class-tree)
-       ,(when (cdr interface-tree) interface-tree)
-       ,(when (cdr enum-tree) enum-tree)
-       ,(when (cdr record-tree) record-tree)
-       ,(when (cdr method-tree) method-tree)))))
+         (class-tree (treesit-induce-sparse-tree
+                      node "^class_declaration$" nil 1000))
+         (interface-tree (treesit-induce-sparse-tree
+                          node "^interface_declaration$" nil 1000))
+         (enum-tree (treesit-induce-sparse-tree
+                     node "^enum_declaration$" nil 1000))
+         (record-tree (treesit-induce-sparse-tree
+                       node "^record_declaration$"  nil 1000))
+         (method-tree (treesit-induce-sparse-tree
+                       node "^method_declaration$" nil 1000))
+         (class-index (java-ts-mode--imenu-1 class-tree))
+         (interface-index (java-ts-mode--imenu-1 interface-tree))
+         (enum-index (java-ts-mode--imenu-1 enum-tree))
+         (record-index (java-ts-mode--imenu-1 record-tree))
+         (method-index (java-ts-mode--imenu-1 method-tree)))
+    (append
+     (when class-index `(("Class" . ,class-index)))
+     (when interface-index `(("Interface" . ,interface-index)))
+     (when enum-index `(("Enum" . ,enum-index)))
+     (when record-index `(("Record" . ,record-index)))
+     (when method-index `(("Method" . ,method-index))))))
 
 ;;;###autoload
 (define-derived-mode java-ts-mode prog-mode "Java"
@@ -307,9 +314,9 @@ the subtrees."
   ;; Font-lock.
   (setq-local treesit-font-lock-settings java-ts-mode--font-lock-settings)
   (setq-local treesit-font-lock-feature-list
-              '((basic comment keyword constant string operator)
+              '((comment keyword constant string)
                 (type definition expression literal annotation)
-                ()))
+                (bracket delimiter operator)))
 
   ;; Imenu.
   (setq-local imenu-create-index-function #'java-ts-mode--imenu)

lisp/progmodes/sh-script.el

@@ -1469,57 +1469,12 @@ When the region is active, send the region instead."
 (defvar sh-mode--treesit-settings)
 
 ;;;###autoload
-(define-derived-mode sh-mode prog-mode "Shell-script"
-  "Major mode for editing shell scripts.
-This mode works for many shells, since they all have roughly the same syntax,
-as far as commands, arguments, variables, pipes, comments etc. are concerned.
-Unless the file's magic number indicates the shell, your usual shell is
-assumed.  Since filenames rarely give a clue, they are not further analyzed.
+(define-derived-mode sh-base-mode prog-mode "Shell-script"
+  "Generic major mode for editing shell scripts.
 
-This mode adapts to the variations between shells (see `sh-set-shell') by
-means of an inheritance based feature lookup (see `sh-feature').  This
-mechanism applies to all variables (including skeletons) that pertain to
-shell-specific features.  Shell script files can use the `sh-shell' local
-variable to indicate the shell variant to be used for the file.
-
-The default style of this mode is that of Rosenblatt's Korn shell book.
-The syntax of the statements varies with the shell being used.  The
-following commands are available, based on the current shell's syntax:
-\\<sh-mode-map>
-\\[sh-case]	 case statement
-\\[sh-for]	 for loop
-\\[sh-function]	 function definition
-\\[sh-if]	 if statement
-\\[sh-indexed-loop]	 indexed loop from 1 to n
-\\[sh-while-getopts]	 while getopts loop
-\\[sh-repeat]	 repeat loop
-\\[sh-select]	 select loop
-\\[sh-until]	 until loop
-\\[sh-while]	 while loop
-
-For sh and rc shells indentation commands are:
-\\[smie-config-show-indent]	Show the rules controlling this line's indentation.
-\\[smie-config-set-indent]	Change the rules controlling this line's indentation.
-\\[smie-config-guess]  Try to tweak the indentation rules so the
-buffer indents as it currently is indented.
-
-
-\\[backward-delete-char-untabify]	 Delete backward one position, even if it was a tab.
-\\[sh-end-of-command]	 Go to end of successive commands.
-\\[sh-beginning-of-command]	 Go to beginning of successive commands.
-\\[sh-set-shell]	 Set this buffer's shell, and maybe its magic number.
-\\[sh-execute-region]	 Have optional header and region be executed in a subshell.
-
-`sh-electric-here-document-mode' controls whether insertion of two
-unquoted < insert a here document.  You can control this behavior by
-modifying `sh-mode-hook'.
-
-If you generally program a shell different from your login shell you can
-set `sh-shell-file' accordingly.  If your shell's file name doesn't correctly
-indicate what shell it is use `sh-alias-alist' to translate.
-
-If your shell gives error messages with line numbers, you can use \\[executable-interpret]
-with your script for an edit-interpret-debug cycle."
+This is a generic major mode intended to be inherited by concrete
+implementations.  Currently there are two: `sh-mode' and
+`bash-ts-mode'."
   (make-local-variable 'sh-shell-file)
   (make-local-variable 'sh-shell)
 
@@ -1583,31 +1538,82 @@ with your script for an edit-interpret-debug cycle."
    nil nil)
   (add-hook 'flymake-diagnostic-functions #'sh-shellcheck-flymake nil t)
   (add-hook 'hack-local-variables-hook
-    #'sh-after-hack-local-variables nil t)
+            #'sh-after-hack-local-variables nil t))
 
-  (cond
-   ;; Tree-sitter.  If the shell is bash, we can enable tree-sitter.
-   ((treesit-ready-p sh-shell)
+;;;###autoload
+(define-derived-mode sh-mode sh-base-mode "Shell-script"
+  "Major mode for editing shell scripts.
+This mode works for many shells, since they all have roughly the same syntax,
+as far as commands, arguments, variables, pipes, comments etc. are concerned.
+Unless the file's magic number indicates the shell, your usual shell is
+assumed.  Since filenames rarely give a clue, they are not further analyzed.
+
+This mode adapts to the variations between shells (see `sh-set-shell') by
+means of an inheritance based feature lookup (see `sh-feature').  This
+mechanism applies to all variables (including skeletons) that pertain to
+shell-specific features.  Shell script files can use the `sh-shell' local
+variable to indicate the shell variant to be used for the file.
+
+The default style of this mode is that of Rosenblatt's Korn shell book.
+The syntax of the statements varies with the shell being used.  The
+following commands are available, based on the current shell's syntax:
+\\<sh-mode-map>
+\\[sh-case]	 case statement
+\\[sh-for]	 for loop
+\\[sh-function]	 function definition
+\\[sh-if]	 if statement
+\\[sh-indexed-loop]	 indexed loop from 1 to n
+\\[sh-while-getopts]	 while getopts loop
+\\[sh-repeat]	 repeat loop
+\\[sh-select]	 select loop
+\\[sh-until]	 until loop
+\\[sh-while]	 while loop
+
+For sh and rc shells indentation commands are:
+\\[smie-config-show-indent]	Show the rules controlling this line's indentation.
+\\[smie-config-set-indent]	Change the rules controlling this line's indentation.
+\\[smie-config-guess]  Try to tweak the indentation rules so the
+buffer indents as it currently is indented.
+
+
+\\[backward-delete-char-untabify]	 Delete backward one position, even if it was a tab.
+\\[sh-end-of-command]	 Go to end of successive commands.
+\\[sh-beginning-of-command]	 Go to beginning of successive commands.
+\\[sh-set-shell]	 Set this buffer's shell, and maybe its magic number.
+\\[sh-execute-region]	 Have optional header and region be executed in a subshell.
+
+`sh-electric-here-document-mode' controls whether insertion of two
+unquoted < insert a here document.  You can control this behavior by
+modifying `sh-mode-hook'.
+
+If you generally program a shell different from your login shell you can
+set `sh-shell-file' accordingly.  If your shell's file name doesn't correctly
+indicate what shell it is use `sh-alias-alist' to translate.
+
+If your shell gives error messages with line numbers, you can use \\[executable-interpret]
+with your script for an edit-interpret-debug cycle."
+  (setq font-lock-defaults
+        `((sh-font-lock-keywords
+           sh-font-lock-keywords-1 sh-font-lock-keywords-2)
+          nil nil
+          ((?/ . "w") (?~ . "w") (?. . "w") (?- . "w") (?_ . "w")) nil
+          (font-lock-syntactic-face-function
+           . ,#'sh-font-lock-syntactic-face-function))))
+
+;;;###autoload
+(defalias 'shell-script-mode 'sh-mode)
+
+;;;###autoload
+(define-derived-mode bash-ts-mode sh-base-mode "Bash"
+  "Major mode for editing Bash shell scripts."
+  (when (treesit-ready-p 'bash)
     (setq-local treesit-font-lock-feature-list
                 '((comment function string heredoc)
                   (variable keyword command declaration-command)
                   (constant operator builtin-variable)))
     (setq-local treesit-font-lock-settings
                 sh-mode--treesit-settings)
-    (treesit-major-mode-setup))
-   ;; Elisp.
-   (t
-    (setq font-lock-defaults
-          `((sh-font-lock-keywords
-             sh-font-lock-keywords-1 sh-font-lock-keywords-2)
-            nil nil
-            ((?/ . "w") (?~ . "w") (?. . "w") (?- . "w") (?_ . "w")) nil
-            (font-lock-syntactic-face-function
-             . ,#'sh-font-lock-syntactic-face-function))))))
-
-;;;###autoload
-(defalias 'shell-script-mode 'sh-mode)
-
+    (treesit-major-mode-setup)))
 
 (defun sh-font-lock-keywords (&optional keywords)
   "Function to get simple fontification based on `sh-font-lock-keywords'.

lisp/progmodes/ts-mode.el

@@ -210,18 +210,18 @@
    :language 'tsx
    :override t
    :feature 'property
-   `((pair key: (property_identifier) @font-lock-variable-name-face)
+   `((pair key: (property_identifier) @font-lock-property-face)
 
      (pair value: (identifier) @font-lock-variable-name-face)
 
      (pair
-      key: (property_identifier) @font-lock-function-name-face
+      key: (property_identifier) @font-lock-property-face
       value: [(function) (arrow_function)])
 
      (property_signature
-      name: (property_identifier) @font-lock-variable-name-face)
+      name: (property_identifier) @font-lock-property-face)
 
-     ((shorthand_property_identifier) @font-lock-variable-name-face)
+     ((shorthand_property_identifier) @font-lock-property-face)
 
      ((shorthand_property_identifier_pattern)
       @font-lock-variable-name-face))
@@ -230,7 +230,7 @@
    :override t
    :feature 'pattern
    `((pair_pattern
-      key: (property_identifier) @font-lock-variable-name-face)
+      key: (property_identifier) @font-lock-property-face)
 
      (array_pattern (identifier) @font-lock-variable-name-face))
 
@@ -249,7 +249,14 @@
       [(nested_identifier (identifier)) (identifier)]
       @font-lock-function-name-face)
 
-     (jsx_attribute (property_identifier) @font-lock-constant-face)))
+     (jsx_attribute (property_identifier) @font-lock-constant-face))
+   :language 'tsx
+   :feature 'bracket
+   '((["(" ")" "[" "]" "{" "}"]) @font-lock-bracket-face)
+
+   :language 'tsx
+   :feature 'delimiter
+   '((["," ":" ";"]) @font-lock-delimiter-face))
   "Tree-sitter font-lock settings.")
 
 ;;;###autoload
@@ -297,7 +304,7 @@
     (setq-local treesit-font-lock-feature-list
                 '((comment declaration)
                   (string keyword identifier expression constant)
-                  (property pattern jsx)))
+                  (property pattern jsx bracket delimiter)))
     ;; Imenu.
     (setq-local imenu-create-index-function #'js--treesit-imenu)
 

lisp/treesit.el

@@ -95,7 +95,12 @@ indent, imenu, etc."
   :group 'tools
   :version "29.1")
 
-(defcustom treesit-max-buffer-size (* 4 1024 1024)
+(defcustom treesit-max-buffer-size
+  (let ((mb (* 1024 1024)))
+    ;; 40MB for 64-bit systems, 15 for 32-bit.
+    (if (> most-positive-fixnum (* 4 1024 mb))
+        (* 40 mb)
+      (* 15 mb)))
   "Maximum buffer size for enabling tree-sitter parsing (in bytes)."
   :type 'integer
   :version "29.1")
@@ -689,16 +694,23 @@ name, it is ignored."
   "If non-nil, print debug messages when fontifying.")
 
 (defun treesit-font-lock-recompute-features (&optional add-list remove-list)
-  "Enable/disable font-lock settings according to decoration level.
+  "Enable/disable font-lock features.
 
-First compute the enabled features according to
-`treesit-font-lock-feature-list' and `font-lock-maximum-decoration',
-then, if ADD-LIST or REMOVE-LIST are not omitted, further add and
-remove features accordingly.
+Enable each feature in ADD-LIST, disable each feature in
+REMOVE-LIST.
+
+If both ADD-LIST and REMOVE-LIST are omitted, recompute each
+feature according to `treesit-font-lock-feature-list' and
+`font-lock-maximum-decoration'.  Let N be the value of
+`font-lock-maximum-decoration', features in the first Nth sublist
+of `treesit-font-lock-feature-list' are enabled, and the rest
+features are disabled.  If `font-lock-maximum-decoration' is t,
+all features in `treesit-font-lock-feature-list' are enabled, and
+the rest are disabled.
 
 ADD-LIST and REMOVE-LIST are each a list of feature symbols.  The
 same feature symbol cannot appear in both lists; the function
-signals the `treesit-font-lock-error' error if so."
+signals the `treesit-font-lock-error' error if that happens."
   (when-let ((intersection (cl-intersection add-list remove-list)))
     (signal 'treesit-font-lock-error
             (list "ADD-LIST and REMOVE-LIST contain the same feature"
@@ -712,13 +724,23 @@ signals the `treesit-font-lock-error' error if so."
                                 (>= level (1+ idx)))
                          append features))
          (features (cl-set-difference (cl-union base-features add-list)
-                                      remove-list)))
+                                      remove-list))
+         ;; If additive non-nil, we are configuring on top of the
+         ;; existing configuration, if nil, we are resetting
+         ;; everything according to `treesit-font-lock-feature-list'.
+         (additive (or add-list remove-list)))
     (cl-loop for idx = 0 then (1+ idx)
              for setting in treesit-font-lock-settings
              for feature = (nth 2 setting)
+             for current-value = (nth 1 setting)
              ;; Set the ENABLE flag for the setting.
              do (setf (nth 1 (nth idx treesit-font-lock-settings))
-                      (if (memq feature features) t nil)))))
+                      (cond
+                       ((not additive)
+                        (if (memq feature features) t nil))
+                       ((memq feature add-list) t)
+                       ((memq feature remove-list) nil)
+                       (t current-value))))))
 
 (defun treesit-fontify-with-override (start end face override)
   "Apply FACE to the region between START and END.
@@ -1761,10 +1783,11 @@ to the offending pattern and highlight the pattern."
 
 (defun treesit--explorer--nodes-to-highlight (language)
   "Return nodes for LANGUAGE covered in region.
-This function tries to return the largest node possible.  So it
-will return a single large node rather than a bunch of small
-nodes.  If it end up returning multiple small nodes, it only
-returns the first and last node, and omits the ones in between."
+This function tries to return the largest node possible.  If the
+region covers exactly one node, that node is returned (in a
+list).  If the region covers more than one node, two nodes are
+returned: the very first one in the region and the very last one
+in the region."
   (let* ((beg (region-beginning))
          (end (region-end))
          (node (treesit-node-on beg end language))
@@ -1883,13 +1906,10 @@ Return the start of the syntax tree text corresponding to NODE."
 
 (defun treesit--explorer-draw-node (node)
   "Draw the syntax tree of NODE.
-If NODE and NODE-HIGHLIGHT are the same node, highlight it.
 
-When this function is called, point should be at an empty line,
-when appropriate indent in front of point.  When this function
-returns, it leaves point at the end of the last line of NODE.
-
-Return the start position of NODE-HIGHLIGHT in the buffer, if any."
+When this function is called, point should be at the position
+where the node should start.  When this function returns, it
+leaves point at the end of the last line of NODE."
   (let* ((type (treesit-node-type node))
          (field-name (treesit-node-field-name node))
          (children (treesit-node-children node))