Fix dependency between packages and ELPA checkout

Git update not working -- it was always running

.gitignore

@@ -148,6 +148,12 @@ test/manual/etags/ETAGS
 test/manual/etags/CTAGS
 test/manual/indent/*.new
 
+# ELPA
+/elpa/elpa-git/
+/elpa/packages/
+/lisp/elpa/
+/test/lisp/elpa/
+
 # ctags, etags.
 TAGS
 !admin/notes/tags
@@ -284,4 +290,4 @@ lib/SYS/
 nt/emacs.rc
 nt/emacsclient.rc
 src/gdb.ini
-/var/
+/var/

Makefile.in

@@ -108,6 +108,9 @@ am__v_at_ = $(am__v_at_@AM_DEFAULT_V@)
 am__v_at_0 = @
 am__v_at_1 =
 
+ELPA_IN_CORE=@ELPA_IN_CORE@
+ELPA_LOCAL_PATH=@ELPA_LOCAL_PATH@
+
 # ==================== Where To Install Things ====================
 
 # Location to install Emacs.app under GNUstep / macOS.
@@ -311,7 +314,7 @@ EMACS = ${EMACS_NAME}${EXEEXT}
 EMACSFULL = `echo emacs-${version} | sed '$(TRANSFORM)'`${EXEEXT}
 
 # Subdirectories to make recursively.
-SUBDIR = $(NTDIR) lib lib-src src lisp
+SUBDIR = $(NTDIR) elpa lib lib-src src lisp
 
 # The subdir makefiles created by config.status.
 SUBDIR_MAKEFILES_IN = @SUBDIR_MAKEFILES_IN@
@@ -405,6 +408,9 @@ src: lib-src
 # We need to build 'emacs' in 'src' to compile the *.elc files in 'lisp'.
 lisp: src
 
+# We need to fetch 'elpa' before we build lisp
+lisp: elpa
+
 # These targets should be "${SUBDIR} without 'src'".
 lib lib-src lisp nt: Makefile
 	$(MAKE) -C $@ all
@@ -422,6 +428,12 @@ VCSWITNESS = $(if $(wildcard $(srcdir)/$(dirstate)),$$(srcdir)/../$(dirstate))
 src: Makefile
 	$(MAKE) -C $@ VCSWITNESS='$(VCSWITNESS)' all
 
+elpa:
+  ifeq ($(ELPA_IN_CORE),yes)
+	$(MAKE) -C $@ VCSWITNESS='$(VCSWITNESS)' all
+  endif
+
+
 blessmail: Makefile src
 	$(MAKE) -C lib-src maybe-blessmail
 
@@ -830,7 +842,7 @@ endef
 ###      target for GCC does not delete 'libgcc.a', because recompiling it
 ###      is rarely necessary and takes a lot of time.
 mostlyclean_dirs = src oldXMenu lwlib lib lib-src nt doc/emacs doc/misc \
-  doc/lispref doc/lispintro
+  doc/lispref doc/lispintro elpa
 
 $(foreach dir,$(mostlyclean_dirs),$(eval $(call submake_template,$(dir),mostlyclean)))
 

configure.ac

@@ -542,6 +542,15 @@ elif test "${enableval}" != "yes"; then
   locallisppath=${enableval} locallisppathset=yes
 fi)
 
+AC_ARG_ENABLE(elpa,
+[AS_HELP_STRING([--enable-elpa],
+		[Copy Selected ELPA files into core])],
+  ELPA_IN_CORE=yes
+  ELPA_LOCAL_PATH=$enableval,
+  ELPA_IN_CORE=
+  ELPA_LOCAL_PATH=
+)
+
 AC_ARG_ENABLE(checking,
 [AS_HELP_STRING([--enable-checking@<:@=LIST@:>@],
 		[enable expensive run-time checks.  With LIST,
@@ -5169,6 +5178,8 @@ AC_SUBST(ns_appresdir)
 AC_SUBST(ns_appsrc)
 AC_SUBST(GNU_OBJC_CFLAGS)
 AC_SUBST(OTHER_FILES)
+AC_SUBST(ELPA_IN_CORE)
+AC_SUBST(ELPA_LOCAL_PATH)
 
 if test -n "${term_header}"; then
     AC_DEFINE_UNQUOTED(TERM_HEADER, "${term_header}",
@@ -5679,7 +5690,7 @@ dnl config.status treats $srcdir specially, so I think this is ok...
 AC_CONFIG_FILES([$srcdir/doc/man/emacs.1])
 
 m4_define([subdir_makefiles],
-  [lib/Makefile lib-src/Makefile oldXMenu/Makefile doc/emacs/Makefile doc/misc/Makefile doc/lispintro/Makefile doc/lispref/Makefile src/Makefile lwlib/Makefile lisp/Makefile leim/Makefile nextstep/Makefile nt/Makefile])
+  [lib/Makefile lib-src/Makefile oldXMenu/Makefile doc/emacs/Makefile doc/misc/Makefile doc/lispintro/Makefile doc/lispref/Makefile src/Makefile lwlib/Makefile lisp/Makefile leim/Makefile nextstep/Makefile nt/Makefile elpa/Makefile])
 SUBDIR_MAKEFILES="subdir_makefiles"
 AC_CONFIG_FILES(subdir_makefiles)
 

elpa/Makefile.in

@@ -0,0 +1,52 @@
+ELPA_LOCAL_PATH=@ELPA_LOCAL_PATH@
+
+ifdef ELPA_LOCAL_PATH
+	ELPA_PATH=$(ELPA_LOCAL_PATH)
+else
+	ELPA_PATH=elpa-git
+endif
+
+## Get this stuff from above
+dirstate = .git/logs/HEAD
+VCSWITNESS = $(if $(wildcard $(srcdir)/$(dirstate)),$$(srcdir)/../$(dirstate))
+
+all: all-packages
+
+../lisp/elpa:
+	mkdir ../lisp/elpa
+
+../test/lisp/elpa:
+	mkdir ../test/lisp/elpa
+
+packages:
+	mkdir packages
+
+$(ELPA_PATH):
+	git clone --mirror https://git.savannah.gnu.org/git/emacs/elpa.git $(ELPA_PATH)
+
+$(ELPA_PATH)/FETCH_HEAD: $(ELPA_PATH) Makefile
+	cd $(ELPA_PATH);git fetch --all
+
+PACKAGES=
+define package_template
+  packages/$(1)-$(2)/$(1): $(ELPA_PATH)/FETCH_HEAD
+	./bin/extract-package.sh -g $(ELPA_PATH) -p $(1) -s $(2) $(3)
+	$$(MAKE) -C packages/$(1)-$(2)/$(1) -f package-makefile.mk deploy PACKAGE=$(1)
+
+  $(1): packages/$(1)-$(2)/$(1)
+
+  .PHONY: $(1)
+  PACKAGES:=$$(PACKAGES) packages/$(1)-$(2)/$(1)
+endef
+
+$(eval $(call package_template,pabbrev,d28cf8632d2691dc93afbb28500126242d37961c,-e))
+$(eval $(call package_template,darkroom,73701d95133ba6581a95cec9489224b87f483cc0))
+$(eval $(call package_template,queue,cf2001d3e83e05af5820174e9fa1f9638a4f8c08))
+
+
+all-packages: $(PACKAGES)
+
+clean:
+	rm -rf packages
+	rm -rf ../lisp/elpa
+	rm -rf ../test/lisp/elpa

elpa/bin/extract-package.sh

@@ -0,0 +1,55 @@
+#!/bin/bash
+
+function grab_external {
+    rm -rf packages/$PACKAGE*
+    mkdir --parents $PACKAGE_LOC
+    cwd=`pwd`
+    cd $GIT_LOC
+    git archive $SHA \
+        | tar xv -C $cwd/$PACKAGE_LOC
+    cd $cwd
+    cp --no-clobber bin/package-makefile.mk $PACKAGE_LOC
+}
+
+
+function grab_subtree {
+    rm -rf packages/*$PACKAGE
+    mkdir --parents $PACKAGE_LOC
+    cwd=`pwd`
+    cd $GIT_LOC
+    git archive $SHA packages/$PACKAGE \
+        | tar xv  --strip-components=2 -C $cwd/$PACKAGE_LOC
+    cd $cwd
+    cp --no-clobber bin/package-makefile.mk $PACKAGE_LOC
+}
+
+SHA=
+PACKAGE=
+EXTERNAL=0
+GIT_LOC=
+while getopts "g:s:p:e" opt; do
+    case $opt in
+        s)
+            SHA=$OPTARG
+            ;;
+        p)
+            PACKAGE=$OPTARG
+            ;;
+        e)
+            EXTERNAL=1
+            ;;
+        g)
+            GIT_LOC=$OPTARG
+            ;;
+    esac
+done
+
+PACKAGE_LOC=packages/$PACKAGE-$SHA/$PACKAGE
+
+if (($EXTERNAL))
+then
+    grab_external
+    exit 0
+fi
+
+grab_subtree

elpa/bin/package-makefile.mk

@@ -0,0 +1,36 @@
+FILE = $(realpath $(lastword $(MAKEFILE_LIST)))
+DIR = $(dir $(FILE))
+DIRS = $(subst /, ,$(DIR))
+PACKAGE=$(lastword $(DIRS))
+
+TEST=$(wildcard *-test.el)
+TEST:=$(wildcard *-tests.el) $(TEST)
+LISP=$(filter-out $(TEST), $(wildcard *.el))
+
+LISP_DIR=../../../../lisp/elpa/$(PACKAGE)/
+TEST_DIR=../../../../test/lisp/elpa/$(PACKAGE)/
+
+LISP_TARGETS=$(addprefix $(LISP_DIR),$(LISP))
+TEST_TARGETS=$(addprefix $(TEST_DIR),$(TEST))
+
+deploy: $(LISP_TARGETS) $(TEST_TARGETS)
+
+define lisp_template
+  $(LISP_DIR)$(1): $(1) $(LISP_DIR)
+	cp $$< $$@
+endef
+
+$(foreach prereq, $(LISP), $(eval $(call lisp_template,$(prereq))))
+
+define test_template
+	  $(TEST_DIR)$(1): $(1) $(TEST_DIR)
+	cp $$< $$@
+endef
+
+$(foreach prereq, $(TEST), $(eval $(call test_template,$(prereq))))
+
+$(LISP_DIR):
+	mkdir --parents $(LISP_DIR)
+
+$(TEST_DIR):
+	mkdir --parents $(TEST_DIR)

notes.org

@@ -0,0 +1,84 @@
+
+
+* Working on
+
+Have package checkout working for external
+Next get it working for non external so we can template
+
+Then how to get the submakes working
+
+
+
+* To Fix
+
+Think we are running git fetch --all on every package. Is there
+something more intelligent we can do here? Lookup VCS Witness
+elsewhere.
+
+* Notes
+
+1) Scripting?
+
+Guess we have to use sh
+
+2) How to get specific directory of a repository
+
+
+
+git archive --remote=phillord@git.savannah.gnu.org:/srv/git/emacs/elpa.git master packages/hydra | tar xv --strip-components=1
+
+Currently, this requires a member login -- it doesn't work with http
+or the current git server.
+
+Would also be possible to configure this from local.
+
+3) How to work out when a directory is out of date wrt to the make
+   file
+
+unpack the directory with the commit
+
+SHAID-hydra
+
+Then when the SHA is updated to point somewhere else, magically, the
+directory will run out of space
+
+
+
+4) Heuristics for working out which files to copy to right place
+
+*-test.el
+*-tests.el
+/test/*.el
+
+to test
+
+All other *el to lisp
+
+Nothing else
+
+5) Work out what is the right place to copy to
+
+lisp/elpa/package-name
+test/lisp/elpa/package-name
+
+is my prefered option. Eli may complain in which case "package-name"
+can be dropped.
+
+
+6) How to enable ELPA packages to customize the process
+
+Add the option to run an core-deploy.sh file placed into the root of a
+package.
+
+
+7) What about make bootstrap
+
+I think we would add another command called make clean-elpa which
+would delete the elpa extracted files. I'd have to think very
+carefully about which clean things I wanted to attach to. a priori
+there is no reason to ever check the files out more than once.
+
+
+8) Support other repos
+
+Org-mode effectively does this already