Directory reorg: move code into redo/, generate binaries in bin/.

It's time to start preparing for a version of redo that doesn't work
unless we build it first (because it will rely on C modules, and
eventually be rewritten in C altogether).

To get rolling, remove the old-style symlinks to the main programs, and
rename those programs from redo-*.py to redo/cmd_*.py.  We'll also move
all library functions into the redo/ dir, which is a more python-style
naming convention.

Previously, install.do was generating wrappers for installing in
/usr/bin, which extend sys.path and then import+run the right file.
This made "installed" redo work quite differently from running redo
inside its source tree.  Instead, let's always generate the wrappers in
bin/, and not make anything executable except those wrappers.

Since we're generating wrappers anyway, let's actually auto-detect the
right version of python for the running system; distros can't seem to
agree on what to call their python2 binaries (sigh). We'll fill in the
right #! shebang lines.  Since we're doing that, we can stop using
/usr/bin/env, which will a) make things slightly faster, and b) let us
use "python -S", which tells python not to load a bunch of extra crap
we're not using, thus improving startup times.

Annoyingly, we now have to build redo using minimal/do, then run the
tests using bin/redo.  To make this less annoying, we add a toplevel
./do script that knows the right steps, and a Makefile (whee!) for
people who are used to typing 'make' and 'make test' and 'make clean'.

.gitignore

@@ -4,8 +4,6 @@
 /minimal/.do_built
 /minimal/.do_built.dir
 *~
-*.pyc
 *.tmp
-/redo-sh
 *.did
-/website
+/docs.out

Documentation/md-to-man.do

@@ -1,8 +0,0 @@
-redo-ifchange md2man.py
-if ./md2man.py /dev/null /dev/null >/dev/null; then
-	echo './md2man.py $2.md.tmp $2.html'
-else
-	echo "Warning: md2man.py missing modules; can't generate manpages." >&2
-	echo "Warning: try this: sudo easy_install markdown BeautifulSoup" >&2
-	echo 'echo Skipping: $2.1 >&2'
-fi

Makefile

@@ -1,10 +1,21 @@
+default: all
+
+build:
+	+./do build
+
 all:
+	+./do all
+
+test:
+	+./do test
 
-Makefile:
-	@
+clean:
+	+./do clean
 
-%: FORCE
-	+./redo $@
+install:
+	+./do install
 
-.PHONY: FORCE
+env:
+	env
 
+.PHONY: build test clean env

README.md

@@ -11,6 +11,9 @@ This version, sometimes called apenwarr/redo, is probably the most advanced
 one, including support for parallel builds, improved logging, and helpful
 debugging features.
 
+To build and test redo, run `./do -j10 test`.  To install it, run
+`DESTDIR=/tmp/testinstall PREFIX=/usr/local ./do -j10 install`.
+
 ---
 
 - View the [documentation](https://redo.rtfd.io) via readthedocs.org

_all.do

@@ -3,5 +3,7 @@ if [ "$1,$2" != "_all,_all" ]; then
 	exit 1
 fi
 
-redo-ifchange redo-sh
-redo-ifchange version/all Documentation/all
+# Do this first, to ensure we're using a good shell
+redo-ifchange redo/sh
+
+redo-ifchange bin/all docs/all

all.do

@@ -1,2 +1,2 @@
 redo-ifchange _all
-echo "Nothing much to do.  Try 'redo t/all' or 'redo test'" >&2
+echo "Nothing much to do.  Try 'bin/redo -j10 test'" >&2

bin/.gitignore

@@ -0,0 +1,3 @@
+list
+redo
+redo-*

bin/all.do

@@ -0,0 +1,3 @@
+exec >&2
+redo-ifchange ../redo/version/all ../redo/python list redo-sh
+xargs redo-ifchange <list

bin/clean.do

@@ -0,0 +1 @@
+rm -f redo redo-* list whichpython

bin/default.do

@@ -0,0 +1,33 @@
+exec >&2
+
+case $1 in
+	redo-sh)
+		redo-ifchange ../redo/sh
+		cat >$3 <<-EOF
+			#!/bin/sh
+			d=\$(dirname "\$0")/..
+			[ -x \$d/lib/redo/sh ] && exec \$d/lib/redo/sh "\$@"
+			[ -x \$d/redo/sh ] && exec \$d/redo/sh "\$@"
+			echo "\$0: fatal: can't find \$d/lib/redo/sh or \$d/redo/sh" >&2
+			exit 98
+		EOF
+		chmod a+x "$3"
+		;;
+	redo|redo-*)
+		redo-ifchange ../redo/whichpython
+		read py <../redo/whichpython
+		cmd=${1#redo-}
+		cat >$3 <<-EOF
+			#!$py -S
+			import sys, os;
+			exe = os.path.realpath(os.path.abspath(sys.argv[0]))
+			exedir = os.path.dirname(exe)
+			sys.path.insert(0, os.path.join(exedir, '../lib'))
+			sys.path.insert(0, os.path.join(exedir, '..'))
+			import redo.cmd_$cmd
+			redo.cmd_$cmd.main()
+		EOF
+		chmod a+x "$3"
+		;;
+	*) echo "$0: don't know how to build '$1'" >&2; exit 99 ;;
+esac

bin/list.do

@@ -0,0 +1,16 @@
+exec >&2
+
+redo-always
+(
+	cd ../redo
+	for d in cmd_*.py; do
+		d=${d#cmd_}
+		d=${d%.py}
+		if [ "$d" = "redo" ]; then
+			echo redo
+		else
+			echo "redo-${d%.py}"
+		fi
+	done
+) >$3
+redo-stamp <$3

clean.do

@@ -1,13 +1,13 @@
 exec >&2
-rm -rf t/.redo redo-sh
+rm -rf t/.redo redo/sh
 if [ -e .do_built ]; then
 	while read x; do
 		[ -d "$x" ] || rm -f "$x"
 	done <.do_built
 fi
 [ -z "$DO_BUILT" ] && rm -rf .do_built .do_built.dir
-rm -rf minimal/.do_built minimal/.do_built.dir website
-redo t/clean Documentation/clean version/clean
+rm -rf minimal/.do_built minimal/.do_built.dir docs.out
+redo t/clean docs/clean redo/clean
 rm -f *~ .*~ */*~ */.*~ *.pyc install.wrapper
-find . -name '*.tmp' -exec rm -fv {} \;
-find . -name '*.did' -exec rm -fv {} \;
+find . -name '*.tmp' -exec rm -f {} \;
+find . -name '*.did' -exec rm -f {} \;

do

@@ -0,0 +1,83 @@
+#!/bin/sh
+#
+# Bootstrap script, so we can build and test redo using (mostly) redo.
+# Before redo is available, we have to use minimal/do to build it.  After
+# that, we switch to real redo.
+#
+# NOTE: Don't use this as a model for your own redo projects!  It's friendly
+# to provide a 'do' script at the top of your project for people who haven't
+# installed redo, but that script is usually just a copy of minimal/do,
+# because your project probably doesn't have the same bootstrap problem that
+# redo itself does.
+#
+
+die() {
+	echo "$0:" "$@" >&2
+	exit 42
+}
+
+usage() {
+	echo "Usage: $0 [redo-args...] <target>" >&2
+	echo "   where valid targets are: build all test install clean" >&2
+	echo "   and redo-args are optional args for redo, like -j10, -x" >&2
+	exit 10
+}
+
+mydir=$(dirname "$0")
+cd "$mydir" || die "can't find self in dir: $mydir"
+
+args=
+while [ "$1" != "${1#-}" ]; do
+	args="$args $1"
+	shift
+done
+
+if [ "$#" -gt 1 ]; then
+	usage
+fi
+
+if [ -n "$args" -a "$#" -lt 1 ]; then
+	usage
+fi
+
+if [ "$#" -lt 1 ]; then
+	# if no extra args given, use a default target
+	target=all
+else
+	target=$1
+fi
+
+
+build() {
+	./minimal/do -c bin/all || die "failed to compile redo."
+	bin/redo bin/all || die "redo failed self test."
+}
+
+clean() {
+	./minimal/do -c clean || die "failed to clean."
+	rm -rf .redo .do_built .do_built.dir
+}
+
+case $target in
+	build)
+		build
+		;;
+	all|install)
+		build && bin/redo $args "$target"
+		;;
+	test)
+		# Test both redo and minimal/do
+		build
+		PATH=$PWD/redo:$PATH minimal/do test || die "minimal/do test failed"
+		clean
+		build
+		bin/redo $args test || die "redo test failed"
+		;;
+	clean)
+		clean
+		;;
+	*)
+		echo "$0: unknown target '$target'" >&2
+		exit 11
+		;;
+esac

Documentation/cookbook/defaults/index.md → docs/cookbook/defaults/index.md

@@ -8,7 +8,7 @@ today's date.
 
 To play with this code on your own machine, get the [redo
 source code](https://github.com/apenwarr/redo) and look in the
-`Documentation/cookbook/defaults/` directory.
+`docs/cookbook/defaults/` directory.
 
 ### Input files
 

Documentation/cookbook/hello/index.md → docs/cookbook/hello/index.md

@@ -7,7 +7,7 @@ compile it.
 
 To play with the code on your own machine, get the [redo
 source code](https://github.com/apenwarr/redo) and look in the
-`Documentation/cookbook/hello/` directory.
+`docs/cookbook/hello/` directory.
 
 ### Compiling the code
 

Documentation/cookbook/latex/index.md → docs/cookbook/latex/index.md

@@ -10,7 +10,7 @@ ggplot2](https://ggplot2.tidyverse.org/).
 
 To play with this code on your own machine, get the [redo
 source code](https://github.com/apenwarr/redo) and look in the
-`Documentation/cookbook/latex/` directory.
+`docs/cookbook/latex/` directory.
 
 
 ### Generating a plot from an R script

Documentation/default.md.tmp.do → docs/default.md.tmp.do

@@ -1,5 +1,5 @@
-redo-ifchange ../version/vars $2.md
-. ../version/vars
+redo-ifchange ../redo/version/vars $2.md
+. ../redo/version/vars
 cat - $2.md <<-EOF
 	% $2(1) Redo $TAG
 	% Avery Pennarun <apenwarr@gmail.com>

Documentation/git-export.do → docs/git-export.do

@@ -8,7 +8,7 @@ rm -f $GIT_INDEX_FILE
 git add -f *.1
 
 MSG="Autogenerated man pages for $(git describe)"
-TREE=$(git write-tree --prefix=Documentation)
+TREE=$(git write-tree --prefix=docs)
 git show-ref refs/heads/man >/dev/null && PARENT="-p refs/heads/man"
 COMMITID=$(echo "$MSG" | git commit-tree $TREE $PARENT)
 

Documentation/index.md → docs/index.md

@@ -170,7 +170,7 @@ easily be linked from this document.  Here are a few open source examples:
   in order to clean up its dependency logic.
 
 * You can also find some curated examples in the
-  [Documentation/cookbook/](https://github.com/apenwarr/redo/tree/master/Documentation/cookbook/) subdir of the redo project itself.
+  [docs/cookbook/](https://github.com/apenwarr/redo/tree/master/docs/cookbook/) subdir of the redo project itself.
 
 * A [github search for all.do](https://github.com/search?p=9&q=path%3A%2F+extension%3Ado+filename%3A%2Fall.do&type=Code)
   shows an ever-growing number of projects using redo.

docs/md-to-man.do

@@ -0,0 +1,9 @@
+redo-ifchange md2man.py ../redo/whichpython
+read py <../redo/whichpython
+if ../redo/python ./md2man.py /dev/null /dev/null >/dev/null; then
+	echo '../redo/python ./md2man.py $2.md.tmp $2.html'
+else
+	echo "Warning: md2man.py missing modules; can't generate manpages." >&2
+	echo "Warning: try this: sudo easy_install markdown BeautifulSoup" >&2
+	echo 'echo Skipping: $2.1 >&2'
+fi

Documentation/md2man.py → docs/md2man.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python2
 import sys, os, markdown, re
 from BeautifulSoup import BeautifulSoup
 

install.do

@@ -1,49 +1,42 @@
 exec >&2
-redo-ifchange _all
 
 : ${INSTALL:=install}
-: ${DESTDIR:=}
+: ${DESTDIR=NONE}
 : ${PREFIX:=/usr}
 : ${MANDIR:=$DESTDIR$PREFIX/share/man}
 : ${DOCDIR:=$DESTDIR$PREFIX/share/doc/redo}
 : ${BINDIR:=$DESTDIR$PREFIX/bin}
 : ${LIBDIR:=$DESTDIR$PREFIX/lib/redo}
 
+if [ "$DESTDIR" = "NONE" ]; then
+	echo "$0: fatal: set DESTDIR before trying to install."
+	exit 99
+fi
+
+redo-ifchange _all redo/whichpython
+read py <redo/whichpython
+
 echo "Installing to: $DESTDIR$PREFIX"
 
 # make dirs
 $INSTALL -d $MANDIR/man1 $DOCDIR $BINDIR $LIBDIR $LIBDIR/version
 
 # docs
-for d in Documentation/*.1; do
-	[ "$d" = "Documentation/*.1" ] && continue
+for d in docs/*.1; do
+	[ "$d" = "docs/*.1" ] && continue
 	$INSTALL -m 0644 $d $MANDIR/man1
 done
 $INSTALL -m 0644 README.md $DOCDIR
 
 # .py files (precompiled to .pyc files for speed)
-for d in *.py version/*.py; do
-	fix=$(echo $d | sed 's,-,_,g')
-	$INSTALL -m 0644 $d $LIBDIR/$fix
-done
-python2 -mcompileall $LIBDIR
+$INSTALL -m 0644 redo/*.py $LIBDIR/
+$INSTALL -m 0644 redo/version/*.py $LIBDIR/version/
+$py -mcompileall $LIBDIR
 
 # It's important for the file to actually be named 'sh'.  Some shells (like
 # bash and zsh) only go into POSIX-compatible mode if they have that name.
-cp -R redo-sh/sh $LIBDIR/sh
+cp -R redo/sh $LIBDIR/sh
 
 # binaries
-for dd in redo*.py; do
-	d=$(basename $dd .py)
-	fix=$(echo $d | sed -e 's,-,_,g')
-	cat >install.wrapper <<-EOF
-		#!/usr/bin/python2
-		import sys, os;
-		exedir = os.path.dirname(os.path.realpath(os.path.abspath(sys.argv[0])))
-		sys.path.insert(0, os.path.join(exedir, '../lib/redo'))
-		import $fix
-		$fix.main()
-	EOF
-	$INSTALL -m 0755 install.wrapper $BINDIR/$d
-done
-rm -f install.wrapper
+bins=$(ls bin/redo* | grep '^bin/redo[-a-z]*$')
+$INSTALL -m 0755 $bins $BINDIR/

mkdocs.yml

@@ -1,10 +1,10 @@
 site_name: "redo: a recursive build system"
 theme: readthedocs
-docs_dir: Documentation
-site_dir: website
+docs_dir: docs
+site_dir: docs.out
 strict: true
 repo_url: https://github.com/apenwarr/redo
-edit_uri: edit/master/Documentation
+edit_uri: edit/master/docs
 extra_javascript:
   - fetchcode.js
 extra_css:

redo

@@ -1 +0,0 @@
-redo.py

redo-always

@@ -1 +0,0 @@
-redo-always.py

redo-ifchange

@@ -1 +0,0 @@
-redo-ifchange.py

redo-ifcreate

@@ -1 +0,0 @@
-redo-ifcreate.py

redo-log

@@ -1 +0,0 @@
-redo-log.py

redo-ood

@@ -1 +0,0 @@
-redo-ood.py

redo-sources

@@ -1 +0,0 @@
-redo-sources.py

redo-stamp

@@ -1 +0,0 @@
-redo-stamp.py

redo-targets

@@ -1 +0,0 @@
-redo-targets.py

redo-unlocked

@@ -1 +0,0 @@
-redo-unlocked.py

redo-whichdo

@@ -1 +0,0 @@
-redo-whichdo.py

redo/.gitignore

@@ -0,0 +1,4 @@
+*.pyc
+sh
+whichpython
+python

redo/clean.do

@@ -0,0 +1,2 @@
+redo version/clean
+rm -f whichpython python *.pyc */*.pyc

redo-always.py → redo/cmd_always.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python2
 import sys, os
 import vars, state
 

redo-ifchange.py → redo/cmd_ifchange.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python2
 import os, sys, traceback
 
 import vars_init