ENH: listall doesn't simply use concatenated task lists.

The simplistic "listall" action implementation just uses _list() on the concatenation of active and done task lists. This has the following shortcomings:

- Task numbers shown for archived tasks from done.txt are invalid.
- As the number of done tasks likely greatly outnumbers the number of active tasks, the task number padding is often larger than expected (e.g. 0005 instead of 05).
- Verbose output lists all tasks as originating from TODO, whereas it should differentiate between TODO: and DONE: sources.

The main challenge is to keep processing all tasks through a single pass of _list(), so that there is a single, unified sorting applied to all tasks. A custom AWK script sets all (originally invalid) task numbers from done.txt to "0", meaning "archived task".
The verbose message from _list() is replaced with a custom message that shows the tasks from todo.txt, done.txt, and totals.

Oh, and added tests for the previously untested "listall" action.

tests/t0001-null.sh

@@ -45,6 +45,8 @@ test_expect_success 'null listpri a' '
 cat > expect <<EOF
 --
 TODO: 0 of 0 tasks shown
+DONE: 0 of 0 tasks shown
+total 0 of 0 tasks shown
 EOF
 
 test_expect_success 'null lsa' '

tests/t1350-listall.sh

@@ -0,0 +1,150 @@
+#!/bin/sh
+
+test_description='listall functionality
+'
+. ./test-lib.sh
+
+cat > todo.txt <<EOF
+smell the uppercase Roses +flowers @outside
+x 2011-08-08 tend the garden @outside
+notice the sunflowers
+x 2011-12-26 go outside +wakeup
+(A) stop
+EOF
+cat > done.txt <<EOF
+x 2011-12-01 eat breakfast
+x 2011-12-05 smell the coffee +wakeup
+EOF
+
+test_todo_session 'basic listall' <<EOF
+>>> todo.sh -p listall
+5 (A) stop
+3 notice the sunflowers
+1 smell the uppercase Roses +flowers @outside
+2 x 2011-08-08 tend the garden @outside
+0 x 2011-12-01 eat breakfast
+0 x 2011-12-05 smell the coffee +wakeup
+4 x 2011-12-26 go outside +wakeup
+--
+TODO: 5 of 5 tasks shown
+DONE: 2 of 2 tasks shown
+total 7 of 7 tasks shown
+EOF
+
+test_todo_session 'listall highlighting' <<EOF
+>>> todo.sh listall
+5 (A) stop
+3 notice the sunflowers
+1 smell the uppercase Roses +flowers @outside
+2 x 2011-08-08 tend the garden @outside
+0 x 2011-12-01 eat breakfast
+0 x 2011-12-05 smell the coffee +wakeup
+4 x 2011-12-26 go outside +wakeup
+--
+TODO: 5 of 5 tasks shown
+DONE: 2 of 2 tasks shown
+total 7 of 7 tasks shown
+EOF
+
+test_todo_session 'listall nonverbose' <<EOF
+>>> TODOTXT_VERBOSE=0 todo.sh -p listall
+5 (A) stop
+3 notice the sunflowers
+1 smell the uppercase Roses +flowers @outside
+2 x 2011-08-08 tend the garden @outside
+0 x 2011-12-01 eat breakfast
+0 x 2011-12-05 smell the coffee +wakeup
+4 x 2011-12-26 go outside +wakeup
+EOF
+
+test_todo_session 'listall filtering' <<EOF
+>>> todo.sh -p listall @outside
+1 smell the uppercase Roses +flowers @outside
+2 x 2011-08-08 tend the garden @outside
+--
+TODO: 2 of 5 tasks shown
+DONE: 0 of 2 tasks shown
+total 2 of 7 tasks shown
+
+>>> todo.sh -p listall the
+3 notice the sunflowers
+1 smell the uppercase Roses +flowers @outside
+2 x 2011-08-08 tend the garden @outside
+0 x 2011-12-05 smell the coffee +wakeup
+--
+TODO: 3 of 5 tasks shown
+DONE: 1 of 2 tasks shown
+total 4 of 7 tasks shown
+
+>>> todo.sh -p listall breakfast
+0 x 2011-12-01 eat breakfast
+--
+TODO: 0 of 5 tasks shown
+DONE: 1 of 2 tasks shown
+total 1 of 7 tasks shown
+
+>>> todo.sh -p listall doesnotmatch
+--
+TODO: 0 of 5 tasks shown
+DONE: 0 of 2 tasks shown
+total 0 of 7 tasks shown
+EOF
+
+cat >> done.txt <<EOF
+x 2010-01-01 old task 1
+x 2010-01-01 old task 2
+x 2010-01-01 old task 3
+x 2010-01-01 old task 4
+EOF
+test_todo_session 'listall number width' <<EOF
+>>> todo.sh -p listall
+5 (A) stop
+3 notice the sunflowers
+1 smell the uppercase Roses +flowers @outside
+0 x 2010-01-01 old task 1
+0 x 2010-01-01 old task 2
+0 x 2010-01-01 old task 3
+0 x 2010-01-01 old task 4
+2 x 2011-08-08 tend the garden @outside
+0 x 2011-12-01 eat breakfast
+0 x 2011-12-05 smell the coffee +wakeup
+4 x 2011-12-26 go outside +wakeup
+--
+TODO: 5 of 5 tasks shown
+DONE: 6 of 6 tasks shown
+total 11 of 11 tasks shown
+
+>>> TODOTXT_VERBOSE=0 todo.sh add new task 1
+
+>>> TODOTXT_VERBOSE=0 todo.sh add new task 2
+
+>>> TODOTXT_VERBOSE=0 todo.sh add new task 3
+
+>>> TODOTXT_VERBOSE=0 todo.sh add new task 4
+
+>>> TODOTXT_VERBOSE=0 todo.sh add new task 5
+
+>>> todo.sh -p listall
+05 (A) stop
+06 new task 1
+07 new task 2
+08 new task 3
+09 new task 4
+10 new task 5
+03 notice the sunflowers
+01 smell the uppercase Roses +flowers @outside
+00 x 2010-01-01 old task 1
+00 x 2010-01-01 old task 2
+00 x 2010-01-01 old task 3
+00 x 2010-01-01 old task 4
+02 x 2011-08-08 tend the garden @outside
+00 x 2011-12-01 eat breakfast
+00 x 2011-12-05 smell the coffee +wakeup
+04 x 2011-12-26 go outside +wakeup
+--
+TODO: 10 of 10 tasks shown
+DONE: 6 of 6 tasks shown
+total 16 of 16 tasks shown
+EOF
+
+test_done

tests/t1900-archive.sh

@@ -1,35 +0,0 @@
-#!/bin/sh
-
-test_description='archive functionality
-
-Ensure we can archive items successfully.
-'
-. ./test-lib.sh
-
-cat > todo.txt <<EOF
-one
-two
-three
-one
-x done
-four
-EOF
-
-test_todo_session 'archive with duplicates' <<EOF
->>> todo.sh archive
-x done
-TODO: $HOME/todo.txt archived.
-EOF
-
-test_todo_session 'list after archive' <<EOF
->>> todo.sh ls
-5 four
-1 one
-4 one
-3 three
-2 two
---
-TODO: 5 of 5 tasks shown
-EOF
-
-test_done

tests/t1910-deduplicate.sh

@@ -1,103 +0,0 @@
-#!/bin/sh
-
-test_description='deduplicate functionality
-
-Ensure we can deduplicate items successfully.
-'
-. ./test-lib.sh
-
-cat > todo.txt <<EOF
-duplicated
-two
-x done
-duplicated
-double task
-double task
-three
-EOF
-
-test_todo_session 'deduplicate and preserve line numbers' <<EOF
->>> todo.sh deduplicate
-TODO: 2 duplicate task(s) removed
-
->>> todo.sh -p ls
-5 double task
-1 duplicated
-7 three
-2 two
-3 x done
---
-TODO: 5 of 5 tasks shown
-EOF
-
-test_todo_session 'deduplicate without duplicates' <<EOF
->>> todo.sh deduplicate
-TODO: No duplicate tasks found
-EOF
-
-cat > todo.txt <<EOF
-duplicated
-two
-x done
-duplicated
-double task
-double task
-three
-EOF
-test_todo_session 'deduplicate and delete lines' <<EOF
->>> todo.sh -n deduplicate
-TODO: 2 duplicate task(s) removed
-
->>> todo.sh -p ls
-4 double task
-1 duplicated
-5 three
-2 two
-3 x done
---
-TODO: 5 of 5 tasks shown
-EOF
-
-cat > todo.txt <<EOF
-one
-duplicated
-three
-duplicated
-duplicated
-six
-duplicated
-EOF
-test_todo_session 'deduplicate more than two occurrences' <<EOF
->>> todo.sh deduplicate
-TODO: 3 duplicate task(s) removed
-
->>> todo.sh -p ls
-2 duplicated
-1 one
-6 six
-3 three
---
-TODO: 4 of 4 tasks shown
-EOF
-
-cat > todo.txt <<EOF
-normal task
-a bold action
-something else
-a bold action
-something more
-EOF
-test_todo_session 'deduplicate with non-printable duplicates' <<EOF
->>> todo.sh deduplicate
-TODO: 1 duplicate task(s) removed
-
->>> todo.sh -p ls
-2 a bold action
-1 normal task
-3 something else
-5 something more
---
-TODO: 4 of 4 tasks shown
-EOF
-
-test_done

tests/t9999-testsuite_example.sh

@@ -85,12 +85,10 @@ TODO: $HOME/todo.txt archived.
 TODO: 5 of 5 tasks shown
 
 >>> todo.sh report
-TODO: $HOME/todo.txt archived.
 TODO: Report file updated.
 2009-02-13-04:40:00 5 1
 
 >>> todo.sh report
-TODO: $HOME/todo.txt archived.
 TODO: Report file updated.
 2009-02-13-04:40:00 5 1
 2009-02-13-04:40:00 5 1

todo.sh

@@ -50,7 +50,6 @@ shorthelp()
 		    append|app ITEM# "TEXT TO APPEND"
 		    archive
 		    command [ACTIONS]
-		    deduplicate
 		    del|rm ITEM# [TERM]
 		    depri|dp ITEM#[, ITEM#, ITEM#, ...]
 		    do ITEM#[, ITEM#, ITEM#, ...]
@@ -181,9 +180,6 @@ help()
 		      Runs the remaining arguments using only todo.sh builtins.
 		      Will not call any .todo.actions.d scripts.
 
-		    deduplicate
-		      Removes duplicate lines from todo.txt.
-
 		    del ITEM# [TERM]
 		    rm ITEM# [TERM]
 		      Deletes the task on line ITEM# in todo.txt.
@@ -357,6 +353,20 @@ getNewtodo()
     [ -z "$newtodo" ] && die "$(getPrefix "$2"): No updated task $item."
 }
 
+archive()
+{
+    #defragment blank lines
+    sed -i.bak -e '/./!d' "$TODO_FILE"
+    [ $TODOTXT_VERBOSE -gt 0 ] && grep "^x " "$TODO_FILE"
+    grep "^x " "$TODO_FILE" >> "$DONE_FILE"
+    sed -i.bak '/^x /d' "$TODO_FILE"
+    cp "$TODO_FILE" "$TMP_FILE"
+    sed -n 'G; s/\n/&&/; /^\([ ~-]*\n\).*\n\1/d; s/\n//; h; P' "$TMP_FILE" > "$TODO_FILE"
+    if [ $TODOTXT_VERBOSE -gt 0 ]; then
+	echo "TODO: $TODO_FILE archived."
+    fi
+}
+
 replaceOrPrepend()
 {
   action=$1; shift
@@ -923,15 +933,7 @@ case $action in
     ;;
 
 "archive" )
-    # defragment blank lines
-    sed -i.bak -e '/./!d' "$TODO_FILE"
-    [ $TODOTXT_VERBOSE -gt 0 ] && grep "^x " "$TODO_FILE"
-    grep "^x " "$TODO_FILE" >> "$DONE_FILE"
-    sed -i.bak '/^x /d' "$TODO_FILE"
-    if [ $TODOTXT_VERBOSE -gt 0 ]; then
-	echo "TODO: $TODO_FILE archived."
-    fi
-    ;;
+    archive;;
 
 "del" | "rm" )
     # replace deleted line with a blank line when TODOTXT_PRESERVE_LINE_NUMBERS is 1
@@ -1033,9 +1035,7 @@ case $action in
     done
 
     if [ $TODOTXT_AUTO_ARCHIVE = 1 ]; then
-        # Recursively invoke the script to allow overriding of the archive
-        # action.
-        "$TODO_FULL_SH" archive
+        archive
     fi
     ;;
 
@@ -1068,7 +1068,20 @@ case $action in
     shift  ## Was lsa; new $1 is first search term
 
     cat "$TODO_FILE" "$DONE_FILE" > "$TMP_FILE"
-    _list "$TMP_FILE" "$@"
+    TOTAL=$( sed -n '$ =' "$TODO_FILE" )
+
+    post_filter_command="awk -v TOTAL=$TOTAL -v PADDING=${#TOTAL} '{ \$1 = sprintf(\"%\" PADDING \"d\", (\$1 > TOTAL ? 0 : \$1)); print }' "
+    TODOTXT_VERBOSE=0 _list "$TMP_FILE" "$@"
+
+    if [ $TODOTXT_VERBOSE -gt 0 ]; then
+        TDONE=$( sed -n '$ =' "$DONE_FILE" )
+        TASKNUM=$(TODOTXT_PLAIN=1 TODOTXT_VERBOSE=0 _list "$TODO_FILE" "$@" | sed -n '$ =')
+        DONENUM=$(TODOTXT_PLAIN=1 TODOTXT_VERBOSE=0 _list "$DONE_FILE" "$@" | sed -n '$ =')
+        echo "--"
+        echo "$(getPrefix "$TODO_FILE"): ${TASKNUM:-0} of ${TOTAL:-0} tasks shown"
+        echo "$(getPrefix "$DONE_FILE"): ${DONENUM:-0} of ${TDONE:-0} tasks shown"
+        echo "total $((TASKNUM + DONENUM)) of $((TOTAL + TDONE)) tasks shown"
+    fi
     ;;
 
 "listfile" | "lf" )
@@ -1187,10 +1200,9 @@ note: PRIORITY must be anywhere from A to Z."
     ;;
 
 "report" )
-    # archive first
-    # Recursively invoke the script to allow overriding of the archive
-    # action.
-    "$TODO_FULL_SH" archive
+    #archive first
+    sed '/^x /!d' "$TODO_FILE" >> "$DONE_FILE"
+    sed -i.bak '/^x /d' "$TODO_FILE"
 
     NUMLINES=$( sed -n '$ =' "$TODO_FILE" )
     if [ ${NUMLINES:-0} = "0" ]; then
@@ -1206,42 +1218,6 @@ note: PRIORITY must be anywhere from A to Z."
     cat "$REPORT_FILE"
     ;;
 
-"deduplicate" )
-    if [ $TODOTXT_PRESERVE_LINE_NUMBERS = 0 ]; then
-        deduplicateSedCommand='d'
-    else
-        deduplicateSedCommand='{ s/^.*//; p; b }'
-    fi
-
-    # To determine the difference when deduplicated lines are preserved, only
-    # non-empty lines must be counted.
-    originalTaskNum=$( sed -e '/./!d' "$TODO_FILE" | sed -n '$ =' )
-
-    # Look for duplicate lines and discard the second occurrence.
-    # We start with an empty hold space on the first line.  For each line:
-    #   G - appends newline + hold space to the pattern space
-    #   s/\n/&&/; - double up the first new line so we catch adjacent dups
-    #   /^\([^\n]*\n\).*\n\1/
-    #       If the first line of the hold space shows up again later as an
-    #       entire line, it's a duplicate.
-    #       d;                 - Delete the current pattern space, quit this line
-    #                            and move on to the next, or:
-    #       { s/^.*//; p; b }; - Clear the task text, print this line and move on
-    #                            to the next.
-    #   s/\n//;   - else, drop the doubled newline
-    #   h;        - replace the hold space with the expanded pattern space
-    #   P;        - print up to the first newline (that is, the input line)
-    sed -i.bak -n 'G; s/\n/&&/; /^\([^\n]*\n\).*\n\1/'"$deduplicateSedCommand"'; s/\n//; h; P' "$TODO_FILE"
-
-    newTaskNum=$( sed -e '/./!d' "$TODO_FILE" | sed -n '$ =' )
-    deduplicateNum=$(( originalTaskNum - newTaskNum ))
-    if [ $deduplicateNum -eq 0 ]; then
-        echo "TODO: No duplicate tasks found"
-    else
-        echo "TODO: $deduplicateNum duplicate task(s) removed"
-    fi
-    ;;
-
 * )
     usage;;
 esac