Pull archive() in-line and delegate via recursive call, also for report.

The report action should delegate to archive; it previously (half) did this via duplicated code (and forgot to defragment empty lines, so the tally could be off, and kept silent about the archiving).

The do action directly invoked archive(); if the user had extended / modified the archive action via an eponymous custom action, it would not run. Therefore, always invoke archive through another call of todo.sh, so that a possible custom action is considered.

tests/t0001-null.sh

@@ -45,8 +45,6 @@ test_expect_success 'null listpri a' '
 cat > expect <<EOF
 --
 TODO: 0 of 0 tasks shown
-DONE: 0 of 0 tasks shown
-total 0 of 0 tasks shown
 EOF
 
 test_expect_success 'null lsa' '

tests/t1350-listall.sh

@@ -1,150 +0,0 @@
-#!/bin/sh
-
-test_description='listall functionality
-'
-. ./test-lib.sh
-
-cat > todo.txt <<EOF
-smell the uppercase Roses +flowers @outside
-x 2011-08-08 tend the garden @outside
-notice the sunflowers
-x 2011-12-26 go outside +wakeup
-(A) stop
-EOF
-cat > done.txt <<EOF
-x 2011-12-01 eat breakfast
-x 2011-12-05 smell the coffee +wakeup
-EOF
-
-test_todo_session 'basic listall' <<EOF
->>> todo.sh -p listall
-5 (A) stop
-3 notice the sunflowers
-1 smell the uppercase Roses +flowers @outside
-2 x 2011-08-08 tend the garden @outside
-0 x 2011-12-01 eat breakfast
-0 x 2011-12-05 smell the coffee +wakeup
-4 x 2011-12-26 go outside +wakeup
---
-TODO: 5 of 5 tasks shown
-DONE: 2 of 2 tasks shown
-total 7 of 7 tasks shown
-EOF
-
-test_todo_session 'listall highlighting' <<EOF
->>> todo.sh listall
-5 (A) stop
-3 notice the sunflowers
-1 smell the uppercase Roses +flowers @outside
-2 x 2011-08-08 tend the garden @outside
-0 x 2011-12-01 eat breakfast
-0 x 2011-12-05 smell the coffee +wakeup
-4 x 2011-12-26 go outside +wakeup
---
-TODO: 5 of 5 tasks shown
-DONE: 2 of 2 tasks shown
-total 7 of 7 tasks shown
-EOF
-
-test_todo_session 'listall nonverbose' <<EOF
->>> TODOTXT_VERBOSE=0 todo.sh -p listall
-5 (A) stop
-3 notice the sunflowers
-1 smell the uppercase Roses +flowers @outside
-2 x 2011-08-08 tend the garden @outside
-0 x 2011-12-01 eat breakfast
-0 x 2011-12-05 smell the coffee +wakeup
-4 x 2011-12-26 go outside +wakeup
-EOF
-
-test_todo_session 'listall filtering' <<EOF
->>> todo.sh -p listall @outside
-1 smell the uppercase Roses +flowers @outside
-2 x 2011-08-08 tend the garden @outside
---
-TODO: 2 of 5 tasks shown
-DONE: 0 of 2 tasks shown
-total 2 of 7 tasks shown
-
->>> todo.sh -p listall the
-3 notice the sunflowers
-1 smell the uppercase Roses +flowers @outside
-2 x 2011-08-08 tend the garden @outside
-0 x 2011-12-05 smell the coffee +wakeup
---
-TODO: 3 of 5 tasks shown
-DONE: 1 of 2 tasks shown
-total 4 of 7 tasks shown
-
->>> todo.sh -p listall breakfast
-0 x 2011-12-01 eat breakfast
---
-TODO: 0 of 5 tasks shown
-DONE: 1 of 2 tasks shown
-total 1 of 7 tasks shown
-
->>> todo.sh -p listall doesnotmatch
---
-TODO: 0 of 5 tasks shown
-DONE: 0 of 2 tasks shown
-total 0 of 7 tasks shown
-EOF
-
-cat >> done.txt <<EOF
-x 2010-01-01 old task 1
-x 2010-01-01 old task 2
-x 2010-01-01 old task 3
-x 2010-01-01 old task 4
-EOF
-test_todo_session 'listall number width' <<EOF
->>> todo.sh -p listall
-5 (A) stop
-3 notice the sunflowers
-1 smell the uppercase Roses +flowers @outside
-0 x 2010-01-01 old task 1
-0 x 2010-01-01 old task 2
-0 x 2010-01-01 old task 3
-0 x 2010-01-01 old task 4
-2 x 2011-08-08 tend the garden @outside
-0 x 2011-12-01 eat breakfast
-0 x 2011-12-05 smell the coffee +wakeup
-4 x 2011-12-26 go outside +wakeup
---
-TODO: 5 of 5 tasks shown
-DONE: 6 of 6 tasks shown
-total 11 of 11 tasks shown
-
->>> TODOTXT_VERBOSE=0 todo.sh add new task 1
-
->>> TODOTXT_VERBOSE=0 todo.sh add new task 2
-
->>> TODOTXT_VERBOSE=0 todo.sh add new task 3
-
->>> TODOTXT_VERBOSE=0 todo.sh add new task 4
-
->>> TODOTXT_VERBOSE=0 todo.sh add new task 5
-
->>> todo.sh -p listall
-05 (A) stop
-06 new task 1
-07 new task 2
-08 new task 3
-09 new task 4
-10 new task 5
-03 notice the sunflowers
-01 smell the uppercase Roses +flowers @outside
-00 x 2010-01-01 old task 1
-00 x 2010-01-01 old task 2
-00 x 2010-01-01 old task 3
-00 x 2010-01-01 old task 4
-02 x 2011-08-08 tend the garden @outside
-00 x 2011-12-01 eat breakfast
-00 x 2011-12-05 smell the coffee +wakeup
-04 x 2011-12-26 go outside +wakeup
---
-TODO: 10 of 10 tasks shown
-DONE: 6 of 6 tasks shown
-total 16 of 16 tasks shown
-EOF
-
-test_done

tests/t1900-archive.sh

@@ -0,0 +1,35 @@
+#!/bin/sh
+
+test_description='archive functionality
+
+Ensure we can archive items successfully.
+'
+. ./test-lib.sh
+
+cat > todo.txt <<EOF
+one
+two
+three
+one
+x done
+four
+EOF
+
+test_todo_session 'archive with duplicates' <<EOF
+>>> todo.sh archive
+x done
+TODO: $HOME/todo.txt archived.
+EOF
+
+test_todo_session 'list after archive' <<EOF
+>>> todo.sh ls
+5 four
+1 one
+4 one
+3 three
+2 two
+--
+TODO: 5 of 5 tasks shown
+EOF
+
+test_done

tests/t1910-deduplicate.sh

@@ -0,0 +1,103 @@
+#!/bin/sh
+
+test_description='deduplicate functionality
+
+Ensure we can deduplicate items successfully.
+'
+. ./test-lib.sh
+
+cat > todo.txt <<EOF
+duplicated
+two
+x done
+duplicated
+double task
+double task
+three
+EOF
+
+test_todo_session 'deduplicate and preserve line numbers' <<EOF
+>>> todo.sh deduplicate
+TODO: 2 duplicate task(s) removed
+
+>>> todo.sh -p ls
+5 double task
+1 duplicated
+7 three
+2 two
+3 x done
+--
+TODO: 5 of 5 tasks shown
+EOF
+
+test_todo_session 'deduplicate without duplicates' <<EOF
+>>> todo.sh deduplicate
+TODO: No duplicate tasks found
+EOF
+
+cat > todo.txt <<EOF
+duplicated
+two
+x done
+duplicated
+double task
+double task
+three
+EOF
+test_todo_session 'deduplicate and delete lines' <<EOF
+>>> todo.sh -n deduplicate
+TODO: 2 duplicate task(s) removed
+
+>>> todo.sh -p ls
+4 double task
+1 duplicated
+5 three
+2 two
+3 x done
+--
+TODO: 5 of 5 tasks shown
+EOF
+
+cat > todo.txt <<EOF
+one
+duplicated
+three
+duplicated
+duplicated
+six
+duplicated
+EOF
+test_todo_session 'deduplicate more than two occurrences' <<EOF
+>>> todo.sh deduplicate
+TODO: 3 duplicate task(s) removed
+
+>>> todo.sh -p ls
+2 duplicated
+1 one
+6 six
+3 three
+--
+TODO: 4 of 4 tasks shown
+EOF
+
+cat > todo.txt <<EOF
+normal task
+a bold action
+something else
+a bold action
+something more
+EOF
+test_todo_session 'deduplicate with non-printable duplicates' <<EOF
+>>> todo.sh deduplicate
+TODO: 1 duplicate task(s) removed
+
+>>> todo.sh -p ls
+2 a bold action
+1 normal task
+3 something else
+5 something more
+--
+TODO: 4 of 4 tasks shown
+EOF
+
+test_done

tests/t9999-testsuite_example.sh

@@ -85,10 +85,12 @@ TODO: $HOME/todo.txt archived.
 TODO: 5 of 5 tasks shown
 
 >>> todo.sh report
+TODO: $HOME/todo.txt archived.
 TODO: Report file updated.
 2009-02-13-04:40:00 5 1
 
 >>> todo.sh report
+TODO: $HOME/todo.txt archived.
 TODO: Report file updated.
 2009-02-13-04:40:00 5 1
 2009-02-13-04:40:00 5 1

todo.sh

@@ -50,6 +50,7 @@ shorthelp()
 		    append|app ITEM# "TEXT TO APPEND"
 		    archive
 		    command [ACTIONS]
+		    deduplicate
 		    del|rm ITEM# [TERM]
 		    depri|dp ITEM#[, ITEM#, ITEM#, ...]
 		    do ITEM#[, ITEM#, ITEM#, ...]
@@ -180,6 +181,9 @@ help()
 		      Runs the remaining arguments using only todo.sh builtins.
 		      Will not call any .todo.actions.d scripts.
 
+		    deduplicate
+		      Removes duplicate lines from todo.txt.
+
 		    del ITEM# [TERM]
 		    rm ITEM# [TERM]
 		      Deletes the task on line ITEM# in todo.txt.
@@ -353,20 +357,6 @@ getNewtodo()
     [ -z "$newtodo" ] && die "$(getPrefix "$2"): No updated task $item."
 }
 
-archive()
-{
-    #defragment blank lines
-    sed -i.bak -e '/./!d' "$TODO_FILE"
-    [ $TODOTXT_VERBOSE -gt 0 ] && grep "^x " "$TODO_FILE"
-    grep "^x " "$TODO_FILE" >> "$DONE_FILE"
-    sed -i.bak '/^x /d' "$TODO_FILE"
-    cp "$TODO_FILE" "$TMP_FILE"
-    sed -n 'G; s/\n/&&/; /^\([ ~-]*\n\).*\n\1/d; s/\n//; h; P' "$TMP_FILE" > "$TODO_FILE"
-    if [ $TODOTXT_VERBOSE -gt 0 ]; then
-	echo "TODO: $TODO_FILE archived."
-    fi
-}
-
 replaceOrPrepend()
 {
   action=$1; shift
@@ -933,7 +923,15 @@ case $action in
     ;;
 
 "archive" )
-    archive;;
+    # defragment blank lines
+    sed -i.bak -e '/./!d' "$TODO_FILE"
+    [ $TODOTXT_VERBOSE -gt 0 ] && grep "^x " "$TODO_FILE"
+    grep "^x " "$TODO_FILE" >> "$DONE_FILE"
+    sed -i.bak '/^x /d' "$TODO_FILE"
+    if [ $TODOTXT_VERBOSE -gt 0 ]; then
+	echo "TODO: $TODO_FILE archived."
+    fi
+    ;;
 
 "del" | "rm" )
     # replace deleted line with a blank line when TODOTXT_PRESERVE_LINE_NUMBERS is 1
@@ -1035,7 +1033,9 @@ case $action in
     done
 
     if [ $TODOTXT_AUTO_ARCHIVE = 1 ]; then
-        archive
+        # Recursively invoke the script to allow overriding of the archive
+        # action.
+        "$TODO_FULL_SH" archive
     fi
     ;;
 
@@ -1068,20 +1068,7 @@ case $action in
     shift  ## Was lsa; new $1 is first search term
 
     cat "$TODO_FILE" "$DONE_FILE" > "$TMP_FILE"
-    TOTAL=$( sed -n '$ =' "$TODO_FILE" )
-
-    post_filter_command="awk -v TOTAL=$TOTAL -v PADDING=${#TOTAL} '{ \$1 = sprintf(\"%\" PADDING \"d\", (\$1 > TOTAL ? 0 : \$1)); print }' "
-    TODOTXT_VERBOSE=0 _list "$TMP_FILE" "$@"
-
-    if [ $TODOTXT_VERBOSE -gt 0 ]; then
-        TDONE=$( sed -n '$ =' "$DONE_FILE" )
-        TASKNUM=$(TODOTXT_PLAIN=1 TODOTXT_VERBOSE=0 _list "$TODO_FILE" "$@" | sed -n '$ =')
-        DONENUM=$(TODOTXT_PLAIN=1 TODOTXT_VERBOSE=0 _list "$DONE_FILE" "$@" | sed -n '$ =')
-        echo "--"
-        echo "$(getPrefix "$TODO_FILE"): ${TASKNUM:-0} of ${TOTAL:-0} tasks shown"
-        echo "$(getPrefix "$DONE_FILE"): ${DONENUM:-0} of ${TDONE:-0} tasks shown"
-        echo "total $((TASKNUM + DONENUM)) of $((TOTAL + TDONE)) tasks shown"
-    fi
+    _list "$TMP_FILE" "$@"
     ;;
 
 "listfile" | "lf" )
@@ -1200,9 +1187,10 @@ note: PRIORITY must be anywhere from A to Z."
     ;;
 
 "report" )
-    #archive first
-    sed '/^x /!d' "$TODO_FILE" >> "$DONE_FILE"
-    sed -i.bak '/^x /d' "$TODO_FILE"
+    # archive first
+    # Recursively invoke the script to allow overriding of the archive
+    # action.
+    "$TODO_FULL_SH" archive
 
     NUMLINES=$( sed -n '$ =' "$TODO_FILE" )
     if [ ${NUMLINES:-0} = "0" ]; then
@@ -1218,6 +1206,42 @@ note: PRIORITY must be anywhere from A to Z."
     cat "$REPORT_FILE"
     ;;
 
+"deduplicate" )
+    if [ $TODOTXT_PRESERVE_LINE_NUMBERS = 0 ]; then
+        deduplicateSedCommand='d'
+    else
+        deduplicateSedCommand='{ s/^.*//; p; b }'
+    fi
+
+    # To determine the difference when deduplicated lines are preserved, only
+    # non-empty lines must be counted.
+    originalTaskNum=$( sed -e '/./!d' "$TODO_FILE" | sed -n '$ =' )
+
+    # Look for duplicate lines and discard the second occurrence.
+    # We start with an empty hold space on the first line.  For each line:
+    #   G - appends newline + hold space to the pattern space
+    #   s/\n/&&/; - double up the first new line so we catch adjacent dups
+    #   /^\([^\n]*\n\).*\n\1/
+    #       If the first line of the hold space shows up again later as an
+    #       entire line, it's a duplicate.
+    #       d;                 - Delete the current pattern space, quit this line
+    #                            and move on to the next, or:
+    #       { s/^.*//; p; b }; - Clear the task text, print this line and move on
+    #                            to the next.
+    #   s/\n//;   - else, drop the doubled newline
+    #   h;        - replace the hold space with the expanded pattern space
+    #   P;        - print up to the first newline (that is, the input line)
+    sed -i.bak -n 'G; s/\n/&&/; /^\([^\n]*\n\).*\n\1/'"$deduplicateSedCommand"'; s/\n//; h; P' "$TODO_FILE"
+
+    newTaskNum=$( sed -e '/./!d' "$TODO_FILE" | sed -n '$ =' )
+    deduplicateNum=$(( originalTaskNum - newTaskNum ))
+    if [ $deduplicateNum -eq 0 ]; then
+        echo "TODO: No duplicate tasks found"
+    else
+        echo "TODO: $deduplicateNum duplicate task(s) removed"
+    fi
+    ;;
+
 * )
     usage;;
 esac