Also get rid of TMP_FILE in todo.cfg.

There's a slight chance that some add-on has used this (undocumented, unofficial) configuration value for its own purposes (and maybe also relied on the unexposed cleanup() infrastructure), but detecting and fixing that problem (by moving the cleanup into the add-on itself) is pretty straightforward.

tests/t1900-archive.sh

@@ -0,0 +1,35 @@
+#!/bin/sh
+
+test_description='archive functionality
+
+Ensure we can archive items successfully.
+'
+. ./test-lib.sh
+
+cat > todo.txt <<EOF
+one
+two
+three
+one
+x done
+four
+EOF
+
+test_todo_session 'archive with duplicates' <<EOF
+>>> todo.sh archive
+x done
+TODO: $HOME/todo.txt archived.
+EOF
+
+test_todo_session 'list after archive' <<EOF
+>>> todo.sh ls
+5 four
+1 one
+4 one
+3 three
+2 two
+--
+TODO: 5 of 5 tasks shown
+EOF
+
+test_done

tests/t1910-deduplicate.sh

@@ -0,0 +1,103 @@
+#!/bin/sh
+
+test_description='deduplicate functionality
+
+Ensure we can deduplicate items successfully.
+'
+. ./test-lib.sh
+
+cat > todo.txt <<EOF
+duplicated
+two
+x done
+duplicated
+double task
+double task
+three
+EOF
+
+test_todo_session 'deduplicate and preserve line numbers' <<EOF
+>>> todo.sh deduplicate
+TODO: 2 duplicate task(s) removed
+
+>>> todo.sh -p ls
+5 double task
+1 duplicated
+7 three
+2 two
+3 x done
+--
+TODO: 5 of 5 tasks shown
+EOF
+
+test_todo_session 'deduplicate without duplicates' <<EOF
+>>> todo.sh deduplicate
+TODO: No duplicate tasks found
+EOF
+
+cat > todo.txt <<EOF
+duplicated
+two
+x done
+duplicated
+double task
+double task
+three
+EOF
+test_todo_session 'deduplicate and delete lines' <<EOF
+>>> todo.sh -n deduplicate
+TODO: 2 duplicate task(s) removed
+
+>>> todo.sh -p ls
+4 double task
+1 duplicated
+5 three
+2 two
+3 x done
+--
+TODO: 5 of 5 tasks shown
+EOF
+
+cat > todo.txt <<EOF
+one
+duplicated
+three
+duplicated
+duplicated
+six
+duplicated
+EOF
+test_todo_session 'deduplicate more than two occurrences' <<EOF
+>>> todo.sh deduplicate
+TODO: 3 duplicate task(s) removed
+
+>>> todo.sh -p ls
+2 duplicated
+1 one
+6 six
+3 three
+--
+TODO: 4 of 4 tasks shown
+EOF
+
+cat > todo.txt <<EOF
+normal task
+a bold task
+something else
+a bold task
+something more
+EOF
+test_todo_session 'deduplicate with non-printable duplicates' <<EOF
+>>> todo.sh deduplicate
+TODO: 1 duplicate task(s) removed
+
+>>> todo.sh -p ls
+2 a bold task
+1 normal task
+3 something else
+5 something more
+--
+TODO: 4 of 4 tasks shown
+EOF
+
+test_done

tests/t1950-report.sh

@@ -0,0 +1,96 @@
+#!/bin/sh
+
+test_description='report functionality
+
+This test checks the reporting and the format of the report file.
+'
+. ./test-lib.sh
+
+cat > todo.txt <<EOF
+(B) smell the uppercase Roses +flowers @outside
+stop and think
+smell the coffee +wakeup
+make the coffee +wakeup
+visit http://example.com
+EOF
+
+test_todo_session 'create new report' <<EOF
+>>> todo.sh report
+TODO: $HOME/todo.txt archived.
+2009-02-13T04:40:00 5 0
+TODO: Report file updated.
+
+>>> todo.sh -p list
+1 (B) smell the uppercase Roses +flowers @outside
+4 make the coffee +wakeup
+3 smell the coffee +wakeup
+2 stop and think
+5 visit http://example.com
+--
+TODO: 5 of 5 tasks shown
+EOF
+
+test_todo_session 'report of done tasks' <<EOF
+>>> todo.sh -A do 3
+3 x 2009-02-13 smell the coffee +wakeup
+TODO: 3 marked as done.
+x 2009-02-13 smell the coffee +wakeup
+TODO: $HOME/todo.txt archived.
+
+>>> todo.sh report
+TODO: $HOME/todo.txt archived.
+2009-02-13T04:40:00 4 1
+TODO: Report file updated.
+
+>>> todo.sh -p list
+1 (B) smell the uppercase Roses +flowers @outside
+3 make the coffee +wakeup
+2 stop and think
+4 visit http://example.com
+--
+TODO: 4 of 4 tasks shown
+EOF
+
+test_todo_session 'report performs archiving' <<EOF
+>>> todo.sh -a do 3
+3 x 2009-02-13 make the coffee +wakeup
+TODO: 3 marked as done.
+
+>>> todo.sh report
+x 2009-02-13 make the coffee +wakeup
+TODO: $HOME/todo.txt archived.
+2009-02-13T04:40:00 3 2
+TODO: Report file updated.
+
+>>> todo.sh -p list
+1 (B) smell the uppercase Roses +flowers @outside
+2 stop and think
+3 visit http://example.com
+--
+TODO: 3 of 3 tasks shown
+
+>>> todo.sh -p listfile done.txt
+2 x 2009-02-13 make the coffee +wakeup
+1 x 2009-02-13 smell the coffee +wakeup
+--
+DONE: 2 of 2 tasks shown
+EOF
+
+test_todo_session 'report is unchanged when no changes' <<EOF
+>>> cat report.txt
+2009-02-13T04:40:00 5 0
+2009-02-13T04:40:00 4 1
+2009-02-13T04:40:00 3 2
+
+>>> todo.sh report
+TODO: $HOME/todo.txt archived.
+2009-02-13T04:40:00 3 2
+TODO: Report file is up-to-date.
+
+>>> cat report.txt
+2009-02-13T04:40:00 5 0
+2009-02-13T04:40:00 4 1
+2009-02-13T04:40:00 3 2
+EOF
+
+test_done

tests/t9999-testsuite_example.sh

@@ -85,13 +85,9 @@ TODO: $HOME/todo.txt archived.
 TODO: 5 of 5 tasks shown
 
 >>> todo.sh report
+TODO: $HOME/todo.txt archived.
+2009-02-13T04:40:00 5 1
 TODO: Report file updated.
-2009-02-13-04:40:00 5 1
-
->>> todo.sh report
-TODO: Report file updated.
-2009-02-13-04:40:00 5 1
-2009-02-13-04:40:00 5 1
 
 >>> todo.sh append g a
 usage: todo.sh append ITEM# "TEXT TO APPEND"

todo.cfg

@@ -8,7 +8,6 @@ export TODO_DIR=`dirname "$0"`
 export TODO_FILE="$TODO_DIR/todo.txt"
 export DONE_FILE="$TODO_DIR/done.txt"
 export REPORT_FILE="$TODO_DIR/report.txt"
-export TMP_FILE="$TODO_DIR/todo.tmp"
 
 # You can customize your actions directory location
 #export TODO_ACTIONS_DIR="$HOME/.todo.actions.d"

todo.sh

@@ -50,6 +50,7 @@ shorthelp()
 		    append|app ITEM# "TEXT TO APPEND"
 		    archive
 		    command [ACTIONS]
+		    deduplicate
 		    del|rm ITEM# [TERM]
 		    depri|dp ITEM#[, ITEM#, ITEM#, ...]
 		    do ITEM#[, ITEM#, ITEM#, ...]
@@ -181,6 +182,9 @@ help()
 		      Runs the remaining arguments using only todo.sh builtins.
 		      Will not call any .todo.actions.d scripts.
 
+		    deduplicate
+		      Removes duplicate lines from todo.txt.
+
 		    del ITEM# [TERM]
 		    rm ITEM# [TERM]
 		      Deletes the task on line ITEM# in todo.txt.
@@ -298,12 +302,6 @@ die()
     exit 1
 }
 
-cleanup()
-{
-    [ -f "$TMP_FILE" ] && rm "$TMP_FILE"
-    return 0
-}
-
 cleaninput()
 {
     # Parameters:    When $1 = "for sed", performs additional escaping for use
@@ -363,20 +361,6 @@ getNewtodo()
     [ -z "$newtodo" ] && die "$(getPrefix "$2"): No updated task $item."
 }
 
-archive()
-{
-    #defragment blank lines
-    sed -i.bak -e '/./!d' "$TODO_FILE"
-    [ $TODOTXT_VERBOSE -gt 0 ] && grep "^x " "$TODO_FILE"
-    grep "^x " "$TODO_FILE" >> "$DONE_FILE"
-    sed -i.bak '/^x /d' "$TODO_FILE"
-    cp "$TODO_FILE" "$TMP_FILE"
-    sed -n 'G; s/\n/&&/; /^\([ ~-]*\n\).*\n\1/d; s/\n//; h; P' "$TMP_FILE" > "$TODO_FILE"
-    if [ $TODOTXT_VERBOSE -gt 0 ]; then
-	echo "TODO: $TODO_FILE archived."
-    fi
-}
-
 replaceOrPrepend()
 {
   action=$1; shift
@@ -680,7 +664,6 @@ ACTION=${1:-$TODOTXT_DEFAULT_ACTION}
 [ -d "$TODO_DIR" ]  || die "Fatal Error: $TODO_DIR is not a directory"
 ( cd "$TODO_DIR" )  || die "Fatal Error: Unable to cd to $TODO_DIR"
 
-[ -w "$TMP_FILE"  ] || echo -n > "$TMP_FILE" || die "Fatal Error: Unable to write to $TMP_FILE"
 [ -f "$TODO_FILE" ] || cp /dev/null "$TODO_FILE"
 [ -f "$DONE_FILE" ] || cp /dev/null "$DONE_FILE"
 [ -f "$REPORT_FILE" ] || cp /dev/null "$REPORT_FILE"
@@ -770,13 +753,32 @@ _list() {
     ## Get our search arguments, if any
     shift ## was file name, new $1 is first search term
 
-    ## Build the filter.
+    _format "$src" '' "$@"
-    filter_command=$(filtercommand "${pre_filter_command:-}" "${post_filter_command:-}" "$@")
 
-    ## Figure out how much padding we need to use
+    if [ $TODOTXT_VERBOSE -gt 0 ]; then
-    ## We need one level of padding for each power of 10 $LINES uses
+        echo "--"
-    LINES=$( sed -n '$ =' "$src" )
+        echo "$(getPrefix "$src"): ${NUMTASKS:-0} of ${TOTALTASKS:-0} tasks shown"
-    PADDING=${#LINES}
+    fi
+}
+getPadding()
+{
+    ## We need one level of padding for each power of 10 $LINES uses.
+    LINES=$(sed -n '$ =' "${1:-$TODO_FILE}")
+    printf %s ${#LINES}
+}
+_format()
+{
+    # Parameters:    $1: todo input file; when empty formats stdin
+    #                $2: ITEM# number width; if empty auto-detects from $1 / $TODO_FILE.
+    # Precondition:  None
+    # Postcondition: $NUMTASKS and $TOTALTASKS contain statistics (unless $TODOTXT_VERBOSE=0).
+
+    FILE=$1
+    shift
+
+    ## Figure out how much padding we need to use, unless this was passed to us.
+    PADDING=${1:-$(getPadding "$FILE")}
+    shift
 
     ## Number the file, then run the filter command,
     ## then sort and mangle output some more
@@ -784,7 +786,11 @@ _list() {
         TODOTXT_FINAL_FILTER="cat"
     fi
     items=$(
-        sed = "$src"                                            \
+        if [ "$FILE" ]; then
+            sed = "$FILE"
+        else
+            sed =
+        fi                                                      \
         | sed -e '''
             N
             s/^/     /
@@ -792,6 +798,9 @@ _list() {
             /^[ 0-9]\{1,\} *$/d
          '''
     )
+
+    ## Build and apply the filter.
+    filter_command=$(filtercommand "${pre_filter_command:-}" "${post_filter_command:-}" "$@")
     if [ "${filter_command}" ]; then
         filtered_items=$(echo -n "$items" | eval "${filter_command}")
     else
@@ -837,16 +846,13 @@ _list() {
     if [ $TODOTXT_VERBOSE -gt 0 ]; then
         NUMTASKS=$( echo -n "$filtered_items" | sed -n '$ =' )
         TOTALTASKS=$( echo -n "$items" | sed -n '$ =' )
-
-        echo "--"
-        echo "$(getPrefix "$FILE"): ${NUMTASKS:-0} of ${TOTALTASKS:-0} tasks shown"
     fi
     if [ $TODOTXT_VERBOSE -gt 1 ]; then
         echo "TODO DEBUG: Filter Command was: ${filter_command:-cat}"
     fi
 }
 
-export -f cleaninput getPrefix getTodo getNewtodo shellquote filtercommand _list die
+export -f cleaninput getPrefix getTodo getNewtodo shellquote filtercommand _list getPadding _format die
 
 # == HANDLE ACTION ==
 action=$( printf "%s\n" "$ACTION" | tr 'A-Z' 'a-z' )
@@ -864,9 +870,7 @@ then
 elif [ -d "$TODO_ACTIONS_DIR" -a -x "$TODO_ACTIONS_DIR/$action" ]
 then
     "$TODO_ACTIONS_DIR/$action" "$@"
-    status=$?
+    exit $?
-    cleanup
-    exit $status
 fi
 
 ## Only run if $action isn't found in .todo.actions.d
@@ -948,7 +952,15 @@ case $action in
     ;;
 
 "archive" )
-    archive;;
+    # defragment blank lines
+    sed -i.bak -e '/./!d' "$TODO_FILE"
+    [ $TODOTXT_VERBOSE -gt 0 ] && grep "^x " "$TODO_FILE"
+    grep "^x " "$TODO_FILE" >> "$DONE_FILE"
+    sed -i.bak '/^x /d' "$TODO_FILE"
+    if [ $TODOTXT_VERBOSE -gt 0 ]; then
+	echo "TODO: $TODO_FILE archived."
+    fi
+    ;;
 
 "del" | "rm" )
     # replace deleted line with a blank line when TODOTXT_PRESERVE_LINE_NUMBERS is 1
@@ -1050,7 +1062,9 @@ case $action in
     done
 
     if [ $TODOTXT_AUTO_ARCHIVE = 1 ]; then
-        archive
+        # Recursively invoke the script to allow overriding of the archive
+        # action.
+        "$TODO_FULL_SH" archive
     fi
     ;;
 
@@ -1082,16 +1096,16 @@ case $action in
 "listall" | "lsa" )
     shift  ## Was lsa; new $1 is first search term
 
-    cat "$TODO_FILE" "$DONE_FILE" > "$TMP_FILE"
     TOTAL=$( sed -n '$ =' "$TODO_FILE" )
+    PADDING=${#TOTAL}
 
-    post_filter_command="awk -v TOTAL=$TOTAL -v PADDING=${#TOTAL} '{ \$1 = sprintf(\"%\" PADDING \"d\", (\$1 > TOTAL ? 0 : \$1)); print }' "
+    post_filter_command="awk -v TOTAL=$TOTAL -v PADDING=$PADDING '{ \$1 = sprintf(\"%\" PADDING \"d\", (\$1 > TOTAL ? 0 : \$1)); print }' "
-    TODOTXT_VERBOSE=0 _list "$TMP_FILE" "$@"
+    cat "$TODO_FILE" "$DONE_FILE" | TODOTXT_VERBOSE=0 _format '' "$PADDING" "$@"
 
     if [ $TODOTXT_VERBOSE -gt 0 ]; then
         TDONE=$( sed -n '$ =' "$DONE_FILE" )
-        TASKNUM=$(TODOTXT_PLAIN=1 TODOTXT_VERBOSE=0 _list "$TODO_FILE" "$@" | sed -n '$ =')
+        TASKNUM=$(TODOTXT_PLAIN=1 TODOTXT_VERBOSE=0 _format "$TODO_FILE" 1 "$@" | sed -n '$ =')
-        DONENUM=$(TODOTXT_PLAIN=1 TODOTXT_VERBOSE=0 _list "$DONE_FILE" "$@" | sed -n '$ =')
+        DONENUM=$(TODOTXT_PLAIN=1 TODOTXT_VERBOSE=0 _format "$DONE_FILE" 1 "$@" | sed -n '$ =')
         echo "--"
         echo "$(getPrefix "$TODO_FILE"): ${TASKNUM:-0} of ${TOTAL:-0} tasks shown"
         echo "$(getPrefix "$DONE_FILE"): ${DONENUM:-0} of ${TDONE:-0} tasks shown"
@@ -1215,26 +1229,71 @@ note: PRIORITY must be anywhere from A to Z."
     ;;
 
 "report" )
-    #archive first
+    # archive first
-    sed '/^x /!d' "$TODO_FILE" >> "$DONE_FILE"
+    # Recursively invoke the script to allow overriding of the archive
-    sed -i.bak '/^x /d' "$TODO_FILE"
+    # action.
+    "$TODO_FULL_SH" archive
 
-    NUMLINES=$( sed -n '$ =' "$TODO_FILE" )
-    if [ ${NUMLINES:-0} = "0" ]; then
-         echo "datetime todos dones" >> "$REPORT_FILE"
-    fi
-    #now report
     TOTAL=$( sed -n '$ =' "$TODO_FILE" )
     TDONE=$( sed -n '$ =' "$DONE_FILE" )
-    TECHO=$(echo $(date +%Y-%m-%d-%T); echo ' '; echo ${TOTAL:-0}; echo ' ';
+    NEWDATA="${TOTAL:-0} ${TDONE:-0}"
-    echo ${TDONE:-0})
+    LASTREPORT=$(sed -ne '$p' "$REPORT_FILE")
-    echo $TECHO >> "$REPORT_FILE"
+    LASTDATA=${LASTREPORT#* }   # Strip timestamp.
+    if [ "$LASTDATA" = "$NEWDATA" ]; then
+        echo "$LASTREPORT"
+        [ $TODOTXT_VERBOSE -gt 0 ] && echo "TODO: Report file is up-to-date."
+    else
+        NEWREPORT="$(date +%Y-%m-%dT%T) ${NEWDATA}"
+        echo "${NEWREPORT}" >> "$REPORT_FILE"
+        echo "${NEWREPORT}"
         [ $TODOTXT_VERBOSE -gt 0 ] && echo "TODO: Report file updated."
-    cat "$REPORT_FILE"
+    fi
+    ;;
+
+"deduplicate" )
+    if [ $TODOTXT_PRESERVE_LINE_NUMBERS = 0 ]; then
+        deduplicateSedCommand='d'
+    else
+        deduplicateSedCommand='s/^.*//; p'
+    fi
+
+    # To determine the difference when deduplicated lines are preserved, only
+    # non-empty lines must be counted.
+    originalTaskNum=$( sed -e '/./!d' "$TODO_FILE" | sed -n '$ =' )
+
+    # Look for duplicate lines and discard the second occurrence.
+    # We start with an empty hold space on the first line.  For each line:
+    #   G - appends newline + hold space to the pattern space
+    #   s/\n/&&/; - double up the first new line so we catch adjacent dups
+    #   /^\([^\n]*\n\).*\n\1/b dedup
+    #       If the first line of the hold space shows up again later as an
+    #       entire line, it's a duplicate. Jump to the "dedup" label, where
+    #       either of the following is executed, depending on whether empty
+    #       lines should be preserved:
+    #       d           - Delete the current pattern space, quit this line and
+    #                     move on to the next, or:
+    #       s/^.*//; p  - Clear the task text, print this line and move on to
+    #                     the next.
+    #   s/\n//;   - else (no duplicate), drop the doubled newline
+    #   h;        - replace the hold space with the expanded pattern space
+    #   P;        - print up to the first newline (that is, the input line)
+    #   b         - end processing of the current line
+    sed -i.bak -n \
+        -e 'G; s/\n/&&/; /^\([^\n]*\n\).*\n\1/b dedup' \
+        -e 's/\n//; h; P; b' \
+        -e ':dedup' \
+        -e "$deduplicateSedCommand" \
+        "$TODO_FILE"
+
+    newTaskNum=$( sed -e '/./!d' "$TODO_FILE" | sed -n '$ =' )
+    deduplicateNum=$(( originalTaskNum - newTaskNum ))
+    if [ $deduplicateNum -eq 0 ]; then
+        echo "TODO: No duplicate tasks found"
+    else
+        echo "TODO: $deduplicateNum duplicate task(s) removed"
+    fi
     ;;
 
 * )
     usage;;
 esac
-
-cleanup