Rework fixed archive deduplication into new deduplicate action.

As per discussion on the mailing list (http://tech.groups.yahoo.com/group/todotxt/message/3775), the automatic deduplication during archiving is unexpected and difficult to enforce in other implementations. Rather, make this a separate (optional) action.

tests/t1900-archive.sh

@@ -6,7 +6,7 @@ Ensure we can archive items successfully.
 '
 . ./test-lib.sh
 
-cat >> todo.txt<<EOF
+cat > todo.txt <<EOF
 one
 two
 three
@@ -23,12 +23,13 @@ EOF
 
 test_todo_session 'list after archive' <<EOF
 >>> todo.sh ls
-4 four
+5 four
 1 one
+4 one
 3 three
 2 two
 --
-TODO: 4 of 4 tasks shown
+TODO: 5 of 5 tasks shown
 EOF
 
 test_done

tests/t1910-deduplicate.sh

@@ -0,0 +1,38 @@
+#!/bin/sh
+
+test_description='deduplicate functionality
+
+Ensure we can deduplicate items successfully.
+'
+. ./test-lib.sh
+
+cat >> todo.txt<<EOF
+duplicated
+two
+x done
+duplicated
+double task
+double task
+three
+EOF
+
+test_todo_session 'deduplicate with duplicates' <<EOF
+>>> todo.sh deduplicate
+TODO: 2 duplicate task(s) removed
+
+>>> todo.sh -p ls
+4 double task
+1 duplicated
+5 three
+2 two
+3 x done
+--
+TODO: 5 of 5 tasks shown
+EOF
+
+test_todo_session 'deduplicate without duplicates' <<EOF
+>>> todo.sh deduplicate
+TODO: No duplicate tasks found
+EOF
+
+test_done

todo.sh

@@ -50,6 +50,7 @@ shorthelp()
 		    append|app ITEM# "TEXT TO APPEND"
 		    archive
 		    command [ACTIONS]
+		    deduplicate
 		    del|rm ITEM# [TERM]
 		    depri|dp ITEM#[, ITEM#, ITEM#, ...]
 		    do ITEM#[, ITEM#, ITEM#, ...]
@@ -180,6 +181,9 @@ help()
 		      Runs the remaining arguments using only todo.sh builtins.
 		      Will not call any .todo.actions.d scripts.
 
+		    deduplicate
+		      Removes duplicate lines from todo.txt.
+
 		    del ITEM# [TERM]
 		    rm ITEM# [TERM]
 		      Deletes the task on line ITEM# in todo.txt.
@@ -360,21 +364,6 @@ archive()
     [ $TODOTXT_VERBOSE -gt 0 ] && grep "^x " "$TODO_FILE"
     grep "^x " "$TODO_FILE" >> "$DONE_FILE"
     sed -i.bak '/^x /d' "$TODO_FILE"
-    cp "$TODO_FILE" "$TMP_FILE"
-	# look for duplicate lines with printable chars (i.e. duplicate tasks)
-	# and discard the second occurrence
-	# we start with an empty hold space on the first line.  For each line:
-	#   G - appends newline + hold space to the pattern space
-	#   s/\n/&&/; - double up the first new line so we catch adjacent dups
-	#   /^\([ -~]*\n\).*\n\1/d; 
-	#       If the first line (i.e. the new input line) of the hold space
-	#       is printable, and if that same pattern shows up again later as
-	#       an entire line, it's a duplicate.  Delete the current pattern space,
-	#	    quit this line and move on to the next
-	#   s/\n//;   - else, drop the doubled newline
-	#   h;        - replace the hold space with the expanded pattern space
-	#   P;        - print up to the first newline (that is, the input line)
-    sed -n 'G; s/\n/&&/; /^\([ -~]*\n\).*\n\1/d; s/\n//; h; P' "$TMP_FILE" > "$TODO_FILE"
     if [ $TODOTXT_VERBOSE -gt 0 ]; then
 	echo "TODO: $TODO_FILE archived."
     fi
@@ -1218,6 +1207,31 @@ note: PRIORITY must be anywhere from A to Z."
     cat "$REPORT_FILE"
     ;;
 
+"deduplicate" )
+    originalTaskNum=$( sed -n '$ =' "$TODO_FILE" )
+
+    # Look for duplicate lines and discard the second occurrence.
+    # We start with an empty hold space on the first line.  For each line:
+    #   G - appends newline + hold space to the pattern space
+    #   s/\n/&&/; - double up the first new line so we catch adjacent dups
+    #   /^\([ -~]*\n\).*\n\1/d;
+    #       If the first line (i.e. the new input line) of the hold space
+    #       is printable, and if that same pattern shows up again later as
+    #       an entire line, it's a duplicate.  Delete the current pattern space,
+    #       quit this line and move on to the next
+    #   s/\n//;   - else, drop the doubled newline
+    #   h;        - replace the hold space with the expanded pattern space
+    #   P;        - print up to the first newline (that is, the input line)
+    sed -i.bak -n 'G; s/\n/&&/; /^\([ -~]*\n\).*\n\1/d; s/\n//; h; P' "$TODO_FILE"
+    newTaskNum=$( sed -n '$ =' "$TODO_FILE" )
+    deduplicateNum=$(( originalTaskNum - newTaskNum ))
+    if [ $deduplicateNum -eq 0 ]; then
+        echo "TODO: No duplicate tasks found"
+    else
+        echo "TODO: $deduplicateNum duplicate task(s) removed"
+    fi
+    ;;
+
 * )
     usage;;
 esac