From fe5cdcb13ab70d76f47f9de61ef35e9f314ec22c Mon Sep 17 00:00:00 2001
From: Ingo Karkat <swdev@ingo-karkat.de>
Date: Fri, 13 Jan 2012 22:44:35 +0100
Subject: [PATCH] Fix deduplicate for non-printable (and non-ASCII) characters.

---
 tests/t1910-deduplicate.sh | 22 +++++++++++++++++++++-
 todo.sh                    |  9 ++++-----
 2 files changed, 25 insertions(+), 6 deletions(-)

diff --git a/tests/t1910-deduplicate.sh b/tests/t1910-deduplicate.sh
index 425418a..16dc999 100755
--- a/tests/t1910-deduplicate.sh
+++ b/tests/t1910-deduplicate.sh
@@ -6,7 +6,7 @@ Ensure we can deduplicate items successfully.
 '
 . ./test-lib.sh
 
-cat >> todo.txt<<EOF
+cat > todo.txt <<EOF
 duplicated
 two
 x done
@@ -35,4 +35,24 @@ test_todo_session 'deduplicate without duplicates' <<EOF
 TODO: No duplicate tasks found
 EOF
 
+cat > todo.txt <<EOF
+normal task
+a bold action
+something else
+a bold action
+something more
+EOF
+test_todo_session 'deduplicate with non-printable duplicates' <<EOF
+>>> todo.sh deduplicate
+TODO: 1 duplicate task(s) removed
+
+>>> todo.sh -p ls
+2 a bold action
+1 normal task
+3 something else
+4 something more
+--
+TODO: 4 of 4 tasks shown
+EOF
+
 test_done
diff --git a/todo.sh b/todo.sh
index 4a6dde7..e2a3056 100755
--- a/todo.sh
+++ b/todo.sh
@@ -1242,15 +1242,14 @@ note: PRIORITY must be anywhere from A to Z."
     # We start with an empty hold space on the first line.  For each line:
     #   G - appends newline + hold space to the pattern space
     #   s/\n/&&/; - double up the first new line so we catch adjacent dups
-    #   /^\([ -~]*\n\).*\n\1/d;
-    #       If the first line (i.e. the new input line) of the hold space
-    #       is printable, and if that same pattern shows up again later as
-    #       an entire line, it's a duplicate.  Delete the current pattern space,
+    #   /^\([^\n]*\n\).*\n\1/d;
+    #       If the first line of the hold space shows up again later as an
+    #       entire line, it's a duplicate.  Delete the current pattern space,
     #       quit this line and move on to the next
     #   s/\n//;   - else, drop the doubled newline
     #   h;        - replace the hold space with the expanded pattern space
     #   P;        - print up to the first newline (that is, the input line)
-    sed -i.bak -n 'G; s/\n/&&/; /^\([ -~]*\n\).*\n\1/d; s/\n//; h; P' "$TODO_FILE"
+    sed -i.bak -n 'G; s/\n/&&/; /^\([^\n]*\n\).*\n\1/d; s/\n//; h; P' "$TODO_FILE"
     newTaskNum=$( sed -n '$ =' "$TODO_FILE" )
     deduplicateNum=$(( originalTaskNum - newTaskNum ))
     if [ $deduplicateNum -eq 0 ]; then