? tests/GraphemeBreakTest.txt
? tests/SentenceBreakTest.txt
? tests/WordBreakTest.txt
? tests/testboundaries_tr29.c
Index: pango/break.c
===================================================================
RCS file: /cvs/gnome/pango/pango/break.c,v
retrieving revision 1.52
diff -u -p -d -r1.52 break.c
--- pango/break.c	18 Sep 2006 19:57:16 -0000	1.52
+++ pango/break.c	13 Dec 2006 20:49:09 -0000
@@ -528,7 +528,7 @@ pango_default_break (const gchar   *text
   JamoType prev_jamo;
 
   GUnicodeBreakType next_break_type;
-  GUnicodeType prev_type;
+  GUnicodeType prev_type, prev_sent_type;
   GUnicodeBreakType prev_break_type; /* skips spaces */
   gboolean prev_was_break_space;
 
@@ -551,7 +551,8 @@ pango_default_break (const gchar   *text
 
   next = text;
   
-  prev_type = (GUnicodeType) -1;
+  prev_type = G_UNICODE_PARAGRAPH_SEPARATOR;
+  prev_sent_type = G_UNICODE_LINE_SEPARATOR;
   prev_break_type = G_UNICODE_BREAK_UNKNOWN;
   prev_was_break_space = FALSE;
   prev_wc = 0;
@@ -567,8 +568,8 @@ pango_default_break (const gchar   *text
 
   for (i = 0; !done ; i++)
     {
-      GUnicodeType type;
       gunichar wc;
+      GUnicodeType type, orig_type;
       GUnicodeBreakType break_type;
       BreakOpportunity break_op;
       JamoType jamo;
@@ -630,6 +631,10 @@ pango_default_break (const gchar   *text
 
       /* ---- Cursor position breaks (Grapheme breaks) ---- */
 
+      /* FIXME:  Unless we have NFC text (which we don't),  the 
+       * Grapheme_Extend stuff form TR29 should be implemented.
+       */
+
       if (wc == '\n')
         {
           /* Break before line feed unless prev char is a CR */
@@ -1036,11 +1041,11 @@ pango_default_break (const gchar   *text
 
       /* ---- Sentence breaks ---- */
       
-      /* The Unicode spec specifies sentence breakpoints, so that a piece of
-       * text would be partitioned into sentences, and all characters would
-       * be inside some sentence. This code implements that for is_sentence_boundary,
-       * but tries to keep leading/trailing whitespace out of sentences for
-       * the start/end flags
+      /* The Unicode spec specifies sentence breakpoints, so that a piece
+       * of text would be partitioned into sentences, and all characters
+       * would be inside some sentence. This code implements that for
+       * is_sentence_boundary, but tries to keep leading/trailing
+       * whitespace out of sentences for the start/end flags
        */
 
       /* The Unicode spec seems to say that one trailing line/para
@@ -1049,409 +1054,431 @@ pango_default_break (const gchar   *text
        * so am allowing one to be tacked onto a sentence ending in period.
        */
 
-#define MAYBE_START_NEW_SENTENCE                                \
-              switch (type)                                     \
-                {                                               \
-                case G_UNICODE_LINE_SEPARATOR:                  \
-                case G_UNICODE_PARAGRAPH_SEPARATOR:             \
-                case G_UNICODE_CONTROL:                         \
-                case G_UNICODE_FORMAT:                          \
-                case G_UNICODE_SPACE_SEPARATOR:                 \
-                  sentence_state = STATE_SENTENCE_OUTSIDE;      \
-                  break;                                        \
-                                                                \
-                default:                                        \
-                  sentence_state = STATE_SENTENCE_BODY;         \
-                  attrs[i].is_sentence_start = TRUE;            \
-                  break;                                        \
-                }
-      
-      /* No sentence break at the start of the text */
 
+
+      /* 
+       * We are going to modify `type' such that it woudl match the
+       * desired categories of the sentence boundaries section in
+       * the Unicode spec, specifically, UAX#29, Section 5.
+       *
+       */
+      orig_type = type;
+      prev_type = prev_sent_type;
+
+      /* Sentence separators:
+       * Sep = ==U+000A || ==U+000D || ==U+0085 || ==U+2028 || ==U+2029;
+       * Check with: type == G_UNICODE_LINE_SEPARATOR
+       *
+       * U+000A LINE FEED (LF)
+       * U+000D CARRIAGE RETURN (CR)
+       * U+0085 NEXT LINE (NEL)
+       * U+2028 LINE SEPARATOR (LS)
+       * U+2029 PARAGRAPH SEPARATOR (PS)
+       */
+      if (wc == 0x000A || wc == 0x000D || wc == 0x0085 || wc == 0x0029)
+        type = G_UNICODE_LINE_SEPARATOR;
+
+      /* Format:
+       * Format = General_Category==Cf && != U+200C && != U+200D;
+       * Check with: type == G_UNICODE_FORMAT
+       *
+       * U+200C ZERO WIDTH NON-JOINER (ZWNJ)
+       * U+200D ZERO WIDTH JOINER (ZWJ)
+       */
+      if (wc == 0x200C || wc == 0x200D)
+        type = G_UNICODE_OTHER_SYMBOL;
+
+      /* Spaces:
+       * Sp = White_Space==TRUE && !Sep && !=U+00A0;
+       * Check with: type == G_UNICODE_SPACE_SEPARATOR
+       *
+       * U+0009..U+000C are the only White_Space==TRUE ones
+       * that are not of General_Category Zs and are !Sep.
+       *
+       * U+00A0 NO-BREAK SPACE (NBSP)  (handled in Other letter)
+       */
+      if (0x0009 <= wc && wc <= 0x000C)
+        type = G_UNICODE_OTHER_SYMBOL;
+
+      /* Other letter:
+       * OLetter = (General_Category==Lo || ==U+00A0 || ==U+05F3) &&
+       *           !Lower && !Upper;
+       * Check with: type == G_UNICODE_OTHER_LETTER
+       *
+       * U+00A0 NO-BREAK SPACE (NBSP)
+       * U+05F3 HEBREW PUNCTUATION GERESH
+       */
+      if (wc == 0x00A0 || wc == 0x05F3)
+        type = G_UNICODE_OTHER_LETTER;
+
+
+
+#define IS_SENTENCE_OPENER(type,wc)                             \
+	(type == G_UNICODE_LINE_SEPARATOR &&                    \
+	 type == G_UNICODE_SPACE_SEPARATOR)
+
+
+#define MAYBE_START_NEW_SENTENCE	G_STMT_START{           \
+	  if (IS_SENTENCE_OPENER (type, wc))                    \
+	    {                                                   \
+	      sentence_state = STATE_SENTENCE_BODY;             \
+	      attrs[i].is_sentence_start = TRUE;                \
+	    } else {                                            \
+	      sentence_state = STATE_SENTENCE_OUTSIDE;          \
+	    }                                                   \
+	}G_STMT_END
+      
       /* default to not a sentence breakpoint */
       attrs[i].is_sentence_boundary = FALSE;
       attrs[i].is_sentence_start = FALSE;
       attrs[i].is_sentence_end = FALSE;
-      
-      /* FIXME the Unicode spec lumps control/format chars with
-       * line/para separators in descriptive text, but not in the
-       * character class specs, in table 5-6, so who knows whether you
-       * are actually supposed to break on control/format
-       * characters. Seems semi-broken to break on tabs...
-       */
 
+      /* Break at the end of text.
+       */
+      if (done)
+	attrs[i].is_sentence_boundary = TRUE;
+      
+#define WAS_SENTENCE_TERMINATOR(prev_type,prev_wc,wc)           \
+	(prev_type == G_UNICODE_LINE_SEPARATOR &&               \
+	 !(prev_wc == '\r' && wc == '\n') || prev_wc == '\r')
+	 	
       /* Break after line/para separators except carriage return
        * followed by newline
        */
-      switch (prev_type)
-        {
-        case G_UNICODE_LINE_SEPARATOR:
-        case G_UNICODE_PARAGRAPH_SEPARATOR:
-        case G_UNICODE_CONTROL:
-        case G_UNICODE_FORMAT:
-          if (wc == '\r')
-            {
-              if (next_wc != '\n')
-                attrs[i].is_sentence_boundary = TRUE;
-            }
-          else
-            attrs[i].is_sentence_boundary = TRUE;
-          break;
-
-        default:
-          break;
-        }
+      if (WAS_SENTENCE_TERMINATOR (prev_type, prev_wc, wc))
+	attrs[i].is_sentence_boundary = TRUE;
 
-      /* break before para/line separators except newline following
-       * carriage return
+      /* Format characters and those in the middle of a grpaheme
+       * cluster are skipped.
        */
-      switch (type)
-        {
-        case G_UNICODE_LINE_SEPARATOR:
-        case G_UNICODE_PARAGRAPH_SEPARATOR:
-        case G_UNICODE_CONTROL:
-        case G_UNICODE_FORMAT:
-          if (wc == '\n')
-            {
-              if (prev_wc != '\r')
-                attrs[i].is_sentence_boundary = TRUE;
-            }
-          else
-            attrs[i].is_sentence_boundary = TRUE;
-          break;
+      if (type != G_UNICODE_FORMAT && attrs[i].is_cursor_position)
+	switch (sentence_state)
+	  {
+	  case STATE_SENTENCE_OUTSIDE:
+	    /* Start sentence if we have non-whitespace/separator */
+	    if (IS_SENTENCE_OPENER (type, wc))
+	      {
+		attrs[i].is_sentence_start = TRUE;
+		sentence_state = STATE_SENTENCE_BODY;
+	      }
+	    break;
 
-        default:
-          break;
-        }
+	  case STATE_SENTENCE_BODY:
+	    /* If we already broke here due to separators, end the sentence. */
+	    if (attrs[i].is_sentence_boundary)
+	      {
+		attrs[i].is_sentence_end = TRUE;
 
-      switch (sentence_state)
-        {
-        case STATE_SENTENCE_OUTSIDE:
-          /* Start sentence if we have non-whitespace/format/control */
-          switch (type)
-            {
-            case G_UNICODE_LINE_SEPARATOR:
-            case G_UNICODE_PARAGRAPH_SEPARATOR:
-            case G_UNICODE_CONTROL:
-            case G_UNICODE_FORMAT:
-            case G_UNICODE_SPACE_SEPARATOR:
-              break;
+		MAYBE_START_NEW_SENTENCE;
+	      }
+	    else
+	      {
+		if (wc == '.')
+		  sentence_state = STATE_SENTENCE_DOT;
+		else if (wc == '?' || wc == '!')
+		  sentence_state = STATE_SENTENCE_TERM;
+	      }
+	    break;
 
-            default:
-              attrs[i].is_sentence_start = TRUE;
-              sentence_state = STATE_SENTENCE_BODY;
-              break;
-            }
-          break;
+	  case STATE_SENTENCE_TERM:
+	    /* End sentence on anything but close punctuation and some
+	     * loosely-specified OTHER_PUNCTUATION such as period,
+	     * comma, etc.; follow Unicode rules for breaks
+	     */
+	    switch (type)
+	      {
+	      case G_UNICODE_OTHER_PUNCTUATION:
+	      case G_UNICODE_CLOSE_PUNCTUATION:
+		if (type == G_UNICODE_CLOSE_PUNCTUATION ||
+		    wc == '.' ||
+		    wc == ',' ||
+		    wc == '?' ||
+		    wc == '!')
+		  sentence_state = STATE_SENTENCE_POST_TERM_CLOSE;
+		else
+		  {
+		    attrs[i].is_sentence_end = TRUE;
+		    attrs[i].is_sentence_boundary = TRUE;
 
-        case STATE_SENTENCE_BODY:
-          /* If we already broke here due to separators, end the sentence. */
-          if (attrs[i].is_sentence_boundary)
-            {
-              attrs[i].is_sentence_end = TRUE;
+		    MAYBE_START_NEW_SENTENCE;
+		  }
+		break;
 
-              MAYBE_START_NEW_SENTENCE;
-            }
-          else
-            {
-              if (wc == '.')
-                sentence_state = STATE_SENTENCE_DOT;
-              else if (wc == '?' || wc == '!')
-                sentence_state = STATE_SENTENCE_TERM;
-            }
-          break;
+	      case G_UNICODE_SPACE_SEPARATOR:
+		attrs[i].is_sentence_end = TRUE;
+		sentence_state = STATE_SENTENCE_POST_TERM_SPACE;
+		break;
 
-        case STATE_SENTENCE_TERM:
-          /* End sentence on anything but close punctuation and some
-           * loosely-specified OTHER_PUNCTUATION such as period,
-           * comma, etc.; follow Unicode rules for breaks
-           */
-          switch (type)
-            {
-            case G_UNICODE_OTHER_PUNCTUATION:
-            case G_UNICODE_CLOSE_PUNCTUATION:
-              if (type == G_UNICODE_CLOSE_PUNCTUATION ||
-                  wc == '.' ||
-                  wc == ',' ||
-                  wc == '?' ||
-                  wc == '!')
-                sentence_state = STATE_SENTENCE_POST_TERM_CLOSE;
-              else
-                {
-                  attrs[i].is_sentence_end = TRUE;
-                  attrs[i].is_sentence_boundary = TRUE;
+	      case G_UNICODE_LINE_SEPARATOR:
+	      case G_UNICODE_PARAGRAPH_SEPARATOR:
+		attrs[i].is_sentence_end = TRUE;
+		sentence_state = STATE_SENTENCE_POST_TERM_SEP;
+		break;
 
-                  MAYBE_START_NEW_SENTENCE;
-                }
-              break;
+	      default:
+		attrs[i].is_sentence_end = TRUE;
+		attrs[i].is_sentence_boundary = TRUE;
 
-            case G_UNICODE_SPACE_SEPARATOR:
-              attrs[i].is_sentence_end = TRUE;
-              sentence_state = STATE_SENTENCE_POST_TERM_SPACE;
-              break;
+		MAYBE_START_NEW_SENTENCE;
 
-            case G_UNICODE_LINE_SEPARATOR:
-            case G_UNICODE_PARAGRAPH_SEPARATOR:
-              attrs[i].is_sentence_end = TRUE;
-              sentence_state = STATE_SENTENCE_POST_TERM_SEP;
-              break;
+		break;
+	      }
+	    break;
 
-            default:
-              attrs[i].is_sentence_end = TRUE;
-              attrs[i].is_sentence_boundary = TRUE;
+	  case STATE_SENTENCE_POST_TERM_CLOSE:
+	    /* End sentence on anything besides more punctuation; follow
+	     * rules for breaks
+	     */
+	    switch (type)
+	      {
+	      case G_UNICODE_OTHER_PUNCTUATION:
+	      case G_UNICODE_CLOSE_PUNCTUATION:
+		if (type == G_UNICODE_CLOSE_PUNCTUATION ||
+		    wc == '.' ||
+		    wc == ',' ||
+		    wc == '?' ||
+		    wc == '!')
+		  /* continue in this state */
+		  ;
+		else
+		  {
+		    attrs[i].is_sentence_end = TRUE;
+		    attrs[i].is_sentence_boundary = TRUE;
 
-              MAYBE_START_NEW_SENTENCE;
+		    MAYBE_START_NEW_SENTENCE;
+		  }
+		break;
 
-              break;
-            }
-          break;
+	      case G_UNICODE_SPACE_SEPARATOR:
+		attrs[i].is_sentence_end = TRUE;
+		sentence_state = STATE_SENTENCE_POST_TERM_SPACE;
+		break;
 
-        case STATE_SENTENCE_POST_TERM_CLOSE:
-          /* End sentence on anything besides more punctuation; follow
-           * rules for breaks
-           */
-          switch (type)
-            {
-            case G_UNICODE_OTHER_PUNCTUATION:
-            case G_UNICODE_CLOSE_PUNCTUATION:
-              if (type == G_UNICODE_CLOSE_PUNCTUATION ||
-                  wc == '.' ||
-                  wc == ',' ||
-                  wc == '?' ||
-                  wc == '!')
-                /* continue in this state */
-                ;
-              else
-                {
-                  attrs[i].is_sentence_end = TRUE;
-                  attrs[i].is_sentence_boundary = TRUE;
+	      case G_UNICODE_LINE_SEPARATOR:
+	      case G_UNICODE_PARAGRAPH_SEPARATOR:
+		attrs[i].is_sentence_end = TRUE;
+		/* undo the unconditional break-at-all-line/para-separators
+		 * from above; I'm not sure this is what the Unicode spec
+		 * intends, but it seems right - we get to include
+		 * a single line/para separator in the sentence according
+		 * to their rules
+		 */
+		attrs[i].is_sentence_boundary = FALSE;
+		sentence_state = STATE_SENTENCE_POST_TERM_SEP;
+		break;
 
-                  MAYBE_START_NEW_SENTENCE;
-                }
-              break;
+	      default:
+		attrs[i].is_sentence_end = TRUE;
+		attrs[i].is_sentence_boundary = TRUE;
 
-            case G_UNICODE_SPACE_SEPARATOR:
-              attrs[i].is_sentence_end = TRUE;
-              sentence_state = STATE_SENTENCE_POST_TERM_SPACE;
-              break;
+		MAYBE_START_NEW_SENTENCE;
 
-            case G_UNICODE_LINE_SEPARATOR:
-            case G_UNICODE_PARAGRAPH_SEPARATOR:
-              attrs[i].is_sentence_end = TRUE;
-              /* undo the unconditional break-at-all-line/para-separators
-               * from above; I'm not sure this is what the Unicode spec
-               * intends, but it seems right - we get to include
-               * a single line/para separator in the sentence according
-               * to their rules
-               */
-              attrs[i].is_sentence_boundary = FALSE;
-              sentence_state = STATE_SENTENCE_POST_TERM_SEP;
-              break;
+		break;
+	      }
+	    break;
 
-            default:
-              attrs[i].is_sentence_end = TRUE;
-              attrs[i].is_sentence_boundary = TRUE;
+	  case STATE_SENTENCE_POST_TERM_SPACE:
 
-              MAYBE_START_NEW_SENTENCE;
+	    /* Sentence is definitely already ended; to enter this state
+	     * we had to see a space, which ends the sentence.
+	     */
 
-              break;
-            }
-          break;
+	    switch (type)
+	      {
+	      case G_UNICODE_SPACE_SEPARATOR:
+		/* continue in this state */
+		break;
 
-        case STATE_SENTENCE_POST_TERM_SPACE:
+	      case G_UNICODE_LINE_SEPARATOR:
+	      case G_UNICODE_PARAGRAPH_SEPARATOR:
+		/* undo the unconditional break-at-all-line/para-separators
+		 * from above; I'm not sure this is what the Unicode spec
+		 * intends, but it seems right
+		 */
+		attrs[i].is_sentence_boundary = FALSE;
+		sentence_state = STATE_SENTENCE_POST_TERM_SEP;
+		break;
 
-          /* Sentence is definitely already ended; to enter this state
-           * we had to see a space, which ends the sentence.
-           */
+	      default:
+		attrs[i].is_sentence_boundary = TRUE;
 
-          switch (type)
-            {
-            case G_UNICODE_SPACE_SEPARATOR:
-              /* continue in this state */
-              break;
+		MAYBE_START_NEW_SENTENCE;
 
-            case G_UNICODE_LINE_SEPARATOR:
-            case G_UNICODE_PARAGRAPH_SEPARATOR:
-              /* undo the unconditional break-at-all-line/para-separators
-               * from above; I'm not sure this is what the Unicode spec
-               * intends, but it seems right
-               */
-              attrs[i].is_sentence_boundary = FALSE;
-              sentence_state = STATE_SENTENCE_POST_TERM_SEP;
-              break;
+		break;
+	      }
+	    break;
 
-            default:
-              attrs[i].is_sentence_boundary = TRUE;
+	  case STATE_SENTENCE_POST_TERM_SEP:
+	    /* Break is forced at this point, unless we're a newline
+	     * after a CR, then we will break after the newline on the
+	     * next iteration. Only a single Sep can be in the
+	     * sentence.
+	     */
+	    if (!(prev_wc == '\r' && wc == '\n'))
+	      attrs[i].is_sentence_boundary = TRUE;
 
-              MAYBE_START_NEW_SENTENCE;
+	    MAYBE_START_NEW_SENTENCE;
 
-              break;
-            }
-          break;
+	    break;
 
-        case STATE_SENTENCE_POST_TERM_SEP:
-          /* Break is forced at this point, unless we're a newline
-           * after a CR, then we will break after the newline on the
-           * next iteration. Only a single Sep can be in the
-           * sentence.
-           */
-          if (!(prev_wc == '\r' && wc == '\n'))
-            attrs[i].is_sentence_boundary = TRUE;
+	  case STATE_SENTENCE_DOT:
+	    switch (type)
+	      {
+	      case G_UNICODE_CLOSE_PUNCTUATION:
+		sentence_state = STATE_SENTENCE_POST_DOT_CLOSE;
+		break;
 
-          MAYBE_START_NEW_SENTENCE;
+	      case G_UNICODE_SPACE_SEPARATOR:
+		possible_sentence_end = i;
+		sentence_state = STATE_SENTENCE_POST_DOT_SPACE;
+		break;
 
-          break;
+	      default:
+		/* If we broke on a control/format char, end the
+		 * sentence; else this was not a sentence end, since
+		 * we didn't enter the POST_DOT_SPACE state.
+		 */
+		if (attrs[i].is_sentence_boundary)
+		  {
+		    attrs[i].is_sentence_end = TRUE;
 
-        case STATE_SENTENCE_DOT:
-          switch (type)
-            {
-            case G_UNICODE_CLOSE_PUNCTUATION:
-              sentence_state = STATE_SENTENCE_POST_DOT_CLOSE;
-              break;
+		    MAYBE_START_NEW_SENTENCE;
+		  }
+		else
+		  sentence_state = STATE_SENTENCE_BODY;
+		break;
+	      }
+	    break;
 
-            case G_UNICODE_SPACE_SEPARATOR:
-              possible_sentence_end = i;
-              sentence_state = STATE_SENTENCE_POST_DOT_SPACE;
-              break;
+	  case STATE_SENTENCE_POST_DOT_CLOSE:
+	    switch (type)
+	      {
+	      case G_UNICODE_SPACE_SEPARATOR:
+		possible_sentence_end = i;
+		sentence_state = STATE_SENTENCE_POST_DOT_SPACE;
+		break;
 
-            default:
-              /* If we broke on a control/format char, end the
-               * sentence; else this was not a sentence end, since
-               * we didn't enter the POST_DOT_SPACE state.
-               */
-              if (attrs[i].is_sentence_boundary)
-                {
-                  attrs[i].is_sentence_end = TRUE;
+	      default:
+		/* If we broke on a control/format char, end the
+		 * sentence; else this was not a sentence end, since
+		 * we didn't enter the POST_DOT_SPACE state.
+		 */
+		if (attrs[i].is_sentence_boundary)
+		  {
+		    attrs[i].is_sentence_end = TRUE;
 
-                  MAYBE_START_NEW_SENTENCE;
-                }
-              else
-                sentence_state = STATE_SENTENCE_BODY;
-              break;
-            }
-          break;
+		    MAYBE_START_NEW_SENTENCE;
+		  }
+		else
+		  sentence_state = STATE_SENTENCE_BODY;
+		break;
+	      }
+	    break;
 
-        case STATE_SENTENCE_POST_DOT_CLOSE:
-          switch (type)
-            {
-            case G_UNICODE_SPACE_SEPARATOR:
-              possible_sentence_end = i;
-              sentence_state = STATE_SENTENCE_POST_DOT_SPACE;
-              break;
+	  case STATE_SENTENCE_POST_DOT_SPACE:
 
-            default:
-              /* If we broke on a control/format char, end the
-               * sentence; else this was not a sentence end, since
-               * we didn't enter the POST_DOT_SPACE state.
-               */
-              if (attrs[i].is_sentence_boundary)
-                {
-                  attrs[i].is_sentence_end = TRUE;
+	    possible_sentence_boundary = i;
 
-                  MAYBE_START_NEW_SENTENCE;
-                }
-              else
-                sentence_state = STATE_SENTENCE_BODY;
-              break;
-            }
-          break;
+	    switch (type)
+	      {
+	      case G_UNICODE_SPACE_SEPARATOR:
+		/* remain in current state */
+		break;
 
-        case STATE_SENTENCE_POST_DOT_SPACE:
+	      case G_UNICODE_OPEN_PUNCTUATION:
+		sentence_state = STATE_SENTENCE_POST_DOT_OPEN;
+		break;
 
-          possible_sentence_boundary = i;
+	      case G_UNICODE_LOWERCASE_LETTER:
+		/* wasn't a sentence-ending period; so re-enter the sentence
+		 * body
+		 */
+		sentence_state = STATE_SENTENCE_BODY;
+		break;
 
-          switch (type)
-            {
-            case G_UNICODE_SPACE_SEPARATOR:
-              /* remain in current state */
-              break;
+	      default:
+		/* End the sentence, break, maybe start a new one */
 
-            case G_UNICODE_OPEN_PUNCTUATION:
-              sentence_state = STATE_SENTENCE_POST_DOT_OPEN;
-              break;
+		g_assert (possible_sentence_end >= 0);
+		g_assert (possible_sentence_boundary >= 0);
 
-            case G_UNICODE_LOWERCASE_LETTER:
-              /* wasn't a sentence-ending period; so re-enter the sentence
-               * body
-               */
-              sentence_state = STATE_SENTENCE_BODY;
-              break;
+		attrs[possible_sentence_boundary].is_sentence_boundary = TRUE;
+		attrs[possible_sentence_end].is_sentence_end = TRUE;
 
-            default:
-              /* End the sentence, break, maybe start a new one */
+		possible_sentence_end = -1;
+		possible_sentence_boundary = -1;
 
-              g_assert (possible_sentence_end >= 0);
-              g_assert (possible_sentence_boundary >= 0);
+		MAYBE_START_NEW_SENTENCE;
+		
+		break;
+	      }
+	    break;
 
-              attrs[possible_sentence_boundary].is_sentence_boundary = TRUE;
-              attrs[possible_sentence_end].is_sentence_end = TRUE;
+	  case STATE_SENTENCE_POST_DOT_OPEN:
+	    switch (type)
+	      {
+	      case G_UNICODE_OPEN_PUNCTUATION:
+		/* continue in current state */
+		break;
 
-              possible_sentence_end = -1;
-              possible_sentence_boundary = -1;
+	      case G_UNICODE_LOWERCASE_LETTER:
+		/* wasn't a sentence-ending period; so re-enter the sentence
+		 * body
+		 */
+		sentence_state = STATE_SENTENCE_BODY;
+		break;
 
-              MAYBE_START_NEW_SENTENCE;
-              
-              break;
-            }
-          break;
+	      default:
+		/* End the sentence, break, maybe start a new one */
 
-        case STATE_SENTENCE_POST_DOT_OPEN:
-          switch (type)
-            {
-            case G_UNICODE_OPEN_PUNCTUATION:
-              /* continue in current state */
-              break;
+		g_assert (possible_sentence_end >= 0);
+		g_assert (possible_sentence_boundary >= 0);
 
-            case G_UNICODE_LOWERCASE_LETTER:
-              /* wasn't a sentence-ending period; so re-enter the sentence
-               * body
-               */
-              sentence_state = STATE_SENTENCE_BODY;
-              break;
+		attrs[possible_sentence_boundary].is_sentence_boundary = TRUE;
+		attrs[possible_sentence_end].is_sentence_end = TRUE;
 
-            default:
-              /* End the sentence, break, maybe start a new one */
+		possible_sentence_end = -1;
+		possible_sentence_boundary = -1;
 
-              g_assert (possible_sentence_end >= 0);
-              g_assert (possible_sentence_boundary >= 0);
+		MAYBE_START_NEW_SENTENCE;
 
-              attrs[possible_sentence_boundary].is_sentence_boundary = TRUE;
-              attrs[possible_sentence_end].is_sentence_end = TRUE;
+		break;
+	      }
+	    break;
 
-              possible_sentence_end = -1;
-              possible_sentence_boundary = -1;
+	  case STATE_SENTENCE_POST_DOT_SEP:
+	    /* Break is forced at this point, unless we're a newline
+	     * after a CR, then we will break after the newline on the
+	     * next iteration. Only a single Sep can be in the
+	     * sentence.
+	     */
+	    if (!(prev_wc == '\r' && wc == '\n'))
+	      attrs[i].is_sentence_boundary = TRUE;
 
-              MAYBE_START_NEW_SENTENCE;
+	    g_assert (possible_sentence_end >= 0);
+	    g_assert (possible_sentence_boundary >= 0);
 
-              break;
-            }
-          break;
+	    attrs[possible_sentence_end].is_sentence_end = TRUE;
 
-        case STATE_SENTENCE_POST_DOT_SEP:
-          /* Break is forced at this point, unless we're a newline
-           * after a CR, then we will break after the newline on the
-           * next iteration. Only a single Sep can be in the
-           * sentence.
-           */
-          if (!(prev_wc == '\r' && wc == '\n'))
-            attrs[i].is_sentence_boundary = TRUE;
+	    possible_sentence_end = -1;
+	    possible_sentence_boundary = -1;
 
-          g_assert (possible_sentence_end >= 0);
-          g_assert (possible_sentence_boundary >= 0);
+	    MAYBE_START_NEW_SENTENCE;
 
-          attrs[possible_sentence_end].is_sentence_end = TRUE;
+	    break;
+
+	  default:
+	    g_assert_not_reached ();
+	    break;
+	  }
+
+      /* set type back */
+      prev_sent_type = type;
+      type = orig_type;
 
-          possible_sentence_end = -1;
-          possible_sentence_boundary = -1;
 
-          MAYBE_START_NEW_SENTENCE;
 
-          break;
 
-        default:
-          g_assert_not_reached ();
-          break;
-        }
 
       prev_type = type;
       prev_wc = wc;
