? foo.diff
? mirror_char.diff
? glib/unicode-4.01b
? gobject/testifaceinit
? gobject/testoverride
? tests/NormalizationTest.txt
Index: glib/gunicode.h
===================================================================
RCS file: /cvs/gnome/glib/glib/gunicode.h,v
retrieving revision 1.28
diff -u -p -u -r1.28 gunicode.h
--- glib/gunicode.h	12 Sep 2003 17:54:43 -0000	1.28
+++ glib/gunicode.h	14 Nov 2003 16:17:19 -0000
@@ -105,6 +105,8 @@ typedef enum
   G_UNICODE_BREAK_WORD_JOINER
 } GUnicodeBreakType;
 
+typedef struct _GNormalizeIter GNormalizeIter;
+
 /* Returns TRUE if current locale uses UTF-8 charset.  If CHARSET is
  * not null, sets *CHARSET to the name of the current locale's
  * charset.  This value is statically allocated, and should be copied
@@ -273,6 +275,15 @@ typedef enum {
 gchar *g_utf8_normalize (const gchar   *str,
 			 gssize         len,
 			 GNormalizeMode mode);
+
+GNormalizeIter          *g_normalize_iter_new         (const gchar *str);
+gboolean                 g_normalize_iter_has_more    (GNormalizeIter *iter);
+gboolean                 g_normalize_iter_next        (GNormalizeIter *iter);
+G_CONST_RETURN gunichar *g_normalize_iter_get         (GNormalizeIter *iter, 
+                                                       GNormalizeMode  mode);
+G_CONST_RETURN gchar    *g_normalize_iter_position    (GNormalizeIter *iter);
+gint                     g_normalize_iter_unit_length (GNormalizeIter *iter);
+void                     g_normalize_iter_free        (GNormalizeIter *iter);
 
 gint   g_utf8_collate     (const gchar *str1,
 			   const gchar *str2);
Index: glib/gunidecomp.c
===================================================================
RCS file: /cvs/gnome/glib/glib/gunidecomp.c,v
retrieving revision 1.18
diff -u -p -u -r1.18 gunidecomp.c
--- glib/gunidecomp.c	10 Sep 2003 16:55:36 -0000	1.18
+++ glib/gunidecomp.c	14 Nov 2003 16:17:19 -0000
@@ -203,6 +203,8 @@ combine (gunichar  a,
 {
   gushort index_a, index_b;
 
+  /* g_printerr ("combine %04X %04X\n", a, b); */
+
   index_a = COMPOSE_INDEX(a);
 
   if (index_a >= COMPOSE_FIRST_SINGLE_START && index_a < COMPOSE_SECOND_START)
@@ -402,6 +404,7 @@ g_utf8_normalize (const gchar    *str,
 		  gssize          len,
 		  GNormalizeMode  mode)
 {
+#if 0
   gunichar *result_wc = _g_utf8_normalize_wc (str, len, mode);
   gchar *result;
 
@@ -409,4 +412,498 @@ g_utf8_normalize (const gchar    *str,
   g_free (result_wc);
 
   return result;
+#endif
+
+  GNormalizeIter *iter;
+  gchar *complete_string;
+  GString *gs = g_string_new (NULL);
+  const gunichar *wcs;
+
+  if (len == -1)
+    complete_string = (gchar *) str;
+  else
+    complete_string = g_strndup (str, len);
+
+  iter = g_normalize_iter_new (complete_string);
+
+  while (g_normalize_iter_has_more (iter))
+    {
+      gint i;
+
+      wcs = g_normalize_iter_get (iter, mode);
+
+      for (i = 0; wcs[i] != 0; i++)
+        g_string_append_unichar (gs, wcs[i]);
+
+      g_normalize_iter_next (iter);
+    }
+
+  g_normalize_iter_free (iter);
+  if (len != -1)
+    g_free (complete_string);
+
+  return g_string_free (gs, FALSE);
+}
+
+struct _GNormalizeIter
+{
+  const gchar *str;       /* points to the current position in the string */
+  gunichar    *nfd;
+  gssize       nfd_size;
+  gunichar    *nfkd;
+  gssize       nfkd_size;
+  gunichar    *nfc;
+  gssize       nfc_size;
+  gunichar    *nfkc;
+  gssize       nfkc_size;
+};
+
+/**
+ * g_normalize_iter_new:
+ * @str: a UTF-8 encoded string. Must not be modified or freed until the
+ * caller is finished using the returned #GNormalizeIter.
+ * 
+ * Return value: a new #GNormalizeIter.
+ **/
+GNormalizeIter *
+g_normalize_iter_new (const gchar *str)
+{
+  GNormalizeIter *iter = g_new (GNormalizeIter, 1);
+
+  g_assert (str != NULL);
+
+  iter->str = str;
+  iter->nfd = NULL;
+  iter->nfd_size = -1;
+  iter->nfkd = NULL;
+  iter->nfkd_size = -1;
+  iter->nfc = NULL;
+  iter->nfc_size = -1;
+  iter->nfkc = NULL;
+  iter->nfkc_size = -1;
+
+  return iter;
+}
+
+/**
+ * g_normalize_iter_next:
+ * @iter: the #GNormalizeIter 
+ *
+ * Advances a #GNormalizeIter to the start of the next combining character
+ * sequence. If the iterator is already at the end, it is left unchanged
+ * and %FALSE is returned.
+ * 
+ * Return value: %TRUE if the iter was successfully advanced.
+ **/
+gboolean 
+g_normalize_iter_next (GNormalizeIter *iter)
+{
+  gunichar wc;
+
+  if (*iter->str == 0)
+    return FALSE;
+
+  do
+    {
+      iter->str = g_utf8_next_char (iter->str);
+      wc = g_utf8_get_char (iter->str);
+    }
+  while (*iter->str != 0 && _g_unichar_combining_class (wc) != 0);
+
+  if (iter->nfd)
+    iter->nfd[0] = 0;
+  if (iter->nfkd)
+    iter->nfkd[0] = 0;
+  if (iter->nfc)
+    iter->nfc[0] = 0;
+  if (iter->nfkc)
+    iter->nfkc[0] = 0;
+
+  return TRUE;
+}
+
+/**
+ * g_normalize_iter_has_more:
+ * @iter: the #GNormalizeIter 
+ * 
+ * Return value: %TRUE if the iterator is not at the end of the string.
+ **/
+gboolean
+g_normalize_iter_has_more (GNormalizeIter *iter)
+{
+  return *iter->str != 0;
+}
+
+/**
+ * g_normalize_iter_get:
+ * @iter: the #GNormalizeIter
+ * @mode: the type of normalization to perform.
+ *
+ * Normalizes the combining character sequence at the iterator's current
+ * position.
+ *
+ * Return value: pointer to the zero-terminated array of #gunichar with the
+ * normalized combining character sequence, or %NULL if the iterator is at
+ * the end of the string. The return value is owned by @iter and should not
+ * be modified or freed.
+ **/
+G_CONST_RETURN gunichar *
+g_normalize_iter_get (GNormalizeIter *iter, 
+                      GNormalizeMode  mode)
+{
+  gboolean do_compat = (mode == G_NORMALIZE_NFKC ||
+                        mode == G_NORMALIZE_NFKD);
+  gboolean do_compose = (mode == G_NORMALIZE_NFC ||
+                         mode == G_NORMALIZE_NFKC);
+  gunichar **decomp_buf, **comp_buf; /* keep the address in case */
+  gssize *decomp_size, *comp_size;   /* we need to realloc */
+
+  if (*iter->str == '\0')
+    return NULL;
+
+  if (do_compat)
+    {
+      decomp_buf = &iter->nfkd;
+      comp_buf = &iter->nfkc;
+      decomp_size = &iter->nfkd_size;
+      comp_size = &iter->nfkc_size;
+    }
+  else
+    {
+      decomp_buf = &iter->nfd;
+      comp_buf = &iter->nfc;
+      decomp_size = &iter->nfd_size;
+      comp_size = &iter->nfc_size;
+    }
+
+  if (*decomp_buf == NULL || **decomp_buf == 0x0000)
+    {
+      gssize decomp_len = 0;
+      gunichar wc;
+      const gchar *p = iter->str;
+      gint i;
+
+      g_print ("decomposing (%s) ", do_compat ? "NFKD" : "NFD");
+
+      wc = g_utf8_get_char (p);
+      do
+        {
+          const gchar *decomp = find_decomposition (wc, do_compat);
+
+          g_print ("%04X ", wc);
+
+          if (decomp)
+            decomp_len += g_utf8_strlen (decomp, -1);
+          else
+            decomp_len++;
+
+          p = g_utf8_next_char (p);
+          wc = g_utf8_get_char (p);
+        }
+      while (*p != 0 && _g_unichar_combining_class (wc) != 0);
+
+      /* + 1 for terminating null character */
+      if (decomp_len + 1 > *decomp_size)
+        {
+          *decomp_size = decomp_len + 1;
+          *decomp_buf = g_realloc (*decomp_buf, *decomp_size * sizeof (gunichar));
+        }
+
+      p = iter->str;
+      i = 0;
+      wc = g_utf8_get_char (p);
+      do
+        {
+          const gchar *decomp = find_decomposition (wc, do_compat);
+
+          if (decomp)
+            {
+              const gchar *pd;
+              for (pd = decomp; *pd != '\0'; pd = g_utf8_next_char (pd))
+                (*decomp_buf)[i++] = g_utf8_get_char (pd);
+            }
+          else
+            (*decomp_buf)[i++] = wc;
+
+          p = g_utf8_next_char (p);
+          wc = g_utf8_get_char (p);
+        }
+      while (*p != 0 && _g_unichar_combining_class (wc) != 0);
+
+      g_assert (i == decomp_len);
+      (*decomp_buf)[i] = 0x0000;
+
+      gint k;
+      g_print (" = ");
+      for (k = 0; (*decomp_buf)[k]; k++)
+        g_print ("%04X ", (*decomp_buf)[k]);
+      g_print ("\n");
+
+      g_unicode_canonical_ordering (*decomp_buf, decomp_len);
+    }
+
+  /* All decomposed and reordered */
+
+  if (do_compose && (*comp_buf == NULL || **comp_buf == 0x0000))
+    {
+      gint i, j;
+      gint last_start;
+      gint cc, last_cc;
+      gboolean cc_is_blocked;
+
+      g_print ("composing ");
+      for (i = 0; (*decomp_buf)[i]; i++)
+        g_print ("%04X ", (*decomp_buf)[i]);
+      g_print ("\n");
+
+      /* for lack of a more accurate size */
+      if (*decomp_size > *comp_size)
+        {
+          *comp_size = *decomp_size;
+          *comp_buf = g_realloc (*comp_buf, *comp_size * sizeof (gunichar));
+        }
+
+      last_start = 0;
+      j = 1;
+      last_cc = 0;
+      cc_is_blocked = FALSE;
+      (*comp_buf)[0] = (*decomp_buf)[0];
+
+      for (i = 1; (*decomp_buf)[i] != 0; i++)
+        {
+          cc = _g_unichar_combining_class ((*decomp_buf)[i]);
+          g_print (" %04X + %04X = ", (*comp_buf)[last_start], (*decomp_buf)[i]);
+
+          if (cc == last_cc && cc_is_blocked)
+            {
+              g_print ("not trying it, blocked\n");
+              (*comp_buf)[j++] = (*decomp_buf)[i];
+            }
+          else
+            {
+              gboolean combined = combine ((*comp_buf)[last_start], (*decomp_buf)[i], *comp_buf + last_start);
+
+              if (!combined)
+                {
+                  (*comp_buf)[j] = (*decomp_buf)[i];
+
+                  g_print ("nothing\n");
+
+                  if (cc == 0)
+                    {
+                      last_start = j;
+                      cc_is_blocked = FALSE;
+                    }
+                  else
+                    cc_is_blocked = TRUE;
+
+                  j++;
+                }
+              else
+                g_print ("%04X\n", (*comp_buf)[last_start]);
+
+              last_cc = cc;
+            }
+        }
+
+      (*comp_buf)[j] = 0;
+    }
+
+  if (do_compose)
+    return *comp_buf;
+  else
+    return *decomp_buf;
+}
+
+#if 0
+      gint last_start;
+      gint i, j, last_cc;
+      gboolean cc_is_blocked;
+
+      g_print ("composing ");
+      for (i = 0; (*decomp_buf)[i]; i++)
+        g_print ("%04X ", (*decomp_buf)[i]);
+      g_print ("\n ");
+      sleep (1);
+
+      /* for lack of a more accurate size */
+      if (*decomp_size > *comp_size)
+        {
+          *comp_size = *decomp_size;
+          *comp_buf = g_realloc (*comp_buf, *comp_size * sizeof (gunichar));
+        }
+
+      last_start = 0;
+      j = 0;
+      cc_is_blocked = FALSE;
+      last_cc = 0;
+      for (i = 0; (*decomp_buf)[i] != 0; i++)
+        {
+          gint cc = _g_unichar_combining_class ((*decomp_buf)[i]);
+
+          if (cc != last_cc)
+            cc_is_blocked = FALSE;
+
+          if (cc == 0)
+            {
+              g_print ("  %04X: combining class is 0, it's a new starter\n", (*comp_buf)[last_start], (*decomp_buf)[j]);
+              last_start = j;
+              (*comp_buf)[j++] = (*decomp_buf)[i];
+              cc_is_blocked = FALSE;
+            }
+          else if (cc == last_cc && cc_is_blocked)
+            {
+              g_print ("  %04X + %04X = not trying it, blocked\n", (*comp_buf)[last_start], (*decomp_buf)[j]);
+              (*comp_buf)[j++] = (*decomp_buf)[i];
+            }
+          else
+            {
+              gunichar wc;
+              gboolean combined = combine ((*comp_buf)[last_start], (*decomp_buf)[i], &wc);
+
+              if (combined)
+                {
+                  g_print ("  %04X + %04X = %04X\n", (*comp_buf)[last_start], (*decomp_buf)[i], wc);
+                  (*comp_buf)[last_start] = wc;
+                  g_assert (!cc_is_blocked);
+                }
+              else
+                {
+                  g_print ("  %04X + %04X = nothing\n", (*comp_buf)[last_start], (*decomp_buf)[i]);
+                  cc_is_blocked = TRUE;
+                  (*comp_buf)[j++] = (*decomp_buf)[i];
+                }
+            }
+
+          last_cc = cc;
+        }
+
+      (*comp_buf)[j] = 0x0000;
+#endif
+#if 0
+      gint i, j, last_cc;
+      gboolean cc_is_blocked;
+
+      g_print ("composing ");
+      for (i = 0; (*decomp_buf)[i]; i++)
+        g_print ("%04X ", (*decomp_buf)[i]);
+      g_print ("\n ");
+
+      /* for lack of a more accurate size */
+      if (*decomp_size > *comp_size)
+        {
+          *comp_size = *decomp_size;
+          *comp_buf = g_realloc (*comp_buf, *comp_size * sizeof (gunichar));
+        }
+
+      (*comp_buf)[0] = (*decomp_buf)[0];
+      last_cc = 0;
+      i = 1;
+      for (j = 1; (*decomp_buf)[j] != 0; j++)
+        {
+          gint cc = _g_unichar_combining_class ((*decomp_buf)[j]);
+
+          if (cc != last_cc)
+            cc_is_blocked = FALSE;
+          /* try combining if not blocked; if it fails just copy it */
+#if 0
+          if (cc == last_cc || !combine ((*comp_buf)[0], (*decomp_buf)[j], *comp_buf))
+            (*comp_buf)[i++] = (*decomp_buf)[j];
+#endif
+          if (cc == last_cc && cc_is_blocked)
+            {
+              g_print ("%04X + %04X = not trying it, blocked\n", (*comp_buf)[0], (*decomp_buf)[j]);
+              (*comp_buf)[i++] = (*decomp_buf)[j];
+            }
+          else
+            {
+              gunichar wc;
+              gboolean combined = combine ((*comp_buf)[0], (*decomp_buf)[j], &wc);
+
+              if (combined)
+                {
+                  g_print ("%04X + %04X = %04X\n", (*comp_buf)[0], (*decomp_buf)[j], wc);
+                  (*comp_buf)[0] = wc;
+                }
+              else
+                {
+                  cc_is_blocked = TRUE;
+                  g_print ("%04X + %04X = nothing\n", (*comp_buf)[0], (*decomp_buf)[j]);
+                  (*comp_buf)[i++] = (*decomp_buf)[j];
+                }
+            }
+
+          last_cc = cc;
+        }
+
+      (*comp_buf)[i] = 0x0000;
+
+      g_print ("\n");
+    }
+#endif
+
+/**
+ * g_normalize_iter_position:
+ * @iter: the #GNormalizeIter
+ *
+ * Return value: pointer to the iterator's current position in the string
+ * being normalized.
+ **/
+/* FIXME: should this return null if we're at the end? */
+G_CONST_RETURN gchar *
+g_normalize_iter_position (GNormalizeIter *iter)
+{
+  return iter->str;
+}
+
+/**
+ * g_normalize_iter_unit_length:
+ * @iter: the #GNormalizeIter
+ *
+ * Return value: the length in bytes of the combining character sequence at
+ * the iterator's current position in the string.
+ **/
+/* FIXME: should this return a gs?size? should it return something other
+ * than 0 if we're at the end? */
+gint 
+g_normalize_iter_unit_length (GNormalizeIter *iter)
+{
+  gunichar wc;
+  const gchar *p = iter->str;
+
+  if (*p == 0)
+    return 0;
+
+  do
+    {
+      p = g_utf8_next_char (p);
+      wc = g_utf8_get_char (p);
+    }
+  while (*p != 0 && _g_unichar_combining_class (wc) != 0);
+
+  return p - iter->str;
+}
+
+/**
+ * g_normalize_iter_free:
+ * @iter: the #GNormalizeIter
+ * 
+ * Frees a #GNormalizeIter created with g_normalize_iter_new().
+ **/
+void
+g_normalize_iter_free (GNormalizeIter *iter)
+{
+  if (iter->nfd)
+    g_free (iter->nfd);
+
+  if (iter->nfkd)
+    g_free (iter->nfkd);
+
+  if (iter->nfc)
+    g_free (iter->nfc);
+
+  if (iter->nfkc)
+    g_free (iter->nfkc);
+
+  g_free (iter);
 }
Index: tests/unicode-normalize.c
===================================================================
RCS file: /cvs/gnome/glib/tests/unicode-normalize.c,v
retrieving revision 1.10
diff -u -p -u -r1.10 unicode-normalize.c
--- tests/unicode-normalize.c	5 Aug 2003 03:41:34 -0000	1.10
+++ tests/unicode-normalize.c	14 Nov 2003 16:17:19 -0000
@@ -49,6 +49,23 @@ const char *names[4] = {
   "NFKC"
 };
 
+static gchar *
+make_codepoint_string (gchar *str)
+{
+  gunichar wc;
+  GString *gs = g_string_new (NULL);
+  gchar *p = str;
+
+  while (*p)
+    {
+      wc = g_utf8_get_char (p);
+      g_string_append_printf (gs, "%04X ", wc);
+      p = g_utf8_next_char (p);
+    }
+
+  return g_string_free (gs, FALSE);
+}
+
 static void
 test_form (int            line,
 	   GNormalizeMode mode,
@@ -69,9 +86,9 @@ test_form (int            line,
 	  char *result = g_utf8_normalize (c[i], -1, mode);
 	  if (strcmp (result, c[expected]) != 0)
 	    {
-	      fprintf (stderr, "\nFailure: %d/%d: %s\n", line, i + 1, raw[5]);
-	      fprintf (stderr, "  g_utf8_normalize (%s, %s) != %s\n",
-		   raw[i], names[mode], raw[expected]);
+	      fprintf (stderr, "Failure: %d/%d: %s\n", line, i + 1, raw[5]);
+	      fprintf (stderr, "  g_utf8_normalize (%s, %s) is %s should be %s\n\n",
+		   raw[i], names[mode], make_codepoint_string (result), raw[expected]);
 	      success = FALSE;
 	    }
 	  
@@ -85,9 +102,9 @@ test_form (int            line,
 	  char *result = g_utf8_normalize (c[i], -1, mode);
 	  if (strcmp (result, c[expected]) != 0)
 	    {
-	      fprintf (stderr, "\nFailure: %d/%d: %s\n", line, i, raw[5]);
-	      fprintf (stderr, "  g_utf8_normalize (%s, %s) != %s\n",
-		   raw[i], names[mode], raw[expected]);
+	      fprintf (stderr, "Failure: %d/%d: %s\n", line, i, raw[5]);
+	      fprintf (stderr, "  g_utf8_normalize (%s, %s) is %s should be %s\n\n",
+		   raw[i], names[mode], make_codepoint_string (result), raw[expected]);
 	      success = FALSE;
 	    }
 	  
