/***************************************************************************
 *   Copyright (C) 2006 by Guillermo Ballester Valor                       *
 *   gbv@oxixares.com                                                      *
 *                                                                         *
 *   This program is free software; you can redistribute it and/or modify  *
 *   it under the terms of the GNU General Public License as published by  *
 *   the Free Software Foundation; either version 2 of the License, or     *
 *   (at your option) any later version.                                   *
 *                                                                         *
 *   This program is distributed in the hope that it will be useful,       *
 *   but WITHOUT ANY WARRANTY; without even the implied warranty of        *
 *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the         *
 *   GNU General Public License for more details.                          *
 *                                                                         *
 *   You should have received a copy of the GNU General Public License     *
 *   along with this program; if not, write to the                         *
 *   Free Software Foundation, Inc.,                                       *
 *   59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.             *
 ***************************************************************************/
/*
 
    This program try to fix pages with mixed code UTF-8 and ISO-8859-1 
    (latin-1). This is a typical problem in some web pages. 
 
    Note that the conversion is not sure in 100% of cases, but I'm pretty 
    sure we can filter hundred of thoushand lines before to find a character
    improperly transformed. 
 
    Input is on stdin, output to stdout.
 
    A tipical use:
 
    cat mixed.html | mix2utf8 > utf8.html
 
    'latin.html' file should have all problematic 8-bits iso-8859-1 characters 
    in 'mixed.html' converted to utf-8. The rest of chars remain unaltered.  
    In this case, the web page should display the characters properly if it has
    in header a meta code like
 
    <meta http-equiv="Content-Type" content="text/html;charset=utf-8">
 
    Some needed info:
 
    utf-8 is a way to encode unicode. Two bytes utf-8 code have the form
 
    110xxxxx 10yyyyyy 
 
    and it is the encoded unicode with binary format
   
    00000xxxxxyyyyyy
 
    In this code we are dealing with possible utf-8 pair bytes where
 
    xxxxx = 0001x    
    
    It can be converted directly to iso-8859-1 with format
 
    1xyyyyyy
 
    Reversely, an iso-8859-1 code with binary format
 
    1xyyyyyy 
 
    are transformed to an utf-8 pair bytes as
  
    1100001x 10yyyyyy
   
    You can read more with 'man utf-8' command
*/
/*
    Este programa trata de transformar las paginas con codigo mixto
    UTF-8 y ISO-8859-1 a UTF-8
 
    Se supone que el codigo UTF puede estar contenido e 8 bits. 
 
    Aunque el algoritmo no es 100% seguro, estoy convencido que se pueden
    filtrar cientos de miles de lineas antes de convertir impropiamente un
    caracter
 
    La entrada es por stdin, la salida en stdout
 
    Uso tipico:
 
    cat mixed.html | mix2utf8 > utf8.html
 
    'latin.html' contiene todos los caracteres problematicos de 8-bits 
    iso-8859-1 en 'mixed.html' convertidos a utf-8, menteniendo el resto 
    como estaban.
 
    En este caso, deberian verse los caracteres correctamente si la pagina
    tiene en su cabecera un codigo como este
 
    <meta http-equiv="Content-Type" content="text/html;charset=utf-8">  
*/


#include "stdlib.h"
#include "string.h"
#include "stdio.h"


int mask0 = 31; /* mask to extract bits from utf byte 0 */
int mask1 = 63; /* mask to extract bits from utf byte 1 */

int main(int argc, char *argv[])
{

  int c0; /* stored readed char (as int) awaiting a decision */
  int c1; /* current readed char (as int) */
  /*
     flags to set if just readed byte is a probable first or second 
     utf-8 byte 
  */
  int utf0, utf1; /*  Same for awaiting char */
  int lutf0, lutf1;
  int empty; /* flag to mark we don't have a byte wating for output */
  int latin; /* flag to mark the most significant bit of readed byte is 1 */
  unsigned char b0; /* stored uchar */
  unsigned char b1; /* recent readed uchar */
  int aux;


  lutf0 = 0;
  lutf1 = 0;
  c0 = 0;
  c1 = 0;
  empty = 1;

  while ( (fread ((void *) &b1, sizeof( unsigned char), 1, stdin)) == 1)
    {
      c1 = (int) b1;

      /* needed test for readed char */
      utf0 = ( (c1 >> 5) == 6)? 1 : 0;
      utf1 = ( (c1 >> 6) == 2)? 1 : 0;
      latin = ( c1 > 127) ? 1 : 0;

      if (latin)
        {
          /* most significant bit is 1 */
          if (utf1)
            {
              if (lutf0)
                {
                  /* write stored and current byte */
                  fwrite( &b0, sizeof(unsigned char), 1, stdout);
                  fwrite( &b1, sizeof(unsigned char), 1, stdout);
                  empty = 1;
                }
              else
                {
                  /* it is not an utf, we have to convert and write them */
                  c0 = ((c1 >> 6) | 192);
                  aux = ((c1 & mask1) | 128);
                  b0 = (unsigned char) c0;
                  b1 = (unsigned char) aux;
                  fwrite( &b0, sizeof(unsigned char), 1, stdout);
                  fwrite( &b1, sizeof(unsigned char), 1, stdout);
                  empty = 1;
                }

            }
          else if (utf0)
            {
              /* we kept the char to see what happens in the following char */
              c0 = c1;
              b0 = b1;
              empty = 0;
            }
          else
            {
              /* it is not an utf, we have to convert and write them */
              c0 = ((c1 >> 6) | 192);
              aux = ((c1 & mask1) | 128);
              b0 = (unsigned char) c0;
              b1 = (unsigned char) aux;
              fwrite( &b0, sizeof(unsigned char), 1, stdout);
              fwrite( &b1, sizeof(unsigned char), 1, stdout);
              empty = 1;
            }
        }
      else
        {
          /* check if buffer is not empty. In that case we before convert
             and write them */
          if (empty == 0)
            {
              aux = ((c0 & mask1) | 128);
              b0 = (unsigned char) ((c0 >> 6) | 192);
              b1 = (unsigned char) aux;
              fwrite( &b0, sizeof(unsigned char), 1, stdout);
              fwrite( &b1, sizeof(unsigned char), 1, stdout);
              b1 = (unsigned char) c1;
              empty = 1;
            }
          /* is a 7-bits char, we then directly write it */
          fwrite( &b1, sizeof(unsigned char), 1, stdout);
          empty = 1;
        }
    }
  /* check if buffer is not empty before exit*/
  if (empty == 0)
    {
      aux = ((c0 & mask1) | 128);
      b0 = (unsigned char) ((c0 >> 6) | 192);
      b1 = (unsigned char) aux;
      fwrite( &b0, sizeof(unsigned char), 1, stdout);
      fwrite( &b1, sizeof(unsigned char), 1, stdout);
    }
  return EXIT_SUCCESS;
}
