/***************************************************************************
 *   Copyright (C) 2006 by Guillermo Ballester Valor                       *
 *   gbv@oxixares.com                                                      *
 *                                                                         *
 *   This program is free software; you can redistribute it and/or modify  *
 *   it under the terms of the GNU General Public License as published by  *
 *   the Free Software Foundation; either version 2 of the License, or     *
 *   (at your option) any later version.                                   *
 *                                                                         *
 *   This program is distributed in the hope that it will be useful,       *
 *   but WITHOUT ANY WARRANTY; without even the implied warranty of        *
 *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the         *
 *   GNU General Public License for more details.                          *
 *                                                                         *
 *   You should have received a copy of the GNU General Public License     *
 *   along with this program; if not, write to the                         *
 *   Free Software Foundation, Inc.,                                       *
 *   59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.             *
 ***************************************************************************/
/*
    This program try to fix pages with mixed code UTF-8 and ISO-8859-1 
    (latin-1). This is a typical problem in some web pages. We assume
    that the utf8 code are from unicode with only less significant 8 bits,
    what is common in most european-occidental languages
 
    Note that the conversion is not sure in 100% of cases, but I'm pretty 
    sure we can filter hundred of thoushand lines before to find a character
    improperly transformed. 
 
    Input is on stdin, output to stdout.
 
    A tipical use:
 
    cat mixed.html | mix2latin1 > latin.html
 
    'latin.html' file should have all problematic utf8 characters in 
    'mixed.html' converted to iso-8859-1. The rest of chars remain unaltered.  
    In this case, the web page should display the characters properly if it has
    in header a meta code like
 
    <meta http-equiv="Content-Type" content="text/html;charset=iso-8859-1">
 
    Some needed info:
 
    utf-8 is a way to encode unicode. Two bytes utf-8 code have the form
 
    110xxxxx 10yyyyyy 
 
    and it is the encoded unicode with binary format
   
    00000xxxxxyyyyyy
 
    In this code we are dealing with possible utf-8 pair bytes where
 
    xxxxx = 0001x    
    
    It can be converted directly to iso-8859-1 with format
 
    1xyyyyyy
 
    Reversely, an iso-8859-1 code with binary format
 
    1xyyyyyy 
 
    are transformed to an utf-8 pair bytes as
  
    1100001x 10yyyyyy
   
    You can read more with 'man utf-8' command
*/
/*
    Este programa trata de transformar las paginas con codigo mixto
    UTF-8 y ISO-8859-1 a ISO-8859-1
 
    Se supone que el codigo UTF puede estar contenido en 8 bits. 
 
    Aunque el algoritmo no es 100% seguro, estoy convencido que se pueden
    filtrar cientos de miles de lineas antes de convertir impropiamente un
    caracter
 
    La entrada es por stdin, la salida en stdout
 
    Un ejemplo tipico de uso es
 
    cat mixed.html | mix2latin1 > latin.html
 
    'latin.html' contiene todos los caracteres problematicos utf8 en 
    'mixed.html' convertidos a iso-8859-1. El resto permanece inalterado.
    En este caso, deberian verse los caracteres correctamente si la pagina
    tiene en su cabecera un codigo como este
 
    <meta http-equiv="Content-Type" content="text/html;charset=iso-8859-1">  
*/
#include "stdlib.h"
#include "string.h"
#include "stdio.h"


int mask0 = 31; /* mask to extract bits from utf byte 0 */
int mask1 = 63; /* mask to extract bits from utf byte 1 */

int main(int argc, char *argv[])
{
  int c0; /* stored readed char (as int) awaiting a decision */
  int c1; /* current readed char (as int) */
  /*
     flags to set if just readed byte is a probable first or second 
     utf-8 byte 
  */
  int utf0, utf1; /*  Same for awaiting char */
  int lutf0, lutf1;
  int empty; /* flag to mark we don't have a byte wating for output */
  unsigned char b0; /* stored uchar */
  unsigned char b1; /* recent readed uchar */
  int aux;


  lutf0 = 0; /* flag for first utf of last read byte */
  lutf1 = 0; /* flag for second utf of last read byte */
  c0 = 0; /* prior data readed */
  c1 = 0; /* last data readed */
  empty = 1; /* flag to mark we don't have a byte wating for output */

  while ( (fread ((void *) &b1, sizeof( unsigned char), 1, stdin)) == 1)
    {
      c1 = (int) b1;

      /* needed test about byte readed now */
      utf0 = ( (c1 >> 5) == 6)? 1 : 0;
      utf1 = ( (c1 >> 6) == 2)? 1 : 0;
      if (utf1)
        {
          /* is a probable second utf8 byte */
          if (lutf0)
            {
              /*
              likely they are a two bytes utf-8 code. Then convert prior
              and current byte to an iso-8859-1 char and write it 
              */
              aux = ((c0 & mask0) << 6) | (c1 & mask1);
              b0 = (unsigned char) aux;
              fwrite( &b0, sizeof(unsigned char), 1, stdout);
              empty = 1;
            }
          else
            {
              /* Prior was not a first utf-byte, so we can write the
              current byte as is. It is an 8-bits iso-8859-1 char.
              */
              fwrite( &b1, sizeof(unsigned char), 1, stdout);
              empty = 1;
            }
        }
      else if (utf0)
        {
          /* we don't write the current char waiting for the following */
          c0 = c1;
          empty = 0;
        }
      else
        {
          /* we write the byte as iso-8859-1 */
          fwrite( &b1, sizeof(unsigned char), 1, stdout);
          empty = 1;
        }

      lutf0 = utf0;
      lutf1 = utf1;
    }
  /* output last char */
  if (empty == 0)
    {
      b0 = (unsigned char) c0;
      fwrite( &b0, sizeof(unsigned char), 1, stdout);
    }
  return EXIT_SUCCESS;
}
