cpconv.c

// Conversion betweeen different charmaps and/or utf8.


// github.com/michael105/codepage_converter
// miSc Michael Myer, 2023, GPL
//
// Don't blame me for erratic conversions.
//
// BOM characters are not converted (!).
// I wouldn't recommend using this code in security relevant contexts;
// e.g. the missing BOM conversion could lead to trouble.
// (Very extrapolated worst case)
//
// local conversion might be safe, however,
// especially for using this hack as an input filter.
// That's why I wrote this.
// Yet I didn't need BOM characters, I write with latex.
//
// The codepage "guessing" might work only for German, it looks 
// for umlauts.
// If you'd need other languages, add the typical (extended ascii) characters
// below to the constant charmap cp.
//


// uncomment, to abort at (some) invalid UTF8 characters.
// (They might still be filtered out)
#define UTF8_STRICT 

// Buffer sizes
#define BUF 64000
#define OBUF (BUF*2)

// memory used for the precomputed cache of the unicode translation table (in Bytes)
// (Mostly) All unicode chars defined per default are below 0x2600
#define UNITABLE 0x2800

// The default target encoding
// (count cp from 0, 0=cpe4002a, 2=cp850, 3=cp1252, .. 15=utf8)
#ifndef DEFAULT_CP 
#define DEFAULT_CP 3
#endif


#include "charmaps.h"

#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <string.h>
#include <errno.h>

typedef unsigned char uchar;

typedef struct { 
	const unsigned short *map;
	const char esign; // used for unconvertible chars
	const char *name;
	const unsigned char *chars;
} charmap;

#define MAP(name,esign,chars) { name, 0x##esign, #name, (unsigned char*) chars }

static const short unsigned int utf8[128]; // empty

// chars, which are likely to be in literal texts.
// It would be helpful using a dictionary.
// But this works out as well, for my needs.
//
// Try to guess by the used characters.
// This eventually works best with german and english literal texts,
// I only added few mathematical and special other chars
// ( (C),  +-, quotation marks, Euro, ..)
// For recognition of other languages, 
// the typically used characters (and cp's) would need to be added below.
const charmap cp[] = {
	MAP( cpe4002a,a8, "\x81\x84\x8e\x94\x9a\x99" // umlaute
	                  "\xe1\xe3\xe4\x9b\x9c\xe6" ),
	MAP( cp850,  a8,  "\x81\x84\x8e\x94\x9a\x99" // umlaute
   	               "\xe1\x9c\xe6\xf4\xf5\xf8\xb8" ),
	MAP( cp437,  a8,  "\x81\x84\x8e\x94\x9a\x99" // umlaute
	                  "\xe1\xe3\xe4\x9b\x9c\xe6" ),
	MAP( cp1252, b6,  "\xe4\xf6\xfc\xdc\xd5\xc4" // "üöäÖÜÄ" // Umlauts
                     "\x80\xb6\xa7\x93\x94\xa9"
                     "\xf7\xa9\xb0\xb1\xd7\xb5\xae" ),
	MAP( cp1250, b6, "" ),
	MAP( cp1251, b6, "" ),
	MAP( cp1253, b6, "" ),
	MAP( cp1255, b6, "" ),
	MAP( cp1256, b6, "" ),
	MAP( cp1257, b6, "" ),
	MAP( cp1258, b6, "" ),
	MAP( atarist, b6, "" ),
	MAP( macintosh, b6, "" ),
	MAP( mac_centraleurope, b6, "" ),
	MAP( iso8859_15, b6, "" ),
	MAP( utf8,7e,""),
	{0,0,0,0},
};


#define UTF8 (sizeof(cp)/sizeof(charmap)-2)

#define NUMCP (sizeof(cp)/sizeof(charmap)-1)


enum { OPT_v=1, OPT_s=2, OPT_u=4, OPT_x=8, OPT_d=16, OPT_Q=32, OPT_c=64, OPT_C=128 } OPT_enum;

#define OPT(x) (opts&OPT_##x)

int opts;


#define W(msg) fprintf(stderr,msg)   //write(2,msg,sizeof(msg))
// verbose
#define V(msg) ({ if ( OPT(v) ) W(msg); 1; })
#define v(...) { if ( OPT(v) ) fprintf(stderr,__VA_ARGS__); }
// not silent
#define E(msg) { if ( !(OPT(s)) ) W(msg); }
#define e(...) { if ( !(OPT(s)) ) fprintf(stderr,__VA_ARGS__); }
// debug
#define D(msg) { if ( OPT(d) ) W(msg); }
#define d(...) { if ( OPT(d) ) fprintf(stderr,__VA_ARGS__); }



void usage(){
	W("convert stdin to stdout\n"
			"Usage: cpconv [-hslxud] [tocp [fromcp]]\n\n"
			"Examples: cat text.txt | cpconv cp437 cp850\n"
			"         (convert from cp850 to cp437)\n\n"
			"          cat text.txt | cpconv\n"
			"         (convert to "); 
	W(cp[DEFAULT_CP].name); 
	W(", guessing source charset\n\n"
			"Without any options, try to guess the charset and convert to ");
   W(cp[DEFAULT_CP].name);
	W(		"\n(change the default in the source, if needed)\n\n"
			"options: -s : silence, no messages to stderr\n"
			"         -v : verbose\n"
			"         -l : list codepages\n"
			"         -U : dump umlaute, converted\n"
			"         -c : convert input to cstring notation\n"
			"         -C : convert input to cstring notation, including linebreaks\n"
			"         -x : display non convertible chars in hexadecimal\n"
			"         -u : display non convertible chars as utf8\n"
			"         -d : print debug information\n"
			"         -h : show this help\n"
			"\n"
			"(I'm using this as input filter for vi, and to copy text between terminal and x clipboard\n"
			" the conversion is done automatically, if needed)\n"
			"\nmiSc 23, GPL\n\n"
			);
	exit(1);
}


void listcodepages(){
	fprintf(stderr,"Supported (extended ASCII) encodings:\n");
	for ( const charmap *cm = cp; cm->name != 0; cm++ ){
		fprintf(stderr,"\n%s",cm->name);
		if ( cp+DEFAULT_CP == cm )
			fprintf(stderr," (default)");
	}
	fprintf(stderr,"\n");

	exit(0);
}

// handle stuttering pipes and signals (renamed to nmread - this has nothing with those folks to do)
static int nmread(uchar *buf, int len){
	uchar *b = buf;
	uchar *e = buf+len;
	int ret;

	if ( opts & OPT_Q )
		return(0);

	do {
		errno = 0;
		ret = read(0,b,(e-b));
		if ( ret<=0 ){
			if ( errno == EAGAIN || errno == EINTR )
				continue;
			return( b-buf ); // rw bytes (if), or error code
		}
		b+=ret;
	} while ( b < e );

	return( len );
}


// according to rfc 3629, these values are not valid
int valid_uc_utf8(uint unicode){ 
	if ( (unicode>0xd800)&&(unicode<0xdfff) ){
#ifdef UTF8_STRICT
		e("invalid utf8 conversion: %x\n",unicode);
		exit(1);
#endif
		return(0);
	}
	return(1);
}


int guess_charmap(const unsigned char *buf, int len){
	int guess[NUMCP];
	memset(guess,0,NUMCP*sizeof(int));
	int n = 0; // ascii 32-127
	int so=0; // chars, extended ascii ( 128-192 )
	int co = 0; // control chars.
	int ext = 0; // chars 128-255
	int a = 0;

	while ( a<len ){
		switch (buf[a]) {
			case 0 ... 31:
				co++;
				break;
			case 128 ... 255:
				ext++;
				for ( int b = 0; b<NUMCP-1; b++ ){
					for ( const unsigned char *c = cp[b].chars; *c; c++ ){
						if ( *c == buf[a] ){
							guess[b] = guess[b] + 1;
							break;
						}
					}
				}

				if ( buf[a] < 192 ){ // 128-192 = no utf8
					so++;
					//utf -= 4; // I'm uncertain about mixed encodings
				} else { // coul'd be  utf8
					if ( (buf[a] & 0xe0) == 0xc0 ){ // initial Byte 2Byte utf8
						if ( (a+1<len) && ( (buf[a+1] & 0xc0) == 0x80 ) ){ 
							guess[UTF8] ++;
							a++;
						}
					} else if ( (buf[a] & 0xf0) == 0xe0 ){ // 3byte
						if ( (a+2 < len ) && ( (buf[a+1] & 0xc0) == 0x80 ) && 
								( (buf[a+2] & 0xc0) == 0x80 )){
							guess[UTF8] += 2;
							a+=2;
						}
					} else if ( (buf[a] & 0xf8) == 0xf0 ){ //4byte
						if ( ( a+3<len ) && ( (buf[a+1] & 0xc0) == 0x80 ) &&
								( (buf[a+2] & 0xc0) == 0x80 ) &&
								( (buf[a+3] & 0xc0) == 0x80 ) ){
							guess[UTF8] += 3;
							a+=3;
						}
					}

				}

			break;

			default: // ascii char 32-127
				n++;
		}

		a++;
	}

	if ( !ext ){
		V("No extended Ascii/UTF present.\n");
		return(-1);
	}

	if ( n<len-len/4 ){
		V("This looks like binary data.\nContinuing anyways\n");
	}

	int max = 0;
	int guessed = -1;
	for ( int a = 0; a<NUMCP; a++ )
		if ( guess[a] > max ){
			max = guess[a];
			guessed = a;
		}

	v("Guess: %s\n(%d chars match)\n",cp[guessed].name,max);
	d("  chars: %d\n   utf8: %d\n   0-31: %d\n"
	  " 32-127: %d\n128-191: %d\n192-255: %d\n",len,guess[UTF8],co,n,so,ext);

	return(guessed);
}


int main(int argc, char **argv ){

	unsigned char buf[BUF],obuf[OBUF];
	int len = 0;
	int from = -1;
	int to = -1;
	//opts = OPT_v; // verbose
					 
	// parse options
	for ( *argv++; *argv && *argv[0] == '-'; *argv++ ){
		argc--;
		for ( char *o = *argv+1; *o; o++ ){
			switch (*o) {
				case 'l':
					listcodepages();
				case 's':
					opts|=OPT_s;
					opts&=~OPT_v; // reset bit verbose
					break;
				case 'u':
					opts|=OPT_u; // write utf8 for nonconvertible chars
					break;
				case 'x':
					opts|=OPT_x; // write hex unicode for nonconv. chars
				break;
				case 'd':
					opts|=OPT_d; // debugging (count of chars)
				case 'v':
					 opts|=OPT_v;
				break;
				case 'C':
					 opts|=OPT_C;
				case 'c':
					 opts|=OPT_c;
				break;
				case 'U':
					# define _COPY(d,s) memcpy(d,s,sizeof(s)); len=sizeof(s) -1
					_COPY(buf, "\nUmlaute\n\n\x84\x94\x81\n---\n\x8e\x99\x9a\n\n");
					to = 0; // gets to from
					opts|=OPT_Q; // don't read from stdin
					break;
				default: // -h, --help, ..
					usage();
			}
		}
	}
	argc--;

	if ( argc>2 )
		usage();


	// parse from and to codepage (if)
	while ( argc>0 ){
		from = to;
		 const charmap *c = cp;
		 while ( strcmp(c->name,argv[argc-1]) != 0 ){
			 c++;
			 if ( c->name == 0 ){
				 fprintf(stderr,"Unknown codepage: %s\n",argv[argc-1]);
				 exit(1);
			 }
		 }
		 to = c - cp;
		 argc--;
	}

	if ( to==-1 )
		to = DEFAULT_CP;

	// create reverse table
	unsigned char ocp[UNITABLE];
	memset( ocp, 0 , UNITABLE );
	for ( int a=0; a<128; a++ ){
		if ( cp[to].map[a] < UNITABLE )
			ocp[ cp[to].map[a] ] = a + 128;
	}

	if ( !len )
		len = nmread(buf,BUF);

	if ( !OPT(c) && (from == -1) ){
		V("Guessing charset\n");
		from = guess_charmap(buf,len);
	}

	if ( !OPT(c) && ((from == -1) || // no conversion possible, no extended ascii
			( (from==to) && ( V("Source and destination codepage are the same\n")) )) ){ 		
		// write stdin to stdout (we are a filter)
		do{ write(1,buf,len); } while (( len=nmread(buf,BUF) ));
		exit(0);
	}

	v("Converting from %s to %s\n", cp[from].name,cp[to].name);

	do { // mainloop
		int p = 0;
		int a = 0; 
		while ( a<len ){

			if ( OPT(c) ) { // cstring
				if ( buf[a] <32 || buf[a] >127 )
					if ( buf[a] == '\n' ){
						if ( OPT(C) ){
							obuf[p++] = '\\';
							obuf[p++] = 'n';
						} else
							obuf[p++] = '\n';
					} else
						p+= sprintf( (char*)obuf+p,"\\x%02x",buf[a] );
				else 
					obuf[p++] = buf[a];

			} else if ( buf[a] <128 )
				obuf[p++] = buf[a];

			else { // char > 127
							// get unicode point
				uint uc; 
				// convert utf-8 to unicode
				if ( from == UTF8 ){
					if ( a+3>=len ){ // refill the buffer if needed, prevent incomplete sequences
						len -= a;
						memmove(buf,(buf+a),len);
						a = 0;
						int l = nmread((buf+len),BUF-len);
						if ( l>0 )
							len += l;
					}

					int tmp = a;
					// convert utf8
					if ( (a+1<len) && ( (buf[a+1] & 0xc0) == 0x80 ) ){ 
						uc = ( (buf[a] & 0x1f) << 6 ) | (buf[a+1] & 0x3f);
						if ( (buf[a] & 0xe0) == 0xc0 ){ // initial Byte 2Byte utf8
							a++;
						} else if ( (a+2<len) && ( (buf[a+2] & 0xc0) == 0x80 ) ){ 
							uc = ( uc << 6 ) | (buf[a+2] & 0x3f);
							if ( (buf[a] & 0xf0) == 0xe0 ){ // initial Byte 3Byte utf8
								a+=2;
							} else if ( (a+3<len) && ( (buf[a+3] & 0xc0) == 0x80 ) ){ 
								if ( (buf[a] & 0xf8) == 0xf0 ){ // 4byte
									uc = ((uc<<6) & 0x1FFFFF ) | ( buf[a+3] & 0x3f );
									a+=3;
								}
							}
						} 
					} 

					if ( (tmp == a) || !( valid_uc_utf8(uc) ) ){ // error. invalid utf8
						e("Invalid utf8 sequence: %02x%02x\n",buf[a],buf[a+1]);
						uc = buf[a]; // for "mixed" encodings (e.g. cp1252, and utf-8)
						// I'm a bit uncertain. In theory, the whole document's conversion
						// is "invalid". In practice - maybe, some umlauts fall through
						// I did say, John, you don't use umlauts for the calculation of the escape velocity.
						// Just don't do it. Well.
					}
					  
				} else { // codepage table conversion to unicode
					uc = cp[from].map[buf[a]-128];
				}

				// convert unicode to the destination encoding =================
				int t=p;
				if ( to == UTF8 ){ // convert to utf8
					if ( !valid_uc_utf8(uc) )
						goto ERR_UTF8;
					// invalid values (rfc3629
				
					char initb=(char)0xc0;
					if ( uc >= 65536 ){
						obuf[t+3] = (uc & 0x3f) | 0x80;
						initb >>= (char)1;
						uc >>= 6;
						p++;
					}
					if ( uc >= 2048 ){
						obuf[t+2] = (uc & 0x3f) | 0x80;
						initb >>= (char)1;
						uc >>= 6;
						p++;
					}
					obuf[t+1] = (uc & 0x3f) | 0x80;
					obuf[t] = (uc>>6) | initb;
					p += 2;
					

				} else {
					// convert to codepage
					if ( (uc<UNITABLE) && ( ocp[ uc ] != 0 ) ){
						obuf[p++] = ocp[ uc ]; 
					} else if ( uc>=UNITABLE ){
						for ( int a = 0; a<128; a++ )
							if ( cp[to].map[a] == uc ){
								obuf[p++] = a+128;
								break;
							}
					} 
				}

				if ( t == p ){ // no conversion possible
ERR_UTF8:
					e( "Cannot convert: %s: %d, unicode: %x\n",
							cp[from].name, buf[a], uc );
					obuf[p++] = cp[to].esign;
					if ( OPT(x) ){
						p+= sprintf( (char*)obuf+p,"<%x>",uc );
					}
				}
			}
			if ( p > OBUF - 16 ){
				write(1,obuf,p);
				p=0;
			}
			a++;
		}

		write(1,obuf,p);

	} while ( (len = nmread(buf,BUF)) );

	exit(0);
}