>From fdd615e6e48ce09e9b9c6c025a2d8e3ba0ba39f8 Mon Sep 17 00:00:00 2001 From: Lukas Slebodnik Date: Wed, 6 Nov 2013 14:56:10 +0100 Subject: [PATCH] UTF fix --- ini/ini_fileobj.c | 135 +++++++++++++++++++++++++++++++++++------------------- 1 file changed, 88 insertions(+), 47 deletions(-) diff --git a/ini/ini_fileobj.c b/ini/ini_fileobj.c index d7a42ed7627db12694434076f3e00b9439af9f71..036c5f7d3b1726b4a51c18df28429fd629447fd4 100644 --- a/ini/ini_fileobj.c +++ b/ini/ini_fileobj.c @@ -77,26 +77,26 @@ void ini_config_file_destroy(struct ini_cfgfile *file_ctx) } /* How much I plan to read? */ -size_t how_much_to_read(size_t left, size_t increment) +static size_t how_much_to_read(size_t left, size_t increment) { if(left > increment) return increment; else return left; } -enum index_utf_t check_bom(enum index_utf_t ind, - unsigned char *buffer, - size_t len, - size_t *bom_shift) +static enum index_utf_t check_bom(enum index_utf_t ind, + unsigned char *buffer, + size_t len, + size_t *_bom_shift) { TRACE_FLOW_ENTRY(); - if (len > BOM4_SIZE) { + if (len >= BOM4_SIZE) { if ((buffer[0] == 0x00) && (buffer[1] == 0x00) && (buffer[2] == 0xFE) && (buffer[3] == 0xFF)) { TRACE_FLOW_RETURN(INDEX_UTF32BE); - *bom_shift = BOM4_SIZE; + *_bom_shift = BOM4_SIZE; return INDEX_UTF32BE; } else if ((buffer[0] == 0xFF) && @@ -104,32 +104,32 @@ enum index_utf_t check_bom(enum index_utf_t ind, (buffer[2] == 0x00) && (buffer[3] == 0x00)) { TRACE_FLOW_RETURN(INDEX_UTF32LE); - *bom_shift = BOM4_SIZE; + *_bom_shift = BOM4_SIZE; return INDEX_UTF32LE; } } - if (len > BOM3_SIZE) { + if (len >= BOM3_SIZE) { if ((buffer[0] == 0xEF) && (buffer[1] == 0xBB) && (buffer[2] == 0xBF)) { TRACE_FLOW_RETURN(INDEX_UTF8); - *bom_shift = BOM3_SIZE; + *_bom_shift = BOM3_SIZE; return INDEX_UTF8; } } - if (len > BOM2_SIZE) { + if (len >= BOM2_SIZE) { if ((buffer[0] == 0xFE) && (buffer[1] == 0xFF)) { TRACE_FLOW_RETURN(INDEX_UTF16BE); - *bom_shift = BOM2_SIZE; + *_bom_shift = BOM2_SIZE; return INDEX_UTF16BE; } else if ((buffer[0] == 0xFF) && (buffer[1] == 0xFE)) { TRACE_FLOW_RETURN(INDEX_UTF16LE); - *bom_shift = BOM2_SIZE; + *_bom_shift = BOM2_SIZE; return INDEX_UTF16LE; } } @@ -139,7 +139,7 @@ enum index_utf_t check_bom(enum index_utf_t ind, } static int read_chunk(int raw_file, size_t left, size_t increment, - char *position, size_t *read_num) + char *position, size_t *_read_num) { int error = EOK; size_t to_read = 0; @@ -164,7 +164,7 @@ static int read_chunk(int raw_file, size_t left, size_t increment, return error; } - *read_num = read_cnt; + *_read_num = read_cnt; TRACE_FLOW_EXIT(); return error; @@ -172,7 +172,7 @@ static int read_chunk(int raw_file, size_t left, size_t increment, /* Function useful for debugging */ /* -void print_buffer(char *read_buffer, int len) +static void print_buffer(char *read_buffer, int len) { int i; for (i=0; i < len; i++) { @@ -182,6 +182,68 @@ void print_buffer(char *read_buffer, int len) } */ +static int read_file_header(int raw_file, + struct ini_cfgfile *file_ctx, + char *read_buf, + size_t *_read_cnt, + size_t *_left_in_buffer, + iconv_t *_conv) +{ + int error; + enum index_utf_t ind = INDEX_UTF8; + size_t read_cnt = 0; + size_t bom_shift = 0; + size_t left_bytes = 0; + iconv_t conv = (iconv_t)-1; + const char *encodings[] = { "UTF-32BE", + "UTF-32LE", + "UTF-16BE", + "UTF-16LE", + "UTF-8" }; + + /* First time do some initialization */ + TRACE_INFO_STRING("Reading first time.","Checking BOM"); + + error = read_chunk(raw_file, + file_ctx->file_stats.st_size, + 4, /* 4 bytes to check bom */ + read_buf, + &read_cnt); + if (error) { + TRACE_ERROR_NUMBER("Failed to read initial chunk", error); + return error; + } + + ind = check_bom(ind, + (unsigned char *)read_buf, + read_cnt, + &bom_shift); + + TRACE_INFO_STRING("Converting to", encodings[INDEX_UTF8]); + TRACE_INFO_STRING("Converting from", encodings[ind]); + + errno = 0; + conv = iconv_open(encodings[INDEX_UTF8], encodings[ind]); + if (conv == (iconv_t) -1) { + error = errno; + TRACE_ERROR_NUMBER("Failed to create converter", error); + return error; + } + + /* move left bytes to beginning of buffer*/ + if (read_cnt - bom_shift > 0) { + left_bytes = read_cnt - bom_shift; + memmove(read_buf, read_buf + bom_shift, left_bytes); + } + + *_read_cnt = read_cnt; + *_left_in_buffer = left_bytes; + *_conv = conv; + + return EOK; +} + + /* Internal conversion part */ static int common_file_convert(int raw_file, struct ini_cfgfile *file_ctx) { @@ -189,7 +251,6 @@ static int common_file_convert(int raw_file, struct ini_cfgfile *file_ctx) size_t read_cnt = 0; size_t total_read = 0; size_t in_buffer = 0; - enum index_utf_t ind = INDEX_UTF8; iconv_t conv = (iconv_t)-1; size_t conv_res = 0; char read_buf[ICONV_BUFFER+1]; @@ -197,15 +258,18 @@ static int common_file_convert(int raw_file, struct ini_cfgfile *file_ctx) char *src, *dest; size_t to_convert = 0; size_t room_left = 0; - size_t bom_shift = 0; - const char *encodings[] = { "UTF-32BE", - "UTF-32LE", - "UTF-16BE", - "UTF-16LE", - "UTF-8" }; TRACE_FLOW_ENTRY(); + error = read_file_header(raw_file, file_ctx, read_buf, + &read_cnt, &in_buffer, &conv); + if (error) { + TRACE_ERROR_NUMBER("Failed to read file header", error); + return error; + } + + total_read += read_cnt; + do { /* print_buffer(read_buf, ICONV_BUFFER); */ error = read_chunk(raw_file, @@ -226,29 +290,6 @@ static int common_file_convert(int raw_file, struct ini_cfgfile *file_ctx) in_buffer = 0; /* First time do some initialization */ - if (total_read == 0) { - TRACE_INFO_STRING("Reading first time.","Checking BOM"); - - ind = check_bom(ind, - (unsigned char *)read_buf, - read_cnt, - &bom_shift); - - /* Skip BOM the first time */ - src += bom_shift; - to_convert -= bom_shift; - - TRACE_INFO_STRING("Converting to", encodings[INDEX_UTF8]); - TRACE_INFO_STRING("Converting from", encodings[ind]); - - errno = 0; - conv = iconv_open(encodings[INDEX_UTF8], encodings[ind]); - if (conv == (iconv_t) -1) { - error = errno; - TRACE_ERROR_NUMBER("Failed to create converter", error); - return error; - } - } total_read += read_cnt; TRACE_INFO_NUMBER("Total read", total_read); @@ -388,10 +429,10 @@ static int common_file_init(struct ini_cfgfile *file_ctx) */ if (file_ctx->file_stats.st_size) { error = common_file_convert(raw_file, file_ctx); + close(raw_file); if (error) { TRACE_ERROR_NUMBER("Failed to convert file", error); - close(raw_file); return error; } } -- 1.8.3.1