ucnvmbcs.h (23835B)
1 // © 2016 and later: Unicode, Inc. and others. 2 // License & terms of use: http://www.unicode.org/copyright.html 3 /* 4 ****************************************************************************** 5 * 6 * Copyright (C) 2000-2013, International Business Machines 7 * Corporation and others. All Rights Reserved. 8 * 9 ****************************************************************************** 10 * file name: ucnvmbcs.h 11 * encoding: UTF-8 12 * tab size: 8 (not used) 13 * indentation:4 14 * 15 * created on: 2000jul07 16 * created by: Markus W. Scherer 17 */ 18 19 #ifndef __UCNVMBCS_H__ 20 #define __UCNVMBCS_H__ 21 22 #include "unicode/utypes.h" 23 24 #if !UCONFIG_NO_CONVERSION 25 26 #include "unicode/ucnv.h" 27 #include "ucnv_cnv.h" 28 #include "ucnv_ext.h" 29 30 /** 31 * ICU conversion (.cnv) data file structure, following the usual UDataInfo 32 * header. 33 * 34 * Format version: 6.2 35 * 36 * struct UConverterStaticData -- struct containing the converter name, IBM CCSID, 37 * min/max bytes per character, etc. 38 * see ucnv_bld.h 39 * 40 * -------------------- 41 * 42 * The static data is followed by conversionType-specific data structures. 43 * At the moment, there are only variations of MBCS converters. They all have 44 * the same toUnicode structures, while the fromUnicode structures for SBCS 45 * differ from those for other MBCS-style converters. 46 * 47 * _MBCSHeader.version 5 is optional and not backward-compatible 48 * (as usual for changes in the major version field). 49 * 50 * Versions 5.m work like versions 4.m except: 51 * - The _MBCSHeader has variable length (and is always longer than in version 4). 52 * See the struct _MBCSHeader further description below. 53 * - There is a set of flags which indicate further incompatible changes. 54 * (Reader code must reject the file if it does not recognize them all.) 55 * - In particular, one of these flags indicates that most of the fromUnicode 56 * data is missing and must be reconstituted from the toUnicode data 57 * and from the utf8Friendly mbcsIndex at load time. 58 * (This only works with a utf8Friendly table.) 59 * In this case, makeconv may increase maxFastUChar automatically to U+FFFF. 60 * 61 * The first of these versions is 5.3, which is like 4.3 except for the differences above. 62 * 63 * When possible, makeconv continues to generate version 4.m files. 64 * 65 * _MBCSHeader.version 5.4/4.4 supports "good one-way" mappings (|4) 66 * in the extension tables (fromUTableValues bit 30). See ucnv_ext.h for details. 67 * 68 * _MBCSHeader.version 4.3 optionally modifies the fromUnicode data structures 69 * slightly and optionally adds a table for conversion to MBCS (non-SBCS) 70 * charsets. 71 * 72 * The modifications are to make the data utf8Friendly. Not every 4.3 file 73 * file contains utf8Friendly data. 74 * It is utf8Friendly if _MBCSHeader.version[2]!=0. 75 * In this case, the data structures are utf8Friendly up to the code point 76 * maxFastUChar=((_MBCSHeader.version[2]<<8)|0xff) 77 * 78 * A utf8Friendly file has fromUnicode stage 3 entries for code points up to 79 * maxFastUChar allocated in blocks of 64 for indexing with the 6 bits from 80 * a UTF-8 trail byte. ASCII is allocated linearly with 128 contiguous entries. 81 * 82 * In addition, a utf8Friendly MBCS file contains an additional 83 * uint16_t mbcsIndex[(maxFastUChar+1)>>6]; 84 * which replaces the stage 1 and 2 tables for indexing with bits from the 85 * UTF-8 lead byte and middle trail byte. Unlike the older MBCS stage 2 table, 86 * the mbcsIndex does not contain roundtrip flags. Therefore, all fallbacks 87 * from code points up to maxFastUChar (and roundtrips to 0x00) are moved to 88 * the extension data structure. This also allows for faster roundtrip 89 * conversion from UTF-16. 90 * 91 * SBCS files do not contain an additional sbcsIndex[] array because the 92 * proportional size increase would be noticeable, but the runtime 93 * code builds one for the code point range for which the runtime conversion 94 * code is optimized. 95 * 96 * For SBCS, maxFastUChar should be at least U+0FFF. The initial makeconv 97 * implementation sets it to U+1FFF. Because the sbcsIndex is not stored in 98 * the file, a larger maxFastUChar only affects stage 3 block allocation size 99 * and is free in empty blocks. (Larger blocks with sparse contents cause larger 100 * files.) U+1FFF includes almost all of the small scripts. 101 * U+0FFF covers UTF-8 two-byte sequences and three-byte sequences starting with 102 * 0xe0. This includes most scripts with legacy SBCS charsets. 103 * The initial runtime implementation using 4.3 files only builds an sbcsIndex 104 * for code points up to U+0FFF. 105 * 106 * For MBCS, maxFastUChar should be at least U+D7FF (=initial value). 107 * This boundary is convenient because practically all of the commonly used 108 * characters are below it, and because it is the boundary to surrogate 109 * code points, above which special handling is necessary anyway. 110 * (Surrogate pair assembly for UTF-16, validity checking for UTF-8.) 111 * 112 * maxFastUChar could be up to U+FFFF to cover the whole BMP, which could be 113 * useful especially for conversion from UTF-8 when the input can be assumed 114 * to be valid, because the surrogate range would then not have to be 115 * checked. 116 * (With maxFastUChar=0xffff, makeconv would have to check for mbcsIndex value 117 * overflow because with the all-unassigned block 0 and nearly full mappings 118 * from the BMP it is theoretically possible that an index into stage 3 119 * exceeds 16 bits.) 120 * 121 * _MBCSHeader.version 4.2 adds an optional conversion extension data structure. 122 * If it is present, then an ICU version reading header versions 4.0 or 4.1 123 * will be able to use the base table and ignore the extension. 124 * 125 * The unicodeMask in the static data is part of the base table data structure. 126 * Especially, the UCNV_HAS_SUPPLEMENTARY flag determines the length of the 127 * fromUnicode stage 1 array. 128 * The static data unicodeMask refers only to the base table's properties if 129 * a base table is included. 130 * In an extension-only file, the static data unicodeMask is 0. 131 * The extension data indexes have a separate field with the unicodeMask flags. 132 * 133 * MBCS-style data structure following the static data. 134 * Offsets are counted in bytes from the beginning of the MBCS header structure. 135 * Details about usage in comments in ucnvmbcs.c. 136 * 137 * struct _MBCSHeader (see the definition in this header file below) 138 * contains 32-bit fields as follows: 139 * 8 values: 140 * 0 uint8_t[4] MBCS version in UVersionInfo format (currently 4.3.x.0) 141 * 1 uint32_t countStates 142 * 2 uint32_t countToUFallbacks 143 * 3 uint32_t offsetToUCodeUnits 144 * 4 uint32_t offsetFromUTable 145 * 5 uint32_t offsetFromUBytes 146 * 6 uint32_t flags, bits: 147 * 31.. 8 offsetExtension -- _MBCSHeader.version 4.2 (ICU 2.8) and higher 148 * 0 for older versions and if 149 * there is not extension structure 150 * 7.. 0 outputType 151 * 7 uint32_t fromUBytesLength -- _MBCSHeader.version 4.1 (ICU 2.4) and higher 152 * counts bytes in fromUBytes[] 153 * 154 * New and required in version 5: 155 * 8 uint32_t options, bits: 156 * 31..16 reserved for flags that can be added without breaking 157 * backward compatibility 158 * 15.. 6 reserved for flags whose addition will break 159 * backward compatibility 160 * 6 MBCS_OPT_FROM_U -- if set, 161 * then most of the fromUnicode data is omitted; 162 * fullStage2Length is present and the missing 163 * bottom part of stage 2 must be reconstituted from 164 * the toUnicode data; 165 * stage 3 is missing completely as well; 166 * not used for SBCS tables 167 * 5.. 0 length of the _MBCSHeader (number of uint32_t) 168 * 169 * New and optional in version 5: 170 * 9 uint32_t fullStage2Length: used if MBCS_OPT_FROM_U is set 171 * specifies the full length of stage 2 172 * including the omitted part 173 * 174 * if(outputType==MBCS_OUTPUT_EXT_ONLY) { 175 * -- base table name for extension-only table 176 * char baseTableName[variable]; -- with NUL plus padding for 4-alignment 177 * 178 * -- all _MBCSHeader fields except for version and flags are 0 179 * } else { 180 * -- normal base table with optional extension 181 * 182 * int32_t stateTable[countStates][256]; 183 * 184 * struct _MBCSToUFallback { (fallbacks are sorted by offset) 185 * uint32_t offset; 186 * UChar32 codePoint; 187 * } toUFallbacks[countToUFallbacks]; 188 * 189 * uint16_t unicodeCodeUnits[(offsetFromUTable-offsetToUCodeUnits)/2]; 190 * (padded to an even number of units) 191 * 192 * -- stage 1 tables 193 * if(staticData.unicodeMask&UCNV_HAS_SUPPLEMENTARY) { 194 * -- stage 1 table for all of Unicode 195 * uint16_t fromUTable[0x440]; (32-bit-aligned) 196 * } else { 197 * -- BMP-only tables have a smaller stage 1 table 198 * uint16_t fromUTable[0x40]; (32-bit-aligned) 199 * } 200 * 201 * -- stage 2 tables 202 * length determined by top of stage 1 and bottom of stage 3 tables 203 * if(outputType==MBCS_OUTPUT_1) { 204 * -- SBCS: pure indexes 205 * uint16_t stage 2 indexes[?]; 206 * } else { 207 * -- DBCS, MBCS, EBCDIC_STATEFUL, ...: roundtrip flags and indexes 208 * uint32_t stage 2 flags and indexes[?]; 209 * if(options&MBCS_OPT_NO_FROM_U) { 210 * stage 2 really has length fullStage2Length 211 * and the omitted lower part must be reconstituted from 212 * the toUnicode data 213 * } 214 * } 215 * 216 * -- stage 3 tables with byte results 217 * if(outputType==MBCS_OUTPUT_1) { 218 * -- SBCS: each 16-bit result contains flags and the result byte, see ucnvmbcs.c 219 * uint16_t fromUBytes[fromUBytesLength/2]; 220 * } else if(!(options&MBCS_OPT_NO_FROM_U)) { 221 * -- DBCS, MBCS, EBCDIC_STATEFUL, ... 2/3/4 bytes result, see ucnvmbcs.c 222 * uint8_t fromUBytes[fromUBytesLength]; or 223 * uint16_t fromUBytes[fromUBytesLength/2]; or 224 * uint32_t fromUBytes[fromUBytesLength/4]; 225 * } else { 226 * fromUBytes[] must be reconstituted from the toUnicode data 227 * } 228 * 229 * -- optional utf8Friendly mbcsIndex -- _MBCSHeader.version 4.3 (ICU 3.8) and higher 230 * if(outputType!=MBCS_OUTPUT_1 && 231 * _MBCSHeader.version[1]>=3 && 232 * (maxFastUChar=_MBCSHeader.version[2])!=0 233 * ) { 234 * maxFastUChar=(maxFastUChar<<8)|0xff; 235 * uint16_t mbcsIndex[(maxFastUChar+1)>>6]; 236 * } 237 * } 238 * 239 * -- extension table, details see ucnv_ext.h 240 * int32_t indexes[>=32]; ... 241 */ 242 243 /* MBCS converter data and state -------------------------------------------- */ 244 245 enum { 246 MBCS_MAX_STATE_COUNT=128 247 }; 248 249 /** 250 * MBCS action codes for conversions to Unicode. 251 * These values are in bits 23..20 of the state table entries. 252 */ 253 enum { 254 MBCS_STATE_VALID_DIRECT_16, 255 MBCS_STATE_VALID_DIRECT_20, 256 257 MBCS_STATE_FALLBACK_DIRECT_16, 258 MBCS_STATE_FALLBACK_DIRECT_20, 259 260 MBCS_STATE_VALID_16, 261 MBCS_STATE_VALID_16_PAIR, 262 263 MBCS_STATE_UNASSIGNED, 264 MBCS_STATE_ILLEGAL, 265 266 MBCS_STATE_CHANGE_ONLY 267 }; 268 269 /* Macros for state table entries */ 270 #define MBCS_ENTRY_TRANSITION(state, offset) (int32_t)(((int32_t)(state)<<24L)|(offset)) 271 #define MBCS_ENTRY_TRANSITION_SET_OFFSET(entry, offset) (int32_t)(((entry)&0xff000000)|(offset)) 272 #define MBCS_ENTRY_TRANSITION_ADD_OFFSET(entry, offset) (int32_t)((entry)+(offset)) 273 274 #define MBCS_ENTRY_FINAL(state, action, value) (int32_t)(0x80000000|((int32_t)(state)<<24L)|((action)<<20L)|(value)) 275 #define MBCS_ENTRY_SET_FINAL(entry) (int32_t)((entry)|0x80000000) 276 #define MBCS_ENTRY_FINAL_SET_ACTION(entry, action) (int32_t)(((entry)&0xff0fffff)|((int32_t)(action)<<20L)) 277 #define MBCS_ENTRY_FINAL_SET_VALUE(entry, value) (int32_t)(((entry)&0xfff00000)|(value)) 278 #define MBCS_ENTRY_FINAL_SET_ACTION_VALUE(entry, action, value) (int32_t)(((entry)&0xff000000)|((int32_t)(action)<<20L)|(value)) 279 280 #define MBCS_ENTRY_SET_STATE(entry, state) (int32_t)(((entry)&0x80ffffff)|((int32_t)(state)<<24L)) 281 282 #define MBCS_ENTRY_STATE(entry) ((((uint32_t)entry)>>24)&0x7f) 283 284 #define MBCS_ENTRY_IS_TRANSITION(entry) ((entry)>=0) 285 #define MBCS_ENTRY_IS_FINAL(entry) ((entry)<0) 286 287 #define MBCS_ENTRY_TRANSITION_STATE(entry) (((uint32_t)entry)>>24) 288 #define MBCS_ENTRY_TRANSITION_OFFSET(entry) ((entry)&0xffffff) 289 290 #define MBCS_ENTRY_FINAL_STATE(entry) ((((uint32_t)entry)>>24)&0x7f) 291 #define MBCS_ENTRY_FINAL_IS_VALID_DIRECT_16(entry) ((entry)<(int32_t)0x80100000) 292 #define MBCS_ENTRY_FINAL_ACTION(entry) ((((uint32_t)entry)>>20)&0xf) 293 #define MBCS_ENTRY_FINAL_VALUE(entry) ((entry)&0xfffff) 294 #define MBCS_ENTRY_FINAL_VALUE_16(entry) (uint16_t)(entry) 295 296 #define IS_ASCII_ROUNDTRIP(b, asciiRoundtrips) (((asciiRoundtrips) & (1<<((b)>>2)))!=0) 297 298 /* single-byte fromUnicode: get the 16-bit result word */ 299 #define MBCS_SINGLE_RESULT_FROM_U(table, results, c) (results)[ (table)[ (table)[(c)>>10] +(((c)>>4)&0x3f) ] +((c)&0xf) ] 300 301 /* single-byte fromUnicode using the sbcsIndex */ 302 #define SBCS_RESULT_FROM_LOW_BMP(table, results, c) (results)[ (table)[(c)>>6] +((c)&0x3f) ] 303 304 /* single-byte fromUTF8 using the sbcsIndex; l and t must be masked externally; can be l=0 and t<=0x7f */ 305 #define SBCS_RESULT_FROM_UTF8(table, results, l, t) (results)[ (table)[l] +(t) ] 306 307 /* multi-byte fromUnicode: get the 32-bit stage 2 entry */ 308 #define MBCS_STAGE_2_FROM_U(table, c) ((const uint32_t *)(table))[ (table)[(c)>>10] +(((c)>>4)&0x3f) ] 309 #define MBCS_FROM_U_IS_ROUNDTRIP(stage2Entry, c) ( ((stage2Entry) & ((uint32_t)1<< (16+((c)&0xf)) )) !=0) 310 311 #define MBCS_VALUE_2_FROM_STAGE_2(bytes, stage2Entry, c) ((uint16_t *)(bytes))[16*(uint32_t)(uint16_t)(stage2Entry)+((c)&0xf)] 312 #define MBCS_VALUE_4_FROM_STAGE_2(bytes, stage2Entry, c) ((uint32_t *)(bytes))[16*(uint32_t)(uint16_t)(stage2Entry)+((c)&0xf)] 313 314 #define MBCS_POINTER_3_FROM_STAGE_2(bytes, stage2Entry, c) ((bytes)+(16*(uint32_t)(uint16_t)(stage2Entry)+((c)&0xf))*3) 315 316 /* double-byte fromUnicode using the mbcsIndex */ 317 #define DBCS_RESULT_FROM_MOST_BMP(table, results, c) (results)[ (table)[(c)>>6] +((c)&0x3f) ] 318 319 /* double-byte fromUTF8 using the mbcsIndex; l and t1 combined into lt1; lt1 and t2 must be masked externally */ 320 #define DBCS_RESULT_FROM_UTF8(table, results, lt1, t2) (results)[ (table)[lt1] +(t2) ] 321 322 323 /** 324 * MBCS output types for conversions from Unicode. 325 * These per-converter types determine the storage method in stage 3 of the lookup table, 326 * mostly how many bytes are stored per entry. 327 */ 328 enum { 329 MBCS_OUTPUT_1, /* 0 */ 330 MBCS_OUTPUT_2, /* 1 */ 331 MBCS_OUTPUT_3, /* 2 */ 332 MBCS_OUTPUT_4, /* 3 */ 333 334 MBCS_OUTPUT_3_EUC=8, /* 8 */ 335 MBCS_OUTPUT_4_EUC, /* 9 */ 336 337 MBCS_OUTPUT_2_SISO=12, /* c */ 338 MBCS_OUTPUT_2_HZ, /* d */ 339 340 MBCS_OUTPUT_EXT_ONLY, /* e */ 341 342 MBCS_OUTPUT_COUNT, 343 344 MBCS_OUTPUT_DBCS_ONLY=0xdb /* runtime-only type for DBCS-only handling of SISO tables */ 345 }; 346 347 /** 348 * Fallbacks to Unicode are stored outside the normal state table and code point structures 349 * in a vector of items of this type. They are sorted by offset. 350 */ 351 typedef struct { 352 uint32_t offset; 353 UChar32 codePoint; 354 } _MBCSToUFallback; 355 356 /** Constants for fast and UTF-8-friendly conversion. */ 357 enum { 358 SBCS_FAST_MAX=0x0fff, /* maximum code point with UTF-8-friendly SBCS runtime code, see makeconv SBCS_UTF8_MAX */ 359 SBCS_FAST_LIMIT=SBCS_FAST_MAX+1, /* =0x1000 */ 360 MBCS_FAST_MAX=0xd7ff, /* maximum code point with UTF-8-friendly MBCS runtime code, see makeconv MBCS_UTF8_MAX */ 361 MBCS_FAST_LIMIT=MBCS_FAST_MAX+1 /* =0xd800 */ 362 }; 363 364 /** 365 * This is the MBCS part of the UConverterTable union (a runtime data structure). 366 * It keeps all the per-converter data and points into the loaded mapping tables. 367 * 368 * utf8Friendly data structures added with _MBCSHeader.version 4.3 369 */ 370 typedef struct UConverterMBCSTable { 371 /* toUnicode */ 372 uint8_t countStates, dbcsOnlyState, stateTableOwned; 373 uint32_t countToUFallbacks; 374 375 const int32_t (*stateTable)/*[countStates]*/[256]; 376 int32_t (*swapLFNLStateTable)/*[countStates]*/[256]; /* for swaplfnl */ 377 const uint16_t *unicodeCodeUnits/*[countUnicodeResults]*/; 378 const _MBCSToUFallback *toUFallbacks; 379 380 /* fromUnicode */ 381 const uint16_t *fromUnicodeTable; 382 const uint16_t *mbcsIndex; /* for fast conversion from most of BMP to MBCS (utf8Friendly data) */ 383 uint16_t sbcsIndex[SBCS_FAST_LIMIT>>6]; /* for fast conversion from low BMP to SBCS (utf8Friendly data) */ 384 const uint8_t *fromUnicodeBytes; 385 uint8_t *swapLFNLFromUnicodeBytes; /* for swaplfnl */ 386 uint32_t fromUBytesLength; 387 uint8_t outputType, unicodeMask; 388 UBool utf8Friendly; /* for utf8Friendly data */ 389 UChar maxFastUChar; /* for utf8Friendly data */ 390 391 /* roundtrips */ 392 uint32_t asciiRoundtrips; 393 394 /* reconstituted data that was omitted from the .cnv file */ 395 uint8_t *reconstitutedData; 396 397 /* converter name for swaplfnl */ 398 char *swapLFNLName; 399 400 /* extension data */ 401 struct UConverterSharedData *baseSharedData; 402 const int32_t *extIndexes; 403 } UConverterMBCSTable; 404 405 #define UCNV_MBCS_TABLE_INITIALIZER { \ 406 /* toUnicode */ \ 407 0, 0, 0, \ 408 0, \ 409 \ 410 NULL, \ 411 NULL, \ 412 NULL, \ 413 NULL, \ 414 \ 415 /* fromUnicode */ \ 416 NULL, \ 417 NULL, \ 418 { 0 }, \ 419 NULL, \ 420 NULL, \ 421 0, \ 422 0, 0, \ 423 false, \ 424 0, \ 425 \ 426 /* roundtrips */ \ 427 0, \ 428 \ 429 /* reconstituted data that was omitted from the .cnv file */ \ 430 NULL, \ 431 \ 432 /* converter name for swaplfnl */ \ 433 NULL, \ 434 \ 435 /* extension data */ \ 436 NULL, \ 437 NULL \ 438 } 439 440 enum { 441 MBCS_OPT_LENGTH_MASK=0x3f, 442 MBCS_OPT_NO_FROM_U=0x40, 443 /* 444 * If any of the following options bits are set, 445 * then the file must be rejected. 446 */ 447 MBCS_OPT_INCOMPATIBLE_MASK=0xffc0, 448 /* 449 * Remove bits from this mask as more options are recognized 450 * by all implementations that use this constant. 451 */ 452 MBCS_OPT_UNKNOWN_INCOMPATIBLE_MASK=0xff80 453 }; 454 455 enum { 456 MBCS_HEADER_V4_LENGTH=8, 457 MBCS_HEADER_V5_MIN_LENGTH=9 458 }; 459 460 /** 461 * MBCS data header. See data format description above. 462 */ 463 typedef struct { 464 UVersionInfo version; 465 uint32_t countStates, 466 countToUFallbacks, 467 offsetToUCodeUnits, 468 offsetFromUTable, 469 offsetFromUBytes, 470 flags, 471 fromUBytesLength; 472 473 /* new and required in version 5 */ 474 uint32_t options; 475 476 /* new and optional in version 5; used if options&MBCS_OPT_NO_FROM_U */ 477 uint32_t fullStage2Length; /* number of 32-bit units */ 478 } _MBCSHeader; 479 480 #define UCNV_MBCS_HEADER_INITIALIZER { { 0 }, 0, 0, 0, 0, 0, 0, 0, 0, 0 } 481 482 /* 483 * This is a simple version of _MBCSGetNextUChar() that is used 484 * by other converter implementations. 485 * It only returns an "assigned" result if it consumes the entire input. 486 * It does not use state from the converter, nor error codes. 487 * It does not handle the EBCDIC swaplfnl option (set in UConverter). 488 * It handles conversion extensions but not GB 18030. 489 * 490 * Return value: 491 * U+fffe unassigned 492 * U+ffff illegal 493 * otherwise the Unicode code point 494 */ 495 U_CFUNC UChar32 496 ucnv_MBCSSimpleGetNextUChar(UConverterSharedData *sharedData, 497 const char *source, int32_t length, 498 UBool useFallback); 499 500 /** 501 * This version of _MBCSSimpleGetNextUChar() is optimized for single-byte, single-state codepages. 502 * It does not handle the EBCDIC swaplfnl option (set in UConverter). 503 * It does not handle conversion extensions (_extToU()). 504 */ 505 U_CFUNC UChar32 506 ucnv_MBCSSingleSimpleGetNextUChar(UConverterSharedData *sharedData, 507 uint8_t b, UBool useFallback); 508 509 /** 510 * This macro version of _MBCSSingleSimpleGetNextUChar() gets a code point from a byte. 511 * It works for single-byte, single-state codepages that only map 512 * to and from BMP code points, and it always 513 * returns fallback values. 514 */ 515 #define _MBCS_SINGLE_SIMPLE_GET_NEXT_BMP(sharedData, b) \ 516 (UChar)MBCS_ENTRY_FINAL_VALUE_16((sharedData)->mbcs.stateTable[0][(uint8_t)(b)]) 517 518 /** 519 * This is an internal function that allows other converter implementations 520 * to check whether a byte is a lead byte. 521 */ 522 U_CFUNC UBool 523 ucnv_MBCSIsLeadByte(UConverterSharedData *sharedData, char byte); 524 525 /** This is a macro version of _MBCSIsLeadByte(). */ 526 #define _MBCS_IS_LEAD_BYTE(sharedData, byte) \ 527 (UBool)MBCS_ENTRY_IS_TRANSITION((sharedData)->mbcs.stateTable[0][(uint8_t)(byte)]) 528 529 /* 530 * This is another simple conversion function for internal use by other 531 * conversion implementations. 532 * It does not use the converter state nor call callbacks. 533 * It does not handle the EBCDIC swaplfnl option (set in UConverter). 534 * It handles conversion extensions but not GB 18030. 535 * 536 * It converts one single Unicode code point into codepage bytes, encoded 537 * as one 32-bit value. The function returns the number of bytes in *pValue: 538 * 1..4 the number of bytes in *pValue 539 * 0 unassigned (*pValue undefined) 540 * -1 illegal (currently not used, *pValue undefined) 541 * 542 * *pValue will contain the resulting bytes with the last byte in bits 7..0, 543 * the second to last byte in bits 15..8, etc. 544 * Currently, the function assumes but does not check that 0<=c<=0x10ffff. 545 */ 546 U_CFUNC int32_t 547 ucnv_MBCSFromUChar32(UConverterSharedData *sharedData, 548 UChar32 c, uint32_t *pValue, 549 UBool useFallback); 550 551 /** 552 * This version of _MBCSFromUChar32() is optimized for single-byte codepages. 553 * It does not handle the EBCDIC swaplfnl option (set in UConverter). 554 * 555 * It returns the codepage byte for the code point, or -1 if it is unassigned. 556 */ 557 U_CFUNC int32_t 558 ucnv_MBCSSingleFromUChar32(UConverterSharedData *sharedData, 559 UChar32 c, 560 UBool useFallback); 561 562 /** 563 * SBCS, DBCS, and EBCDIC_STATEFUL are replaced by MBCS, but 564 * we cheat a little about the type, returning the old types if appropriate. 565 */ 566 U_CFUNC UConverterType 567 ucnv_MBCSGetType(const UConverter* converter); 568 569 U_CFUNC void 570 ucnv_MBCSFromUnicodeWithOffsets(UConverterFromUnicodeArgs *pArgs, 571 UErrorCode *pErrorCode); 572 U_CFUNC void 573 ucnv_MBCSToUnicodeWithOffsets(UConverterToUnicodeArgs *pArgs, 574 UErrorCode *pErrorCode); 575 576 /* 577 * Internal function returning a UnicodeSet for toUnicode() conversion. 578 * Currently only used for ISO-2022-CN, and only handles roundtrip mappings. 579 * In the future, if we add support for fallback sets, this function 580 * needs to be updated. 581 * Handles extensions. 582 * Does not empty the set first. 583 */ 584 U_CFUNC void 585 ucnv_MBCSGetUnicodeSetForUnicode(const UConverterSharedData *sharedData, 586 const USetAdder *sa, 587 UConverterUnicodeSet which, 588 UErrorCode *pErrorCode); 589 590 /* 591 * Same as ucnv_MBCSGetUnicodeSetForUnicode() but 592 * the set can be filtered by encoding scheme. 593 * Used by stateful converters which share regular conversion tables 594 * but only use a subset of their mappings. 595 */ 596 U_CFUNC void 597 ucnv_MBCSGetFilteredUnicodeSetForUnicode(const UConverterSharedData *sharedData, 598 const USetAdder *sa, 599 UConverterUnicodeSet which, 600 UConverterSetFilter filter, 601 UErrorCode *pErrorCode); 602 603 #endif 604 605 #endif