gencnvex.c (35007B)
1 // © 2016 and later: Unicode, Inc. and others. 2 // License & terms of use: http://www.unicode.org/copyright.html 3 /* 4 ******************************************************************************* 5 * 6 * Copyright (C) 2003-2014, International Business Machines 7 * Corporation and others. All Rights Reserved. 8 * 9 ******************************************************************************* 10 * file name: gencnvex.c 11 * encoding: UTF-8 12 * tab size: 8 (not used) 13 * indentation:4 14 * 15 * created on: 2003oct12 16 * created by: Markus W. Scherer 17 */ 18 19 #include <stdbool.h> 20 #include <stdio.h> 21 #include "unicode/utypes.h" 22 #include "unicode/ustring.h" 23 #include "cstring.h" 24 #include "cmemory.h" 25 #include "ucnv_cnv.h" 26 #include "ucnvmbcs.h" 27 #include "toolutil.h" 28 #include "unewdata.h" 29 #include "ucm.h" 30 #include "makeconv.h" 31 #include "genmbcs.h" 32 33 static void 34 CnvExtClose(NewConverter *cnvData); 35 36 static UBool 37 CnvExtIsValid(NewConverter *cnvData, 38 const uint8_t *bytes, int32_t length); 39 40 static UBool 41 CnvExtAddTable(NewConverter *cnvData, UCMTable *table, UConverterStaticData *staticData); 42 43 static uint32_t 44 CnvExtWrite(NewConverter *cnvData, const UConverterStaticData *staticData, 45 UNewDataMemory *pData, int32_t tableType); 46 47 typedef struct CnvExtData { 48 NewConverter newConverter; 49 50 UCMFile *ucm; 51 52 /* toUnicode (state table in ucm->states) */ 53 UToolMemory *toUTable, *toUUChars; 54 55 /* fromUnicode */ 56 UToolMemory *fromUTableUChars, *fromUTableValues, *fromUBytes; 57 58 uint16_t stage1[MBCS_STAGE_1_SIZE]; 59 uint16_t stage2[MBCS_STAGE_2_SIZE]; 60 uint16_t stage3[0x10000<<UCNV_EXT_STAGE_2_LEFT_SHIFT]; /* 0x10000 because of 16-bit stage 2/3 indexes */ 61 uint32_t stage3b[0x10000]; 62 63 int32_t stage1Top, stage2Top, stage3Top, stage3bTop; 64 65 /* for stage3 compaction of <subchar1> |2 mappings */ 66 uint16_t stage3Sub1Block; 67 68 /* statistics */ 69 int32_t 70 maxInBytes, maxOutBytes, maxBytesPerUChar, 71 maxInUChars, maxOutUChars, maxUCharsPerByte; 72 } CnvExtData; 73 74 NewConverter * 75 CnvExtOpen(UCMFile *ucm) { 76 CnvExtData *extData; 77 78 extData=(CnvExtData *)uprv_malloc(sizeof(CnvExtData)); 79 if(extData==NULL) { 80 printf("out of memory\n"); 81 exit(U_MEMORY_ALLOCATION_ERROR); 82 } 83 uprv_memset(extData, 0, sizeof(CnvExtData)); 84 85 extData->ucm=ucm; /* aliased, not owned */ 86 87 extData->newConverter.close=CnvExtClose; 88 extData->newConverter.isValid=CnvExtIsValid; 89 extData->newConverter.addTable=CnvExtAddTable; 90 extData->newConverter.write=CnvExtWrite; 91 return &extData->newConverter; 92 } 93 94 static void 95 CnvExtClose(NewConverter *cnvData) { 96 CnvExtData *extData=(CnvExtData *)cnvData; 97 if(extData!=NULL) { 98 utm_close(extData->toUTable); 99 utm_close(extData->toUUChars); 100 utm_close(extData->fromUTableUChars); 101 utm_close(extData->fromUTableValues); 102 utm_close(extData->fromUBytes); 103 uprv_free(extData); 104 } 105 } 106 107 /* we do not expect this to be called */ 108 static UBool 109 CnvExtIsValid(NewConverter *cnvData, 110 const uint8_t *bytes, int32_t length) { 111 // suppress compiler warnings about unused variables 112 (void)cnvData; 113 (void)bytes; 114 (void)length; 115 return false; 116 } 117 118 static uint32_t 119 CnvExtWrite(NewConverter *cnvData, const UConverterStaticData *staticData, 120 UNewDataMemory *pData, int32_t tableType) { 121 (void) staticData; // suppress compiler warnings about unused variable 122 CnvExtData *extData=(CnvExtData *)cnvData; 123 int32_t length, top, headerSize; 124 125 int32_t indexes[UCNV_EXT_INDEXES_MIN_LENGTH]={ 0 }; 126 127 if(tableType&TABLE_BASE) { 128 headerSize=0; 129 } else { 130 _MBCSHeader header={ { 0, 0, 0, 0 }, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; 131 132 /* write the header and base table name for an extension-only table */ 133 length=(int32_t)uprv_strlen(extData->ucm->baseName)+1; 134 while(length&3) { 135 /* add padding */ 136 extData->ucm->baseName[length++]=0; 137 } 138 139 headerSize=MBCS_HEADER_V4_LENGTH*4+length; 140 141 /* fill the header */ 142 header.version[0]=4; 143 header.version[1]=2; 144 header.flags=(uint32_t)((headerSize<<8)|MBCS_OUTPUT_EXT_ONLY); 145 146 /* write the header and the base table name */ 147 udata_writeBlock(pData, &header, MBCS_HEADER_V4_LENGTH*4); 148 udata_writeBlock(pData, extData->ucm->baseName, length); 149 } 150 151 /* fill indexes[] - offsets/indexes are in units of the target array */ 152 top=0; 153 154 indexes[UCNV_EXT_INDEXES_LENGTH]=length=UCNV_EXT_INDEXES_MIN_LENGTH; 155 top+=length*4; 156 157 indexes[UCNV_EXT_TO_U_INDEX]=top; 158 indexes[UCNV_EXT_TO_U_LENGTH]=length=utm_countItems(extData->toUTable); 159 top+=length*4; 160 161 indexes[UCNV_EXT_TO_U_UCHARS_INDEX]=top; 162 indexes[UCNV_EXT_TO_U_UCHARS_LENGTH]=length=utm_countItems(extData->toUUChars); 163 top+=length*2; 164 165 indexes[UCNV_EXT_FROM_U_UCHARS_INDEX]=top; 166 length=utm_countItems(extData->fromUTableUChars); 167 top+=length*2; 168 169 if(top&3) { 170 /* add padding */ 171 *((UChar *)utm_alloc(extData->fromUTableUChars))=0; 172 *((uint32_t *)utm_alloc(extData->fromUTableValues))=0; 173 ++length; 174 top+=2; 175 } 176 indexes[UCNV_EXT_FROM_U_LENGTH]=length; 177 178 indexes[UCNV_EXT_FROM_U_VALUES_INDEX]=top; 179 top+=length*4; 180 181 indexes[UCNV_EXT_FROM_U_BYTES_INDEX]=top; 182 length=utm_countItems(extData->fromUBytes); 183 top+=length; 184 185 if(top&1) { 186 /* add padding */ 187 *((uint8_t *)utm_alloc(extData->fromUBytes))=0; 188 ++length; 189 ++top; 190 } 191 indexes[UCNV_EXT_FROM_U_BYTES_LENGTH]=length; 192 193 indexes[UCNV_EXT_FROM_U_STAGE_12_INDEX]=top; 194 indexes[UCNV_EXT_FROM_U_STAGE_1_LENGTH]=length=extData->stage1Top; 195 indexes[UCNV_EXT_FROM_U_STAGE_12_LENGTH]=length+=extData->stage2Top; 196 top+=length*2; 197 198 indexes[UCNV_EXT_FROM_U_STAGE_3_INDEX]=top; 199 length=extData->stage3Top; 200 top+=length*2; 201 202 if(top&3) { 203 /* add padding */ 204 extData->stage3[extData->stage3Top++]=0; 205 ++length; 206 top+=2; 207 } 208 indexes[UCNV_EXT_FROM_U_STAGE_3_LENGTH]=length; 209 210 indexes[UCNV_EXT_FROM_U_STAGE_3B_INDEX]=top; 211 indexes[UCNV_EXT_FROM_U_STAGE_3B_LENGTH]=length=extData->stage3bTop; 212 top+=length*4; 213 214 indexes[UCNV_EXT_SIZE]=top; 215 216 /* statistics */ 217 indexes[UCNV_EXT_COUNT_BYTES]= 218 (extData->maxInBytes<<16)| 219 (extData->maxOutBytes<<8)| 220 extData->maxBytesPerUChar; 221 indexes[UCNV_EXT_COUNT_UCHARS]= 222 (extData->maxInUChars<<16)| 223 (extData->maxOutUChars<<8)| 224 extData->maxUCharsPerByte; 225 226 indexes[UCNV_EXT_FLAGS]=extData->ucm->ext->unicodeMask; 227 228 /* write the extension data */ 229 udata_writeBlock(pData, indexes, sizeof(indexes)); 230 udata_writeBlock(pData, utm_getStart(extData->toUTable), indexes[UCNV_EXT_TO_U_LENGTH]*4); 231 udata_writeBlock(pData, utm_getStart(extData->toUUChars), indexes[UCNV_EXT_TO_U_UCHARS_LENGTH]*2); 232 233 udata_writeBlock(pData, utm_getStart(extData->fromUTableUChars), indexes[UCNV_EXT_FROM_U_LENGTH]*2); 234 udata_writeBlock(pData, utm_getStart(extData->fromUTableValues), indexes[UCNV_EXT_FROM_U_LENGTH]*4); 235 udata_writeBlock(pData, utm_getStart(extData->fromUBytes), indexes[UCNV_EXT_FROM_U_BYTES_LENGTH]); 236 237 udata_writeBlock(pData, extData->stage1, extData->stage1Top*2); 238 udata_writeBlock(pData, extData->stage2, extData->stage2Top*2); 239 udata_writeBlock(pData, extData->stage3, extData->stage3Top*2); 240 udata_writeBlock(pData, extData->stage3b, extData->stage3bTop*4); 241 242 #if 0 243 { 244 int32_t i, j; 245 246 length=extData->stage1Top; 247 printf("\nstage1[%x]:\n", length); 248 249 for(i=0; i<length; ++i) { 250 if(extData->stage1[i]!=length) { 251 printf("stage1[%04x]=%04x\n", i, extData->stage1[i]); 252 } 253 } 254 255 j=length; 256 length=extData->stage2Top; 257 printf("\nstage2[%x]:\n", length); 258 259 for(i=0; i<length; ++j, ++i) { 260 if(extData->stage2[i]!=0) { 261 printf("stage12[%04x]=%04x\n", j, extData->stage2[i]); 262 } 263 } 264 265 length=extData->stage3Top; 266 printf("\nstage3[%x]:\n", length); 267 268 for(i=0; i<length; ++i) { 269 if(extData->stage3[i]!=0) { 270 printf("stage3[%04x]=%04x\n", i, extData->stage3[i]); 271 } 272 } 273 274 length=extData->stage3bTop; 275 printf("\nstage3b[%x]:\n", length); 276 277 for(i=0; i<length; ++i) { 278 if(extData->stage3b[i]!=0) { 279 printf("stage3b[%04x]=%08x\n", i, extData->stage3b[i]); 280 } 281 } 282 } 283 #endif 284 285 if(VERBOSE) { 286 printf("size of extension data: %ld\n", (long)top); 287 } 288 289 /* return the number of bytes that should have been written */ 290 return (uint32_t)(headerSize+top); 291 } 292 293 /* to Unicode --------------------------------------------------------------- */ 294 295 /* 296 * Remove fromUnicode fallbacks and SUB mappings which are irrelevant for 297 * the toUnicode table. 298 * This includes mappings with MBCS_FROM_U_EXT_FLAG which were suitable 299 * for the base toUnicode table but not for the base fromUnicode table. 300 * The table must be sorted. 301 * Modifies previous data in the reverseMap. 302 */ 303 static int32_t 304 reduceToUMappings(UCMTable *table) { 305 UCMapping *mappings; 306 int32_t *map; 307 int32_t i, j, count; 308 int8_t flag; 309 310 mappings=table->mappings; 311 map=table->reverseMap; 312 count=table->mappingsLength; 313 314 /* leave the map alone for the initial mappings with desired flags */ 315 for(i=j=0; i<count; ++i) { 316 flag=mappings[map[i]].f; 317 if(flag!=0 && flag!=3) { 318 break; 319 } 320 } 321 322 /* reduce from here to the rest */ 323 for(j=i; i<count; ++i) { 324 flag=mappings[map[i]].f; 325 if(flag==0 || flag==3) { 326 map[j++]=map[i]; 327 } 328 } 329 330 return j; 331 } 332 333 static uint32_t 334 getToUnicodeValue(CnvExtData *extData, UCMTable *table, UCMapping *m) { 335 UChar32 *u32; 336 UChar *u; 337 uint32_t value; 338 int32_t u16Length, ratio; 339 UErrorCode errorCode; 340 341 /* write the Unicode result code point or string index */ 342 if(m->uLen==1) { 343 u16Length=U16_LENGTH(m->u); 344 value=(uint32_t)(UCNV_EXT_TO_U_MIN_CODE_POINT+m->u); 345 } else { 346 /* the parser enforces m->uLen<=UCNV_EXT_MAX_UCHARS */ 347 348 /* get the result code point string and its 16-bit string length */ 349 u32=UCM_GET_CODE_POINTS(table, m); 350 errorCode=U_ZERO_ERROR; 351 u_strFromUTF32(NULL, 0, &u16Length, u32, m->uLen, &errorCode); 352 if(U_FAILURE(errorCode) && errorCode!=U_BUFFER_OVERFLOW_ERROR) { 353 exit(errorCode); 354 } 355 356 /* allocate it and put its length and index into the value */ 357 value= 358 (((uint32_t)u16Length+UCNV_EXT_TO_U_LENGTH_OFFSET)<<UCNV_EXT_TO_U_LENGTH_SHIFT)| 359 ((uint32_t)utm_countItems(extData->toUUChars)); 360 u=utm_allocN(extData->toUUChars, u16Length); 361 362 /* write the result 16-bit string */ 363 errorCode=U_ZERO_ERROR; 364 u_strFromUTF32(u, u16Length, NULL, u32, m->uLen, &errorCode); 365 if(U_FAILURE(errorCode) && errorCode!=U_BUFFER_OVERFLOW_ERROR) { 366 exit(errorCode); 367 } 368 } 369 if(m->f==0) { 370 value|=UCNV_EXT_TO_U_ROUNDTRIP_FLAG; 371 } 372 373 /* update statistics */ 374 if(m->bLen>extData->maxInBytes) { 375 extData->maxInBytes=m->bLen; 376 } 377 if(u16Length>extData->maxOutUChars) { 378 extData->maxOutUChars=u16Length; 379 } 380 381 ratio=(u16Length+(m->bLen-1))/m->bLen; 382 if(ratio>extData->maxUCharsPerByte) { 383 extData->maxUCharsPerByte=ratio; 384 } 385 386 return value; 387 } 388 389 /* 390 * Recursive toUTable generator core function. 391 * Preconditions: 392 * - start<limit (There is at least one mapping.) 393 * - The mappings are sorted lexically. (Access is through the reverseMap.) 394 * - All mappings between start and limit have input sequences that share 395 * the same prefix of unitIndex length, and therefore all of these sequences 396 * are at least unitIndex+1 long. 397 * - There are only relevant mappings available through the reverseMap, 398 * see reduceToUMappings(). 399 * 400 * One function invocation generates one section table. 401 * 402 * Steps: 403 * 1. Count the number of unique unit values and get the low/high unit values 404 * that occur at unitIndex. 405 * 2. Allocate the section table with possible optimization for linear access. 406 * 3. Write temporary version of the section table with start indexes of 407 * subsections, each corresponding to one unit value at unitIndex. 408 * 4. Iterate through the table once more, and depending on the subsection length: 409 * 0: write 0 as a result value (unused byte in linear-access section table) 410 * >0: if there is one mapping with an input unit sequence of unitIndex+1 411 * then defaultValue=compute the mapping result for this whole sequence 412 * else defaultValue=0 413 * 414 * recurse into the subsection 415 */ 416 static UBool 417 generateToUTable(CnvExtData *extData, UCMTable *table, 418 int32_t start, int32_t limit, int32_t unitIndex, 419 uint32_t defaultValue) { 420 UCMapping *mappings, *m; 421 int32_t *map; 422 int32_t i, j, uniqueCount, count, subStart, subLimit; 423 424 uint8_t *bytes; 425 int32_t low, high, prev; 426 427 uint32_t *section; 428 429 mappings=table->mappings; 430 map=table->reverseMap; 431 432 /* step 1: examine the input units; set low, high, uniqueCount */ 433 m=mappings+map[start]; 434 bytes=UCM_GET_BYTES(table, m); 435 low=bytes[unitIndex]; 436 uniqueCount=1; 437 438 prev=high=low; 439 for(i=start+1; i<limit; ++i) { 440 m=mappings+map[i]; 441 bytes=UCM_GET_BYTES(table, m); 442 high=bytes[unitIndex]; 443 444 if(high!=prev) { 445 prev=high; 446 ++uniqueCount; 447 } 448 } 449 450 /* step 2: allocate the section; set count, section */ 451 count=(high-low)+1; 452 if(count<0x100 && (unitIndex==0 || uniqueCount>=(3*count)/4)) { 453 /* 454 * for the root table and for fairly full tables: 455 * allocate for direct, linear array access 456 * by keeping count, to write an entry for each unit value 457 * from low to high 458 * exception: use a compact table if count==0x100 because 459 * that cannot be encoded in the length byte 460 */ 461 } else { 462 count=uniqueCount; 463 } 464 465 if(count>=0x100) { 466 fprintf(stderr, "error: toUnicode extension table section overflow: %ld section entries\n", (long)count); 467 return false; 468 } 469 470 /* allocate the section: 1 entry for the header + count for the items */ 471 section=(uint32_t *)utm_allocN(extData->toUTable, 1+count); 472 473 /* write the section header */ 474 *section++=((uint32_t)count<<UCNV_EXT_TO_U_BYTE_SHIFT)|defaultValue; 475 476 /* step 3: write temporary section table with subsection starts */ 477 prev=low-1; /* just before low to prevent empty subsections before low */ 478 j=0; /* section table index */ 479 for(i=start; i<limit; ++i) { 480 m=mappings+map[i]; 481 bytes=UCM_GET_BYTES(table, m); 482 high=bytes[unitIndex]; 483 484 if(high!=prev) { 485 /* start of a new subsection for unit high */ 486 if(count>uniqueCount) { 487 /* write empty subsections for unused units in a linear table */ 488 while(++prev<high) { 489 section[j++]=((uint32_t)prev<<UCNV_EXT_TO_U_BYTE_SHIFT)|(uint32_t)i; 490 } 491 } else { 492 prev=high; 493 } 494 495 /* write the entry with the subsection start */ 496 section[j++]=((uint32_t)high<<UCNV_EXT_TO_U_BYTE_SHIFT)|(uint32_t)i; 497 } 498 } 499 /* assert(j==count) */ 500 501 /* step 4: recurse and write results */ 502 subLimit=UCNV_EXT_TO_U_GET_VALUE(section[0]); 503 for(j=0; j<count; ++j) { 504 subStart=subLimit; 505 subLimit= (j+1)<count ? (int32_t)UCNV_EXT_TO_U_GET_VALUE(section[j+1]) : limit; 506 507 /* remove the subStart temporary value */ 508 section[j]&=~UCNV_EXT_TO_U_VALUE_MASK; 509 510 if(subStart==subLimit) { 511 /* leave the value zero: empty subsection for unused unit in a linear table */ 512 continue; 513 } 514 515 /* see if there is exactly one input unit sequence of length unitIndex+1 */ 516 defaultValue=0; 517 m=mappings+map[subStart]; 518 if(m->bLen==unitIndex+1) { 519 /* do not include this in generateToUTable() */ 520 ++subStart; 521 522 if(subStart<subLimit && mappings[map[subStart]].bLen==unitIndex+1) { 523 /* print error for multiple same-input-sequence mappings */ 524 fprintf(stderr, "error: multiple mappings from same bytes\n"); 525 ucm_printMapping(table, m, stderr); 526 ucm_printMapping(table, mappings+map[subStart], stderr); 527 return false; 528 } 529 530 defaultValue=getToUnicodeValue(extData, table, m); 531 } 532 533 if(subStart==subLimit) { 534 /* write the result for the input sequence ending here */ 535 section[j]|=defaultValue; 536 } else { 537 /* write the index to the subsection table */ 538 section[j]|=(uint32_t)utm_countItems(extData->toUTable); 539 540 /* recurse */ 541 if(!generateToUTable(extData, table, subStart, subLimit, unitIndex+1, defaultValue)) { 542 return false; 543 } 544 } 545 } 546 return true; 547 } 548 549 /* 550 * Generate the toUTable and toUUChars from the input table. 551 * The input table must be sorted, and all precision flags must be 0..3. 552 * This function will modify the table's reverseMap. 553 */ 554 static UBool 555 makeToUTable(CnvExtData *extData, UCMTable *table) { 556 int32_t toUCount; 557 558 toUCount=reduceToUMappings(table); 559 560 extData->toUTable=utm_open("cnv extension toUTable", 0x10000, UCNV_EXT_TO_U_MIN_CODE_POINT, 4); 561 extData->toUUChars=utm_open("cnv extension toUUChars", 0x10000, UCNV_EXT_TO_U_INDEX_MASK+1, 2); 562 563 return generateToUTable(extData, table, 0, toUCount, 0, 0); 564 } 565 566 /* from Unicode ------------------------------------------------------------- */ 567 568 /* 569 * preprocessing: 570 * rebuild reverseMap with mapping indexes for mappings relevant for from Unicode 571 * change each Unicode string to encode all but the first code point in 16-bit form 572 * 573 * generation: 574 * for each unique code point 575 * write an entry in the 3-stage trie 576 * check that there is only one single-code point sequence 577 * start recursion for following 16-bit input units 578 */ 579 580 /* 581 * Remove toUnicode fallbacks and non-<subchar1> SUB mappings 582 * which are irrelevant for the fromUnicode extension table. 583 * Remove MBCS_FROM_U_EXT_FLAG bits. 584 * Overwrite the reverseMap with an index array to the relevant mappings. 585 * Modify the code point sequences to a generator-friendly format where 586 * the first code points remains unchanged but the following are recoded 587 * into 16-bit Unicode string form. 588 * The table must be sorted. 589 * Destroys previous data in the reverseMap. 590 */ 591 static int32_t 592 prepareFromUMappings(UCMTable *table) { 593 UCMapping *mappings, *m; 594 int32_t *map; 595 int32_t i, j, count; 596 int8_t flag; 597 598 mappings=table->mappings; 599 map=table->reverseMap; 600 count=table->mappingsLength; 601 602 /* 603 * we do not go through the map on input because the mappings are 604 * sorted lexically 605 */ 606 m=mappings; 607 608 for(i=j=0; i<count; ++m, ++i) { 609 flag=m->f; 610 if(flag>=0) { 611 flag&=MBCS_FROM_U_EXT_MASK; 612 m->f=flag; 613 } 614 if(flag==0 || flag==1 || (flag==2 && m->bLen==1) || flag==4) { 615 map[j++]=i; 616 617 if(m->uLen>1) { 618 /* recode all but the first code point to 16-bit Unicode */ 619 UChar32 *u32; 620 UChar *u; 621 UChar32 c; 622 int32_t q, r; 623 624 u32=UCM_GET_CODE_POINTS(table, m); 625 u=(UChar *)u32; /* destructive in-place recoding */ 626 for(r=2, q=1; q<m->uLen; ++q) { 627 c=u32[q]; 628 U16_APPEND_UNSAFE(u, r, c); 629 } 630 631 /* counts the first code point always at 2 - the first 16-bit unit is at 16-bit index 2 */ 632 m->uLen=(int8_t)r; 633 } 634 } 635 } 636 637 return j; 638 } 639 640 static uint32_t 641 getFromUBytesValue(CnvExtData *extData, UCMTable *table, UCMapping *m) { 642 uint8_t *bytes, *resultBytes; 643 uint32_t value; 644 int32_t u16Length, ratio; 645 646 if(m->f==2) { 647 /* 648 * no mapping, <subchar1> preferred 649 * 650 * no need to count in statistics because the subchars are already 651 * counted for maxOutBytes and maxBytesPerUChar in UConverterStaticData, 652 * and this non-mapping does not count for maxInUChars which are always 653 * trivially at least two if counting unmappable supplementary code points 654 */ 655 return UCNV_EXT_FROM_U_SUBCHAR1; 656 } 657 658 bytes=UCM_GET_BYTES(table, m); 659 value=0; 660 switch(m->bLen) { 661 /* 1..3: store the bytes in the value word */ 662 case 3: 663 value=((uint32_t)*bytes++)<<16; 664 U_FALLTHROUGH; 665 case 2: 666 value|=((uint32_t)*bytes++)<<8; 667 U_FALLTHROUGH; 668 case 1: 669 value|=*bytes; 670 break; 671 default: 672 /* the parser enforces m->bLen<=UCNV_EXT_MAX_BYTES */ 673 /* store the bytes in fromUBytes[] and the index in the value word */ 674 value=(uint32_t)utm_countItems(extData->fromUBytes); 675 resultBytes=utm_allocN(extData->fromUBytes, m->bLen); 676 uprv_memcpy(resultBytes, bytes, m->bLen); 677 break; 678 } 679 value|=(uint32_t)m->bLen<<UCNV_EXT_FROM_U_LENGTH_SHIFT; 680 if(m->f==0) { 681 value|=UCNV_EXT_FROM_U_ROUNDTRIP_FLAG; 682 } else if(m->f==4) { 683 value|=UCNV_EXT_FROM_U_GOOD_ONE_WAY_FLAG; 684 } 685 686 /* calculate the real UTF-16 length (see recoding in prepareFromUMappings()) */ 687 if(m->uLen==1) { 688 u16Length=U16_LENGTH(m->u); 689 } else { 690 u16Length=U16_LENGTH(UCM_GET_CODE_POINTS(table, m)[0])+(m->uLen-2); 691 } 692 693 /* update statistics */ 694 if(u16Length>extData->maxInUChars) { 695 extData->maxInUChars=u16Length; 696 } 697 if(m->bLen>extData->maxOutBytes) { 698 extData->maxOutBytes=m->bLen; 699 } 700 701 ratio=(m->bLen+(u16Length-1))/u16Length; 702 if(ratio>extData->maxBytesPerUChar) { 703 extData->maxBytesPerUChar=ratio; 704 } 705 706 return value; 707 } 708 709 /* 710 * works like generateToUTable(), except that the 711 * output section consists of two arrays, one for input UChars and one 712 * for result values 713 * 714 * also, fromUTable sections are always stored in a compact form for 715 * access via binary search 716 */ 717 static UBool 718 generateFromUTable(CnvExtData *extData, UCMTable *table, 719 int32_t start, int32_t limit, int32_t unitIndex, 720 uint32_t defaultValue) { 721 UCMapping *mappings, *m; 722 int32_t *map; 723 int32_t i, j, uniqueCount, count, subStart, subLimit; 724 725 UChar *uchars; 726 UChar32 low, high, prev; 727 728 UChar *sectionUChars; 729 uint32_t *sectionValues; 730 731 mappings=table->mappings; 732 map=table->reverseMap; 733 734 /* step 1: examine the input units; set low, high, uniqueCount */ 735 m=mappings+map[start]; 736 uchars=(UChar *)UCM_GET_CODE_POINTS(table, m); 737 low=uchars[unitIndex]; 738 uniqueCount=1; 739 740 prev=high=low; 741 for(i=start+1; i<limit; ++i) { 742 m=mappings+map[i]; 743 uchars=(UChar *)UCM_GET_CODE_POINTS(table, m); 744 high=uchars[unitIndex]; 745 746 if(high!=prev) { 747 prev=high; 748 ++uniqueCount; 749 } 750 } 751 752 /* step 2: allocate the section; set count, section */ 753 /* the fromUTable always stores for access via binary search */ 754 count=uniqueCount; 755 756 /* allocate the section: 1 entry for the header + count for the items */ 757 sectionUChars=(UChar *)utm_allocN(extData->fromUTableUChars, 1+count); 758 sectionValues=(uint32_t *)utm_allocN(extData->fromUTableValues, 1+count); 759 760 /* write the section header */ 761 *sectionUChars++=(UChar)count; 762 *sectionValues++=defaultValue; 763 764 /* step 3: write temporary section table with subsection starts */ 765 prev=low-1; /* just before low to prevent empty subsections before low */ 766 j=0; /* section table index */ 767 for(i=start; i<limit; ++i) { 768 m=mappings+map[i]; 769 uchars=(UChar *)UCM_GET_CODE_POINTS(table, m); 770 high=uchars[unitIndex]; 771 772 if(high!=prev) { 773 /* start of a new subsection for unit high */ 774 prev=high; 775 776 /* write the entry with the subsection start */ 777 sectionUChars[j]=(UChar)high; 778 sectionValues[j]=(uint32_t)i; 779 ++j; 780 } 781 } 782 /* assert(j==count) */ 783 784 /* step 4: recurse and write results */ 785 subLimit=(int32_t)(sectionValues[0]); 786 for(j=0; j<count; ++j) { 787 subStart=subLimit; 788 subLimit= (j+1)<count ? (int32_t)(sectionValues[j+1]) : limit; 789 790 /* see if there is exactly one input unit sequence of length unitIndex+1 */ 791 defaultValue=0; 792 m=mappings+map[subStart]; 793 if(m->uLen==unitIndex+1) { 794 /* do not include this in generateToUTable() */ 795 ++subStart; 796 797 if(subStart<subLimit && mappings[map[subStart]].uLen==unitIndex+1) { 798 /* print error for multiple same-input-sequence mappings */ 799 fprintf(stderr, "error: multiple mappings from same Unicode code points\n"); 800 ucm_printMapping(table, m, stderr); 801 ucm_printMapping(table, mappings+map[subStart], stderr); 802 return false; 803 } 804 805 defaultValue=getFromUBytesValue(extData, table, m); 806 } 807 808 if(subStart==subLimit) { 809 /* write the result for the input sequence ending here */ 810 sectionValues[j]=defaultValue; 811 } else { 812 /* write the index to the subsection table */ 813 sectionValues[j]=(uint32_t)utm_countItems(extData->fromUTableValues); 814 815 /* recurse */ 816 if(!generateFromUTable(extData, table, subStart, subLimit, unitIndex+1, defaultValue)) { 817 return false; 818 } 819 } 820 } 821 return true; 822 } 823 824 /* 825 * add entries to the fromUnicode trie, 826 * assume to be called with code points in ascending order 827 * and use that to build the trie in precompacted form 828 */ 829 static void 830 addFromUTrieEntry(CnvExtData *extData, UChar32 c, uint32_t value) { 831 int32_t i1, i2, i3, i3b, nextOffset, min, newBlock; 832 833 if(value==0) { 834 return; 835 } 836 837 /* 838 * compute the index for each stage, 839 * allocate a stage block if necessary, 840 * and write the stage value 841 */ 842 i1=c>>10; 843 if(i1>=extData->stage1Top) { 844 extData->stage1Top=i1+1; 845 } 846 847 nextOffset=(c>>4)&0x3f; 848 849 if(extData->stage1[i1]==0) { 850 /* allocate another block in stage 2; overlap with the previous block */ 851 newBlock=extData->stage2Top; 852 min=newBlock-nextOffset; /* minimum block start with overlap */ 853 while(min<newBlock && extData->stage2[newBlock-1]==0) { 854 --newBlock; 855 } 856 857 extData->stage1[i1]=(uint16_t)newBlock; 858 extData->stage2Top=newBlock+MBCS_STAGE_2_BLOCK_SIZE; 859 if(extData->stage2Top>UPRV_LENGTHOF(extData->stage2)) { 860 fprintf(stderr, "error: too many stage 2 entries at U+%04x\n", (int)c); 861 exit(U_MEMORY_ALLOCATION_ERROR); 862 } 863 } 864 865 i2=extData->stage1[i1]+nextOffset; 866 nextOffset=c&0xf; 867 868 if(extData->stage2[i2]==0) { 869 /* allocate another block in stage 3; overlap with the previous block */ 870 newBlock=extData->stage3Top; 871 min=newBlock-nextOffset; /* minimum block start with overlap */ 872 while(min<newBlock && extData->stage3[newBlock-1]==0) { 873 --newBlock; 874 } 875 876 /* round up to a multiple of stage 3 granularity >1 (similar to utrie.c) */ 877 newBlock=(newBlock+(UCNV_EXT_STAGE_3_GRANULARITY-1))&~(UCNV_EXT_STAGE_3_GRANULARITY-1); 878 extData->stage2[i2]=(uint16_t)(newBlock>>UCNV_EXT_STAGE_2_LEFT_SHIFT); 879 880 extData->stage3Top=newBlock+MBCS_STAGE_3_BLOCK_SIZE; 881 if(extData->stage3Top>UPRV_LENGTHOF(extData->stage3)) { 882 fprintf(stderr, "error: too many stage 3 entries at U+%04x\n", (int)c); 883 exit(U_MEMORY_ALLOCATION_ERROR); 884 } 885 } 886 887 i3=((int32_t)extData->stage2[i2]<<UCNV_EXT_STAGE_2_LEFT_SHIFT)+nextOffset; 888 /* 889 * assume extData->stage3[i3]==0 because we get 890 * code points in strictly ascending order 891 */ 892 893 if(value==UCNV_EXT_FROM_U_SUBCHAR1) { 894 /* <subchar1> SUB mapping, see getFromUBytesValue() and prepareFromUMappings() */ 895 extData->stage3[i3]=1; 896 897 /* 898 * precompaction is not optimal for <subchar1> |2 mappings because 899 * stage3 values for them are all the same, unlike for other mappings 900 * which all have unique values; 901 * use a simple compaction of reusing a whole block filled with these 902 * mappings 903 */ 904 905 /* is the entire block filled with <subchar1> |2 mappings? */ 906 if(nextOffset==MBCS_STAGE_3_BLOCK_SIZE-1) { 907 for(min=i3-nextOffset; 908 min<i3 && extData->stage3[min]==1; 909 ++min) {} 910 911 if(min==i3) { 912 /* the entire block is filled with these mappings */ 913 if(extData->stage3Sub1Block!=0) { 914 /* point to the previous such block and remove this block from stage3 */ 915 extData->stage2[i2]=extData->stage3Sub1Block; 916 extData->stage3Top-=MBCS_STAGE_3_BLOCK_SIZE; 917 uprv_memset(extData->stage3+extData->stage3Top, 0, MBCS_STAGE_3_BLOCK_SIZE*2); 918 } else { 919 /* remember this block's stage2 entry */ 920 extData->stage3Sub1Block=extData->stage2[i2]; 921 } 922 } 923 } 924 } else { 925 if((i3b=extData->stage3bTop++)>=UPRV_LENGTHOF(extData->stage3b)) { 926 fprintf(stderr, "error: too many stage 3b entries at U+%04x\n", (int)c); 927 exit(U_MEMORY_ALLOCATION_ERROR); 928 } 929 930 /* roundtrip or fallback mapping */ 931 extData->stage3[i3]=(uint16_t)i3b; 932 extData->stage3b[i3b]=value; 933 } 934 } 935 936 static UBool 937 generateFromUTrie(CnvExtData *extData, UCMTable *table, int32_t mapLength) { 938 UCMapping *mappings, *m; 939 int32_t *map; 940 uint32_t value; 941 int32_t subStart, subLimit; 942 943 UChar32 *codePoints; 944 UChar32 c, next; 945 946 if(mapLength==0) { 947 return true; 948 } 949 950 mappings=table->mappings; 951 map=table->reverseMap; 952 953 /* 954 * iterate over same-initial-code point mappings, 955 * enter the initial code point into the trie, 956 * and start a recursion on the corresponding mappings section 957 * with generateFromUTable() 958 */ 959 m=mappings+map[0]; 960 codePoints=UCM_GET_CODE_POINTS(table, m); 961 next=codePoints[0]; 962 subLimit=0; 963 while(subLimit<mapLength) { 964 /* get a new subsection of mappings starting with the same code point */ 965 subStart=subLimit; 966 c=next; 967 while(next==c && ++subLimit<mapLength) { 968 m=mappings+map[subLimit]; 969 codePoints=UCM_GET_CODE_POINTS(table, m); 970 next=codePoints[0]; 971 } 972 973 /* 974 * compute the value for this code point; 975 * if there is a mapping for this code point alone, it is at subStart 976 * because the table is sorted lexically 977 */ 978 value=0; 979 m=mappings+map[subStart]; 980 codePoints=UCM_GET_CODE_POINTS(table, m); 981 if(m->uLen==1) { 982 /* do not include this in generateFromUTable() */ 983 ++subStart; 984 985 if(subStart<subLimit && mappings[map[subStart]].uLen==1) { 986 /* print error for multiple same-input-sequence mappings */ 987 fprintf(stderr, "error: multiple mappings from same Unicode code points\n"); 988 ucm_printMapping(table, m, stderr); 989 ucm_printMapping(table, mappings+map[subStart], stderr); 990 return false; 991 } 992 993 value=getFromUBytesValue(extData, table, m); 994 } 995 996 if(subStart==subLimit) { 997 /* write the result for this one code point */ 998 addFromUTrieEntry(extData, c, value); 999 } else { 1000 /* write the index to the subsection table */ 1001 addFromUTrieEntry(extData, c, (uint32_t)utm_countItems(extData->fromUTableValues)); 1002 1003 /* recurse, starting from 16-bit-unit index 2, the first 16-bit unit after c */ 1004 if(!generateFromUTable(extData, table, subStart, subLimit, 2, value)) { 1005 return false; 1006 } 1007 } 1008 } 1009 return true; 1010 } 1011 1012 /* 1013 * Generate the fromU data structures from the input table. 1014 * The input table must be sorted, and all precision flags must be 0..3. 1015 * This function will modify the table's reverseMap. 1016 */ 1017 static UBool 1018 makeFromUTable(CnvExtData *extData, UCMTable *table) { 1019 uint16_t *stage1; 1020 int32_t i, stage1Top, fromUCount; 1021 1022 fromUCount=prepareFromUMappings(table); 1023 1024 extData->fromUTableUChars=utm_open("cnv extension fromUTableUChars", 0x10000, UCNV_EXT_FROM_U_DATA_MASK+1, 2); 1025 extData->fromUTableValues=utm_open("cnv extension fromUTableValues", 0x10000, UCNV_EXT_FROM_U_DATA_MASK+1, 4); 1026 extData->fromUBytes=utm_open("cnv extension fromUBytes", 0x10000, UCNV_EXT_FROM_U_DATA_MASK+1, 1); 1027 1028 /* allocate all-unassigned stage blocks */ 1029 extData->stage2Top=MBCS_STAGE_2_FIRST_ASSIGNED; 1030 extData->stage3Top=MBCS_STAGE_3_FIRST_ASSIGNED; 1031 1032 /* 1033 * stage 3b stores only unique values, and in 1034 * index 0: 0 for "no mapping" 1035 * index 1: "no mapping" with preference for <subchar1> rather than <subchar> 1036 */ 1037 extData->stage3b[1]=UCNV_EXT_FROM_U_SUBCHAR1; 1038 extData->stage3bTop=2; 1039 1040 /* allocate the first entry in the fromUTable because index 0 means "no result" */ 1041 utm_alloc(extData->fromUTableUChars); 1042 utm_alloc(extData->fromUTableValues); 1043 1044 if(!generateFromUTrie(extData, table, fromUCount)) { 1045 return false; 1046 } 1047 1048 /* 1049 * offset the stage 1 trie entries by stage1Top because they will 1050 * be stored in a single array 1051 */ 1052 stage1=extData->stage1; 1053 stage1Top=extData->stage1Top; 1054 for(i=0; i<stage1Top; ++i) { 1055 stage1[i]=(uint16_t)(stage1[i]+stage1Top); 1056 } 1057 1058 return true; 1059 } 1060 1061 /* -------------------------------------------------------------------------- */ 1062 1063 static UBool 1064 CnvExtAddTable(NewConverter *cnvData, UCMTable *table, UConverterStaticData *staticData) { 1065 CnvExtData *extData; 1066 1067 if(table->unicodeMask&UCNV_HAS_SURROGATES) { 1068 fprintf(stderr, "error: contains mappings for surrogate code points\n"); 1069 return false; 1070 } 1071 1072 staticData->conversionType=UCNV_MBCS; 1073 1074 extData=(CnvExtData *)cnvData; 1075 1076 /* 1077 * assume that the table is sorted 1078 * 1079 * call the functions in this order because 1080 * makeToUTable() modifies the original reverseMap, 1081 * makeFromUTable() writes a whole new mapping into reverseMap 1082 */ 1083 return 1084 makeToUTable(extData, table) && 1085 makeFromUTable(extData, table); 1086 }