package.cpp (41549B)
1 // © 2016 and later: Unicode, Inc. and others. 2 // License & terms of use: http://www.unicode.org/copyright.html 3 /* 4 ******************************************************************************* 5 * 6 * Copyright (C) 1999-2015, International Business Machines 7 * Corporation and others. All Rights Reserved. 8 * 9 ******************************************************************************* 10 * file name: package.cpp 11 * encoding: UTF-8 12 * tab size: 8 (not used) 13 * indentation:4 14 * 15 * created on: 2005aug25 16 * created by: Markus W. Scherer 17 * 18 * Read, modify, and write ICU .dat data package files. 19 * This is an integral part of the icupkg tool, moved to the toolutil library 20 * because parts of tool implementations tend to be later shared by 21 * other tools. 22 * Subsumes functionality and implementation code from 23 * gencmn, decmn, and icuswap tools. 24 */ 25 26 #include "unicode/utypes.h" 27 #include "unicode/putil.h" 28 #include "unicode/udata.h" 29 #include "cstring.h" 30 #include "uarrsort.h" 31 #include "ucmndata.h" 32 #include "udataswp.h" 33 #include "swapimpl.h" 34 #include "toolutil.h" 35 #include "package.h" 36 #include "cmemory.h" 37 38 #include <stdio.h> 39 #include <stdlib.h> 40 #include <string.h> 41 42 43 static const int32_t kItemsChunk = 256; /* How much to increase the filesarray by each time */ 44 45 // general definitions ----------------------------------------------------- *** 46 47 /* UDataInfo cf. udata.h */ 48 static const UDataInfo dataInfo={ 49 static_cast<uint16_t>(sizeof(UDataInfo)), 50 0, 51 52 U_IS_BIG_ENDIAN, 53 U_CHARSET_FAMILY, 54 static_cast<uint8_t>(sizeof(char16_t)), 55 0, 56 57 {0x43, 0x6d, 0x6e, 0x44}, /* dataFormat="CmnD" */ 58 {1, 0, 0, 0}, /* formatVersion */ 59 {3, 0, 0, 0} /* dataVersion */ 60 }; 61 62 U_CDECL_BEGIN 63 static void U_CALLCONV 64 printPackageError(void *context, const char *fmt, va_list args) { 65 vfprintf((FILE *)context, fmt, args); 66 } 67 U_CDECL_END 68 69 static uint16_t 70 readSwapUInt16(uint16_t x) { 71 return static_cast<uint16_t>((x << 8) | (x >> 8)); 72 } 73 74 // platform types ---------------------------------------------------------- *** 75 76 static const char *types="lb?e"; 77 78 enum { TYPE_L, TYPE_B, TYPE_LE, TYPE_E, TYPE_COUNT }; 79 80 static inline int32_t 81 makeTypeEnum(uint8_t charset, UBool isBigEndian) { 82 return 2 * static_cast<int32_t>(charset) + isBigEndian; 83 } 84 85 static inline int32_t 86 makeTypeEnum(char type) { 87 return 88 type == 'l' ? TYPE_L : 89 type == 'b' ? TYPE_B : 90 type == 'e' ? TYPE_E : 91 -1; 92 } 93 94 static inline char 95 makeTypeLetter(uint8_t charset, UBool isBigEndian) { 96 return types[makeTypeEnum(charset, isBigEndian)]; 97 } 98 99 static inline char 100 makeTypeLetter(int32_t typeEnum) { 101 return types[typeEnum]; 102 } 103 104 static void 105 makeTypeProps(char type, uint8_t &charset, UBool &isBigEndian) { 106 int32_t typeEnum=makeTypeEnum(type); 107 charset = static_cast<uint8_t>(typeEnum >> 1); 108 isBigEndian = static_cast<UBool>(typeEnum & 1); 109 } 110 111 U_CFUNC const UDataInfo * 112 getDataInfo(const uint8_t *data, int32_t length, 113 int32_t &infoLength, int32_t &headerLength, 114 UErrorCode *pErrorCode) { 115 const DataHeader *pHeader; 116 const UDataInfo *pInfo; 117 118 if(pErrorCode==nullptr || U_FAILURE(*pErrorCode)) { 119 return nullptr; 120 } 121 if( data==nullptr || 122 (length>=0 && length<(int32_t)sizeof(DataHeader)) 123 ) { 124 *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; 125 return nullptr; 126 } 127 128 pHeader=(const DataHeader *)data; 129 pInfo=&pHeader->info; 130 if( (length>=0 && length<(int32_t)sizeof(DataHeader)) || 131 pHeader->dataHeader.magic1!=0xda || 132 pHeader->dataHeader.magic2!=0x27 || 133 pInfo->sizeofUChar!=2 134 ) { 135 *pErrorCode=U_UNSUPPORTED_ERROR; 136 return nullptr; 137 } 138 139 if(pInfo->isBigEndian==U_IS_BIG_ENDIAN) { 140 headerLength=pHeader->dataHeader.headerSize; 141 infoLength=pInfo->size; 142 } else { 143 headerLength=readSwapUInt16(pHeader->dataHeader.headerSize); 144 infoLength=readSwapUInt16(pInfo->size); 145 } 146 147 if( headerLength<(int32_t)sizeof(DataHeader) || 148 infoLength<(int32_t)sizeof(UDataInfo) || 149 headerLength<(int32_t)(sizeof(pHeader->dataHeader)+infoLength) || 150 (length>=0 && length<headerLength) 151 ) { 152 *pErrorCode=U_UNSUPPORTED_ERROR; 153 return nullptr; 154 } 155 156 return pInfo; 157 } 158 159 static int32_t 160 getTypeEnumForInputData(const uint8_t *data, int32_t length, 161 UErrorCode *pErrorCode) { 162 const UDataInfo *pInfo; 163 int32_t infoLength, headerLength; 164 165 /* getDataInfo() checks for illegal arguments */ 166 pInfo=getDataInfo(data, length, infoLength, headerLength, pErrorCode); 167 if(pInfo==nullptr) { 168 return -1; 169 } 170 171 return makeTypeEnum(pInfo->charsetFamily, static_cast<UBool>(pInfo->isBigEndian)); 172 } 173 174 // file handling ----------------------------------------------------------- *** 175 176 static void 177 extractPackageName(const char *filename, 178 char pkg[], int32_t capacity) { 179 const char *basename; 180 int32_t len; 181 182 basename=findBasename(filename); 183 len = static_cast<int32_t>(strlen(basename)) - 4; /* -4: subtract the length of ".dat" */ 184 185 if(len<=0 || 0!=strcmp(basename+len, ".dat")) { 186 fprintf(stderr, "icupkg: \"%s\" is not recognized as a package filename (must end with .dat)\n", 187 basename); 188 exit(U_ILLEGAL_ARGUMENT_ERROR); 189 } 190 191 if(len>=capacity) { 192 fprintf(stderr, "icupkg: the package name \"%s\" is too long (>=%ld)\n", 193 basename, static_cast<long>(capacity)); 194 exit(U_ILLEGAL_ARGUMENT_ERROR); 195 } 196 197 memcpy(pkg, basename, len); 198 pkg[len]=0; 199 } 200 201 static int32_t 202 getFileLength(FILE *f) { 203 int32_t length; 204 205 fseek(f, 0, SEEK_END); 206 length = static_cast<int32_t>(ftell(f)); 207 fseek(f, 0, SEEK_SET); 208 return length; 209 } 210 211 /* 212 * Turn tree separators and alternate file separators into normal file separators. 213 */ 214 #if U_TREE_ENTRY_SEP_CHAR==U_FILE_SEP_CHAR && U_FILE_ALT_SEP_CHAR==U_FILE_SEP_CHAR 215 #define treeToPath(s) 216 #else 217 static void 218 treeToPath(char *s) { 219 char *t; 220 221 for(t=s; *t!=0; ++t) { 222 if(*t==U_TREE_ENTRY_SEP_CHAR || *t==U_FILE_ALT_SEP_CHAR) { 223 *t=U_FILE_SEP_CHAR; 224 } 225 } 226 } 227 #endif 228 229 /* 230 * Turn file separators into tree separators. 231 */ 232 #if U_TREE_ENTRY_SEP_CHAR==U_FILE_SEP_CHAR && U_FILE_ALT_SEP_CHAR==U_FILE_SEP_CHAR 233 #define pathToTree(s) 234 #else 235 static void 236 pathToTree(char *s) { 237 char *t; 238 239 for(t=s; *t!=0; ++t) { 240 if(*t==U_FILE_SEP_CHAR || *t==U_FILE_ALT_SEP_CHAR) { 241 *t=U_TREE_ENTRY_SEP_CHAR; 242 } 243 } 244 } 245 #endif 246 247 /* 248 * Prepend the path (if any) to the name and run the name through treeToName(). 249 */ 250 static void 251 makeFullFilename(const char *path, const char *name, 252 char *filename, int32_t capacity) { 253 char *s; 254 255 // prepend the path unless nullptr or empty 256 if(path!=nullptr && path[0]!=0) { 257 if (static_cast<int32_t>(strlen(path) + 1) >= capacity) { 258 fprintf(stderr, "pathname too long: \"%s\"\n", path); 259 exit(U_BUFFER_OVERFLOW_ERROR); 260 } 261 strcpy(filename, path); 262 263 // make sure the path ends with a file separator 264 s=strchr(filename, 0); 265 if(*(s-1)!=U_FILE_SEP_CHAR && *(s-1)!=U_FILE_ALT_SEP_CHAR) { 266 *s++=U_FILE_SEP_CHAR; 267 } 268 } else { 269 s=filename; 270 } 271 272 // turn the name into a filename, turn tree separators into file separators 273 if (static_cast<int32_t>((s - filename) + strlen(name)) >= capacity) { 274 fprintf(stderr, "path/filename too long: \"%s%s\"\n", filename, name); 275 exit(U_BUFFER_OVERFLOW_ERROR); 276 } 277 strcpy(s, name); 278 treeToPath(s); 279 } 280 281 static void 282 makeFullFilenameAndDirs(const char *path, const char *name, 283 char *filename, int32_t capacity) { 284 char *sep; 285 UErrorCode errorCode; 286 287 makeFullFilename(path, name, filename, capacity); 288 289 // make tree directories 290 errorCode=U_ZERO_ERROR; 291 sep=strchr(filename, 0)-strlen(name); 292 while((sep=strchr(sep, U_FILE_SEP_CHAR))!=nullptr) { 293 if(sep!=filename) { 294 *sep=0; // truncate temporarily 295 uprv_mkdir(filename, &errorCode); 296 if(U_FAILURE(errorCode)) { 297 fprintf(stderr, "icupkg: unable to create tree directory \"%s\"\n", filename); 298 exit(U_FILE_ACCESS_ERROR); 299 } 300 } 301 *sep++=U_FILE_SEP_CHAR; // restore file separator character 302 } 303 } 304 305 static uint8_t * 306 readFile(const char *path, const char *name, int32_t &length, char &type) { 307 char filename[1024]; 308 FILE *file; 309 UErrorCode errorCode; 310 int32_t fileLength, typeEnum; 311 312 makeFullFilename(path, name, filename, static_cast<int32_t>(sizeof(filename))); 313 314 /* open the input file, get its length, allocate memory for it, read the file */ 315 file=fopen(filename, "rb"); 316 if(file==nullptr) { 317 fprintf(stderr, "icupkg: unable to open input file \"%s\"\n", filename); 318 exit(U_FILE_ACCESS_ERROR); 319 } 320 321 /* get the file length */ 322 fileLength=getFileLength(file); 323 if(ferror(file) || fileLength<=0) { 324 fprintf(stderr, "icupkg: empty input file \"%s\"\n", filename); 325 fclose(file); 326 exit(U_FILE_ACCESS_ERROR); 327 } 328 329 /* allocate the buffer, pad to multiple of 16 */ 330 length=(fileLength+0xf)&~0xf; 331 icu::LocalMemory<uint8_t> data(static_cast<uint8_t*>(uprv_malloc(length))); 332 if(data.isNull()) { 333 fclose(file); 334 fprintf(stderr, "icupkg: malloc error allocating %d bytes.\n", static_cast<int>(length)); 335 exit(U_MEMORY_ALLOCATION_ERROR); 336 } 337 338 /* read the file */ 339 if (fileLength != static_cast<int32_t>(fread(data.getAlias(), 1, fileLength, file))) { 340 fprintf(stderr, "icupkg: error reading \"%s\"\n", filename); 341 fclose(file); 342 exit(U_FILE_ACCESS_ERROR); 343 } 344 345 /* pad the file to a multiple of 16 using the usual padding byte */ 346 if(fileLength<length) { 347 memset(data.getAlias()+fileLength, 0xaa, length-fileLength); 348 } 349 350 fclose(file); 351 352 // minimum check for ICU-format data 353 errorCode=U_ZERO_ERROR; 354 typeEnum=getTypeEnumForInputData(data.getAlias(), length, &errorCode); 355 if(typeEnum<0 || U_FAILURE(errorCode)) { 356 fprintf(stderr, "icupkg: not an ICU data file: \"%s\"\n", filename); 357 #if !UCONFIG_NO_LEGACY_CONVERSION 358 exit(U_INVALID_FORMAT_ERROR); 359 #else 360 fprintf(stderr, "U_INVALID_FORMAT_ERROR occurred but UCONFIG_NO_LEGACY_CONVERSION is on so this is expected.\n"); 361 exit(0); 362 #endif 363 } 364 type=makeTypeLetter(typeEnum); 365 366 return data.orphan(); 367 } 368 369 // .dat package file representation ---------------------------------------- *** 370 371 U_CDECL_BEGIN 372 373 static int32_t U_CALLCONV 374 compareItems(const void * /*context*/, const void *left, const void *right) { 375 U_NAMESPACE_USE 376 377 return (int32_t)strcmp(((Item *)left)->name, ((Item *)right)->name); 378 } 379 380 U_CDECL_END 381 382 U_NAMESPACE_BEGIN 383 384 Package::Package() 385 : doAutoPrefix(false), prefixEndsWithType(false) { 386 inPkgName[0]=0; 387 pkgPrefix[0]=0; 388 inData=nullptr; 389 inLength=0; 390 inCharset=U_CHARSET_FAMILY; 391 inIsBigEndian=U_IS_BIG_ENDIAN; 392 393 itemCount=0; 394 itemMax=0; 395 items=nullptr; 396 397 inStringTop=outStringTop=0; 398 399 matchMode=0; 400 findPrefix=findSuffix=nullptr; 401 findPrefixLength=findSuffixLength=0; 402 findNextIndex=-1; 403 404 // create a header for an empty package 405 DataHeader *pHeader; 406 pHeader = reinterpret_cast<DataHeader*>(header); 407 pHeader->dataHeader.magic1=0xda; 408 pHeader->dataHeader.magic2=0x27; 409 memcpy(&pHeader->info, &dataInfo, sizeof(dataInfo)); 410 headerLength = static_cast<int32_t>(4 + sizeof(dataInfo)); 411 if(headerLength&0xf) { 412 /* NUL-pad the header to a multiple of 16 */ 413 int32_t length=(headerLength+0xf)&~0xf; 414 memset(header+headerLength, 0, length-headerLength); 415 headerLength=length; 416 } 417 pHeader->dataHeader.headerSize = static_cast<uint16_t>(headerLength); 418 } 419 420 Package::~Package() { 421 int32_t idx; 422 423 uprv_free(inData); 424 425 for(idx=0; idx<itemCount; ++idx) { 426 if(items[idx].isDataOwned) { 427 uprv_free(items[idx].data); 428 } 429 } 430 431 uprv_free((void*)items); 432 } 433 434 void 435 Package::setPrefix(const char *p) { 436 if(strlen(p)>=sizeof(pkgPrefix)) { 437 fprintf(stderr, "icupkg: --toc_prefix %s too long\n", p); 438 exit(U_ILLEGAL_ARGUMENT_ERROR); 439 } 440 strcpy(pkgPrefix, p); 441 } 442 443 void 444 Package::readPackage(const char *filename) { 445 UDataSwapper *ds; 446 const UDataInfo *pInfo; 447 UErrorCode errorCode; 448 449 const uint8_t *inBytes; 450 451 int32_t length, offset, i; 452 int32_t itemLength, typeEnum; 453 char type; 454 455 const UDataOffsetTOCEntry *inEntries; 456 457 extractPackageName(filename, inPkgName, static_cast<int32_t>(sizeof(inPkgName))); 458 459 /* read the file */ 460 inData=readFile(nullptr, filename, inLength, type); 461 length=inLength; 462 463 /* 464 * swap the header - even if the swapping itself is a no-op 465 * because it tells us the header length 466 */ 467 errorCode=U_ZERO_ERROR; 468 makeTypeProps(type, inCharset, inIsBigEndian); 469 ds=udata_openSwapper(inIsBigEndian, inCharset, U_IS_BIG_ENDIAN, U_CHARSET_FAMILY, &errorCode); 470 if(U_FAILURE(errorCode)) { 471 fprintf(stderr, "icupkg: udata_openSwapper(\"%s\") failed - %s\n", 472 filename, u_errorName(errorCode)); 473 exit(errorCode); 474 } 475 476 ds->printError=printPackageError; 477 ds->printErrorContext=stderr; 478 479 headerLength=sizeof(header); 480 if(length<headerLength) { 481 headerLength=length; 482 } 483 headerLength=udata_swapDataHeader(ds, inData, headerLength, header, &errorCode); 484 if(U_FAILURE(errorCode)) { 485 exit(errorCode); 486 } 487 488 /* check data format and format version */ 489 pInfo = reinterpret_cast<const UDataInfo*>(reinterpret_cast<const char*>(inData) + 4); 490 if(!( 491 pInfo->dataFormat[0]==0x43 && /* dataFormat="CmnD" */ 492 pInfo->dataFormat[1]==0x6d && 493 pInfo->dataFormat[2]==0x6e && 494 pInfo->dataFormat[3]==0x44 && 495 pInfo->formatVersion[0]==1 496 )) { 497 fprintf(stderr, "icupkg: data format %02x.%02x.%02x.%02x (format version %02x) is not recognized as an ICU .dat package\n", 498 pInfo->dataFormat[0], pInfo->dataFormat[1], 499 pInfo->dataFormat[2], pInfo->dataFormat[3], 500 pInfo->formatVersion[0]); 501 exit(U_UNSUPPORTED_ERROR); 502 } 503 inIsBigEndian = static_cast<UBool>(pInfo->isBigEndian); 504 inCharset=pInfo->charsetFamily; 505 506 inBytes=(const uint8_t *)inData+headerLength; 507 inEntries = reinterpret_cast<const UDataOffsetTOCEntry*>(inBytes + 4); 508 509 /* check that the itemCount fits, then the ToC table, then at least the header of the last item */ 510 length-=headerLength; 511 if(length<4) { 512 /* itemCount does not fit */ 513 offset=0x7fffffff; 514 } else { 515 itemCount = udata_readInt32(ds, *reinterpret_cast<const int32_t*>(inBytes)); 516 setItemCapacity(itemCount); /* resize so there's space */ 517 if(itemCount==0) { 518 offset=4; 519 } else if(length<(4+8*itemCount)) { 520 /* ToC table does not fit */ 521 offset=0x7fffffff; 522 } else { 523 /* offset of the last item plus at least 20 bytes for its header */ 524 offset = 20 + static_cast<int32_t>(ds->readUInt32(inEntries[itemCount - 1].dataOffset)); 525 } 526 } 527 if(length<offset) { 528 fprintf(stderr, "icupkg: too few bytes (%ld after header) for a .dat package\n", 529 static_cast<long>(length)); 530 exit(U_INDEX_OUTOFBOUNDS_ERROR); 531 } 532 /* do not modify the package length variable until the last item's length is set */ 533 534 if(itemCount<=0) { 535 if(doAutoPrefix) { 536 fprintf(stderr, "icupkg: --auto_toc_prefix[_with_type] but the input package is empty\n"); 537 exit(U_INVALID_FORMAT_ERROR); 538 } 539 } else { 540 char prefix[MAX_PKG_NAME_LENGTH+4]; 541 char *s, *inItemStrings; 542 543 if(itemCount>itemMax) { 544 fprintf(stderr, "icupkg: too many items, maximum is %d\n", itemMax); 545 exit(U_BUFFER_OVERFLOW_ERROR); 546 } 547 548 /* swap the item name strings */ 549 int32_t stringsOffset=4+8*itemCount; 550 itemLength = static_cast<int32_t>(ds->readUInt32(inEntries[0].dataOffset)) - stringsOffset; 551 552 // don't include padding bytes at the end of the item names 553 while(itemLength>0 && inBytes[stringsOffset+itemLength-1]!=0) { 554 --itemLength; 555 } 556 557 if((inStringTop+itemLength)>STRING_STORE_SIZE) { 558 fprintf(stderr, "icupkg: total length of item name strings too long\n"); 559 exit(U_BUFFER_OVERFLOW_ERROR); 560 } 561 562 inItemStrings=inStrings+inStringTop; 563 ds->swapInvChars(ds, inBytes+stringsOffset, itemLength, inItemStrings, &errorCode); 564 if(U_FAILURE(errorCode)) { 565 fprintf(stderr, "icupkg failed to swap the input .dat package item name strings\n"); 566 exit(U_INVALID_FORMAT_ERROR); 567 } 568 inStringTop+=itemLength; 569 570 // reset the Item entries 571 memset(items, 0, itemCount*sizeof(Item)); 572 573 /* 574 * Get the common prefix of the items. 575 * New-style ICU .dat packages use tree separators ('/') between package names, 576 * tree names, and item names, 577 * while old-style ICU .dat packages (before multi-tree support) 578 * use an underscore ('_') between package and item names. 579 */ 580 offset = static_cast<int32_t>(ds->readUInt32(inEntries[0].nameOffset)) - stringsOffset; 581 s=inItemStrings+offset; // name of the first entry 582 int32_t prefixLength; 583 if(doAutoPrefix) { 584 // Use the first entry's prefix. Must be a new-style package. 585 const char *prefixLimit=strchr(s, U_TREE_ENTRY_SEP_CHAR); 586 if(prefixLimit==nullptr) { 587 fprintf(stderr, 588 "icupkg: --auto_toc_prefix[_with_type] but " 589 "the first entry \"%s\" does not contain a '%c'\n", 590 s, U_TREE_ENTRY_SEP_CHAR); 591 exit(U_INVALID_FORMAT_ERROR); 592 } 593 prefixLength = static_cast<int32_t>(prefixLimit - s); 594 if(prefixLength==0 || prefixLength>=UPRV_LENGTHOF(pkgPrefix)) { 595 fprintf(stderr, 596 "icupkg: --auto_toc_prefix[_with_type] but " 597 "the prefix of the first entry \"%s\" is empty or too long\n", 598 s); 599 exit(U_INVALID_FORMAT_ERROR); 600 } 601 if(prefixEndsWithType && s[prefixLength-1]!=type) { 602 fprintf(stderr, 603 "icupkg: --auto_toc_prefix_with_type but " 604 "the prefix of the first entry \"%s\" does not end with '%c'\n", 605 s, type); 606 exit(U_INVALID_FORMAT_ERROR); 607 } 608 memcpy(pkgPrefix, s, prefixLength); 609 pkgPrefix[prefixLength]=0; 610 memcpy(prefix, s, ++prefixLength); // include the / 611 } else { 612 // Use the package basename as prefix. 613 int32_t inPkgNameLength= static_cast<int32_t>(strlen(inPkgName)); 614 memcpy(prefix, inPkgName, inPkgNameLength); 615 prefixLength=inPkgNameLength; 616 617 if (static_cast<int32_t>(strlen(s)) >= (inPkgNameLength + 2) && 618 0==memcmp(s, inPkgName, inPkgNameLength) && 619 s[inPkgNameLength]=='_' 620 ) { 621 // old-style .dat package 622 prefix[prefixLength++]='_'; 623 } else { 624 // new-style .dat package 625 prefix[prefixLength++]=U_TREE_ENTRY_SEP_CHAR; 626 // if it turns out to not contain U_TREE_ENTRY_SEP_CHAR 627 // then the test in the loop below will fail 628 } 629 } 630 prefix[prefixLength]=0; 631 632 /* read the ToC table */ 633 for(i=0; i<itemCount; ++i) { 634 // skip the package part of the item name, error if it does not match the actual package name 635 // or if nothing follows the package name 636 offset = static_cast<int32_t>(ds->readUInt32(inEntries[i].nameOffset)) - stringsOffset; 637 s=inItemStrings+offset; 638 if(0!=strncmp(s, prefix, prefixLength) || s[prefixLength]==0) { 639 fprintf(stderr, "icupkg: input .dat item name \"%s\" does not start with \"%s\"\n", 640 s, prefix); 641 exit(U_INVALID_FORMAT_ERROR); 642 } 643 items[i].name=s+prefixLength; 644 645 // set the item's data 646 items[i].data = const_cast<uint8_t*>(inBytes) + ds->readUInt32(inEntries[i].dataOffset); 647 if(i>0) { 648 items[i - 1].length = static_cast<int32_t>(items[i].data - items[i - 1].data); 649 650 // set the previous item's platform type 651 typeEnum=getTypeEnumForInputData(items[i-1].data, items[i-1].length, &errorCode); 652 if(typeEnum<0 || U_FAILURE(errorCode)) { 653 fprintf(stderr, "icupkg: not an ICU data file: item \"%s\" in \"%s\"\n", items[i-1].name, filename); 654 exit(U_INVALID_FORMAT_ERROR); 655 } 656 items[i-1].type=makeTypeLetter(typeEnum); 657 } 658 items[i].isDataOwned=false; 659 } 660 // set the last item's length 661 items[itemCount-1].length=length-ds->readUInt32(inEntries[itemCount-1].dataOffset); 662 663 // set the last item's platform type 664 typeEnum=getTypeEnumForInputData(items[itemCount-1].data, items[itemCount-1].length, &errorCode); 665 if(typeEnum<0 || U_FAILURE(errorCode)) { 666 fprintf(stderr, "icupkg: not an ICU data file: item \"%s\" in \"%s\"\n", items[itemCount-1].name, filename); 667 exit(U_INVALID_FORMAT_ERROR); 668 } 669 items[itemCount-1].type=makeTypeLetter(typeEnum); 670 671 if(type!=U_ICUDATA_TYPE_LETTER[0]) { 672 // sort the item names for the local charset 673 sortItems(); 674 } 675 } 676 677 udata_closeSwapper(ds); 678 } 679 680 char 681 Package::getInType() { 682 return makeTypeLetter(inCharset, inIsBigEndian); 683 } 684 685 void 686 Package::writePackage(const char *filename, char outType, const char *comment) { 687 char prefix[MAX_PKG_NAME_LENGTH+4]; 688 UDataOffsetTOCEntry entry; 689 UDataSwapper *dsLocalToOut, *ds[TYPE_COUNT]; 690 FILE *file; 691 Item *pItem; 692 char *name; 693 UErrorCode errorCode; 694 int32_t i, length, prefixLength, maxItemLength, basenameOffset, offset, outInt32; 695 uint8_t outCharset; 696 UBool outIsBigEndian; 697 698 extractPackageName(filename, prefix, MAX_PKG_NAME_LENGTH); 699 700 // if there is an explicit comment, then use it, else use what's in the current header 701 if(comment!=nullptr) { 702 /* get the header size minus the current comment */ 703 DataHeader *pHeader; 704 int32_t length; 705 706 pHeader = reinterpret_cast<DataHeader*>(header); 707 headerLength=4+pHeader->info.size; 708 length = static_cast<int32_t>(strlen(comment)); 709 if ((headerLength + length) >= static_cast<int32_t>(sizeof(header))) { 710 fprintf(stderr, "icupkg: comment too long\n"); 711 exit(U_BUFFER_OVERFLOW_ERROR); 712 } 713 memcpy(header+headerLength, comment, length+1); 714 headerLength+=length; 715 if(headerLength&0xf) { 716 /* NUL-pad the header to a multiple of 16 */ 717 length=(headerLength+0xf)&~0xf; 718 memset(header+headerLength, 0, length-headerLength); 719 headerLength=length; 720 } 721 pHeader->dataHeader.headerSize = static_cast<uint16_t>(headerLength); 722 } 723 724 makeTypeProps(outType, outCharset, outIsBigEndian); 725 726 // open (TYPE_COUNT-2) swappers 727 // one is a no-op for local type==outType 728 // one type (TYPE_LE) is bogus 729 errorCode=U_ZERO_ERROR; 730 i=makeTypeEnum(outType); 731 ds[TYPE_B]= i==TYPE_B ? nullptr : udata_openSwapper(true, U_ASCII_FAMILY, outIsBigEndian, outCharset, &errorCode); 732 ds[TYPE_L]= i==TYPE_L ? nullptr : udata_openSwapper(false, U_ASCII_FAMILY, outIsBigEndian, outCharset, &errorCode); 733 ds[TYPE_LE]=nullptr; 734 ds[TYPE_E]= i==TYPE_E ? nullptr : udata_openSwapper(true, U_EBCDIC_FAMILY, outIsBigEndian, outCharset, &errorCode); 735 if(U_FAILURE(errorCode)) { 736 fprintf(stderr, "icupkg: udata_openSwapper() failed - %s\n", u_errorName(errorCode)); 737 exit(errorCode); 738 } 739 for(i=0; i<TYPE_COUNT; ++i) { 740 if(ds[i]!=nullptr) { 741 ds[i]->printError=printPackageError; 742 ds[i]->printErrorContext=stderr; 743 } 744 } 745 746 dsLocalToOut=ds[makeTypeEnum(U_CHARSET_FAMILY, U_IS_BIG_ENDIAN)]; 747 748 // create the file and write its contents 749 file=fopen(filename, "wb"); 750 if(file==nullptr) { 751 fprintf(stderr, "icupkg: unable to create file \"%s\"\n", filename); 752 exit(U_FILE_ACCESS_ERROR); 753 } 754 755 // swap and write the header 756 if(dsLocalToOut!=nullptr) { 757 udata_swapDataHeader(dsLocalToOut, header, headerLength, header, &errorCode); 758 if(U_FAILURE(errorCode)) { 759 fprintf(stderr, "icupkg: udata_swapDataHeader(local to out) failed - %s\n", u_errorName(errorCode)); 760 exit(errorCode); 761 } 762 } 763 length = static_cast<int32_t>(fwrite(header, 1, headerLength, file)); 764 if(length!=headerLength) { 765 fprintf(stderr, "icupkg: unable to write complete header to file \"%s\"\n", filename); 766 exit(U_FILE_ACCESS_ERROR); 767 } 768 769 // prepare and swap the package name with a tree separator 770 // for prepending to item names 771 if(pkgPrefix[0]==0) { 772 prefixLength = static_cast<int32_t>(strlen(prefix)); 773 } else { 774 prefixLength = static_cast<int32_t>(strlen(pkgPrefix)); 775 memcpy(prefix, pkgPrefix, prefixLength); 776 if(prefixEndsWithType) { 777 prefix[prefixLength-1]=outType; 778 } 779 } 780 prefix[prefixLength++]=U_TREE_ENTRY_SEP_CHAR; 781 prefix[prefixLength]=0; 782 if(dsLocalToOut!=nullptr) { 783 dsLocalToOut->swapInvChars(dsLocalToOut, prefix, prefixLength, prefix, &errorCode); 784 if(U_FAILURE(errorCode)) { 785 fprintf(stderr, "icupkg: swapInvChars(output package name) failed - %s\n", u_errorName(errorCode)); 786 exit(errorCode); 787 } 788 789 // swap and sort the item names (sorting needs to be done in the output charset) 790 dsLocalToOut->swapInvChars(dsLocalToOut, inStrings, inStringTop, inStrings, &errorCode); 791 if(U_FAILURE(errorCode)) { 792 fprintf(stderr, "icupkg: swapInvChars(item names) failed - %s\n", u_errorName(errorCode)); 793 exit(errorCode); 794 } 795 sortItems(); 796 } 797 798 // create the output item names in sorted order, with the package name prepended to each 799 for(i=0; i<itemCount; ++i) { 800 length = static_cast<int32_t>(strlen(items[i].name)); 801 name=allocString(false, length+prefixLength); 802 memcpy(name, prefix, prefixLength); 803 memcpy(name+prefixLength, items[i].name, length+1); 804 items[i].name=name; 805 } 806 807 // calculate offsets for item names and items, pad to 16-align items 808 // align only the first item; each item's length is a multiple of 16 809 basenameOffset=4+8*itemCount; 810 offset=basenameOffset+outStringTop; 811 if((length=(offset&15))!=0) { 812 length=16-length; 813 memset(allocString(false, length-1), 0xaa, length); 814 offset+=length; 815 } 816 817 // write the table of contents 818 // first the itemCount 819 outInt32=itemCount; 820 if(dsLocalToOut!=nullptr) { 821 dsLocalToOut->swapArray32(dsLocalToOut, &outInt32, 4, &outInt32, &errorCode); 822 if(U_FAILURE(errorCode)) { 823 fprintf(stderr, "icupkg: swapArray32(item count) failed - %s\n", u_errorName(errorCode)); 824 exit(errorCode); 825 } 826 } 827 length = static_cast<int32_t>(fwrite(&outInt32, 1, 4, file)); 828 if(length!=4) { 829 fprintf(stderr, "icupkg: unable to write complete item count to file \"%s\"\n", filename); 830 exit(U_FILE_ACCESS_ERROR); 831 } 832 833 // then write the item entries (and collect the maxItemLength) 834 maxItemLength=0; 835 for(i=0; i<itemCount; ++i) { 836 entry.nameOffset = static_cast<uint32_t>(basenameOffset + (items[i].name - outStrings)); 837 entry.dataOffset = static_cast<uint32_t>(offset); 838 if(dsLocalToOut!=nullptr) { 839 dsLocalToOut->swapArray32(dsLocalToOut, &entry, 8, &entry, &errorCode); 840 if(U_FAILURE(errorCode)) { 841 fprintf(stderr, "icupkg: swapArray32(item entry %ld) failed - %s\n", static_cast<long>(i), u_errorName(errorCode)); 842 exit(errorCode); 843 } 844 } 845 length = static_cast<int32_t>(fwrite(&entry, 1, 8, file)); 846 if(length!=8) { 847 fprintf(stderr, "icupkg: unable to write complete item entry %ld to file \"%s\"\n", static_cast<long>(i), filename); 848 exit(U_FILE_ACCESS_ERROR); 849 } 850 851 length=items[i].length; 852 if(length>maxItemLength) { 853 maxItemLength=length; 854 } 855 offset+=length; 856 } 857 858 // write the item names 859 length = static_cast<int32_t>(fwrite(outStrings, 1, outStringTop, file)); 860 if(length!=outStringTop) { 861 fprintf(stderr, "icupkg: unable to write complete item names to file \"%s\"\n", filename); 862 exit(U_FILE_ACCESS_ERROR); 863 } 864 865 // write the items 866 for(pItem=items, i=0; i<itemCount; ++pItem, ++i) { 867 int32_t type=makeTypeEnum(pItem->type); 868 if(ds[type]!=nullptr) { 869 // swap each item from its platform properties to the desired ones 870 udata_swap( 871 ds[type], 872 pItem->data, pItem->length, pItem->data, 873 &errorCode); 874 if(U_FAILURE(errorCode)) { 875 fprintf(stderr, "icupkg: udata_swap(item %ld) failed - %s\n", static_cast<long>(i), u_errorName(errorCode)); 876 exit(errorCode); 877 } 878 } 879 length = static_cast<int32_t>(fwrite(pItem->data, 1, pItem->length, file)); 880 if(length!=pItem->length) { 881 fprintf(stderr, "icupkg: unable to write complete item %ld to file \"%s\"\n", static_cast<long>(i), filename); 882 exit(U_FILE_ACCESS_ERROR); 883 } 884 } 885 886 if(ferror(file)) { 887 fprintf(stderr, "icupkg: unable to write complete file \"%s\"\n", filename); 888 exit(U_FILE_ACCESS_ERROR); 889 } 890 891 fclose(file); 892 for(i=0; i<TYPE_COUNT; ++i) { 893 udata_closeSwapper(ds[i]); 894 } 895 } 896 897 int32_t 898 Package::findItem(const char *name, int32_t length) const { 899 int32_t i, start, limit; 900 int result; 901 902 /* do a binary search for the string */ 903 start=0; 904 limit=itemCount; 905 while(start<limit) { 906 i=(start+limit)/2; 907 if(length>=0) { 908 result=strncmp(name, items[i].name, length); 909 } else { 910 result=strcmp(name, items[i].name); 911 } 912 913 if(result==0) { 914 /* found */ 915 if(length>=0) { 916 /* 917 * if we compared just prefixes, then we may need to back up 918 * to the first item with this prefix 919 */ 920 while(i>0 && 0==strncmp(name, items[i-1].name, length)) { 921 --i; 922 } 923 } 924 return i; 925 } else if(result<0) { 926 limit=i; 927 } else /* result>0 */ { 928 start=i+1; 929 } 930 } 931 932 return ~start; /* not found, return binary-not of the insertion point */ 933 } 934 935 void 936 Package::findItems(const char *pattern) { 937 const char *wild; 938 939 if(pattern==nullptr || *pattern==0) { 940 findNextIndex=-1; 941 return; 942 } 943 944 findPrefix=pattern; 945 findSuffix=nullptr; 946 findSuffixLength=0; 947 948 wild=strchr(pattern, '*'); 949 if(wild==nullptr) { 950 // no wildcard 951 findPrefixLength = static_cast<int32_t>(strlen(pattern)); 952 } else { 953 // one wildcard 954 findPrefixLength = static_cast<int32_t>(wild - pattern); 955 findSuffix=wild+1; 956 findSuffixLength = static_cast<int32_t>(strlen(findSuffix)); 957 if(nullptr!=strchr(findSuffix, '*')) { 958 // two or more wildcards 959 fprintf(stderr, "icupkg: syntax error (more than one '*') in item pattern \"%s\"\n", pattern); 960 exit(U_PARSE_ERROR); 961 } 962 } 963 964 if(findPrefixLength==0) { 965 findNextIndex=0; 966 } else { 967 findNextIndex=findItem(findPrefix, findPrefixLength); 968 } 969 } 970 971 int32_t 972 Package::findNextItem() { 973 const char *name, *middle, *treeSep; 974 int32_t idx, nameLength, middleLength; 975 976 if(findNextIndex<0) { 977 return -1; 978 } 979 980 while(findNextIndex<itemCount) { 981 idx=findNextIndex++; 982 name=items[idx].name; 983 nameLength = static_cast<int32_t>(strlen(name)); 984 if(nameLength<(findPrefixLength+findSuffixLength)) { 985 // item name too short for prefix & suffix 986 continue; 987 } 988 if(findPrefixLength>0 && 0!=memcmp(findPrefix, name, findPrefixLength)) { 989 // left the range of names with this prefix 990 break; 991 } 992 middle=name+findPrefixLength; 993 middleLength=nameLength-findPrefixLength-findSuffixLength; 994 if(findSuffixLength>0 && 0!=memcmp(findSuffix, name+(nameLength-findSuffixLength), findSuffixLength)) { 995 // suffix does not match 996 continue; 997 } 998 // prefix & suffix match 999 1000 if(matchMode&MATCH_NOSLASH) { 1001 treeSep=strchr(middle, U_TREE_ENTRY_SEP_CHAR); 1002 if(treeSep!=nullptr && (treeSep-middle)<middleLength) { 1003 // the middle (matching the * wildcard) contains a tree separator / 1004 continue; 1005 } 1006 } 1007 1008 // found a matching item 1009 return idx; 1010 } 1011 1012 // no more items 1013 findNextIndex=-1; 1014 return -1; 1015 } 1016 1017 void 1018 Package::setMatchMode(uint32_t mode) { 1019 matchMode=mode; 1020 } 1021 1022 void 1023 Package::addItem(const char *name) { 1024 addItem(name, nullptr, 0, false, U_ICUDATA_TYPE_LETTER[0]); 1025 } 1026 1027 void 1028 Package::addItem(const char *name, uint8_t *data, int32_t length, UBool isDataOwned, char type) { 1029 int32_t idx; 1030 1031 idx=findItem(name); 1032 if(idx<0) { 1033 // new item, make space at the insertion point 1034 ensureItemCapacity(); 1035 // move the following items down 1036 idx=~idx; 1037 if(idx<itemCount) { 1038 memmove(items+idx+1, items+idx, (itemCount-idx)*sizeof(Item)); 1039 } 1040 ++itemCount; 1041 1042 // reset this Item entry 1043 memset(items+idx, 0, sizeof(Item)); 1044 1045 // copy the item's name 1046 items[idx].name=allocString(true, static_cast<int32_t>(strlen(name))); 1047 strcpy(items[idx].name, name); 1048 pathToTree(items[idx].name); 1049 } else { 1050 // same-name item found, replace it 1051 if(items[idx].isDataOwned) { 1052 uprv_free(items[idx].data); 1053 } 1054 1055 // keep the item's name since it is the same 1056 } 1057 1058 // set the item's data 1059 items[idx].data=data; 1060 items[idx].length=length; 1061 items[idx].isDataOwned=isDataOwned; 1062 items[idx].type=type; 1063 } 1064 1065 void 1066 Package::addFile(const char *filesPath, const char *name) { 1067 uint8_t *data; 1068 int32_t length; 1069 char type; 1070 1071 data=readFile(filesPath, name, length, type); 1072 // readFile() exits the tool if it fails 1073 addItem(name, data, length, true, type); 1074 } 1075 1076 void 1077 Package::addItems(const Package &listPkg) { 1078 const Item *pItem; 1079 int32_t i; 1080 1081 for(pItem=listPkg.items, i=0; i<listPkg.itemCount; ++pItem, ++i) { 1082 addItem(pItem->name, pItem->data, pItem->length, false, pItem->type); 1083 } 1084 } 1085 1086 void 1087 Package::removeItem(int32_t idx) { 1088 if(idx>=0) { 1089 // remove the item 1090 if(items[idx].isDataOwned) { 1091 uprv_free(items[idx].data); 1092 } 1093 1094 // move the following items up 1095 if((idx+1)<itemCount) { 1096 memmove(items+idx, items+idx+1, (itemCount-(idx+1))*sizeof(Item)); 1097 } 1098 --itemCount; 1099 1100 if(idx<=findNextIndex) { 1101 --findNextIndex; 1102 } 1103 } 1104 } 1105 1106 void 1107 Package::removeItems(const char *pattern) { 1108 int32_t idx; 1109 1110 findItems(pattern); 1111 while((idx=findNextItem())>=0) { 1112 removeItem(idx); 1113 } 1114 } 1115 1116 void 1117 Package::removeItems(const Package &listPkg) { 1118 const Item *pItem; 1119 int32_t i; 1120 1121 for(pItem=listPkg.items, i=0; i<listPkg.itemCount; ++pItem, ++i) { 1122 removeItems(pItem->name); 1123 } 1124 } 1125 1126 void 1127 Package::extractItem(const char *filesPath, const char *outName, int32_t idx, char outType) { 1128 char filename[1024]; 1129 UDataSwapper *ds; 1130 FILE *file; 1131 Item *pItem; 1132 int32_t fileLength; 1133 uint8_t itemCharset, outCharset; 1134 UBool itemIsBigEndian, outIsBigEndian; 1135 1136 if(idx<0 || itemCount<=idx) { 1137 return; 1138 } 1139 pItem=items+idx; 1140 1141 // swap the data to the outType 1142 // outType==0: don't swap 1143 if(outType!=0 && pItem->type!=outType) { 1144 // open the swapper 1145 UErrorCode errorCode=U_ZERO_ERROR; 1146 makeTypeProps(pItem->type, itemCharset, itemIsBigEndian); 1147 makeTypeProps(outType, outCharset, outIsBigEndian); 1148 ds=udata_openSwapper(itemIsBigEndian, itemCharset, outIsBigEndian, outCharset, &errorCode); 1149 if(U_FAILURE(errorCode)) { 1150 fprintf(stderr, "icupkg: udata_openSwapper(item %ld) failed - %s\n", 1151 static_cast<long>(idx), u_errorName(errorCode)); 1152 exit(errorCode); 1153 } 1154 1155 ds->printError=printPackageError; 1156 ds->printErrorContext=stderr; 1157 1158 // swap the item from its platform properties to the desired ones 1159 udata_swap(ds, pItem->data, pItem->length, pItem->data, &errorCode); 1160 if(U_FAILURE(errorCode)) { 1161 fprintf(stderr, "icupkg: udata_swap(item %ld) failed - %s\n", static_cast<long>(idx), u_errorName(errorCode)); 1162 exit(errorCode); 1163 } 1164 udata_closeSwapper(ds); 1165 pItem->type=outType; 1166 } 1167 1168 // create the file and write its contents 1169 makeFullFilenameAndDirs(filesPath, outName, filename, static_cast<int32_t>(sizeof(filename))); 1170 file=fopen(filename, "wb"); 1171 if(file==nullptr) { 1172 fprintf(stderr, "icupkg: unable to create file \"%s\"\n", filename); 1173 exit(U_FILE_ACCESS_ERROR); 1174 } 1175 fileLength = static_cast<int32_t>(fwrite(pItem->data, 1, pItem->length, file)); 1176 1177 if(ferror(file) || fileLength!=pItem->length) { 1178 fprintf(stderr, "icupkg: unable to write complete file \"%s\"\n", filename); 1179 exit(U_FILE_ACCESS_ERROR); 1180 } 1181 fclose(file); 1182 } 1183 1184 void 1185 Package::extractItem(const char *filesPath, int32_t idx, char outType) { 1186 extractItem(filesPath, items[idx].name, idx, outType); 1187 } 1188 1189 void 1190 Package::extractItems(const char *filesPath, const char *pattern, char outType) { 1191 int32_t idx; 1192 1193 findItems(pattern); 1194 while((idx=findNextItem())>=0) { 1195 extractItem(filesPath, idx, outType); 1196 } 1197 } 1198 1199 void 1200 Package::extractItems(const char *filesPath, const Package &listPkg, char outType) { 1201 const Item *pItem; 1202 int32_t i; 1203 1204 for(pItem=listPkg.items, i=0; i<listPkg.itemCount; ++pItem, ++i) { 1205 extractItems(filesPath, pItem->name, outType); 1206 } 1207 } 1208 1209 int32_t 1210 Package::getItemCount() const { 1211 return itemCount; 1212 } 1213 1214 const Item * 1215 Package::getItem(int32_t idx) const { 1216 if (0 <= idx && idx < itemCount) { 1217 return &items[idx]; 1218 } 1219 return nullptr; 1220 } 1221 1222 void 1223 Package::checkDependency(void *context, const char *itemName, const char *targetName) { 1224 // check dependency: make sure the target item is in the package 1225 Package* me = static_cast<Package*>(context); 1226 if(me->findItem(targetName)<0) { 1227 me->isMissingItems=true; 1228 fprintf(stderr, "Item %s depends on missing item %s\n", itemName, targetName); 1229 } 1230 } 1231 1232 UBool 1233 Package::checkDependencies() { 1234 isMissingItems=false; 1235 enumDependencies(this, checkDependency); 1236 return !isMissingItems; 1237 } 1238 1239 void 1240 Package::enumDependencies(void *context, CheckDependency check) { 1241 int32_t i; 1242 1243 for(i=0; i<itemCount; ++i) { 1244 enumDependencies(items+i, context, check); 1245 } 1246 } 1247 1248 char * 1249 Package::allocString(UBool in, int32_t length) { 1250 char *p; 1251 int32_t top; 1252 1253 if(in) { 1254 top=inStringTop; 1255 p=inStrings+top; 1256 } else { 1257 top=outStringTop; 1258 p=outStrings+top; 1259 } 1260 top+=length+1; 1261 1262 if(top>STRING_STORE_SIZE) { 1263 fprintf(stderr, "icupkg: string storage overflow\n"); 1264 exit(U_BUFFER_OVERFLOW_ERROR); 1265 } 1266 if(in) { 1267 inStringTop=top; 1268 } else { 1269 outStringTop=top; 1270 } 1271 return p; 1272 } 1273 1274 void 1275 Package::sortItems() { 1276 UErrorCode errorCode=U_ZERO_ERROR; 1277 uprv_sortArray(items, itemCount, static_cast<int32_t>(sizeof(Item)), compareItems, nullptr, false, &errorCode); 1278 if(U_FAILURE(errorCode)) { 1279 fprintf(stderr, "icupkg: sorting item names failed - %s\n", u_errorName(errorCode)); 1280 exit(errorCode); 1281 } 1282 } 1283 1284 void Package::setItemCapacity(int32_t max) 1285 { 1286 if(max<=itemMax) { 1287 return; 1288 } 1289 Item* newItems = static_cast<Item*>(uprv_malloc(max * sizeof(items[0]))); 1290 Item *oldItems = items; 1291 if(newItems == nullptr) { 1292 fprintf(stderr, "icupkg: Out of memory trying to allocate %lu bytes for %d items\n", 1293 max * sizeof(items[0]), max); 1294 exit(U_MEMORY_ALLOCATION_ERROR); 1295 } 1296 if(items && itemCount>0) { 1297 uprv_memcpy(newItems, items, (size_t)itemCount*sizeof(items[0])); 1298 } 1299 itemMax = max; 1300 items = newItems; 1301 uprv_free(oldItems); 1302 } 1303 1304 void Package::ensureItemCapacity() 1305 { 1306 if((itemCount+1)>itemMax) { 1307 setItemCapacity(itemCount+kItemsChunk); 1308 } 1309 } 1310 1311 U_NAMESPACE_END