genrb.cpp (33476B)
1 // © 2016 and later: Unicode, Inc. and others. 2 // License & terms of use: http://www.unicode.org/copyright.html 3 /* 4 ******************************************************************************* 5 * 6 * Copyright (C) 1998-2016, International Business Machines 7 * Corporation and others. All Rights Reserved. 8 * 9 ******************************************************************************* 10 * 11 * File genrb.cpp 12 * 13 * Modification History: 14 * 15 * Date Name Description 16 * 05/25/99 stephen Creation. 17 * 5/10/01 Ram removed ustdio dependency 18 ******************************************************************************* 19 */ 20 21 #include <fstream> 22 #include <iostream> 23 #include <list> 24 #include <string> 25 26 #include <assert.h> 27 #include "genrb.h" 28 #include "unicode/localpointer.h" 29 #include "unicode/uclean.h" 30 #include "unicode/utf16.h" 31 #include "charstr.h" 32 #include "cmemory.h" 33 #include "filterrb.h" 34 #include "reslist.h" 35 #include "ucmndata.h" /* TODO: for reading the pool bundle */ 36 #include "collationroot.h" 37 38 U_NAMESPACE_USE 39 40 /* Protos */ 41 void processFile(const char *filename, const char* cp, 42 const char *inputDir, const char *outputDir, const char *filterDir, 43 const char *packageName, 44 SRBRoot *newPoolBundle, UBool omitBinaryCollation, UErrorCode &status); 45 static char *make_res_filename(const char *filename, const char *outputDir, 46 const char *packageName, UErrorCode &status); 47 48 /* File suffixes */ 49 #define RES_SUFFIX ".res" 50 #define COL_SUFFIX ".col" 51 52 const char *gCurrentFileName = nullptr; 53 #ifdef XP_MAC_CONSOLE 54 #include <console.h> 55 #endif 56 57 void ResFile::close() { 58 delete[] fBytes; 59 fBytes = nullptr; 60 delete fStrings; 61 fStrings = nullptr; 62 } 63 64 enum 65 { 66 HELP1, 67 HELP2, 68 VERBOSE, 69 QUIET, 70 VERSION, 71 SOURCEDIR, 72 DESTDIR, 73 ENCODING, 74 ICUDATADIR, 75 WRITE_JAVA, 76 COPYRIGHT, 77 JAVA_PACKAGE, 78 BUNDLE_NAME, 79 WRITE_XLIFF, 80 STRICT, 81 NO_BINARY_COLLATION, 82 LANGUAGE, 83 NO_COLLATION_RULES, 84 FORMAT_VERSION, 85 WRITE_POOL_BUNDLE, 86 USE_POOL_BUNDLE, 87 INCLUDE_UNIHAN_COLL, 88 FILTERDIR, 89 ICU4X_MODE, 90 UCADATA 91 }; 92 93 UOption options[]={ 94 UOPTION_HELP_H, 95 UOPTION_HELP_QUESTION_MARK, 96 UOPTION_VERBOSE, 97 UOPTION_QUIET, 98 UOPTION_VERSION, 99 UOPTION_SOURCEDIR, 100 UOPTION_DESTDIR, 101 UOPTION_ENCODING, 102 UOPTION_ICUDATADIR, 103 UOPTION_WRITE_JAVA, 104 UOPTION_COPYRIGHT, 105 UOPTION_DEF("java-package", '\x01', UOPT_REQUIRES_ARG), 106 UOPTION_BUNDLE_NAME, 107 UOPTION_DEF("write-xliff", 'x', UOPT_OPTIONAL_ARG), 108 UOPTION_DEF("strict", 'k', UOPT_NO_ARG), /* 14 */ 109 UOPTION_DEF("noBinaryCollation", 'C', UOPT_NO_ARG),/* 15 */ 110 UOPTION_DEF("language", 'l', UOPT_REQUIRES_ARG), /* 16 */ 111 UOPTION_DEF("omitCollationRules", 'R', UOPT_NO_ARG),/* 17 */ 112 UOPTION_DEF("formatVersion", '\x01', UOPT_REQUIRES_ARG),/* 18 */ 113 UOPTION_DEF("writePoolBundle", '\x01', UOPT_OPTIONAL_ARG),/* 19 */ 114 UOPTION_DEF("usePoolBundle", '\x01', UOPT_OPTIONAL_ARG),/* 20 */ 115 UOPTION_DEF("includeUnihanColl", '\x01', UOPT_NO_ARG),/* 21 */ /* temporary, don't display in usage info */ 116 UOPTION_DEF("filterDir", '\x01', UOPT_OPTIONAL_ARG), /* 22 */ 117 UOPTION_DEF("icu4xMode", 'X', UOPT_NO_ARG),/* 23 */ 118 UOPTION_DEF("ucadata", '\x01', UOPT_REQUIRES_ARG),/* 24 */ 119 }; 120 121 static UBool write_java = false; 122 static UBool write_xliff = false; 123 static const char* outputEnc =""; 124 125 static ResFile poolBundle; 126 127 /*added by Jing*/ 128 static const char* language = nullptr; 129 static const char* xliffOutputFileName = nullptr; 130 int 131 main(int argc, 132 char* argv[]) 133 { 134 UErrorCode status = U_ZERO_ERROR; 135 const char *arg = nullptr; 136 const char *outputDir = nullptr; /* nullptr = no output directory, use current */ 137 const char *inputDir = nullptr; 138 const char *filterDir = nullptr; 139 const char *encoding = ""; 140 int i; 141 UBool illegalArg = false; 142 143 U_MAIN_INIT_ARGS(argc, argv); 144 145 options[JAVA_PACKAGE].value = "com.ibm.icu.impl.data"; 146 options[BUNDLE_NAME].value = "LocaleElements"; 147 argc = u_parseArgs(argc, argv, UPRV_LENGTHOF(options), options); 148 149 /* error handling, printing usage message */ 150 if(argc<0) { 151 fprintf(stderr, "%s: error in command line argument \"%s\"\n", argv[0], argv[-argc]); 152 illegalArg = true; 153 } else if(argc<2) { 154 illegalArg = true; 155 } 156 if(options[WRITE_POOL_BUNDLE].doesOccur && options[USE_POOL_BUNDLE].doesOccur) { 157 fprintf(stderr, "%s: cannot combine --writePoolBundle and --usePoolBundle\n", argv[0]); 158 illegalArg = true; 159 } 160 if (options[ICU4X_MODE].doesOccur && !options[UCADATA].doesOccur) { 161 fprintf(stderr, "%s: --icu4xMode requires --ucadata\n", argv[0]); 162 illegalArg = true; 163 } 164 if(options[FORMAT_VERSION].doesOccur) { 165 const char *s = options[FORMAT_VERSION].value; 166 if(uprv_strlen(s) != 1 || (s[0] < '1' || '3' < s[0])) { 167 fprintf(stderr, "%s: unsupported --formatVersion %s\n", argv[0], s); 168 illegalArg = true; 169 } else if(s[0] == '1' && 170 (options[WRITE_POOL_BUNDLE].doesOccur || options[USE_POOL_BUNDLE].doesOccur) 171 ) { 172 fprintf(stderr, "%s: cannot combine --formatVersion 1 with --writePoolBundle or --usePoolBundle\n", argv[0]); 173 illegalArg = true; 174 } else { 175 setFormatVersion(s[0] - '0'); 176 } 177 } 178 179 if((options[JAVA_PACKAGE].doesOccur || options[BUNDLE_NAME].doesOccur) && 180 !options[WRITE_JAVA].doesOccur) { 181 fprintf(stderr, 182 "%s error: command line argument --java-package or --bundle-name " 183 "without --write-java\n", 184 argv[0]); 185 illegalArg = true; 186 } 187 188 if(options[VERSION].doesOccur) { 189 fprintf(stderr, 190 "%s version %s (ICU version %s).\n" 191 "%s\n", 192 argv[0], GENRB_VERSION, U_ICU_VERSION, U_COPYRIGHT_STRING); 193 if(!illegalArg) { 194 return U_ZERO_ERROR; 195 } 196 } 197 198 if(illegalArg || options[HELP1].doesOccur || options[HELP2].doesOccur) { 199 /* 200 * Broken into chunks because the C89 standard says the minimum 201 * required supported string length is 509 bytes. 202 */ 203 fprintf(stderr, 204 "Usage: %s [OPTIONS] [FILES]\n" 205 "\tReads the list of resource bundle source files and creates\n" 206 "\tbinary version of resource bundles (.res files)\n", 207 argv[0]); 208 fprintf(stderr, 209 "Options:\n" 210 "\t-h or -? or --help this usage text\n" 211 "\t-q or --quiet do not display warnings\n" 212 "\t-v or --verbose print extra information when processing files\n" 213 "\t-V or --version prints out version number and exits\n" 214 "\t-c or --copyright include copyright notice\n"); 215 fprintf(stderr, 216 "\t-e or --encoding encoding of source files\n" 217 "\t-d or --destdir destination directory, followed by the path, defaults to '%s'\n" 218 "\t-s or --sourcedir source directory for files followed by path, defaults to '%s'\n" 219 "\t-i or --icudatadir directory for locating any needed intermediate data files,\n" 220 "\t followed by path, defaults to '%s'\n", 221 u_getDataDirectory(), u_getDataDirectory(), u_getDataDirectory()); 222 fprintf(stderr, 223 "\t-j or --write-java write a Java ListResourceBundle for ICU4J, followed by optional encoding\n" 224 "\t defaults to ASCII and \\uXXXX format.\n" 225 "\t --java-package For --write-java: package name for writing the ListResourceBundle,\n" 226 "\t defaults to com.ibm.icu.impl.data\n"); 227 fprintf(stderr, 228 "\t-b or --bundle-name For --write-java: root resource bundle name for writing the ListResourceBundle,\n" 229 "\t defaults to LocaleElements\n" 230 "\t-x or --write-xliff write an XLIFF file for the resource bundle. Followed by\n" 231 "\t an optional output file name.\n" 232 "\t-k or --strict use pedantic parsing of syntax\n" 233 /*added by Jing*/ 234 "\t-l or --language for XLIFF: language code compliant with BCP 47.\n"); 235 fprintf(stderr, 236 "\t-C or --noBinaryCollation do not generate binary collation image;\n" 237 "\t makes .res file smaller but collator instantiation much slower;\n" 238 "\t maintains ability to get tailoring rules\n" 239 "\t-R or --omitCollationRules do not include collation (tailoring) rules;\n" 240 "\t makes .res file smaller and maintains collator instantiation speed\n" 241 "\t but tailoring rules will not be available (they are rarely used)\n"); 242 fprintf(stderr, 243 "\t --formatVersion write a .res file compatible with the requested formatVersion (single digit);\n" 244 "\t for example, --formatVersion 1\n"); 245 fprintf(stderr, 246 "\t --writePoolBundle [directory] write a pool.res file with all of the keys of all input bundles\n" 247 "\t --usePoolBundle [directory] point to keys from the pool.res keys pool bundle if they are available there;\n" 248 "\t makes .res files smaller but dependent on the pool bundle\n" 249 "\t (--writePoolBundle and --usePoolBundle cannot be combined)\n"); 250 fprintf(stderr, 251 "\t --filterDir Input directory where filter files are available.\n" 252 "\t For more on filter files, see ICU Data Build Tool.\n"); 253 254 return illegalArg ? U_ILLEGAL_ARGUMENT_ERROR : U_ZERO_ERROR; 255 } 256 257 if(options[VERBOSE].doesOccur) { 258 setVerbose(true); 259 } 260 261 if(options[QUIET].doesOccur) { 262 setShowWarning(false); 263 } 264 if(options[STRICT].doesOccur) { 265 setStrict(true); 266 } 267 if(options[COPYRIGHT].doesOccur){ 268 setIncludeCopyright(true); 269 } 270 271 if(options[SOURCEDIR].doesOccur) { 272 inputDir = options[SOURCEDIR].value; 273 } 274 275 if(options[DESTDIR].doesOccur) { 276 outputDir = options[DESTDIR].value; 277 } 278 279 if (options[FILTERDIR].doesOccur) { 280 filterDir = options[FILTERDIR].value; 281 } 282 283 if(options[ENCODING].doesOccur) { 284 encoding = options[ENCODING].value; 285 } 286 287 if(options[ICUDATADIR].doesOccur) { 288 u_setDataDirectory(options[ICUDATADIR].value); 289 } 290 /* Initialize ICU */ 291 u_init(&status); 292 if (U_FAILURE(status) && status != U_FILE_ACCESS_ERROR) { 293 /* Note: u_init() will try to open ICU property data. 294 * failures here are expected when building ICU from scratch. 295 * ignore them. 296 */ 297 fprintf(stderr, "%s: can not initialize ICU. status = %s\n", 298 argv[0], u_errorName(status)); 299 exit(1); 300 } 301 status = U_ZERO_ERROR; 302 if(options[WRITE_JAVA].doesOccur) { 303 write_java = true; 304 outputEnc = options[WRITE_JAVA].value; 305 } 306 307 if(options[WRITE_XLIFF].doesOccur) { 308 write_xliff = true; 309 if(options[WRITE_XLIFF].value != nullptr){ 310 xliffOutputFileName = options[WRITE_XLIFF].value; 311 } 312 } 313 314 if (options[UCADATA].doesOccur) { 315 #if !UCONFIG_NO_COLLATION 316 CollationRoot::forceLoadFromFile(options[UCADATA].value, status); 317 #else 318 fprintf(stderr, "--ucadata was used with UCONFIG_NO_COLLATION\n"); 319 return status; 320 #endif 321 } 322 323 initParser(); 324 325 /*added by Jing*/ 326 if(options[LANGUAGE].doesOccur) { 327 language = options[LANGUAGE].value; 328 } 329 330 LocalPointer<SRBRoot> newPoolBundle; 331 if(options[WRITE_POOL_BUNDLE].doesOccur) { 332 newPoolBundle.adoptInsteadAndCheckErrorCode(new SRBRoot(nullptr, true, status), status); 333 if(U_FAILURE(status)) { 334 fprintf(stderr, "unable to create an empty bundle for the pool keys: %s\n", u_errorName(status)); 335 return status; 336 } else { 337 const char *poolResName = "pool.res"; 338 char *nameWithoutSuffix = static_cast<char *>(uprv_malloc(uprv_strlen(poolResName) + 1)); 339 if (nameWithoutSuffix == nullptr) { 340 fprintf(stderr, "out of memory error\n"); 341 return U_MEMORY_ALLOCATION_ERROR; 342 } 343 uprv_strcpy(nameWithoutSuffix, poolResName); 344 *uprv_strrchr(nameWithoutSuffix, '.') = 0; 345 newPoolBundle->fLocale = nameWithoutSuffix; 346 } 347 } 348 349 if(options[USE_POOL_BUNDLE].doesOccur) { 350 const char *poolResName = "pool.res"; 351 FileStream *poolFile; 352 int32_t poolFileSize; 353 int32_t indexLength; 354 /* 355 * TODO: Consolidate inputDir/filename handling from main() and processFile() 356 * into a common function, and use it here as well. 357 * Try to create toolutil functions for dealing with dir/filenames and 358 * loading ICU data files without udata_open(). 359 * Share code with icupkg? 360 * Also, make_res_filename() seems to be unused. Review and remove. 361 */ 362 CharString poolFileName; 363 if (options[USE_POOL_BUNDLE].value!=nullptr) { 364 poolFileName.append(options[USE_POOL_BUNDLE].value, status); 365 } else if (inputDir) { 366 poolFileName.append(inputDir, status); 367 } 368 poolFileName.appendPathPart(poolResName, status); 369 if (U_FAILURE(status)) { 370 return status; 371 } 372 poolFile = T_FileStream_open(poolFileName.data(), "rb"); 373 if (poolFile == nullptr) { 374 fprintf(stderr, "unable to open pool bundle file %s\n", poolFileName.data()); 375 return 1; 376 } 377 poolFileSize = T_FileStream_size(poolFile); 378 if (poolFileSize < 32) { 379 fprintf(stderr, "the pool bundle file %s is too small\n", poolFileName.data()); 380 return 1; 381 } 382 poolBundle.fBytes = new uint8_t[(poolFileSize + 15) & ~15]; 383 if (poolFileSize > 0 && poolBundle.fBytes == nullptr) { 384 fprintf(stderr, "unable to allocate memory for the pool bundle file %s\n", poolFileName.data()); 385 return U_MEMORY_ALLOCATION_ERROR; 386 } 387 388 UDataSwapper *ds; 389 const DataHeader *header; 390 int32_t bytesRead = T_FileStream_read(poolFile, poolBundle.fBytes, poolFileSize); 391 if (bytesRead != poolFileSize) { 392 fprintf(stderr, "unable to read the pool bundle file %s\n", poolFileName.data()); 393 return 1; 394 } 395 /* 396 * Swap the pool bundle so that a single checked-in file can be used. 397 * The swapper functions also test that the data looks like 398 * a well-formed .res file. 399 */ 400 ds = udata_openSwapperForInputData(poolBundle.fBytes, bytesRead, 401 U_IS_BIG_ENDIAN, U_CHARSET_FAMILY, &status); 402 if (U_FAILURE(status)) { 403 fprintf(stderr, "udata_openSwapperForInputData(pool bundle %s) failed: %s\n", 404 poolFileName.data(), u_errorName(status)); 405 return status; 406 } 407 ures_swap(ds, poolBundle.fBytes, bytesRead, poolBundle.fBytes, &status); 408 udata_closeSwapper(ds); 409 if (U_FAILURE(status)) { 410 fprintf(stderr, "ures_swap(pool bundle %s) failed: %s\n", 411 poolFileName.data(), u_errorName(status)); 412 return status; 413 } 414 header = reinterpret_cast<const DataHeader*>(poolBundle.fBytes); 415 if (header->info.formatVersion[0] < 2) { 416 fprintf(stderr, "invalid format of pool bundle file %s\n", poolFileName.data()); 417 return U_INVALID_FORMAT_ERROR; 418 } 419 const int32_t* pRoot = reinterpret_cast<const int32_t*>( 420 reinterpret_cast<const char*>(header) + header->dataHeader.headerSize); 421 poolBundle.fIndexes = pRoot + 1; 422 indexLength = poolBundle.fIndexes[URES_INDEX_LENGTH] & 0xff; 423 if (indexLength <= URES_INDEX_POOL_CHECKSUM) { 424 fprintf(stderr, "insufficient indexes[] in pool bundle file %s\n", poolFileName.data()); 425 return U_INVALID_FORMAT_ERROR; 426 } 427 int32_t keysBottom = 1 + indexLength; 428 int32_t keysTop = poolBundle.fIndexes[URES_INDEX_KEYS_TOP]; 429 poolBundle.fKeys = reinterpret_cast<const char*>(pRoot + keysBottom); 430 poolBundle.fKeysLength = (keysTop - keysBottom) * 4; 431 poolBundle.fChecksum = poolBundle.fIndexes[URES_INDEX_POOL_CHECKSUM]; 432 433 for (i = 0; i < poolBundle.fKeysLength; ++i) { 434 if (poolBundle.fKeys[i] == 0) { 435 ++poolBundle.fKeysCount; 436 } 437 } 438 439 // 16BitUnits[] begins with strings-v2. 440 // The strings-v2 may optionally be terminated by what looks like 441 // an explicit string length that exceeds the number of remaining 16-bit units. 442 int32_t stringUnitsLength = (poolBundle.fIndexes[URES_INDEX_16BIT_TOP] - keysTop) * 2; 443 if (stringUnitsLength >= 2 && getFormatVersion() >= 3) { 444 poolBundle.fStrings = new PseudoListResource(nullptr, status); 445 if (poolBundle.fStrings == nullptr) { 446 fprintf(stderr, "unable to allocate memory for the pool bundle strings %s\n", 447 poolFileName.data()); 448 return U_MEMORY_ALLOCATION_ERROR; 449 } 450 // The PseudoListResource constructor call did not allocate further memory. 451 assert(U_SUCCESS(status)); 452 const char16_t* p = reinterpret_cast<const char16_t*>(pRoot + keysTop); 453 int32_t remaining = stringUnitsLength; 454 do { 455 int32_t first = *p; 456 int8_t numCharsForLength; 457 int32_t length; 458 if (!U16_IS_TRAIL(first)) { 459 // NUL-terminated 460 numCharsForLength = 0; 461 for (length = 0; 462 length < remaining && p[length] != 0; 463 ++length) {} 464 } else if (first < 0xdfef) { 465 numCharsForLength = 1; 466 length = first & 0x3ff; 467 } else if (first < 0xdfff && remaining >= 2) { 468 numCharsForLength = 2; 469 length = ((first - 0xdfef) << 16) | p[1]; 470 } else if (first == 0xdfff && remaining >= 3) { 471 numCharsForLength = 3; 472 length = (static_cast<int32_t>(p[1]) << 16) | p[2]; 473 } else { 474 break; // overrun 475 } 476 // Check for overrun before changing remaining, 477 // so that it is always accurate after the loop body. 478 if ((numCharsForLength + length) >= remaining || 479 p[numCharsForLength + length] != 0) { 480 break; // overrun or explicitly terminated 481 } 482 int32_t poolStringIndex = stringUnitsLength - remaining; 483 // Maximum pool string index when suffix-sharing the last character. 484 int32_t maxStringIndex = poolStringIndex + numCharsForLength + length - 1; 485 if (maxStringIndex >= RES_MAX_OFFSET) { 486 // pool string index overrun 487 break; 488 } 489 p += numCharsForLength; 490 remaining -= numCharsForLength; 491 if (length != 0) { 492 StringResource *sr = 493 new StringResource(poolStringIndex, numCharsForLength, 494 p, length, status); 495 if (sr == nullptr) { 496 fprintf(stderr, "unable to allocate memory for a pool bundle string %s\n", 497 poolFileName.data()); 498 return U_MEMORY_ALLOCATION_ERROR; 499 } 500 poolBundle.fStrings->add(sr); 501 poolBundle.fStringIndexLimit = maxStringIndex + 1; 502 // The StringResource constructor did not allocate further memory. 503 assert(U_SUCCESS(status)); 504 } 505 p += length + 1; 506 remaining -= length + 1; 507 } while (remaining > 0); 508 if (poolBundle.fStrings->fCount == 0) { 509 delete poolBundle.fStrings; 510 poolBundle.fStrings = nullptr; 511 } 512 } 513 514 T_FileStream_close(poolFile); 515 setUsePoolBundle(true); 516 if (isVerbose() && poolBundle.fStrings != nullptr) { 517 printf("number of shared strings: %d\n", static_cast<int>(poolBundle.fStrings->fCount)); 518 int32_t length = poolBundle.fStringIndexLimit + 1; // incl. last NUL 519 printf("16-bit units for strings: %6d = %6d bytes\n", 520 static_cast<int>(length), static_cast<int>(length) * 2); 521 } 522 } 523 524 if(!options[FORMAT_VERSION].doesOccur && getFormatVersion() == 3 && 525 poolBundle.fStrings == nullptr && 526 !options[WRITE_POOL_BUNDLE].doesOccur) { 527 // If we just default to formatVersion 3 528 // but there are no pool bundle strings to share 529 // and we do not write a pool bundle, 530 // then write formatVersion 2 which is just as good. 531 setFormatVersion(2); 532 } 533 534 if(options[INCLUDE_UNIHAN_COLL].doesOccur) { 535 puts("genrb option --includeUnihanColl ignored: \n" 536 "CLDR 26/ICU 54 unihan data is small, except\n" 537 "the ucadata-unihan.icu version of the collation root data\n" 538 "is about 300kB larger than the ucadata-implicithan.icu version."); 539 } 540 541 if((argc-1)!=1) { 542 printf("genrb number of files: %d\n", argc - 1); 543 } 544 /* generate the binary files */ 545 for(i = 1; i < argc; ++i) { 546 status = U_ZERO_ERROR; 547 arg = getLongPathname(argv[i]); 548 549 CharString theCurrentFileName; 550 if (inputDir) { 551 theCurrentFileName.append(inputDir, status); 552 } 553 theCurrentFileName.appendPathPart(arg, status); 554 if (U_FAILURE(status)) { 555 break; 556 } 557 558 gCurrentFileName = theCurrentFileName.data(); 559 if (isVerbose()) { 560 printf("Processing file \"%s\"\n", theCurrentFileName.data()); 561 } 562 processFile(arg, encoding, inputDir, outputDir, filterDir, nullptr, 563 newPoolBundle.getAlias(), 564 options[NO_BINARY_COLLATION].doesOccur, status); 565 } 566 567 poolBundle.close(); 568 569 if(U_SUCCESS(status) && options[WRITE_POOL_BUNDLE].doesOccur) { 570 const char* writePoolDir; 571 if (options[WRITE_POOL_BUNDLE].value!=nullptr) { 572 writePoolDir = options[WRITE_POOL_BUNDLE].value; 573 } else { 574 writePoolDir = outputDir; 575 } 576 char outputFileName[256]; 577 newPoolBundle->write(writePoolDir, nullptr, outputFileName, sizeof(outputFileName), status); 578 if(U_FAILURE(status)) { 579 fprintf(stderr, "unable to write the pool bundle: %s\n", u_errorName(status)); 580 } 581 } 582 583 u_cleanup(); 584 585 /* Don't return warnings as a failure */ 586 if (U_SUCCESS(status)) { 587 return 0; 588 } 589 590 return status; 591 } 592 593 /* Process a file */ 594 void 595 processFile(const char *filename, const char *cp, 596 const char *inputDir, const char *outputDir, const char *filterDir, 597 const char *packageName, 598 SRBRoot *newPoolBundle, 599 UBool omitBinaryCollation, UErrorCode &status) { 600 LocalPointer<SRBRoot> data; 601 LocalUCHARBUFPointer ucbuf; 602 CharString openFileName; 603 CharString inputDirBuf; 604 605 char outputFileName[256]; 606 int32_t dirlen = 0; 607 608 if (U_FAILURE(status)) { 609 return; 610 } 611 if(filename==nullptr){ 612 status=U_ILLEGAL_ARGUMENT_ERROR; 613 return; 614 } 615 616 if(inputDir == nullptr) { 617 const char *filenameBegin = uprv_strrchr(filename, U_FILE_SEP_CHAR); 618 if (filenameBegin != nullptr) { 619 /* 620 * When a filename ../../../data/root.txt is specified, 621 * we presume that the input directory is ../../../data 622 * This is very important when the resource file includes 623 * another file, like UCARules.txt or thaidict.brk. 624 */ 625 int32_t filenameSize = static_cast<int32_t>(filenameBegin - filename + 1); 626 inputDirBuf.append(filename, filenameSize, status); 627 628 inputDir = inputDirBuf.data(); 629 dirlen = inputDirBuf.length(); 630 } 631 }else{ 632 dirlen = static_cast<int32_t>(uprv_strlen(inputDir)); 633 634 if(inputDir[dirlen-1] != U_FILE_SEP_CHAR) { 635 /* 636 * append the input dir to openFileName if the first char in 637 * filename is not file separation char and the last char input directory is not '.'. 638 * This is to support : 639 * genrb -s. /home/icu/data 640 * genrb -s. icu/data 641 * The user cannot mix notations like 642 * genrb -s. /icu/data --- the absolute path specified. -s redundant 643 * user should use 644 * genrb -s. icu/data --- start from CWD and look in icu/data dir 645 */ 646 if( (filename[0] != U_FILE_SEP_CHAR) && (inputDir[dirlen-1] !='.')){ 647 openFileName.append(inputDir, status); 648 } 649 } else { 650 openFileName.append(inputDir, status); 651 } 652 } 653 openFileName.appendPathPart(filename, status); 654 655 // Test for CharString failure 656 if (U_FAILURE(status)) { 657 return; 658 } 659 660 ucbuf.adoptInstead(ucbuf_open(openFileName.data(), &cp,getShowWarning(),true, &status)); 661 if(status == U_FILE_ACCESS_ERROR) { 662 663 fprintf(stderr, "couldn't open file %s\n", openFileName.data()); 664 return; 665 } 666 if (ucbuf.isNull() || U_FAILURE(status)) { 667 fprintf(stderr, "An error occurred processing file %s. Error: %s\n", 668 openFileName.data(), u_errorName(status)); 669 return; 670 } 671 /* auto detected popular encodings? */ 672 if (cp!=nullptr && isVerbose()) { 673 printf("autodetected encoding %s\n", cp); 674 } 675 /* Parse the data into an SRBRoot */ 676 data.adoptInstead(parse(ucbuf.getAlias(), inputDir, outputDir, filename, 677 !omitBinaryCollation, options[NO_COLLATION_RULES].doesOccur, options[ICU4X_MODE].doesOccur, &status)); 678 679 if (data.isNull() || U_FAILURE(status)) { 680 fprintf(stderr, "couldn't parse the file %s. Error:%s\n", filename, u_errorName(status)); 681 return; 682 } 683 684 // Run filtering before writing pool bundle 685 if (filterDir != nullptr) { 686 CharString filterFileName(filterDir, status); 687 filterFileName.appendPathPart(filename, status); 688 if (U_FAILURE(status)) { 689 return; 690 } 691 692 // Open the file and read it into filter 693 SimpleRuleBasedPathFilter filter; 694 std::ifstream f(filterFileName.data()); 695 if (f.fail()) { 696 std::cerr << "genrb error: unable to open " << filterFileName.data() << std::endl; 697 status = U_FILE_ACCESS_ERROR; 698 return; 699 } 700 std::string currentLine; 701 while (std::getline(f, currentLine)) { 702 // Ignore # comments and empty lines 703 if (currentLine.empty() || currentLine[0] == '#') { 704 continue; 705 } 706 filter.addRule(currentLine, status); 707 if (U_FAILURE(status)) { 708 return; 709 } 710 } 711 712 if (isVerbose()) { 713 filter.print(std::cout); 714 } 715 716 // Apply the filter to the data 717 ResKeyPath path; 718 data->fRoot->applyFilter(filter, path, data.getAlias()); 719 } 720 721 if(options[WRITE_POOL_BUNDLE].doesOccur) { 722 data->fWritePoolBundle = newPoolBundle; 723 data->compactKeys(status); 724 int32_t newKeysLength; 725 const char *newKeys = data->getKeyBytes(&newKeysLength); 726 newPoolBundle->addKeyBytes(newKeys, newKeysLength, status); 727 if(U_FAILURE(status)) { 728 fprintf(stderr, "bundle_compactKeys(%s) or bundle_getKeyBytes() failed: %s\n", 729 filename, u_errorName(status)); 730 return; 731 } 732 /* count the number of just-added key strings */ 733 for(const char *newKeysLimit = newKeys + newKeysLength; newKeys < newKeysLimit; ++newKeys) { 734 if(*newKeys == 0) { 735 ++newPoolBundle->fKeysCount; 736 } 737 } 738 } 739 740 if(options[USE_POOL_BUNDLE].doesOccur) { 741 data->fUsePoolBundle = &poolBundle; 742 } 743 744 /* Determine the target rb filename */ 745 uprv_free(make_res_filename(filename, outputDir, packageName, status)); 746 if(U_FAILURE(status)) { 747 fprintf(stderr, "couldn't make the res fileName for bundle %s. Error:%s\n", 748 filename, u_errorName(status)); 749 return; 750 } 751 if(write_java== true){ 752 bundle_write_java(data.getAlias(), outputDir, outputEnc, 753 outputFileName, sizeof(outputFileName), 754 options[JAVA_PACKAGE].value, options[BUNDLE_NAME].value, &status); 755 }else if(write_xliff ==true){ 756 bundle_write_xml(data.getAlias(), outputDir, outputEnc, 757 filename, outputFileName, sizeof(outputFileName), 758 language, xliffOutputFileName, &status); 759 }else{ 760 /* Write the data to the file */ 761 data->write(outputDir, packageName, outputFileName, sizeof(outputFileName), status); 762 } 763 if (U_FAILURE(status)) { 764 fprintf(stderr, "couldn't write bundle %s. Error:%s\n", outputFileName, u_errorName(status)); 765 } 766 } 767 768 /* Generate the target .res file name from the input file name */ 769 static char* 770 make_res_filename(const char *filename, 771 const char *outputDir, 772 const char *packageName, 773 UErrorCode &status) { 774 char *basename; 775 char *dirname; 776 char *resName; 777 778 int32_t pkgLen = 0; /* length of package prefix */ 779 780 781 if (U_FAILURE(status)) { 782 return nullptr; 783 } 784 785 if(packageName != nullptr) 786 { 787 pkgLen = static_cast<int32_t>(1 + uprv_strlen(packageName)); 788 } 789 790 /* setup */ 791 basename = dirname = resName = nullptr; 792 793 /* determine basename, and compiled file names */ 794 basename = static_cast<char*>(uprv_malloc(sizeof(char) * (uprv_strlen(filename) + 1))); 795 if (basename == nullptr) { 796 status = U_MEMORY_ALLOCATION_ERROR; 797 goto finish; 798 } 799 800 get_basename(basename, filename); 801 802 dirname = static_cast<char*>(uprv_malloc(sizeof(char) * (uprv_strlen(filename) + 1))); 803 if (dirname == nullptr) { 804 status = U_MEMORY_ALLOCATION_ERROR; 805 goto finish; 806 } 807 808 get_dirname(dirname, filename); 809 810 if (outputDir == nullptr) { 811 /* output in same dir as .txt */ 812 resName = static_cast<char*>(uprv_malloc(sizeof(char) * (uprv_strlen(dirname) 813 + pkgLen 814 + uprv_strlen(basename) 815 + uprv_strlen(RES_SUFFIX) + 8))); 816 if (resName == nullptr) { 817 status = U_MEMORY_ALLOCATION_ERROR; 818 goto finish; 819 } 820 821 uprv_strcpy(resName, dirname); 822 823 if(packageName != nullptr) 824 { 825 uprv_strcat(resName, packageName); 826 uprv_strcat(resName, "_"); 827 } 828 829 uprv_strcat(resName, basename); 830 831 } else { 832 int32_t dirlen = static_cast<int32_t>(uprv_strlen(outputDir)); 833 int32_t basenamelen = static_cast<int32_t>(uprv_strlen(basename)); 834 835 resName = static_cast<char*>(uprv_malloc(sizeof(char) * (dirlen + pkgLen + basenamelen + 8))); 836 837 if (resName == nullptr) { 838 status = U_MEMORY_ALLOCATION_ERROR; 839 goto finish; 840 } 841 842 uprv_strcpy(resName, outputDir); 843 844 if(outputDir[dirlen] != U_FILE_SEP_CHAR) { 845 resName[dirlen] = U_FILE_SEP_CHAR; 846 resName[dirlen + 1] = '\0'; 847 } 848 849 if(packageName != nullptr) 850 { 851 uprv_strcat(resName, packageName); 852 uprv_strcat(resName, "_"); 853 } 854 855 uprv_strcat(resName, basename); 856 } 857 858 finish: 859 uprv_free(basename); 860 uprv_free(dirname); 861 862 return resName; 863 } 864 865 /* 866 * Local Variables: 867 * indent-tabs-mode: nil 868 * End: 869 */