gendict.cpp (19369B)
1 // © 2016 and later: Unicode, Inc. and others. 2 // License & terms of use: http://www.unicode.org/copyright.html 3 /* 4 ********************************************************************** 5 * Copyright (C) 2002-2016, International Business Machines 6 * Corporation and others. All Rights Reserved. 7 ********************************************************************** 8 * 9 * File gendict.cpp 10 */ 11 12 #include "unicode/utypes.h" 13 #include "unicode/uchar.h" 14 #include "unicode/ucnv.h" 15 #include "unicode/uniset.h" 16 #include "unicode/unistr.h" 17 #include "unicode/uclean.h" 18 #include "unicode/udata.h" 19 #include "unicode/putil.h" 20 #include "unicode/ucharstriebuilder.h" 21 #include "unicode/bytestriebuilder.h" 22 #include "unicode/ucharstrie.h" 23 #include "unicode/bytestrie.h" 24 #include "unicode/ucnv.h" 25 #include "unicode/ustring.h" 26 #include "unicode/utf16.h" 27 28 #include "charstr.h" 29 #include "dictionarydata.h" 30 #include "uoptions.h" 31 #include "unewdata.h" 32 #include "cmemory.h" 33 #include "uassert.h" 34 #include "ucbuf.h" 35 #include "toolutil.h" 36 #include "cstring.h" 37 #include "writesrc.h" 38 39 #include <stdio.h> 40 #include <stdlib.h> 41 #include <string.h> 42 43 #include "putilimp.h" 44 UDate startTime; 45 46 static int elapsedTime() { 47 return static_cast<int>(uprv_floor((uprv_getRawUTCtime() - startTime) / 1000.0)); 48 } 49 50 U_NAMESPACE_USE 51 52 static char *progName; 53 static UOption options[]={ 54 UOPTION_HELP_H, /* 0 */ 55 UOPTION_HELP_QUESTION_MARK, /* 1 */ 56 UOPTION_VERBOSE, /* 2 */ 57 UOPTION_ICUDATADIR, /* 4 */ 58 UOPTION_COPYRIGHT, /* 5 */ 59 { "uchars", nullptr, nullptr, nullptr, '\1', UOPT_NO_ARG, 0}, /* 6 */ 60 { "bytes", nullptr, nullptr, nullptr, '\1', UOPT_NO_ARG, 0}, /* 7 */ 61 { "transform", nullptr, nullptr, nullptr, '\1', UOPT_REQUIRES_ARG, 0}, /* 8 */ 62 { "toml", nullptr, nullptr, nullptr, '\1', UOPT_NO_ARG, 0}, /* 9 */ 63 UOPTION_QUIET, /* 10 */ 64 }; 65 66 enum arguments { 67 ARG_HELP = 0, 68 ARG_QMARK, 69 ARG_VERBOSE, 70 ARG_ICUDATADIR, 71 ARG_COPYRIGHT, 72 ARG_UCHARS, 73 ARG_BYTES, 74 ARG_TRANSFORM, 75 ARG_TOML, 76 ARG_QUIET 77 }; 78 79 // prints out the standard usage method describing command line arguments, 80 // then bails out with the desired exit code 81 static void usageAndDie(UErrorCode retCode) { 82 fprintf((U_SUCCESS(retCode) ? stdout : stderr), "Usage: %s -trietype [-options] input-dictionary-file output-file\n", progName); 83 fprintf((U_SUCCESS(retCode) ? stdout : stderr), 84 "\tRead in a word list and write out a string trie dictionary\n" 85 "options:\n" 86 "\t-h or -? or --help this usage text\n" 87 "\t-V or --version show a version message\n" 88 "\t-c or --copyright include a copyright notice\n" 89 "\t-v or --verbose turn on verbose output\n" 90 "\t-q or --quiet do not display warnings and progress\n" 91 "\t-i or --icudatadir directory for locating any needed intermediate data files,\n" // TODO: figure out if we need this option 92 "\t followed by path, defaults to %s\n" 93 "\t--uchars output a UCharsTrie (mutually exclusive with -b!)\n" 94 "\t--bytes output a BytesTrie (mutually exclusive with -u!)\n" 95 "\t--transform the kind of transform to use (eg --transform offset-40A3,\n" 96 "\t which specifies an offset transform with constant 0x40A3)\n" 97 "\t--toml output the trie in toml format (default is binary),\n", 98 u_getDataDirectory()); 99 exit(retCode); 100 } 101 102 103 /* UDataInfo cf. udata.h */ 104 static UDataInfo dataInfo = { 105 sizeof(UDataInfo), 106 0, 107 108 U_IS_BIG_ENDIAN, 109 U_CHARSET_FAMILY, 110 U_SIZEOF_UCHAR, 111 0, 112 113 { 0x44, 0x69, 0x63, 0x74 }, /* "Dict" */ 114 { 1, 0, 0, 0 }, /* format version */ 115 { 0, 0, 0, 0 } /* data version */ 116 }; 117 118 #if !UCONFIG_NO_BREAK_ITERATION 119 120 // A wrapper for both BytesTrieBuilder and UCharsTrieBuilder. 121 // may want to put this somewhere in ICU, as it could be useful outside 122 // of this tool? 123 class DataDict { 124 private: 125 BytesTrieBuilder *bt; 126 UCharsTrieBuilder *ut; 127 UChar32 transformConstant; 128 int32_t transformType; 129 public: 130 // constructs a new data dictionary. if there is an error, 131 // it will be returned in status 132 // isBytesTrie != 0 will produce a BytesTrieBuilder, 133 // isBytesTrie == 0 will produce a UCharsTrieBuilder 134 DataDict(UBool isBytesTrie, UErrorCode &status) : bt(nullptr), ut(nullptr), 135 transformConstant(0), transformType(DictionaryData::TRANSFORM_NONE) { 136 if (isBytesTrie) { 137 bt = new BytesTrieBuilder(status); 138 } else { 139 ut = new UCharsTrieBuilder(status); 140 } 141 } 142 143 ~DataDict() { 144 delete bt; 145 delete ut; 146 } 147 148 private: 149 char transform(UChar32 c, UErrorCode &status) { 150 if (transformType == DictionaryData::TRANSFORM_TYPE_OFFSET) { 151 if (c == 0x200D) { return static_cast<char>(0xFF); } 152 else if (c == 0x200C) { return static_cast<char>(0xFE); } 153 int32_t delta = c - transformConstant; 154 if (delta < 0 || 0xFD < delta) { 155 fprintf(stderr, "Codepoint U+%04lx out of range for --transform offset-%04lx!\n", 156 static_cast<long>(c), static_cast<long>(transformConstant)); 157 exit(U_ILLEGAL_ARGUMENT_ERROR); // TODO: should return and print the line number 158 } 159 return static_cast<char>(delta); 160 } else { // no such transform type 161 status = U_INTERNAL_PROGRAM_ERROR; 162 return static_cast<char>(c); // it should be noted this transform type will not generally work 163 } 164 } 165 166 void transform(const UnicodeString &word, CharString &buf, UErrorCode &errorCode) { 167 UChar32 c = 0; 168 int32_t len = word.length(); 169 for (int32_t i = 0; i < len; i += U16_LENGTH(c)) { 170 c = word.char32At(i); 171 buf.append(transform(c, errorCode), errorCode); 172 } 173 } 174 175 public: 176 // sets the desired transformation data. 177 // should be populated from a command line argument 178 // so far the only acceptable format is offset-<hex constant> 179 // eventually others (mask-<hex constant>?) may be enabled 180 // more complex functions may be more difficult 181 void setTransform(const char *t) { 182 if (strncmp(t, "offset-", 7) == 0) { 183 char *end; 184 unsigned long base = uprv_strtoul(t + 7, &end, 16); 185 if (end == (t + 7) || *end != 0 || base > 0x10FF80) { 186 fprintf(stderr, "Syntax for offset value in --transform offset-%s invalid!\n", t + 7); 187 usageAndDie(U_ILLEGAL_ARGUMENT_ERROR); 188 } 189 transformType = DictionaryData::TRANSFORM_TYPE_OFFSET; 190 transformConstant = static_cast<UChar32>(base); 191 } 192 else { 193 fprintf(stderr, "Invalid transform specified: %s\n", t); 194 usageAndDie(U_ILLEGAL_ARGUMENT_ERROR); 195 } 196 } 197 198 // add a word to the trie 199 void addWord(const UnicodeString &word, int32_t value, UErrorCode &status) { 200 if (bt) { 201 CharString buf; 202 transform(word, buf, status); 203 bt->add(buf.toStringPiece(), value, status); 204 } 205 if (ut) { ut->add(word, value, status); } 206 } 207 208 // if we are a bytestrie, give back the StringPiece representing the serialized version of us 209 StringPiece serializeBytes(UErrorCode &status) { 210 return bt->buildStringPiece(USTRINGTRIE_BUILD_SMALL, status); 211 } 212 213 // if we are a ucharstrie, produce the UnicodeString representing the serialized version of us 214 void serializeUChars(UnicodeString &s, UErrorCode &status) { 215 ut->buildUnicodeString(USTRINGTRIE_BUILD_SMALL, s, status); 216 } 217 218 int32_t getTransform() { 219 return (transformType | transformConstant); 220 } 221 }; 222 #endif 223 224 static const char16_t LINEFEED_CHARACTER = 0x000A; 225 static const char16_t CARRIAGE_RETURN_CHARACTER = 0x000D; 226 227 static UBool readLine(UCHARBUF *f, UnicodeString &fileLine, IcuToolErrorCode &errorCode) { 228 int32_t lineLength; 229 const char16_t *line = ucbuf_readline(f, &lineLength, errorCode); 230 if(line == nullptr || errorCode.isFailure()) { return false; } 231 // Strip trailing CR/LF, comments, and spaces. 232 const char16_t *comment = u_memchr(line, 0x23, lineLength); // '#' 233 if(comment != nullptr) { 234 lineLength = static_cast<int32_t>(comment - line); 235 } else { 236 while(lineLength > 0 && (line[lineLength - 1] == CARRIAGE_RETURN_CHARACTER || line[lineLength - 1] == LINEFEED_CHARACTER)) { --lineLength; } 237 } 238 while(lineLength > 0 && u_isspace(line[lineLength - 1])) { --lineLength; } 239 fileLine.setTo(false, line, lineLength); 240 return true; 241 } 242 243 //---------------------------------------------------------------------------- 244 // 245 // main for gendict 246 // 247 //---------------------------------------------------------------------------- 248 int main(int argc, char **argv) { 249 // 250 // Pick up and check the command line arguments, 251 // using the standard ICU tool utils option handling. 252 // 253 U_MAIN_INIT_ARGS(argc, argv); 254 progName = argv[0]; 255 argc=u_parseArgs(argc, argv, UPRV_LENGTHOF(options), options); 256 if(argc<0) { 257 // Unrecognized option 258 fprintf(stderr, "error in command line argument \"%s\"\n", argv[-argc]); 259 usageAndDie(U_ILLEGAL_ARGUMENT_ERROR); 260 } 261 262 if(options[ARG_HELP].doesOccur || options[ARG_QMARK].doesOccur) { 263 // -? or -h for help. 264 usageAndDie(U_ZERO_ERROR); 265 } 266 267 UBool verbose = options[ARG_VERBOSE].doesOccur; 268 UBool quiet = options[ARG_QUIET].doesOccur; 269 270 if (argc < 3) { 271 fprintf(stderr, "input and output file must both be specified.\n"); 272 usageAndDie(U_ILLEGAL_ARGUMENT_ERROR); 273 } 274 const char *outFileName = argv[2]; 275 const char *wordFileName = argv[1]; 276 277 startTime = uprv_getRawUTCtime(); // initialize start timer 278 279 if (options[ARG_ICUDATADIR].doesOccur) { 280 u_setDataDirectory(options[ARG_ICUDATADIR].value); 281 } 282 283 const char *copyright = nullptr; 284 if (options[ARG_COPYRIGHT].doesOccur) { 285 copyright = U_COPYRIGHT_STRING; 286 } 287 288 if (options[ARG_UCHARS].doesOccur == options[ARG_BYTES].doesOccur) { 289 fprintf(stderr, "you must specify exactly one type of trie to output!\n"); 290 usageAndDie(U_ILLEGAL_ARGUMENT_ERROR); 291 } 292 UBool isBytesTrie = options[ARG_BYTES].doesOccur; 293 if (isBytesTrie != options[ARG_TRANSFORM].doesOccur) { 294 fprintf(stderr, "you must provide a transformation for a bytes trie, and must not provide one for a uchars trie!\n"); 295 usageAndDie(U_ILLEGAL_ARGUMENT_ERROR); 296 } 297 298 IcuToolErrorCode status("gendict/main()"); 299 300 UBool isToml = options[ARG_TOML].doesOccur; 301 302 #if UCONFIG_NO_BREAK_ITERATION || UCONFIG_NO_FILE_IO 303 const char* outDir=nullptr; 304 305 UNewDataMemory *pData; 306 char msg[1024]; 307 UErrorCode tempstatus = U_ZERO_ERROR; 308 309 /* write message with just the name */ // potential for a buffer overflow here... 310 snprintf(msg, sizeof(msg), "gendict writes dummy %s because of UCONFIG_NO_BREAK_ITERATION and/or UCONFIG_NO_FILE_IO, see uconfig.h", outFileName); 311 fprintf(stderr, "%s\n", msg); 312 313 /* write the dummy data file */ 314 pData = udata_create(outDir, nullptr, outFileName, &dataInfo, nullptr, &tempstatus); 315 udata_writeBlock(pData, msg, strlen(msg)); 316 udata_finish(pData, &tempstatus); 317 return (int)tempstatus; 318 319 #else 320 // Read in the dictionary source file 321 if (verbose) { printf("Opening file %s...\n", wordFileName); } 322 const char *codepage = "UTF-8"; 323 LocalUCHARBUFPointer f(ucbuf_open(wordFileName, &codepage, true, false, status)); 324 if (status.isFailure()) { 325 fprintf(stderr, "error opening input file: ICU Error \"%s\"\n", status.errorName()); 326 exit(status.reset()); 327 } 328 if (verbose) { printf("Initializing dictionary builder of type %s...\n", (isBytesTrie ? "BytesTrie" : "UCharsTrie")); } 329 DataDict dict(isBytesTrie, status); 330 if (status.isFailure()) { 331 fprintf(stderr, "new DataDict: ICU Error \"%s\"\n", status.errorName()); 332 exit(status.reset()); 333 } 334 if (options[ARG_TRANSFORM].doesOccur) { 335 dict.setTransform(options[ARG_TRANSFORM].value); 336 } 337 338 UnicodeString fileLine; 339 if (verbose) { puts("Adding words to dictionary..."); } 340 UBool hasValues = false; 341 UBool hasValuelessContents = false; 342 int lineCount = 0; 343 int wordCount = 0; 344 int minlen = 255; 345 int maxlen = 0; 346 UBool isOk = true; 347 while (readLine(f.getAlias(), fileLine, status)) { 348 lineCount++; 349 if (fileLine.isEmpty()) continue; 350 351 // Parse word [spaces value]. 352 int32_t keyLen; 353 for (keyLen = 0; keyLen < fileLine.length() && !u_isspace(fileLine[keyLen]); ++keyLen) {} 354 if (keyLen == 0) { 355 fprintf(stderr, "Error: no word on line %i!\n", lineCount); 356 isOk = false; 357 continue; 358 } 359 int32_t valueStart; 360 for (valueStart = keyLen; 361 valueStart < fileLine.length() && u_isspace(fileLine[valueStart]); 362 ++valueStart) {} 363 364 if (keyLen < valueStart) { 365 int32_t valueLength = fileLine.length() - valueStart; 366 if (valueLength > 15) { 367 fprintf(stderr, "Error: value too long on line %i!\n", lineCount); 368 isOk = false; 369 continue; 370 } 371 char s[16]; 372 fileLine.extract(valueStart, valueLength, s, 16, US_INV); 373 char *end; 374 unsigned long value = uprv_strtoul(s, &end, 0); 375 if (end == s || *end != 0 || static_cast<int32_t>(uprv_strlen(s)) != valueLength || value > 0xffffffff) { 376 fprintf(stderr, "Error: value syntax error or value too large on line %i!\n", lineCount); 377 isOk = false; 378 continue; 379 } 380 dict.addWord(fileLine.tempSubString(0, keyLen), static_cast<int32_t>(value), status); 381 hasValues = true; 382 wordCount++; 383 if (keyLen < minlen) minlen = keyLen; 384 if (keyLen > maxlen) maxlen = keyLen; 385 } else { 386 dict.addWord(fileLine.tempSubString(0, keyLen), 0, status); 387 hasValuelessContents = true; 388 wordCount++; 389 if (keyLen < minlen) minlen = keyLen; 390 if (keyLen > maxlen) maxlen = keyLen; 391 } 392 393 if (status.isFailure()) { 394 fprintf(stderr, "ICU Error \"%s\": Failed to add word to trie at input line %d in input file\n", 395 status.errorName(), lineCount); 396 exit(status.reset()); 397 } 398 } 399 if (verbose) { printf("Processed %d lines, added %d words, minlen %d, maxlen %d\n", lineCount, wordCount, minlen, maxlen); } 400 401 if (!isOk && status.isSuccess()) { 402 status.set(U_ILLEGAL_ARGUMENT_ERROR); 403 } 404 if (hasValues && hasValuelessContents) { 405 fprintf(stderr, "warning: file contained both valued and unvalued strings!\n"); 406 } 407 408 if (verbose) { printf("Serializing data...isBytesTrie? %d\n", isBytesTrie); } 409 int32_t outDataSize; 410 const void *outData; 411 UnicodeString usp; 412 if (isBytesTrie) { 413 StringPiece sp = dict.serializeBytes(status); 414 outDataSize = sp.size(); 415 outData = sp.data(); 416 } else { 417 dict.serializeUChars(usp, status); 418 outDataSize = usp.length() * U_SIZEOF_UCHAR; 419 outData = usp.getBuffer(); 420 } 421 if (status.isFailure()) { 422 fprintf(stderr, "gendict: got failure of type %s while serializing, if U_ILLEGAL_ARGUMENT_ERROR possibly due to duplicate dictionary entries\n", status.errorName()); 423 exit(status.reset()); 424 } 425 if (verbose) { puts("Opening output file..."); } 426 427 if (isToml) { 428 FILE* f = fopen(outFileName, "w"); 429 if (f == nullptr) { 430 fprintf(stderr, "gendict: could not open output file \"%s\"\n", outFileName); 431 exit(status.reset()); 432 } 433 fprintf(f, "trie_type = \"%s\"\n", isBytesTrie ? "bytes" : "uchars"); 434 fprintf(f, "has_values = %s\n", hasValues ? "true" : "false"); 435 int32_t transform = dict.getTransform(); 436 bool isOffset = (transform & DictionaryData::TRANSFORM_TYPE_MASK) == DictionaryData::TRANSFORM_TYPE_OFFSET; 437 int32_t offset = transform & DictionaryData::TRANSFORM_OFFSET_MASK; 438 fprintf(f, "transform_type = \"%s\"\n", isOffset ? "offset" : "none"); 439 fprintf(f, "transform_offset = %d\n", offset); 440 441 int32_t outDataWidth = isBytesTrie ? 8 : 16; 442 int32_t outDataLength = isBytesTrie ? outDataSize : outDataSize / U_SIZEOF_UCHAR; 443 usrc_writeArray(f, "trie_data = [\n ", outData, outDataWidth, outDataLength, " ", "\n]\n"); 444 445 446 fclose(f); 447 } else { 448 UNewDataMemory *pData = udata_create(nullptr, nullptr, outFileName, &dataInfo, copyright, status); 449 if (status.isFailure()) { 450 fprintf(stderr, "gendict: could not open output file \"%s\", \"%s\"\n", outFileName, status.errorName()); 451 exit(status.reset()); 452 } 453 454 if (verbose) { puts("Writing to output file..."); } 455 int32_t indexes[DictionaryData::IX_COUNT] = { 456 DictionaryData::IX_COUNT * sizeof(int32_t), 0, 0, 0, 0, 0, 0, 0 457 }; 458 int32_t size = outDataSize + indexes[DictionaryData::IX_STRING_TRIE_OFFSET]; 459 indexes[DictionaryData::IX_RESERVED1_OFFSET] = size; 460 indexes[DictionaryData::IX_RESERVED2_OFFSET] = size; 461 indexes[DictionaryData::IX_TOTAL_SIZE] = size; 462 463 indexes[DictionaryData::IX_TRIE_TYPE] = isBytesTrie ? DictionaryData::TRIE_TYPE_BYTES : DictionaryData::TRIE_TYPE_UCHARS; 464 if (hasValues) { 465 indexes[DictionaryData::IX_TRIE_TYPE] |= DictionaryData::TRIE_HAS_VALUES; 466 } 467 468 indexes[DictionaryData::IX_TRANSFORM] = dict.getTransform(); 469 udata_writeBlock(pData, indexes, sizeof(indexes)); 470 udata_writeBlock(pData, outData, outDataSize); 471 size_t bytesWritten = udata_finish(pData, status); 472 if (status.isFailure()) { 473 fprintf(stderr, "gendict: error \"%s\" writing the output file\n", status.errorName()); 474 exit(status.reset()); 475 } 476 477 if (bytesWritten != static_cast<size_t>(size)) { 478 fprintf(stderr, "Error writing to output file \"%s\"\n", outFileName); 479 exit(U_INTERNAL_PROGRAM_ERROR); 480 } 481 } 482 483 if (!quiet) { printf("%s: done writing\t%s (%ds).\n", progName, outFileName, elapsedTime()); } 484 485 #ifdef TEST_GENDICT 486 if (isBytesTrie) { 487 BytesTrie::Iterator it(outData, outDataSize, status); 488 while (it.hasNext()) { 489 it.next(status); 490 const StringPiece s = it.getString(); 491 int32_t val = it.getValue(); 492 printf("%s -> %i\n", s.data(), val); 493 } 494 } else { 495 UCharsTrie::Iterator it((const char16_t *)outData, outDataSize, status); 496 while (it.hasNext()) { 497 it.next(status); 498 const UnicodeString s = it.getString(); 499 int32_t val = it.getValue(); 500 char tmp[1024]; 501 s.extract(0, s.length(), tmp, 1024); 502 printf("%s -> %i\n", tmp, val); 503 } 504 } 505 #endif 506 507 return 0; 508 #endif /* #if !UCONFIG_NO_BREAK_ITERATION */ 509 }