[ tor-browser ].git.dasho

gencnval.c (39598B)
      1 // © 2016 and later: Unicode, Inc. and others.
      2 // License & terms of use: http://www.unicode.org/copyright.html
      3 /*
      4 *******************************************************************************
      5 *
      6 *   Copyright (C) 1999-2016 International Business Machines
      7 *   Corporation and others.  All Rights Reserved.
      8 *
      9 *******************************************************************************
     10 *   file name:  gencnval.c
     11 *   encoding:   UTF-8
     12 *   tab size:   8 (not used)
     13 *   indentation:4
     14 *
     15 *   created on: 1999nov05
     16 *   created by: Markus W. Scherer
     17 *
     18 *   This program reads convrtrs.txt and writes a memory-mappable
     19 *   converter name alias table to cnvalias.dat .
     20 *
     21 *   This program currently writes version 2.1 of the data format. See
     22 *   ucnv_io.c for more details on the format. Note that version 2.1
     23 *   is written in such a way that a 2.0 reader will be able to use it,
     24 *   and a 2.1 reader will be able to read 2.0.
     25 */
     26 
     27 #include "unicode/utypes.h"
     28 #include "unicode/putil.h"
     29 #include "unicode/ucnv.h" /* ucnv_compareNames() */
     30 #include "ucnv_io.h"
     31 #include "cmemory.h"
     32 #include "cstring.h"
     33 #include "uinvchar.h"
     34 #include "filestrm.h"
     35 #include "toolutil.h"
     36 #include "unicode/uclean.h"
     37 #include "unewdata.h"
     38 #include "uoptions.h"
     39 
     40 #include <ctype.h>
     41 #include <stdbool.h>
     42 #include <stdio.h>
     43 #include <stdlib.h>
     44 
     45 /* TODO: Need to check alias name length is less than UCNV_MAX_CONVERTER_NAME_LENGTH */
     46 
     47 /* STRING_STORE_SIZE + TAG_STORE_SIZE <= ((2^16 - 1) * 2)
     48 That is the maximum size for the string stores combined
     49 because the strings are indexed at 16-bit boundaries by a
     50 16-bit index, and there is only one section for the 
     51 strings.
     52 */
     53 #define STRING_STORE_SIZE 0x1FBFE   /* 130046 */
     54 #define TAG_STORE_SIZE      0x400   /* 1024 */
     55 
     56 /* The combined tag and converter count can affect the number of lists
     57 created.  The size of all lists must be less than (2^17 - 1)
     58 because the lists are indexed as a 16-bit array with a 16-bit index.
     59 */
     60 #define MAX_TAG_COUNT 0x3F      /* 63 */
     61 #define MAX_CONV_COUNT UCNV_CONVERTER_INDEX_MASK
     62 #define MAX_ALIAS_COUNT 0xFFFF  /* 65535 */
     63 
     64 /* The maximum number of aliases that a standard tag/converter combination can have.
     65 At this moment 6/18/2002, IANA has 12 names for ASCII. Don't go below 15 for
     66 this value. I don't recommend more than 31 for this value.
     67 */
     68 #define MAX_TC_ALIAS_COUNT 0x1F    /* 31 */
     69 
     70 #define MAX_LINE_SIZE 0x7FFF    /* 32767 */
     71 #define MAX_LIST_SIZE 0xFFFF    /* 65535 */
     72 
     73 #define DATA_NAME "cnvalias"
     74 #define DATA_TYPE "icu" /* ICU alias table */
     75 
     76 #define ALL_TAG_STR "ALL"
     77 #define ALL_TAG_NUM 1
     78 #define EMPTY_TAG_NUM 0
     79 
     80 /* UDataInfo cf. udata.h */
     81 static const UDataInfo dataInfo={
     82    sizeof(UDataInfo),
     83    0,
     84 
     85    U_IS_BIG_ENDIAN,
     86    U_CHARSET_FAMILY,
     87    sizeof(UChar),
     88    0,
     89 
     90    {0x43, 0x76, 0x41, 0x6c},     /* dataFormat="CvAl" */
     91    {3, 0, 1, 0},                 /* formatVersion */
     92    {1, 4, 2, 0}                  /* dataVersion */
     93 };
     94 
     95 typedef struct {
     96    char *store;
     97    uint32_t top;
     98    uint32_t max;
     99 } StringBlock;
    100 
    101 static char stringStore[STRING_STORE_SIZE];
    102 static StringBlock stringBlock = { stringStore, 0, STRING_STORE_SIZE };
    103 
    104 typedef struct {
    105    uint16_t    aliasCount;
    106    uint16_t    *aliases;     /* Index into stringStore */
    107 } AliasList;
    108 
    109 typedef struct {
    110    uint16_t converter;     /* Index into stringStore */
    111    uint16_t totalAliasCount;    /* Total aliases in this column */
    112 } Converter;
    113 
    114 static Converter converters[MAX_CONV_COUNT];
    115 static uint16_t converterCount=0;
    116 
    117 static char tagStore[TAG_STORE_SIZE];
    118 static StringBlock tagBlock = { tagStore, 0, TAG_STORE_SIZE };
    119 
    120 typedef struct {
    121    uint16_t    tag;        /* Index into tagStore */
    122    uint16_t    totalAliasCount; /* Total aliases in this row */
    123    AliasList   aliasList[MAX_CONV_COUNT];
    124 } Tag;
    125 
    126 /* Think of this as a 3D array. It's tagCount by converterCount by aliasCount */
    127 static Tag tags[MAX_TAG_COUNT];
    128 static uint16_t tagCount = 0;
    129 
    130 /* Used for storing all aliases  */
    131 static uint16_t knownAliases[MAX_ALIAS_COUNT];
    132 static uint16_t knownAliasesCount = 0;
    133 /*static uint16_t duplicateKnownAliasesCount = 0;*/
    134 
    135 /* Used for storing the lists section that point to aliases */
    136 static uint16_t aliasLists[MAX_LIST_SIZE];
    137 static uint16_t aliasListsSize = 0;
    138 
    139 /* Were the standard tags declared before the aliases. */
    140 static UBool standardTagsUsed = false;
    141 static UBool verbose = false;
    142 static UBool quiet = false;
    143 static int lineNum = 1;
    144 
    145 static UConverterAliasOptions tableOptions = {
    146    UCNV_IO_STD_NORMALIZED,
    147    1 /* containsCnvOptionInfo */
    148 };
    149 
    150 
    151 /**
    152 * path to convrtrs.txt
    153 */
    154 const char *path;
    155 
    156 /* prototypes --------------------------------------------------------------- */
    157 
    158 static void
    159 parseLine(const char *line);
    160 
    161 static void
    162 parseFile(FileStream *in);
    163 
    164 static int32_t
    165 chomp(char *line);
    166 
    167 static void
    168 addOfficialTaggedStandards(char *line, int32_t lineLen);
    169 
    170 static uint16_t
    171 addAlias(const char *alias, uint16_t standard, uint16_t converter, UBool defaultName);
    172 
    173 static uint16_t
    174 addConverter(const char *converter);
    175 
    176 static char *
    177 allocString(StringBlock *block, const char *s, int32_t length);
    178 
    179 static uint16_t
    180 addToKnownAliases(const char *alias);
    181 
    182 static int
    183 compareAliases(const void *alias1, const void *alias2);
    184 
    185 static uint16_t
    186 getTagNumber(const char *tag, uint16_t tagLen);
    187 
    188 /*static void
    189 addTaggedAlias(uint16_t tag, const char *alias, uint16_t converter);*/
    190 
    191 static void
    192 writeAliasTable(UNewDataMemory *out);
    193 
    194 /* -------------------------------------------------------------------------- */
    195 
    196 /* Presumes that you used allocString() */
    197 #define GET_ALIAS_STR(index) (stringStore + ((size_t)(index) << 1))
    198 #define GET_TAG_STR(index) (tagStore + ((size_t)(index) << 1))
    199 
    200 /* Presumes that you used allocString() */
    201 #define GET_ALIAS_NUM(str) ((uint16_t)((str - stringStore) >> 1))
    202 #define GET_TAG_NUM(str) ((uint16_t)((str - tagStore) >> 1))
    203 
    204 enum
    205 {
    206    HELP1,
    207    HELP2,
    208    VERBOSE,
    209    COPYRIGHT,
    210    DESTDIR,
    211    SOURCEDIR,
    212    QUIET
    213 };
    214 
    215 static UOption options[]={
    216    UOPTION_HELP_H,
    217    UOPTION_HELP_QUESTION_MARK,
    218    UOPTION_VERBOSE,
    219    UOPTION_COPYRIGHT,
    220    UOPTION_DESTDIR,
    221    UOPTION_SOURCEDIR,
    222    UOPTION_QUIET
    223 };
    224 
    225 extern int
    226 main(int argc, char* argv[]) {
    227    int i, n;
    228    char pathBuf[512];
    229    FileStream *in;
    230    UNewDataMemory *out;
    231    UErrorCode errorCode=U_ZERO_ERROR;
    232 
    233    U_MAIN_INIT_ARGS(argc, argv);
    234 
    235    /* preset then read command line options */
    236    options[DESTDIR].value=options[SOURCEDIR].value=u_getDataDirectory();
    237    argc=u_parseArgs(argc, argv, UPRV_LENGTHOF(options), options);
    238 
    239    /* error handling, printing usage message */
    240    if(argc<0) {
    241        fprintf(stderr,
    242            "error in command line argument \"%s\"\n",
    243            argv[-argc]);
    244    }
    245    if(argc<0 || options[HELP1].doesOccur || options[HELP2].doesOccur) {
    246        fprintf(stderr,
    247            "usage: %s [-options] [convrtrs.txt]\n"
    248            "\tread convrtrs.txt and create " U_ICUDATA_NAME "_" DATA_NAME "." DATA_TYPE "\n"
    249            "options:\n"
    250            "\t-h or -? or --help  this usage text\n"
    251            "\t-v or --verbose     prints out extra information about the alias table\n"
    252            "\t-q or --quiet       do not display warnings and progress\n"
    253            "\t-c or --copyright   include a copyright notice\n"
    254            "\t-d or --destdir     destination directory, followed by the path\n"
    255            "\t-s or --sourcedir   source directory, followed by the path\n",
    256            argv[0]);
    257        return argc<0 ? U_ILLEGAL_ARGUMENT_ERROR : U_ZERO_ERROR;
    258    }
    259 
    260    if(options[VERBOSE].doesOccur) {
    261        verbose = true;
    262    }
    263 
    264    if(options[QUIET].doesOccur) {
    265        quiet = true;
    266    }
    267 
    268    if (argc >= 2) {
    269        path = argv[1];
    270    } else {
    271        path = "convrtrs.txt";
    272    }
    273 
    274    const char* sourcedir = options[SOURCEDIR].value;
    275    if (sourcedir != NULL && *sourcedir != 0) {
    276        if (strlen(sourcedir) + strlen(path) + 2 > sizeof(pathBuf)) {
    277            fprintf(stderr,
    278                    "The source file name is too long, it must be less than %d in bytes.\n",
    279                    (int) sizeof(pathBuf) - 1);
    280            exit(U_ILLEGAL_ARGUMENT_ERROR);
    281        }
    282        char *end;
    283        uprv_strcpy(pathBuf, sourcedir);
    284        end = uprv_strchr(pathBuf, 0);
    285        if(*(end-1)!=U_FILE_SEP_CHAR) {
    286            *(end++)=U_FILE_SEP_CHAR;
    287        }
    288        uprv_strcpy(end, path);
    289        path = pathBuf;
    290    }
    291 
    292    uprv_memset(stringStore, 0, sizeof(stringStore));
    293    uprv_memset(tagStore, 0, sizeof(tagStore));
    294    uprv_memset(converters, 0, sizeof(converters));
    295    uprv_memset(tags, 0, sizeof(tags));
    296    uprv_memset(aliasLists, 0, sizeof(aliasLists));
    297    uprv_memset(knownAliases, 0, sizeof(aliasLists));
    298 
    299 
    300    in=T_FileStream_open(path, "r");
    301    if(in==NULL) {
    302        fprintf(stderr, "gencnval: unable to open input file %s\n", path);
    303        exit(U_FILE_ACCESS_ERROR);
    304    }
    305    parseFile(in);
    306    T_FileStream_close(in);
    307 
    308    /* create the output file */
    309    out=udata_create(options[DESTDIR].value, DATA_TYPE, DATA_NAME, &dataInfo,
    310                     options[COPYRIGHT].doesOccur ? U_COPYRIGHT_STRING : NULL, &errorCode);
    311    if(U_FAILURE(errorCode)) {
    312        fprintf(stderr, "gencnval: unable to open output file - error %s\n", u_errorName(errorCode));
    313        exit(errorCode);
    314    }
    315 
    316    /* write the table of aliases based on a tag/converter name combination */
    317    writeAliasTable(out);
    318 
    319    /* finish */
    320    udata_finish(out, &errorCode);
    321    if(U_FAILURE(errorCode)) {
    322        fprintf(stderr, "gencnval: error finishing output file - %s\n", u_errorName(errorCode));
    323        exit(errorCode);
    324    }
    325 
    326    /* clean up tags */
    327    for (i = 0; i < MAX_TAG_COUNT; i++) {
    328        for (n = 0; n < MAX_CONV_COUNT; n++) {
    329            if (tags[i].aliasList[n].aliases!=NULL) {
    330                uprv_free(tags[i].aliasList[n].aliases);
    331            }
    332        }
    333    }
    334 
    335    return 0;
    336 }
    337 
    338 static void
    339 parseFile(FileStream *in) {
    340    char line[MAX_LINE_SIZE];
    341    char lastLine[MAX_LINE_SIZE];
    342    int32_t lineSize = 0;
    343    int32_t lastLineSize = 0;
    344    UBool validParse = true;
    345 
    346    lineNum = 0;
    347 
    348    /* Add the empty tag, which is for untagged aliases */
    349    getTagNumber("", 0);
    350    getTagNumber(ALL_TAG_STR, 3);
    351    allocString(&stringBlock, "", 0);
    352 
    353    /* read the list of aliases */
    354    while (validParse) {
    355        validParse = false;
    356 
    357        /* Read non-empty lines that don't start with a space character. */
    358        while (T_FileStream_readLine(in, lastLine, MAX_LINE_SIZE) != NULL) {
    359            lastLineSize = chomp(lastLine);
    360            if (lineSize == 0 || (lastLineSize > 0 && isspace((int)*lastLine))) {
    361                uprv_strcpy(line + lineSize, lastLine);
    362                lineSize += lastLineSize;
    363            } else if (lineSize > 0) {
    364                validParse = true;
    365                break;
    366            }
    367            lineNum++;
    368        }
    369 
    370        if (validParse || lineSize > 0) {
    371            if (isspace((int)*line)) {
    372                fprintf(stderr, "%s:%d: error: cannot start an alias with a space\n", path, lineNum-1);
    373                exit(U_PARSE_ERROR);
    374            } else if (line[0] == '{') {
    375                if (!standardTagsUsed && line[lineSize - 1] != '}') {
    376                    fprintf(stderr, "%s:%d: error: alias needs to start with a converter name\n", path, lineNum);
    377                    exit(U_PARSE_ERROR);
    378                }
    379                addOfficialTaggedStandards(line, lineSize);
    380                standardTagsUsed = true;
    381            } else {
    382                if (standardTagsUsed) {
    383                    parseLine(line);
    384                }
    385                else {
    386                    fprintf(stderr, "%s:%d: error: alias table needs to start a list of standard tags\n", path, lineNum);
    387                    exit(U_PARSE_ERROR);
    388                }
    389            }
    390            /* Was the last line consumed */
    391            if (lastLineSize > 0) {
    392                uprv_strcpy(line, lastLine);
    393                lineSize = lastLineSize;
    394            }
    395            else {
    396                lineSize = 0;
    397            }
    398        }
    399        lineNum++;
    400    }
    401 }
    402 
    403 /* This works almost like the Perl chomp.
    404 It removes the newlines, comments and trailing whitespace (not preceding whitespace).
    405 */
    406 static int32_t
    407 chomp(char *line) {
    408    char *s = line;
    409    char *lastNonSpace = line;
    410    while(*s!=0) {
    411        /* truncate at a newline or a comment */
    412        if(*s == '\r' || *s == '\n' || *s == '#') {
    413            *s = 0;
    414            break;
    415        }
    416        if (!isspace((int)*s)) {
    417            lastNonSpace = s;
    418        }
    419        ++s;
    420    }
    421    if (lastNonSpace++ > line) {
    422        *lastNonSpace = 0;
    423        s = lastNonSpace;
    424    }
    425    return (int32_t)(s - line);
    426 }
    427 
    428 static void
    429 parseLine(const char *line) {
    430    uint16_t pos=0, start, limit, length, cnv;
    431    char *converter, *alias;
    432 
    433    /* skip leading white space */
    434    /* There is no whitespace at the beginning anymore */
    435 /*    while(line[pos]!=0 && isspace(line[pos])) {
    436        ++pos;
    437    }
    438 */
    439 
    440    /* is there nothing on this line? */
    441    if(line[pos]==0) {
    442        return;
    443    }
    444 
    445    /* get the converter name */
    446    start=pos;
    447    while(line[pos]!=0 && !isspace((int)line[pos])) {
    448        ++pos;
    449    }
    450    limit=pos;
    451 
    452    /* store the converter name */
    453    length=(uint16_t)(limit-start);
    454    converter=allocString(&stringBlock, line+start, length);
    455 
    456    /* add the converter to the converter table */
    457    cnv=addConverter(converter);
    458 
    459    /* The name itself may be tagged, so let's added it to the aliases list properly */
    460    pos = start;
    461 
    462    /* get all the real aliases */
    463    for(;;) {
    464 
    465        /* skip white space */
    466        while(line[pos]!=0 && isspace((int)line[pos])) {
    467            ++pos;
    468        }
    469 
    470        /* is there no more alias name on this line? */
    471        if(line[pos]==0) {
    472            break;
    473        }
    474 
    475        /* get an alias name */
    476        start=pos;
    477        while(line[pos]!=0 && line[pos]!='{' && !isspace((int)line[pos])) {
    478            ++pos;
    479        }
    480        limit=pos;
    481 
    482        /* store the alias name */
    483        length=(uint16_t)(limit-start);
    484        if (start == 0) {
    485            /* add the converter as its own alias to the alias table */
    486            alias = converter;
    487            addAlias(alias, ALL_TAG_NUM, cnv, true);
    488        }
    489        else {
    490            alias=allocString(&stringBlock, line+start, length);
    491            addAlias(alias, ALL_TAG_NUM, cnv, false);
    492        }
    493        addToKnownAliases(alias);
    494 
    495        /* add the alias/converter pair to the alias table */
    496        /* addAlias(alias, 0, cnv, false);*/
    497 
    498        /* skip whitespace */
    499        while (line[pos] && isspace((int)line[pos])) {
    500            ++pos;
    501        }
    502 
    503        /* handle tags if they are present */
    504        if (line[pos] == '{') {
    505            ++pos;
    506            do {
    507                start = pos;
    508                while (line[pos] && line[pos] != '}' && !isspace((int)line[pos])) {
    509                    ++pos;
    510                }
    511                limit = pos;
    512 
    513                if (start != limit) {
    514                    /* add the tag to the tag table */
    515                    uint16_t tag = getTagNumber(line + start, (uint16_t)(limit - start));
    516                    addAlias(alias, tag, cnv, (UBool)(line[limit-1] == '*'));
    517                }
    518 
    519                while (line[pos] && isspace((int)line[pos])) {
    520                    ++pos;
    521                }
    522            } while (line[pos] && line[pos] != '}');
    523 
    524            if (line[pos] == '}') {
    525                ++pos;
    526            } else {
    527                fprintf(stderr, "%s:%d: Unterminated tag list\n", path, lineNum);
    528                exit(U_UNMATCHED_BRACES);
    529            }
    530        } else {
    531            addAlias(alias, EMPTY_TAG_NUM, cnv, (UBool)(tags[0].aliasList[cnv].aliasCount == 0));
    532        }
    533    }
    534 }
    535 
    536 static uint16_t
    537 getTagNumber(const char *tag, uint16_t tagLen) {
    538    char *atag;
    539    uint16_t t;
    540    UBool preferredName = ((tagLen > 0) ? (tag[tagLen - 1] == '*') : (false));
    541 
    542    if (tagCount >= MAX_TAG_COUNT) {
    543        fprintf(stderr, "%s:%d: too many tags\n", path, lineNum);
    544        exit(U_BUFFER_OVERFLOW_ERROR);
    545    }
    546 
    547    if (preferredName) {
    548 /*        puts(tag);*/
    549        tagLen--;
    550    }
    551 
    552    for (t = 0; t < tagCount; ++t) {
    553        const char *currTag = GET_TAG_STR(tags[t].tag);
    554        if (uprv_strlen(currTag) == tagLen && !uprv_strnicmp(currTag, tag, tagLen)) {
    555            return t;
    556        }
    557    }
    558 
    559    /* we need to add this tag */
    560    if (tagCount >= MAX_TAG_COUNT) {
    561        fprintf(stderr, "%s:%d: error: too many tags\n", path, lineNum);
    562        exit(U_BUFFER_OVERFLOW_ERROR);
    563    }
    564 
    565    /* allocate a new entry in the tag table */
    566    atag = allocString(&tagBlock, tag, tagLen);
    567 
    568    if (standardTagsUsed) {
    569        fprintf(stderr, "%s:%d: error: Tag \"%s\" is not declared at the beginning of the alias table.\n",
    570            path, lineNum, atag);
    571        exit(1);
    572    }
    573    else if (tagLen > 0 && strcmp(tag, ALL_TAG_STR) != 0) {
    574        fprintf(stderr, "%s:%d: warning: Tag \"%s\" was added to the list of standards because it was not declared at beginning of the alias table.\n",
    575            path, lineNum, atag);
    576    }
    577 
    578    /* add the tag to the tag table */
    579    tags[tagCount].tag = GET_TAG_NUM(atag);
    580    /* The aliasList should be set to 0's already */
    581 
    582    return tagCount++;
    583 }
    584 
    585 /*static void
    586 addTaggedAlias(uint16_t tag, const char *alias, uint16_t converter) {
    587    tags[tag].aliases[converter] = alias;
    588 }
    589 */
    590 
    591 static void
    592 addOfficialTaggedStandards(char *line, int32_t lineLen) {
    593    (void) lineLen; // suppress compiler warnings about unused variable
    594    char *atag;
    595    char *endTagExp;
    596    char *tag;
    597    static const char WHITESPACE[] = " \t";
    598 
    599    if (tagCount > UCNV_NUM_RESERVED_TAGS) {
    600        fprintf(stderr, "%s:%d: error: official tags already added\n", path, lineNum);
    601        exit(U_BUFFER_OVERFLOW_ERROR);
    602    }
    603    tag = strchr(line, '{');
    604    if (tag == NULL) {
    605        /* Why were we called? */
    606        fprintf(stderr, "%s:%d: error: Missing start of tag group\n", path, lineNum);
    607        exit(U_PARSE_ERROR);
    608    }
    609    tag++;
    610    endTagExp = strchr(tag, '}');
    611    if (endTagExp == NULL) {
    612        fprintf(stderr, "%s:%d: error: Missing end of tag group\n", path, lineNum);
    613        exit(U_PARSE_ERROR);
    614    }
    615    endTagExp[0] = 0;
    616 
    617    tag = strtok(tag, WHITESPACE);
    618    while (tag != NULL) {
    619 /*        printf("Adding original tag \"%s\"\n", tag);*/
    620 
    621        /* allocate a new entry in the tag table */
    622        atag = allocString(&tagBlock, tag, -1);
    623 
    624        /* add the tag to the tag table */
    625        tags[tagCount++].tag = (uint16_t)((atag - tagStore) >> 1);
    626 
    627        /* The aliasList should already be set to 0's */
    628 
    629        /* Get next tag */
    630        tag = strtok(NULL, WHITESPACE);
    631    }
    632 }
    633 
    634 static uint16_t
    635 addToKnownAliases(const char *alias) {
    636 /*    uint32_t idx; */
    637    /* strict matching */
    638 /*    for (idx = 0; idx < knownAliasesCount; idx++) {
    639        uint16_t num = GET_ALIAS_NUM(alias);
    640        if (knownAliases[idx] != num
    641            && uprv_strcmp(alias, GET_ALIAS_STR(knownAliases[idx])) == 0)
    642        {
    643            fprintf(stderr, "%s:%d: warning: duplicate alias %s and %s found\n", path, 
    644                lineNum, alias, GET_ALIAS_STR(knownAliases[idx]));
    645            duplicateKnownAliasesCount++;
    646            break;
    647        }
    648        else if (knownAliases[idx] != num
    649            && ucnv_compareNames(alias, GET_ALIAS_STR(knownAliases[idx])) == 0)
    650        {
    651            if (verbose) {
    652                fprintf(stderr, "%s:%d: information: duplicate alias %s and %s found\n", path, 
    653                    lineNum, alias, GET_ALIAS_STR(knownAliases[idx]));
    654            }
    655            duplicateKnownAliasesCount++;
    656            break;
    657        }
    658    }
    659 */
    660    if (knownAliasesCount >= MAX_ALIAS_COUNT) {
    661        fprintf(stderr, "%s:%d: warning: Too many aliases defined for all converters\n",
    662            path, lineNum);
    663        exit(U_BUFFER_OVERFLOW_ERROR);
    664    }
    665    /* TODO: We could try to unlist exact duplicates. */
    666    return knownAliases[knownAliasesCount++] = GET_ALIAS_NUM(alias);
    667 }
    668 
    669 /*
    670 @param standard When standard is 0, then it's the "empty" tag.
    671 */
    672 static uint16_t
    673 addAlias(const char *alias, uint16_t standard, uint16_t converter, UBool defaultName) {
    674    uint32_t idx, idx2;
    675    UBool startEmptyWithoutDefault = false;
    676    AliasList *aliasList;
    677 
    678    if(standard>=MAX_TAG_COUNT) {
    679        fprintf(stderr, "%s:%d: error: too many standard tags\n", path, lineNum);
    680        exit(U_BUFFER_OVERFLOW_ERROR);
    681    }
    682    if(converter>=MAX_CONV_COUNT) {
    683        fprintf(stderr, "%s:%d: error: too many converter names\n", path, lineNum);
    684        exit(U_BUFFER_OVERFLOW_ERROR);
    685    }
    686    aliasList = &tags[standard].aliasList[converter];
    687 
    688    if (strchr(alias, '}')) {
    689        fprintf(stderr, "%s:%d: error: unmatched } found\n", path, 
    690            lineNum);
    691    }
    692 
    693    if(aliasList->aliasCount + 1 >= MAX_TC_ALIAS_COUNT) {
    694        fprintf(stderr, "%s:%d: error: too many aliases for alias %s and converter %s\n", path, 
    695            lineNum, alias, GET_ALIAS_STR(converters[converter].converter));
    696        exit(U_BUFFER_OVERFLOW_ERROR);
    697    }
    698 
    699    /* Show this warning only once. All aliases are added to the "ALL" tag. */
    700    if (standard == ALL_TAG_NUM && GET_ALIAS_STR(converters[converter].converter) != alias) {
    701        /* Normally these option values are parsed at runtime, and they can
    702           be discarded when the alias is a default converter. Options should
    703           only be on a converter and not an alias. */
    704        if (uprv_strchr(alias, UCNV_OPTION_SEP_CHAR) != 0)
    705        {
    706            fprintf(stderr, "warning(line %d): alias %s contains a \""UCNV_OPTION_SEP_STRING"\". Options are parsed at run-time and do not need to be in the alias table.\n",
    707                lineNum, alias);
    708        }
    709        if (uprv_strchr(alias, UCNV_VALUE_SEP_CHAR) != 0)
    710        {
    711            fprintf(stderr, "warning(line %d): alias %s contains an \""UCNV_VALUE_SEP_STRING"\". Options are parsed at run-time and do not need to be in the alias table.\n",
    712                lineNum, alias);
    713        }
    714    }
    715 
    716    if (standard != ALL_TAG_NUM) {
    717        /* Check for duplicate aliases for this tag on all converters */
    718        for (idx = 0; idx < converterCount; idx++) {
    719            for (idx2 = 0; idx2 < tags[standard].aliasList[idx].aliasCount; idx2++) {
    720                uint16_t aliasNum = tags[standard].aliasList[idx].aliases[idx2];
    721                if (aliasNum
    722                    && ucnv_compareNames(alias, GET_ALIAS_STR(aliasNum)) == 0)
    723                {
    724                    if (idx == converter) {
    725                        /*
    726                         * (alias, standard) duplicates are harmless if they map to the same converter.
    727                         * Only print a warning in verbose mode, or if the alias is a precise duplicate,
    728                         * not just a lenient-match duplicate.
    729                         */
    730                        if (verbose || 0 == uprv_strcmp(alias, GET_ALIAS_STR(aliasNum))) {
    731                            fprintf(stderr, "%s:%d: warning: duplicate aliases %s and %s found for standard %s and converter %s\n", path, 
    732                                lineNum, alias, GET_ALIAS_STR(aliasNum),
    733                                GET_TAG_STR(tags[standard].tag),
    734                                GET_ALIAS_STR(converters[converter].converter));
    735                        }
    736                    } else {
    737                        fprintf(stderr, "%s:%d: warning: duplicate aliases %s and %s found for standard tag %s between converter %s and converter %s\n", path, 
    738                            lineNum, alias, GET_ALIAS_STR(aliasNum),
    739                            GET_TAG_STR(tags[standard].tag),
    740                            GET_ALIAS_STR(converters[converter].converter),
    741                            GET_ALIAS_STR(converters[idx].converter));
    742                    }
    743                    break;
    744                }
    745            }
    746        }
    747 
    748        /* Check for duplicate default aliases for this converter on all tags */
    749        /* It's okay to have multiple standards prefer the same name */
    750 /*        if (verbose && !dupFound) {
    751            for (idx = 0; idx < tagCount; idx++) {
    752                if (tags[idx].aliasList[converter].aliases) {
    753                    uint16_t aliasNum = tags[idx].aliasList[converter].aliases[0];
    754                    if (aliasNum
    755                        && ucnv_compareNames(alias, GET_ALIAS_STR(aliasNum)) == 0)
    756                    {
    757                        fprintf(stderr, "%s:%d: warning: duplicate alias %s found for converter %s and standard tag %s\n", path, 
    758                            lineNum, alias, GET_ALIAS_STR(converters[converter].converter), GET_TAG_STR(tags[standard].tag));
    759                        break;
    760                    }
    761                }
    762            }
    763        }*/
    764    }
    765 
    766    if (aliasList->aliasCount <= 0) {
    767        aliasList->aliasCount++;
    768        startEmptyWithoutDefault = true;
    769    }
    770    aliasList->aliases = (uint16_t *)uprv_realloc(aliasList->aliases, (aliasList->aliasCount + 1) * sizeof(aliasList->aliases[0]));
    771    if (startEmptyWithoutDefault) {
    772        aliasList->aliases[0] = 0;
    773    }
    774    if (defaultName) {
    775        if (aliasList->aliases[0] != 0) {
    776            fprintf(stderr, "%s:%d: error: Alias %s and %s cannot both be the default alias for standard tag %s and converter %s\n", path, 
    777                lineNum,
    778                alias,
    779                GET_ALIAS_STR(aliasList->aliases[0]),
    780                GET_TAG_STR(tags[standard].tag),
    781                GET_ALIAS_STR(converters[converter].converter));
    782            exit(U_PARSE_ERROR);
    783        }
    784        aliasList->aliases[0] = GET_ALIAS_NUM(alias);
    785    } else {
    786        aliasList->aliases[aliasList->aliasCount++] = GET_ALIAS_NUM(alias);
    787    }
    788 /*    aliasList->converter = converter;*/
    789 
    790    converters[converter].totalAliasCount++; /* One more to the column */
    791    tags[standard].totalAliasCount++; /* One more to the row */
    792 
    793    return aliasList->aliasCount;
    794 }
    795 
    796 static uint16_t
    797 addConverter(const char *converter) {
    798    uint32_t idx;
    799    if(converterCount>=MAX_CONV_COUNT) {
    800        fprintf(stderr, "%s:%d: error: too many converters\n", path, lineNum);
    801        exit(U_BUFFER_OVERFLOW_ERROR);
    802    }
    803 
    804    for (idx = 0; idx < converterCount; idx++) {
    805        if (ucnv_compareNames(converter, GET_ALIAS_STR(converters[idx].converter)) == 0) {
    806            fprintf(stderr, "%s:%d: error: duplicate converter %s found!\n", path, lineNum, converter);
    807            exit(U_PARSE_ERROR);
    808            break;
    809        }
    810    }
    811 
    812    converters[converterCount].converter = GET_ALIAS_NUM(converter);
    813    converters[converterCount].totalAliasCount = 0;
    814 
    815    return converterCount++;
    816 }
    817 
    818 /* resolve this alias based on the prioritization of the standard tags. */
    819 static void
    820 resolveAliasToConverter(uint16_t alias, uint16_t *tagNum, uint16_t *converterNum) {
    821    uint16_t idx, idx2, idx3;
    822 
    823    for (idx = UCNV_NUM_RESERVED_TAGS; idx < tagCount; idx++) {
    824        for (idx2 = 0; idx2 < converterCount; idx2++) {
    825            for (idx3 = 0; idx3 < tags[idx].aliasList[idx2].aliasCount; idx3++) {
    826                uint16_t aliasNum = tags[idx].aliasList[idx2].aliases[idx3];
    827                if (aliasNum == alias) {
    828                    *tagNum = idx;
    829                    *converterNum = idx2;
    830                    return;
    831                }
    832            }
    833        }
    834    }
    835    /* Do the leftovers last, just in case */
    836    /* There is no need to do the ALL tag */
    837    idx = 0;
    838    for (idx2 = 0; idx2 < converterCount; idx2++) {
    839        for (idx3 = 0; idx3 < tags[idx].aliasList[idx2].aliasCount; idx3++) {
    840            uint16_t aliasNum = tags[idx].aliasList[idx2].aliases[idx3];
    841            if (aliasNum == alias) {
    842                *tagNum = idx;
    843                *converterNum = idx2;
    844                return;
    845            }
    846        }
    847    }
    848    *tagNum = UINT16_MAX;
    849    *converterNum = UINT16_MAX;
    850    fprintf(stderr, "%s: warning: alias %s not found\n",
    851        path,
    852        GET_ALIAS_STR(alias));
    853 }
    854 
    855 /* The knownAliases should be sorted before calling this function */
    856 static uint32_t
    857 resolveAliases(uint16_t *uniqueAliasArr, uint16_t *uniqueAliasToConverterArr, uint16_t aliasOffset) {
    858    uint32_t uniqueAliasIdx = 0;
    859    uint32_t idx;
    860    uint16_t currTagNum, oldTagNum;
    861    uint16_t currConvNum, oldConvNum;
    862    const char *lastName;
    863 
    864    if (knownAliasesCount != 0) {
    865      resolveAliasToConverter(knownAliases[0], &oldTagNum, &currConvNum);
    866      uniqueAliasToConverterArr[uniqueAliasIdx] = currConvNum;
    867      oldConvNum = currConvNum;
    868      uniqueAliasArr[uniqueAliasIdx] = knownAliases[0] + aliasOffset;
    869      uniqueAliasIdx++;
    870      lastName = GET_ALIAS_STR(knownAliases[0]);
    871 
    872      for (idx = 1; idx < knownAliasesCount; idx++) {
    873          resolveAliasToConverter(knownAliases[idx], &currTagNum, &currConvNum);
    874          if (ucnv_compareNames(lastName, GET_ALIAS_STR(knownAliases[idx])) == 0) {
    875              /* duplicate found */
    876              if ((currTagNum < oldTagNum && currTagNum >= UCNV_NUM_RESERVED_TAGS)
    877                  || oldTagNum == 0) {
    878                  oldTagNum = currTagNum;
    879                  uniqueAliasToConverterArr[uniqueAliasIdx - 1] = currConvNum;
    880                  uniqueAliasArr[uniqueAliasIdx - 1] = knownAliases[idx] + aliasOffset;
    881                  if (verbose) {
    882                      printf("using %s instead of %s -> %s",
    883                          GET_ALIAS_STR(knownAliases[idx]),
    884                          lastName,
    885                          GET_ALIAS_STR(converters[currConvNum].converter));
    886                      if (oldConvNum != currConvNum) {
    887                          printf(" (alias conflict)");
    888                      }
    889                      puts("");
    890                  }
    891              }
    892              else {
    893                  /* else ignore it */
    894                  if (verbose) {
    895                      printf("folding %s into %s -> %s",
    896                          GET_ALIAS_STR(knownAliases[idx]),
    897                          lastName,
    898                          GET_ALIAS_STR(converters[oldConvNum].converter));
    899                      if (oldConvNum != currConvNum) {
    900                          printf(" (alias conflict)");
    901                      }
    902                      puts("");
    903                  }
    904              }
    905              if (oldConvNum != currConvNum) {
    906                  uniqueAliasToConverterArr[uniqueAliasIdx - 1] |= UCNV_AMBIGUOUS_ALIAS_MAP_BIT;
    907              }
    908          }
    909          else {
    910              uniqueAliasToConverterArr[uniqueAliasIdx] = currConvNum;
    911              oldConvNum = currConvNum;
    912              uniqueAliasArr[uniqueAliasIdx] = knownAliases[idx] + aliasOffset;
    913              uniqueAliasIdx++;
    914              lastName = GET_ALIAS_STR(knownAliases[idx]);
    915              oldTagNum = currTagNum;
    916              /*printf("%s -> %s\n", GET_ALIAS_STR(knownAliases[idx]), GET_ALIAS_STR(converters[currConvNum].converter));*/
    917          }
    918          if (uprv_strchr(GET_ALIAS_STR(converters[currConvNum].converter), UCNV_OPTION_SEP_CHAR) != NULL) {
    919              uniqueAliasToConverterArr[uniqueAliasIdx-1] |= UCNV_CONTAINS_OPTION_BIT;
    920          }
    921      }
    922    }
    923    return uniqueAliasIdx;
    924 }
    925 
    926 static void
    927 createOneAliasList(uint16_t *aliasArrLists, uint32_t tag, uint32_t converter, uint16_t offset) {
    928    uint32_t aliasNum;
    929    AliasList *aliasList = &tags[tag].aliasList[converter];
    930 
    931    if (aliasList->aliasCount == 0) {
    932        aliasArrLists[tag*converterCount + converter] = 0;
    933    }
    934    else {
    935        aliasLists[aliasListsSize++] = aliasList->aliasCount;
    936 
    937        /* write into the array area a 1's based index. */
    938        aliasArrLists[tag*converterCount + converter] = aliasListsSize;
    939 
    940 /*        printf("tag %s converter %s\n",
    941            GET_TAG_STR(tags[tag].tag),
    942            GET_ALIAS_STR(converters[converter].converter));*/
    943        for (aliasNum = 0; aliasNum < aliasList->aliasCount; aliasNum++) {
    944            uint16_t value;
    945 /*            printf("   %s\n",
    946                GET_ALIAS_STR(aliasList->aliases[aliasNum]));*/
    947            if (aliasList->aliases[aliasNum]) {
    948                value = aliasList->aliases[aliasNum] + offset;
    949            } else {
    950                value = 0;
    951                if (tag != 0 && !quiet) { /* Only show the warning when it's not the leftover tag. */
    952                    fprintf(stderr, "%s: warning: tag %s does not have a default alias for %s\n",
    953                            path,
    954                            GET_TAG_STR(tags[tag].tag),
    955                            GET_ALIAS_STR(converters[converter].converter));
    956                }
    957            }
    958            aliasLists[aliasListsSize++] = value;
    959            if (aliasListsSize >= MAX_LIST_SIZE) {
    960                fprintf(stderr, "%s: error: Too many alias lists\n", path);
    961                exit(U_BUFFER_OVERFLOW_ERROR);
    962            }
    963 
    964        }
    965    }
    966 }
    967 
    968 static void
    969 createNormalizedAliasStrings(char *normalizedStrings, const char *origStringBlock, int32_t stringBlockLength) {
    970    int32_t currStrLen;
    971    uprv_memcpy(normalizedStrings, origStringBlock, stringBlockLength);
    972    while ((currStrLen = (int32_t)uprv_strlen(origStringBlock)) < stringBlockLength) {
    973        int32_t currStrSize = currStrLen + 1;
    974        if (currStrLen > 0) {
    975            int32_t normStrLen;
    976            ucnv_io_stripForCompare(normalizedStrings, origStringBlock);
    977            normStrLen = (int32_t)uprv_strlen(normalizedStrings);
    978            if (normStrLen > 0) {
    979                uprv_memset(normalizedStrings + normStrLen, 0, currStrSize - normStrLen);
    980            }
    981        }
    982        stringBlockLength -= currStrSize;
    983        normalizedStrings += currStrSize;
    984        origStringBlock += currStrSize;
    985    }
    986 }
    987 
    988 static void
    989 writeAliasTable(UNewDataMemory *out) {
    990    uint32_t i, j;
    991    uint32_t uniqueAliasesSize;
    992    uint16_t aliasOffset = (uint16_t)(tagBlock.top/sizeof(uint16_t));
    993    uint16_t *aliasArrLists = (uint16_t *)uprv_malloc(tagCount * converterCount * sizeof(uint16_t));
    994    uint16_t *uniqueAliases = (uint16_t *)uprv_malloc(knownAliasesCount * sizeof(uint16_t));
    995    uint16_t *uniqueAliasesToConverter = (uint16_t *)uprv_malloc(knownAliasesCount * sizeof(uint16_t));
    996 
    997    qsort(knownAliases, knownAliasesCount, sizeof(knownAliases[0]), compareAliases);
    998    uniqueAliasesSize = resolveAliases(uniqueAliases, uniqueAliasesToConverter, aliasOffset);
    999 
   1000    /* Array index starts at 1. aliasLists[0] is the size of the lists section. */
   1001    aliasListsSize = 0;
   1002 
   1003    /* write the offsets of all the aliases lists in a 2D array, and create the lists. */
   1004    for (i = 0; i < tagCount; ++i) {
   1005        for (j = 0; j < converterCount; ++j) {
   1006            createOneAliasList(aliasArrLists, i, j, aliasOffset);
   1007        }
   1008    }
   1009 
   1010    /* Write the size of the TOC */
   1011    if (tableOptions.stringNormalizationType == UCNV_IO_UNNORMALIZED) {
   1012        udata_write32(out, 8);
   1013    }
   1014    else {
   1015        udata_write32(out, 9);
   1016    }
   1017 
   1018    /* Write the sizes of each section */
   1019    /* All sizes are the number of uint16_t units, not bytes */
   1020    udata_write32(out, converterCount);
   1021    udata_write32(out, tagCount);
   1022    udata_write32(out, uniqueAliasesSize);  /* list of aliases */
   1023    udata_write32(out, uniqueAliasesSize);  /* The preresolved form of mapping an untagged the alias to a converter */
   1024    udata_write32(out, tagCount * converterCount);
   1025    udata_write32(out, aliasListsSize + 1);
   1026    udata_write32(out, sizeof(tableOptions) / sizeof(uint16_t));
   1027    udata_write32(out, (tagBlock.top + stringBlock.top) / sizeof(uint16_t));
   1028    if (tableOptions.stringNormalizationType != UCNV_IO_UNNORMALIZED) {
   1029        udata_write32(out, (tagBlock.top + stringBlock.top) / sizeof(uint16_t));
   1030    }
   1031 
   1032    /* write the table of converters */
   1033    /* Think of this as the column headers */
   1034    for(i=0; i<converterCount; ++i) {
   1035        udata_write16(out, (uint16_t)(converters[i].converter + aliasOffset));
   1036    }
   1037 
   1038    /* write the table of tags */
   1039    /* Think of this as the row headers */
   1040    for(i=UCNV_NUM_RESERVED_TAGS; i<tagCount; ++i) {
   1041        udata_write16(out, tags[i].tag);
   1042    }
   1043    /* The empty tag is considered the leftover list, and put that at the end of the priority list. */
   1044    udata_write16(out, tags[EMPTY_TAG_NUM].tag);
   1045    udata_write16(out, tags[ALL_TAG_NUM].tag);
   1046 
   1047    /* Write the unique list of aliases */
   1048    udata_writeBlock(out, uniqueAliases, uniqueAliasesSize * sizeof(uint16_t));
   1049 
   1050    /* Write the unique list of aliases */
   1051    udata_writeBlock(out, uniqueAliasesToConverter, uniqueAliasesSize * sizeof(uint16_t));
   1052 
   1053    /* Write the array to the lists */
   1054    udata_writeBlock(out, (const void *)(aliasArrLists + (2*converterCount)), (((tagCount - 2) * converterCount) * sizeof(uint16_t)));
   1055    /* Now write the leftover part of the array for the EMPTY and ALL lists */
   1056    udata_writeBlock(out, (const void *)aliasArrLists, (2 * converterCount * sizeof(uint16_t)));
   1057 
   1058    /* Offset the next array to make the index start at 1. */
   1059    udata_write16(out, 0xDEAD);
   1060 
   1061    /* Write the lists */
   1062    udata_writeBlock(out, (const void *)aliasLists, aliasListsSize * sizeof(uint16_t));
   1063 
   1064    /* Write any options for the alias table. */
   1065    udata_writeBlock(out, (const void *)&tableOptions, sizeof(tableOptions));
   1066 
   1067    /* write the tags strings */
   1068    udata_writeString(out, tagBlock.store, tagBlock.top);
   1069 
   1070    /* write the aliases strings */
   1071    udata_writeString(out, stringBlock.store, stringBlock.top);
   1072 
   1073    /* write the normalized aliases strings */
   1074    if (tableOptions.stringNormalizationType != UCNV_IO_UNNORMALIZED) {
   1075        char *normalizedStrings = (char *)uprv_malloc(tagBlock.top + stringBlock.top);
   1076        createNormalizedAliasStrings(normalizedStrings, tagBlock.store, tagBlock.top);
   1077        createNormalizedAliasStrings(normalizedStrings + tagBlock.top, stringBlock.store, stringBlock.top);
   1078 
   1079        /* Write out the complete normalized array. */
   1080        udata_writeString(out, normalizedStrings, tagBlock.top + stringBlock.top);
   1081        uprv_free(normalizedStrings);
   1082    }
   1083 
   1084    uprv_free(uniqueAliasesToConverter);
   1085    uprv_free(uniqueAliases);
   1086    uprv_free(aliasArrLists);
   1087 }
   1088 
   1089 static char *
   1090 allocString(StringBlock *block, const char *s, int32_t length) {
   1091    uint32_t top;
   1092    char *p;
   1093 
   1094    if(length<0) {
   1095        length=(int32_t)uprv_strlen(s);
   1096    }
   1097 
   1098    /*
   1099     * add 1 for the terminating NUL
   1100     * and round up (+1 &~1)
   1101     * to keep the addresses on a 16-bit boundary
   1102     */
   1103    top=block->top + (uint32_t)((length + 1 + 1) & ~1);
   1104 
   1105    if(top >= block->max) {
   1106        fprintf(stderr, "%s:%d: error: out of memory\n", path, lineNum);
   1107        exit(U_MEMORY_ALLOCATION_ERROR);
   1108    }
   1109 
   1110    /* get the pointer and copy the string */
   1111    p = block->store + block->top;
   1112    uprv_memcpy(p, s, length);
   1113    p[length] = 0; /* NUL-terminate it */
   1114    if((length & 1) == 0) {
   1115        p[length + 1] = 0; /* set the padding byte */
   1116    }
   1117 
   1118    /* check for invariant characters now that we have a NUL-terminated string for easy output */
   1119    if(!uprv_isInvariantString(p, length)) {
   1120        fprintf(stderr, "%s:%d: error: the name %s contains not just invariant characters\n", path, lineNum, p);
   1121        exit(U_INVALID_TABLE_FORMAT);
   1122    }
   1123 
   1124    block->top = top;
   1125    return p;
   1126 }
   1127 
   1128 static int
   1129 compareAliases(const void *alias1, const void *alias2) {
   1130    /* Names like IBM850 and ibm-850 need to be sorted together */
   1131    int result = ucnv_compareNames(GET_ALIAS_STR(*(uint16_t*)alias1), GET_ALIAS_STR(*(uint16_t*)alias2));
   1132    if (!result) {
   1133        /* Sort the shortest first */
   1134        return (int)uprv_strlen(GET_ALIAS_STR(*(uint16_t*)alias1)) - (int)uprv_strlen(GET_ALIAS_STR(*(uint16_t*)alias2));
   1135    }
   1136    return result;
   1137 }
   1138 
   1139 /*
   1140 * Hey, Emacs, please set the following:
   1141 *
   1142 * Local Variables:
   1143 * indent-tabs-mode: nil
   1144 * End:
   1145 *
   1146 */
	tor-browser The Tor Browser
	git clone https://git.dasho.dev/tor-browser.git
	Log \| Files \| Refs \| README \| LICENSE