tor-browser

The Tor Browser
git clone https://git.dasho.dev/tor-browser.git
Log | Files | Refs | README | LICENSE

make-normalize-generateddata-input.py (2929B)

      1 #!/usr/bin/python -B
      2 
      3 """Usage: make-normalize-generateddata-input.py PATH_TO_MOZILLA_CENTRAL
      4 
      5 This script generates test input data for String.prototype.normalize
      6 from intl/icu/source/data/unidata/NormalizationTest.txt
      7 to js/src/tests/non262/String/normalize-generateddata-input.js
      8 """
      9 
     10 import re
     11 import sys
     12 
     13 sep_pat = re.compile(" +")
     14 
     15 
     16 def to_code_list(codes):
     17    return "[" + ", ".join(f"0x{x}" for x in re.split(sep_pat, codes)) + "]"
     18 
     19 
     20 def convert(dir):
     21    ver_pat = re.compile(r"NormalizationTest-([0-9\.]+)\.txt")
     22    part_pat = re.compile("^@(Part([0-9]+) .+)$")
     23    test_pat = re.compile(
     24        "^([0-9A-Fa-f ]+);([0-9A-Fa-f ]+);([0-9A-Fa-f ]+);([0-9A-Fa-f ]+);([0-9A-Fa-f ]+);$"
     25    )
     26    ignore_pat = re.compile("^#|^$")
     27    js_path = "js/src/tests/non262/String/normalize-generateddata-input.js"
     28    txt_path = "intl/icu/source/data/unidata/NormalizationTest.txt"
     29 
     30    part_opened = False
     31    not_empty = False
     32    with open(f"{dir}/{txt_path}") as f:
     33        with open(f"{dir}/{js_path}", "w") as outf:
     34            for line in f:
     35                m = test_pat.search(line)
     36                if m:
     37                    if not_empty:
     38                        outf.write(",")
     39                    outf.write("\n")
     40                    pat = "{{ source: {source}, NFC: {NFC}, NFD: {NFD}, NFKC: {NFKC}, NFKD: {NFKD} }}"  # NOQA: E501
     41                    outf.write(
     42                        pat.format(
     43                            source=to_code_list(m.group(1)),
     44                            NFC=to_code_list(m.group(2)),
     45                            NFD=to_code_list(m.group(3)),
     46                            NFKC=to_code_list(m.group(4)),
     47                            NFKD=to_code_list(m.group(5)),
     48                        )
     49                    )
     50                    not_empty = True
     51                    continue
     52                m = part_pat.search(line)
     53                if m:
     54                    desc = m.group(1)
     55                    part = m.group(2)
     56                    if part_opened:
     57                        outf.write("\n];\n")
     58                    outf.write(f"/* {desc} */\n")
     59                    outf.write(f"var tests_part{part} = [")
     60                    part_opened = True
     61                    not_empty = False
     62                    continue
     63                m = ver_pat.search(line)
     64                if m:
     65                    ver = m.group(1)
     66                    outf.write(f"/* created from NormalizationTest-{ver}.txt */\n")
     67                    continue
     68                m = ignore_pat.search(line)
     69                if m:
     70                    continue
     71                print(f"Unknown line: {line}", file=sys.stderr)
     72            if part_opened:
     73                outf.write("\n];\n")
     74 
     75 
     76 if __name__ == "__main__":
     77    if len(sys.argv) < 2:
     78        print(
     79            "Usage: make-normalize-generateddata-input.py PATH_TO_MOZILLA_CENTRAL",
     80            file=sys.stderr,
     81        )
     82        sys.exit(1)
     83    convert(sys.argv[1])