conversion.txt (79403B)
1 // Copyright (C) 2016 and later: Unicode, Inc. and others. 2 // License & terms of use: http://www.unicode.org/copyright.html 3 //******************************************************************************* 4 // 5 // Copyright (C) 2003-2015, International Business Machines 6 // Corporation and others. All Rights Reserved. 7 // 8 // file name: conversion.txt 9 // encoding: US-ASCII 10 // tab size: 8 (not used) 11 // indentation:4 12 // 13 // created on: 2003jul15 14 // created by: Markus W. Scherer 15 // 16 // ICU resource bundle source file with test data for data-driven conversion tests. 17 // 18 //******************************************************************************* 19 20 conversion:table(nofallback) { 21 Info { 22 Description { "Test data for conversion" } 23 LongDescription { 24 "Test data for data-driven conversion tests in icu/source/test/intltest/convtest.cpp\n" 25 "Run intltest conversion\n" 26 27 "Charset names starting with '*' are for testdata names.\n" 28 "Charset names starting with '+' are for charsets currently not supported in ICU4J.\n" 29 30 "ICU callbacks are specified as strings with pairs of characters, each optional.\n" 31 "Callback function - '?'=Sub '0'=Skip '.'=Stop '&'=Escape\n" 32 "Callback option - a letter is passed in directly as const char * see ucnv_err.h\n" 33 "Empty string: Sub callback with NULL option\n" 34 35 "In order to specify a charset substitution character (for ucnv_setSubstChars()),\n" 36 "add a NUL (U+0000) to the callback string followed by the subchar bytes as Latin-1\n" 37 "characters. For example, for a Sub callback with no option and a subchar of FC FC,\n" 38 "use the string \"?\x00\xFC\xFC\"\n" 39 40 "In order to specify a substitution string (for ucnv_setSubstString()),\n" 41 "add an '=' to the callback string followed by the substitution string.\n" 42 "For example, for a Sub callback with no option and a substitution string\n" 43 "of \"ab\", use the string \"?=ab\"\n" 44 45 "fallbacks: per-direction boolean, currently only for fromUnicode; see Jitterbug 2401\n" 46 47 "errorCode: (empty)==zero | invalid | illegal | truncated | illesc | unsuppesc\n" 48 } 49 } 50 TestData { 51 toUnicode { 52 Headers { "charset", "bytes", "unicode", "offsets", "flush", "fallbacks", "errorCode", "callback", "invalidChars" } 53 Cases { 54 // Test ticket 6789: implement Java-compatible Unicode, UnicodeBig and UnicodeLittle converters 55 // For details about these encodings see convrtrs.txt. 56 // Standard UTF-16 57 { "UTF-16", :bin{ 00610062 }, "ab", :intvector{ 0,2 }, :int{1}, :int{0}, "", "&C", :bin{""} } 58 { "UTF-16", :bin{ feff0061 }, "a", :intvector{ 2 }, :int{1}, :int{0}, "", "&C", :bin{""} } 59 { "UTF-16", :bin{ fffe0061 }, "\u6100", :intvector{ 2 }, :int{1}, :int{0}, "", "&C", :bin{""} } 60 // Java "Unicode" requires a BOM 61 { "+UTF-16,version=1", :bin{ 00610062 }, "\\x00\\x61b", :intvector{ 0,0,0,0,0,0,0,0,2 }, :int{1}, :int{0}, "", "&C", :bin{""} } 62 { "+UTF-16,version=1", :bin{ feff0061 }, "a", :intvector{ 2 }, :int{1}, :int{0}, "", "&C", :bin{""} } 63 { "+UTF-16,version=1", :bin{ fffe0061 }, "\u6100", :intvector{ 2 }, :int{1}, :int{0}, "", "&C", :bin{""} } 64 // Standard UTF-16BE 65 { "UTF-16BE", :bin{ 00610062 }, "ab", :intvector{ 0,2 }, :int{1}, :int{0}, "", "&C", :bin{""} } 66 { "UTF-16BE", :bin{ feff0061 }, "\ufeffa", :intvector{ 0,2 }, :int{1}, :int{0}, "", "&C", :bin{""} } 67 { "UTF-16BE", :bin{ fffe0061 }, "\ufffea", :intvector{ 0,2 }, :int{1}, :int{0}, "", "&C", :bin{""} } 68 // Java "UnicodeBig" requires a BE BOM or no BOM; it consumes the BE BOM 69 { "UTF-16BE,version=1",:bin{ 00610062 }, "ab", :intvector{ 0,2 }, :int{1}, :int{0}, "", "&C", :bin{""} } 70 { "UnicodeBig", :bin{ feff0061 }, "a", :intvector{ 2 }, :int{1}, :int{0}, "", "&C", :bin{""} } 71 { "UnicodeBig", :bin{ fffe0061 }, "\\xFF\\xFEa", :intvector{ 0,0,0,0,0,0,0,0,2 }, :int{1}, :int{0}, "", "&C", :bin{""} } 72 // Standard UTF-16LE 73 { "UTF-16LE", :bin{ 61006200 }, "ab", :intvector{ 0,2 }, :int{1}, :int{0}, "", "&C", :bin{""} } 74 { "UTF-16LE", :bin{ fffe6100 }, "\ufeffa", :intvector{ 0,2 }, :int{1}, :int{0}, "", "&C", :bin{""} } 75 { "UTF-16LE", :bin{ feff6100 }, "\ufffea", :intvector{ 0,2 }, :int{1}, :int{0}, "", "&C", :bin{""} } 76 // Java "UnicodeLittle" requires an LE BOM or no BOM; it consumes the LE BOM 77 { "UTF-16LE,version=1",:bin{ 61006200 }, "ab", :intvector{ 0,2 }, :int{1}, :int{0}, "", "&C", :bin{""} } 78 { "UnicodeLittle", :bin{ fffe6100 }, "a", :intvector{ 2 }, :int{1}, :int{0}, "", "&C", :bin{""} } 79 { "x-UTF-16LE-BOM", :bin{ feff6100 }, "\\xFE\\xFFa", :intvector{ 0,0,0,0,0,0,0,0,2 }, :int{1}, :int{0}, "", "&C", :bin{""} } 80 81 // Test ticket 7704: implement Java-compatible "UTF-16" converter. 82 // Same as standard UTF-16 but fromUnicode always writes big-endian byte stream. 83 { "+UTF-16,version=2", :bin{ 00610062 }, "ab", :intvector{ 0,2 }, :int{1}, :int{0}, "", "&C", :bin{""} } 84 { "+UTF-16,version=2", :bin{ feff0061 }, "a", :intvector{ 2 }, :int{1}, :int{0}, "", "&C", :bin{""} } 85 { "+UTF-16,version=2", :bin{ fffe0061 }, "\u6100", :intvector{ 2 }, :int{1}, :int{0}, "", "&C", :bin{""} } 86 87 // Test ticket 5691: consistent illegal sequences 88 // The following test cases are for illegal character byte sequences. 89 // 90 // Unfortunately, we cannot use the Shift-JIS examples from the ticket 91 // comments because our Shift-JIS table is Windows-compatible and 92 // therefore has no illegal single bytes. Same for GBK. 93 // Instead, we use the stricter GB 18030 also for 2-byte examples. 94 // The byte sequences are generally slightly different from the ticket 95 // comment, simply using assigned characters rather than just 96 // theoretically valid sequences. 97 { 98 "gb18030", 99 :bin{ 618140813c81ff7a }, 100 "a\u4e02\\x81<\\x81\\xFFz", 101 :intvector{ 0,1,3,3,3,3,4,5,5,5,5,5,5,5,5,7 }, 102 :int{1}, :int{0}, "", "&C", :bin{""} 103 } 104 { 105 "EUC-JP", 106 :bin{ 618fb0a98fb03c8f3cb0a97a }, 107 "a\u4e28\\x8F\\xB0<\\x8F<\u9022z", 108 :intvector{ 0,1,4,4,4,4,5,5,5,5,6,7,7,7,7,8,9,11 }, 109 :int{1}, :int{0}, "", "&C", :bin{""} 110 } 111 { // gb18030-2022 changes mapping for 0xFE90 112 "gb18030", 113 :bin{ 618130fc318130fc8181303c3e813cfc817afe90a8bc }, 114 "a\u05ed\\x810\u9f07\\x810<>\\x81<\u9f07z\u9fba\u1e3f", 115 :intvector{ 0,1,5,5,5,5,6,7,9,9,9,9,10,11,12,13,13,13,13,14,15,17,18,20 }, 116 :int{1}, :int{0}, "", "&C", :bin{""} 117 } 118 // GB18030: ICU-22420 adds two reverse fallbacks 119 { 120 "gb18030", 121 :bin{ 80a1a1a2e3a3a0 }, 122 "\u20AC\u3000\u20AC\u3000", 123 :intvector{ 0,1,3,5 }, 124 :int{1}, :int{0}, "", "&C", :bin{""} 125 } 126 { 127 "UTF-8", 128 :bin{ 61f1808182f180813cf18081fff180ff3cf1ff3c3e7a }, 129 "a\U00040042\\xF1\\x80\\x81<\\xF1\\x80\\x81\\xFF\\xF1\\x80\\xFF<\\xF1\\xFF<>z", 130 :intvector{ 0,1,1,5,5,5,5,5,5,5,5,5,5,5,5,8,9,9,9,9,9,9,9,9,9,9,9,9,12,12,12,12,13,13,13,13,13,13,13,13,15,15,15,15,16,17,17,17,17,18,18,18,18,19,20,21 }, 131 :int{1}, :int{0}, "", "&C", :bin{""} 132 } 133 { 134 "ISO-2022-JP", 135 :bin{ 1b24424141af4142affe41431b2842 }, 136 "\u758f\\xAF\u758e\\xAF\\xFE\u790e", 137 :intvector{ 3,5,5,5,5,6,8,8,8,8,8,8,8,8,10 }, 138 :int{1}, :int{0}, "", "&C", :bin{""} 139 } 140 { 141 "ibm-25546", 142 :bin{ 411b242943420e4141af4142affe41430f5a }, 143 "AB\uc88b\\xAF\uc88c\\xAF\\xFE\uc88dZ", 144 :intvector{ 0,5,7,9,9,9,9,10,12,12,12,12,12,12,12,12,14,17 }, 145 :int{1}, :int{0}, "", "&C", :bin{""} 146 } 147 { 148 "ISO-2022-KR", 149 :bin{ 411b242943420e4141af4142affe41430f5a }, 150 "AB\uc88b\\xAF\uc88c\\xAF\\xFE\uc88dZ", 151 :intvector{ 0,5,7,9,9,9,9,10,12,12,12,12,12,12,12,12,14,17 }, 152 :int{1}, :int{0}, "", "&C", :bin{""} 153 } 154 { 155 "ISO-2022-CN", 156 :bin{ 411b242941420e4141af4142affe41430f5a }, 157 "AB\u4eae\\xAF\u8c05\\xAF\\xFE\u64a9Z", 158 :intvector{ 0,5,7,9,9,9,9,10,12,12,12,12,12,12,12,12,14,17 }, 159 :int{1}, :int{0}, "", "&C", :bin{""} 160 } 161 { 162 "ISO-2022-CN-CNS", 163 :bin{ 411b2429470e21702541256f0f }, 164 "A\u00a7\u03c4\u02c7", 165 :intvector{ 0,6,8,10 }, 166 :int{1}, :int{0}, "", "&C", :bin{""} 167 } 168 { 169 "HZ", 170 :bin{ 417e7b4141af4142affe41437e7d5a }, 171 "A\u4eae\\xAF\u8c05\\xAF\\xFE\u64a9Z", 172 :intvector{ 0,3,5,5,5,5,6,8,8,8,8,8,8,8,8,10,14 }, 173 :int{1}, :int{0}, "", "&C", :bin{""} 174 } 175 // Test ticket 5691: consistent illegal sequences 176 // The following test cases are for illegal escape/designator/shift sequences. 177 // 178 // ISO-2022-JP and -CN with illegal escape sequences. 179 { 180 "ISO-2022-JP", 181 :bin{ 611b24201b244241411b283f1b28427a }, 182 "a\\x1B$ \u758f\\x1B\u2538z", 183 :intvector{ 0,1,1,1,1,2,3,7,9,9,9,9,10,15 }, 184 :int{1}, :int{0}, "", "&C", :bin{""} 185 } 186 { 187 "ISO-2022-CN", 188 :bin{ 611b2429201b2429410e41410f7a }, 189 "a\\x1B$) \u4eaez", 190 :intvector{ 0,1,1,1,1,2,3,4,10,13 }, 191 :int{1}, :int{0}, "", "&C", :bin{""} 192 } 193 // Test ticket 5691: ISO-2022-JP-2 with illegal single-shift SS2 and SS3 sequences. 194 // The first ESC N comes before its designator sequence, the last sequence is ESC+space. 195 { 196 "ISO-2022-JP-2", 197 :bin{ 4e1b4e4e1b2e414e1b4e4e4e1b204e }, 198 "N\\x1BNNN\xceN\\x1B N", 199 :intvector{ 0,1,1,1,1,2,3,7,10,11,12,12,12,12,13,14 }, 200 :int{1}, :int{0}, "", "&C", :bin{""} 201 } 202 { 203 "ISO-2022-CN-EXT", 204 :bin{ 4e1b4e4e1b242a484e1b4e4e4e4e1b204e }, 205 "N\\x1BNNN\u8f0eN\\x1B N", 206 :intvector{ 0,1,1,1,1,2,3,8,11,13,14,14,14,14,15,16 }, 207 :int{1}, :int{0}, "", "&C", :bin{""} 208 } 209 /* 210 * ICU 4.4 (ticket #7314) removes mappings for CNS 11643 planes 3..7 211 { 212 "ISO-2022-CN-EXT", 213 :bin{ 4f1b4f4f1b242b494f1b4f4f4f4f1b204f }, 214 "O\\x1BOOO\u492bO\\x1B O", 215 :intvector{ 0,1,1,1,1,2,3,8,11,13,14,14,14,14,15,16 }, 216 :int{1}, :int{0}, "", "&C", :bin{""} 217 } 218 */ 219 // Test ticket 5691: HZ with illegal tilde sequences. 220 { 221 "HZ", 222 :bin{ 417e20427e21437e80447e7b41417e207e41427e7f41437e7d5a }, 223 "A\\x7E B\\x7E!C\\x7E\\x80D\u4eae\\x7E\\x20\\x7E\u8c05\\x7E\\x7F\u64a9Z", 224 :intvector{ 0,1,1,1,1,2,3,4,4,4,4,5,6,7,7,7,7,7,7,7,7,9, // SBCS 225 12,14,14,14,14,14,14,14,14,16,16,16,16,17,19,19,19,19,19,19,19,19,21, // DBCS 226 25 }, // SBCS 227 :int{1}, :int{0}, "", "&C", :bin{""} 228 } 229 // Test ticket 5691: Example from Peter Edberg. 230 { 231 "ISO-2022-JP", 232 :bin{ 1b244230212f7e742630801b284a621b2458631b2842648061 }, 233 "\u4e9c\ufffd\u7199\ufffdb\ufffd$Xcd\ufffda", 234 :intvector{ 3,5,7,9,14,15,16,17,18,22,23,24 }, 235 :int{1}, :int{0}, "", "?", :bin{""} 236 } 237 // Test bug 6071 (2:1 Unicode:charset SBCS mapping). 238 { 239 "*test1bmp", 240 :bin{ 050008 }, 241 "e@uv", 242 :intvector{ 0,1,2,2 }, 243 :int{1}, :int{1}, "", "?", :bin{""} 244 } 245 // test that HZ limits its byte values to lead bytes 21..7d and trail bytes 21..7e 246 { 247 "HZ", 248 :bin{ 7e7b21212120217e217f772100007e217e7e7d207e7e807e0a2b }, 249 "\u3000\ufffd\u3013\ufffd\u9ccc\ufffd\ufffd\u3013 ~\ufffd+", 250 :intvector{ 2,4,6,8,10,12,14,15,19,20,22,25 }, 251 :int{1}, :int{1}, "", "?", :bin{""} 252 } 253 // improve coverage of ISO-2022-JP converter with hardcoded JIS X 0201 and 254 // using the Shift-JIS table for JIS X 0208 (ticket #5797) 255 { 256 "ISO-2022-JP", 257 :bin{ 1b284a7d7e801b2442306c20217f7e21202160217f22202225227f5f211b2842 }, 258 "}\u203e\ufffd\u4e00\ufffd\ufffd\ufffd\ufffd\xf7\ufffd\ufffd\u25b2\ufffd\u6f3e", 259 :intvector{ 3,4,5,9,11,12,14,16,17,19,21,23,25,27 }, 260 :int{1}, :int{1}, "", "?", :bin{""} 261 } 262 // improve coverage of ISO-2022-JP converter by simulating erroneous input 263 { 264 "ISO-2022-JP-2", 265 :bin{ 0f0ed11b2e41461b244141411b4e411b2e4147451b4ed31b2e4641411b4ed2 }, 266 "\u0046\u4eae\u00c1\u6865\u4eae", 267 :intvector{ 6, 10, 14, 18, 26 }, 268 :int{1}, :int{0}, "", "0", :bin{""} 269 } 270 // improve coverage of JIS7 converter by simulating incomplete shifted input 271 { 272 "JIS7", 273 :bin{ 0e11 }, 274 "", 275 :intvector{}, 276 :int{1}, :int{0}, "", "0", :bin{""} 277 } 278 // improve coverage of unrolled loops in ucnvmbcs.c/ucnv_MBCSSingleToBMPWithOffsets() 279 { 280 "ISO-8859-3", 281 :bin{ 000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f2021222324252627 }, 282 "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f !\x22#$%&'", 283 :intvector{ 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39 }, 284 :int{1}, :int{0}, "", "?", :bin{""} 285 } 286 // test that ISO-2022-JP encodes ASCII as itself 287 { 288 "ISO-2022-JP", 289 :bin{ 3f4041424344454647 }, 290 "?@ABCDEFG", 291 :intvector{ 0,1,2,3,4,5,6,7,8 }, 292 :int{1}, :int{1}, "", "?", :bin{""} 293 } 294 // test that ISO-2022-CN encodes ASCII as itself 295 { 296 "ISO-2022-CN", 297 :bin{ 3f4041424344454647 }, 298 "?@ABCDEFG", 299 :intvector{ 0,1,2,3,4,5,6,7,8 }, 300 :int{1}, :int{1}, "", "?", :bin{""} 301 } 302 303 // ISO-2022-KR 304 305 // truncated, partial escape sequence 306 { 307 "ibm-25546", 308 :bin{ 1b }, "", :intvector{}, 309 :int{1}, :int{1}, "truncated", ".", :bin{ 1b } 310 } 311 { 312 "ibm-25546", 313 :bin{ 1b24 }, "", :intvector{}, 314 :int{1}, :int{1}, "truncated", ".", :bin{ 1b24 } 315 } 316 { 317 "ibm-25546", 318 :bin{ 1b2429 }, "", :intvector{}, 319 :int{1}, :int{1}, "truncated", ".", :bin{ 1b2429 } 320 } 321 // complete escape sequence but nothing else 322 { 323 "ibm-25546", 324 :bin{ 1b242943 }, "", :intvector{}, 325 :int{1}, :int{1}, "", ".", :bin{""} 326 } 327 { 328 "ibm-25546", 329 :bin{ 1b2429430e }, "", :intvector{}, 330 :int{1}, :int{1}, "", ".", :bin{""} 331 } 332 // escape plus ASCII character 333 { 334 "ibm-25546", 335 :bin{ 1b24294341 }, "A", :intvector{ 4 }, 336 :int{1}, :int{1}, "", ".", :bin{""} 337 } 338 // escape plus incomplete DBCS character 339 { 340 "ibm-25546", 341 :bin{ 1b2429430e41 }, "", :intvector{}, 342 :int{1}, :int{1}, "truncated", ".", :bin{ 41 } 343 } 344 // all complete with DBCS character 345 { 346 "ibm-25546", 347 :bin{ 1b2429430e4141 }, "\uc88b", :intvector{ 5 }, 348 :int{1}, :int{1}, "", ".", :bin{""} 349 } 350 // more complicated example 351 { 352 "ibm-25546", 353 :bin{ 411b242943420e4141affe0f43 }, 354 "AB\uc88b%XAF%XFEC", 355 :intvector{ 0, 5, 7, 9, 9, 9, 9, 9, 9, 9, 9, 12 }, 356 :int{1}, :int{1}, "", "&", :bin{""} 357 } 358 359 // truncated, partial escape sequence 360 { 361 "ISO-2022-KR", 362 :bin{ 1b }, "", :intvector{}, 363 :int{1}, :int{1}, "truncated", ".", :bin{ 1b } 364 } 365 { 366 "ISO-2022-KR", 367 :bin{ 1b24 }, "", :intvector{}, 368 :int{1}, :int{1}, "truncated", ".", :bin{ 1b24 } 369 } 370 { 371 "ISO-2022-KR", 372 :bin{ 1b2429 }, "", :intvector{}, 373 :int{1}, :int{1}, "truncated", ".", :bin{ 1b2429 } 374 } 375 // complete escape sequence but nothing else 376 { 377 "ISO-2022-KR", 378 :bin{ 1b242943 }, "", :intvector{}, 379 :int{1}, :int{1}, "", ".", :bin{""} 380 } 381 { 382 "ISO-2022-KR", 383 :bin{ 1b2429430e }, "", :intvector{}, 384 :int{1}, :int{1}, "", ".", :bin{""} 385 } 386 // escape plus ASCII character 387 { 388 "ISO-2022-KR", 389 :bin{ 1b24294341 }, "A", :intvector{ 4 }, 390 :int{1}, :int{1}, "", ".", :bin{""} 391 } 392 // escape plus incomplete DBCS character 393 { 394 "ISO-2022-KR", 395 :bin{ 1b2429430e41 }, "", :intvector{}, 396 :int{1}, :int{1}, "truncated", ".", :bin{ 41 } 397 } 398 // all complete with DBCS character 399 { 400 "ISO-2022-KR", 401 :bin{ 1b2429430e4141 }, "\uc88b", :intvector{ 5 }, 402 :int{1}, :int{1}, "", ".", :bin{""} 403 } 404 // more complicated example 405 { 406 "ISO-2022-KR", 407 :bin{ 411b242943420e4141affe0f43 }, 408 "AB\uc88b%XAF%XFEC", 409 :intvector{ 0, 5, 7, 9, 9, 9, 9, 9, 9, 9, 9, 12 }, 410 :int{1}, :int{1}, "", "&", :bin{""} 411 } 412 // empty segment (using substitution and stop) 413 { 414 "ISO-2022-KR", 415 :bin{ 1b242943610e0f620d0a }, 416 "a\uFFFDb\u000D\u000A", 417 :intvector{ 4, 6, 7, 8, 9 }, 418 :int{1}, :int{1}, "", "?", :bin{""} 419 } 420 { 421 "ISO-2022-KR", 422 :bin{ 1b242943610e0f620d0a }, 423 "a", 424 :intvector{ 4 }, 425 :int{1}, :int{1}, "illesc", ".", :bin{"0f"} 426 } 427 428 // ISO-2022-JP 429 430 // truncated, partial escape sequence 431 { 432 "ISO-2022-JP", 433 :bin{ 1b }, "", :intvector{}, 434 :int{1}, :int{1}, "truncated", ".", :bin{ 1b } 435 } 436 { 437 "ISO-2022-JP-2", 438 :bin{ 1b24 }, "", :intvector{}, 439 :int{1}, :int{1}, "truncated", ".", :bin{ 1b24 } 440 } 441 // complete escape sequence but nothing else 442 { 443 "ISO-2022-JP-2", 444 :bin{ 1b2442 }, "", :intvector{}, 445 :int{1}, :int{1}, "", ".", :bin{""} 446 } 447 // escape plus incomplete DBCS character 448 { 449 "ISO-2022-JP-2", 450 :bin{ 1b244241 }, "", :intvector{}, 451 :int{1}, :int{1}, "truncated", ".", :bin{ 41 } 452 } 453 // all complete with DBCS character 454 { 455 "ISO-2022-JP-2", 456 :bin{ 1b24424141 }, "\u758f", :intvector{ 3 }, 457 :int{1}, :int{1}, "", ".", :bin{""} 458 } 459 // test the G2 designator & SS2 shift 460 { 461 "ISO-2022-JP-2", 462 :bin{ 431b2e46461b244241411b4e4e353f }, "CF\u758f\u039e\u7591", :intvector{ 0, 4, 8, 12, 13 }, 463 :int{1}, :int{1}, "", ".", :bin{""} 464 } 465 // JIS7 with Katakana 466 { 467 "JIS7", 468 :bin{ 41420e41420f4142 }, "AB\uff81\uff82AB", :intvector{ 0, 1, 3, 4, 6, 7 }, 469 :int{1}, :int{1}, "", ".", :bin{""} 470 } 471 // JIS8 with Katakana 472 { 473 "JIS8", 474 :bin{ 41c15c1b284a5cc242 }, "A\uff81\\\xa5\uff82B", :intvector{ 0, 1, 2, 6, 7, 8 }, 475 :int{1}, :int{1}, "", ".", :bin{""} 476 } 477 // empty segment (using substitution and stop) 478 { 479 "ISO-2022-JP", 480 :bin{ 61621b24421b284263640d0a }, 481 "ab\uFFFDcd\u000D\u000A", 482 :intvector{ 0, 1, 5, 8, 9, 10, 11 }, 483 :int{1}, :int{1}, "", "?", :bin{""} 484 } 485 { 486 "ISO-2022-JP", 487 :bin{ 61621b24421b284263640d0a }, 488 "ab", 489 :intvector{ 0, 1 }, 490 :int{1}, :int{1}, "illesc", ".", :bin{"1b2842"} 491 } 492 493 // ISO-2022-CN 494 495 // truncated, partial escape sequence 496 { 497 "ISO_2022,locale=zh,version=1", 498 :bin{ 1b }, "", :intvector{}, 499 :int{1}, :int{1}, "truncated", ".", :bin{ 1b } 500 } 501 { 502 "ISO_2022,locale=zh,version=1", 503 :bin{ 1b24 }, "", :intvector{}, 504 :int{1}, :int{1}, "truncated", ".", :bin{ 1b24 } 505 } 506 { 507 "ISO_2022,locale=zh,version=1", 508 :bin{ 1b2429 }, "", :intvector{}, 509 :int{1}, :int{1}, "truncated", ".", :bin{ 1b2429 } 510 } 511 // complete escape sequence but nothing else 512 { 513 "ISO_2022,locale=zh,version=1", 514 :bin{ 1b242941 }, "", :intvector{}, 515 :int{1}, :int{1}, "", ".", :bin{""} 516 } 517 { 518 "ISO_2022,locale=zh,version=1", 519 :bin{ 1b2429410e }, "", :intvector{}, 520 :int{1}, :int{1}, "", ".", :bin{""} 521 } 522 // escape plus ASCII character 523 { 524 "ISO_2022,locale=zh,version=1", 525 :bin{ 1b24294141 }, "\x41", :intvector{ 4 }, 526 :int{1}, :int{1}, "", ".", :bin{""} 527 } 528 // escape plus incomplete DBCS character 529 { 530 "ISO_2022,locale=zh,version=1", 531 :bin{ 1b2429410e41 }, "", :intvector{}, 532 :int{1}, :int{1}, "truncated", ".", :bin{ 41 } 533 } 534 // all complete with DBCS character 535 { 536 "ISO_2022,locale=zh,version=1", 537 :bin{ 1b2429410e4141 }, "\u4eae", :intvector{ 5 }, 538 :int{1}, :int{1}, "", ".", :bin{""} 539 } 540 /* 541 * ICU 4.4 (ticket #7314) removes mappings for CNS 11643 planes 3..7 542 // ISO-2022-CN-EXT with all subcharsets and shifts and with supplementary code points 543 { 544 "ISO-2022-CN-EXT", 545 :bin{ 1b2429411b242a480e41411b2429457e7c1b4e70341b242b4d1b2429477c341b4f664c2421 }, 546 "\u4eae\u9f82\u56cd\u56cc\U0002a6d6\x30", 547 :intvector{ 9, 15, 19, 29, 33, 33, 35 }, 548 :int{1}, :int{1}, "", ".", :bin{""} 549 } 550 */ 551 552 // illegal and unsupported escape sequences 553 // SS2 without designator: illegal 554 { 555 "ISO-2022-CN-EXT", 556 :bin{ 411b4e2121 }, "\x41", :intvector{ 0 }, 557 :int{1}, :int{1}, "illesc", ".", :bin{ 1b } 558 } 559 // G3 designator: recognized, but not supported for -CN (only for -CN-EXT) 560 { 561 "ISO-2022-CN", 562 :bin{ 411b242b491b4f2121 }, "\x41", :intvector{ 0 }, 563 :int{1}, :int{1}, "unsuppesc", ".", :bin{ 1b242b49 } 564 } 565 // empty segment 1 (using substitution and stop) 566 { 567 "ISO-2022-CN", 568 :bin{ 611b242941620e0f1b242a481b4e6a65630d0a }, 569 "ab\uFFFD\u994Cc\u000D\u000A", 570 :intvector{ 0, 5, 7, 14, 16, 17, 18 }, 571 :int{1}, :int{1}, "", "?", :bin{""} 572 } 573 { 574 "ISO-2022-CN", 575 :bin{ 611b242941620e0f1b242a481b4e6a65630d0a }, 576 "ab", 577 :intvector{ 0, 5 }, 578 :int{1}, :int{1}, "illesc", ".", :bin{"0f"} 579 } 580 // empty segment 2 (using substitution and stop) 581 { 582 "ISO-2022-CN", 583 :bin{ 611b242941620e1b24294768640f630d0a }, 584 "ab\uFFFD\u5F70c\u000D\u000A", 585 :intvector{ 0, 5, 7, 11, 14, 15, 16 }, 586 :int{1}, :int{1}, "", "?", :bin{""} 587 } 588 { 589 "ISO-2022-CN", 590 :bin{ 611b242941620e1b24294768640f630d0a }, 591 "ab", 592 :intvector{ 0, 5 }, 593 :int{1}, :int{1}, "illesc", ".", :bin{"1b242947"} 594 } 595 596 // ISO-2022 SBCS 597 // [U_ENABLE_GENERIC_ISO_2022] 598 // The _generic_ ISO-2022 converter is disabled starting 2003-dec-03 (ICU 2.8). 599 // For details see the icu mailing list from 2003-dec-01 and the ucnv2022.c file. 600 // Language-specific variants of ISO-2022 continue to be available as listed below. 601 //{ 602 // "ISO_2022", 603 // :bin{ 0008090a0d1a1c1f203f415c7d7e7f }, 604 // "\x00\x08\t\n\r\x1a\x1c\x1f ?A\\}~\x7f", 605 // :intvector{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14 }, 606 // :int{1}, :int{1}, "", ".", :bin{""} 607 //} 608 609 // HZ-GB-2312 610 611 // empty segment 1 (using substitution and stop) 612 { 613 "HZ-GB-2312", 614 :bin{ 61627e7b7e7d6364 }, 615 "ab\uFFFDcd", 616 :intvector{ 0, 1, 4, 6, 7 }, 617 :int{1}, :int{1}, "", "?", :bin{""} 618 } 619 { 620 "HZ-GB-2312", 621 :bin{ 61627e7b7e7d63640d0a }, 622 "ab", 623 :intvector{ 0, 1 }, 624 :int{1}, :int{1}, "illesc", ".", :bin{"7e7d"} 625 } 626 // empty segment 2 & legal redundant switches (using substitution and stop) 627 { 628 "HZ-GB-2312", 629 :bin{ 61627e7b323b3f557e7b7e7b523b7e7d63647e7d65667e7d7e7d }, 630 "ab\u4E0D\u7A7A\uFFFD\u4E00cdef\uFFFD", 631 :intvector{ 0, 1, 4, 6, 10, 12, 16, 17, 20, 21, 24 }, 632 :int{1}, :int{1}, "", "?", :bin{""} 633 } 634 { 635 "HZ-GB-2312", 636 :bin{ 61627e7b323b3f557e7b7e7b523b7e7d63647e7d65667e7d7e7d }, 637 "ab\u4E0D\u7A7A", 638 :intvector{ 0, 1, 4, 6 }, 639 :int{1}, :int{1}, "illesc", ".", :bin{"7e7b"} 640 } 641 642 // DBCS-only extensions 643 { 644 "ibm-970", 645 :bin{ 617eece9b2eb }, 646 "\x61\x7e\u4e00\ub000", 647 :intvector{ 0, 1, 2, 4 }, 648 :int{1}, :int{1}, "", "?", :bin{""} 649 } 650 651 { 652 "ibm-971", 653 :bin{ 617eece9b2eb }, 654 "\ufffd\u4e00\ub000", 655 :intvector{ 0, 2, 4 }, 656 :int{1}, :int{1}, "", "?", :bin{""} 657 } 658 659 { 660 "ibm-16684", 661 :bin{ 430e4395ecc1404042e1 }, 662 "\ufffd\u30C8\u30C8\u309A\u3000\u20ac", 663 :intvector{ 0, 2, 4, 4, 6, 8 }, 664 :int{1}, :int{0}, "", "?", :bin{""} 665 } 666 667 { 668 "ibm-1399", 669 :bin{ 430e4395ecc140400fe1 }, 670 "\uff62\u30C8\u30C8\u309A\u3000\u20ac", 671 :intvector{ 0, 2, 4, 4, 6, 9 }, 672 :int{1}, :int{0}, "", "?", :bin{""} 673 } 674 675 // extensions 676 { 677 "ibm-1390", 678 :bin{ 430e4395ecc1 }, 679 "\uff63\u30C8\u30C8\u309A", 680 :intvector{ 0, 2, 4, 4 }, 681 :int{1}, :int{0}, "", "?", :bin{""} 682 } 683 684 { 685 "ibm-16684", 686 :bin{ ececec8bec8cec8d4386ecb5ecb6ecb7 }, 687 "\ufffd\u31f6\u31f7\u31f8\u30ab\u304b\u309a\u304d\u309a\u304f\u309a", 688 :intvector{ 0, 2, 4, 6, 8, 10, 10, 12, 12, 14, 14 }, 689 :int{1}, :int{0}, "", "?", :bin{""} 690 } 691 692 { 693 "ibm-1390", 694 :bin{ 43860eececec8bec8cec8d4386ecb5ecb6ecb7ecc10fec }, 695 "\uff63\uff76\ufffd\u31f6\u31f7\u31f8\u30ab\u304b\u309a\u304d\u309a\u304f\u309a\u30C8\u309A\x1a", 696 :intvector{ 0, 1, 3, 5, 7, 9, 11, 13, 13, 15, 15, 17, 17, 19, 19, 22 }, 697 :int{1}, :int{0}, "", "?", :bin{""} 698 } 699 700 { 701 "*test3", 702 :bin{ 00050601020b0701020a01020c }, 703 "\u20ac\x05\x06\x0b\U00101234\U00023456\ufffd", 704 :intvector{ 0, 1, 2, 3, 6, 6, 7, 7, 10 }, 705 :int{1}, :int{0}, "", "?", :bin{""} 706 } 707 708 // test mapping to sequence of multiple Unicode characters which includes nonBMP (ticket #9235) 709 { 710 "*test3", 711 :bin{ 05070001020e050501020c06 }, 712 "\x05\U00101234\U00050005\u00c4\u00c4\U00101234\x05\x06", 713 :intvector{ 0, 1, 1, 1, 1, 7, 7, 7, 7, 7, 11 }, 714 :int{1}, :int{0}, "", "?", :bin{""} 715 } 716 717 // normal conversions 718 { 719 "UTF-16LE", 720 :bin{ 310000d801dc00d902dc320000d8330001dc3400 }, 721 "1\U00010001\U000500022\ufffd3\ufffd4", 722 :intvector{ 0, 2, 2, 6, 6, 10, 12, 14, 16, 18 }, 723 :int{1}, :int{0}, "", "?", :bin{""} 724 } 725 { "UTF-16LE", :bin{ 00 }, "", :intvector{}, :int{1}, :int{0}, "truncated", ".", :bin{ 00 } } 726 { "UTF-16LE", :bin{ 00d800 }, "", :intvector{}, :int{1}, :int{0}, "truncated", ".", :bin{ 00d800 } } 727 728 { 729 "UTF-16BE", 730 :bin{ 0031d800dc01d900dc020032d8000033dc010034 }, 731 "1\U00010001\U000500022\ufffd3\ufffd4", 732 :intvector{ 0, 2, 2, 6, 6, 10, 12, 14, 16, 18 }, 733 :int{1}, :int{0}, "", "?", :bin{""} 734 } 735 { "UTF-16BE", :bin{ 00 }, "", :intvector{}, :int{1}, :int{0}, "truncated", ".", :bin{ 00 } } 736 { "UTF-16BE", :bin{ d800dc }, "", :intvector{}, :int{1}, :int{0}, "truncated", ".", :bin{ d800dc } } 737 738 // e4b8 is a partial sequence 739 { "UTF-8", :bin{ 31e4ba8ce4b8 }, "1\u4e8c", :intvector{ 0, 1 }, :int{1}, :int{0}, "truncated", ".", :bin{ e4b8 } } 740 { "UTF-8", :bin{ 31e4ba8ce4b8 }, "1\u4e8c\ufffd", :intvector{ 0, 1, 4 }, :int{1}, :int{0}, "", "?", :bin{""} } 741 742 // LMBCS with escape callback (1292a0 is unassigned) 743 { 744 "LMBCS", 745 :bin{ 12c9501292a01292a1 }, 746 "\u4e2e%X12%X92%XA0\ue5c4", 747 :intvector{ 0, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 6 }, 748 :int{1}, :int{0}, "", "&", :bin{""} 749 } 750 { 751 "LMBCS", 752 :bin{ 61012981a00f270f91140a7414f60214d84d14dc561088a0 }, 753 "\u0061\u2013\u00fc\u00e1\u0007\u0091\u0a74\u0200\ud84d\udc56\u5516", 754 :intvector{ 0, 1, 3, 4, 5, 7, 9, 12, 15, 18, 21 }, 755 :int{1}, :int{0}, "", ".", :bin{""} 756 } 757 758 // IMAP-mailbox-name with SUB 759 // a<DEL> a&AB~ a&AB\x0c a&AB- a&AB. a&. 760 { 761 "IMAP-mailbox-name", 762 :bin{ 617f612641427e612641420c612641422d612641422e61262e }, 763 "a\ufffda\ufffda\ufffda\ufffda\ufffda\ufffd", 764 :intvector{ 0, 1, 2, 4, 7, 9, 12, 14, 17, 19, 22, 23 }, 765 :int{1}, :int{0}, "", "?", :bin{""} 766 } 767 768 // using testdata_test1.cnv 769 { "*test1", :bin{ 000506070809 }, "\u20ac\x05\x06\U00101234\ufffd\ufffd", :intvector{ 0, 1, 2, 3, 3, 4, 5 }, :int{1}, :int{0}, "", "", :bin{""} } 770 771 // surrogates in CESU-8 772 { "CESU-8", :bin{ eda080eda081edb081 }, "\ud800\U00010401", :intvector{ 0, 3, 6 }, :int{1}, :int{0}, "", "", :bin{""} } 773 // e080 is a partial sequence 774 { "UTF-8", :bin{ 31ffe4ba8ce08061 }, "1\ufffd\u4e8c\ufffd\ufffda", :intvector{ 0, 1, 2, 5, 6, 7 }, :int{0}, :int{0}, "", "", :bin{ 80 } } 775 // fbbfbfbfbf exceedes U+10ffff 776 { "UTF-8", :bin{ 31fbbfbfbfbf61 }, "1\ufffd\ufffd\ufffd\ufffd\ufffda", :intvector{ 0, 1, 2, 3, 4, 5, 6 }, :int{0}, :int{0}, "", "", :bin{ bf } } 777 778 // lead byte a2 without trail byte 779 { "ibm-1363", :bin{ a2aea2 }, "\u00a1", :intvector{ 0 }, :int{1}, :int{0}, "truncated", ".", :bin{ a2 } } 780 { "ibm-1363", :bin{ a2aea2 }, "\u00a1\u001a", :intvector{ 0, 2 }, :int{1}, :int{0}, "", "?", :bin{""} } 781 782 // simple sample, no error handling 783 { "UTF-8", :bin{ 61F48FBFBF }, "a\U0010FFFF", :intvector{ 0, 1, 1 }, :int{1}, :int{0}, "", "", :bin{""} } 784 // Headers { "charset", "bytes", "unicode", "offsets", "flush", "fallbacks", "errorCode", "callback", "invalidChars" } 785 { 786 "iscii-dev", 787 :bin{ EF4BC0E9BFE9E8D80AEF4AC0D4BFD4E8D80AEF4838B30AEF4939B30AEF4A3AB30AEF4B3BB30A}, 788 "\u0A5C\u0A4D\u0A39\u0A5C\u0A4D\u0A39\u000A" /* Gurmukhi test */ 789 "\u0AA2\u0AB5\u0AA1\u0AB5\u0ACD\u0AB9\u000A" /* Gujarati test */ 790 "\u0038\u0C95\u000A" /* Kannada test */ 791 "\u0039\u0D15\u000A" /* Malayalam test */ 792 "\u003A\u0A95\u000A" /* Gujarati test */ 793 "\u003B\u0A15\u000A" /* Punjabi test */, 794 :intvector { 2, 2, 2, 4, 6, 7, 8, 11, 12, 13, 14, 15, 16, 17, 20, 21, 22, 25, 26, 27, 30, 31, 32, 35, 36, 37 }, 795 :int{1}, :int{0}, "", "", :bin{ "" } 796 } 797 { 798 "iscii-gur", 799 :bin{3BB30AC0E9BFE9E8D80AEF43C0E9BFE9E8D80A3BB30AEF403BB30A}, 800 "\u003b\u0a15\u000a" /* Easy characters */ 801 "\u0a5c\u0a4d\u0a39\u0a5c\u0a4d\u0a39\u000a" /* Gurmukhi test */ 802 "\u09dd\u09dc\u09cd\u09b9\u000a" /* Switch script: to Bengali*/ 803 "\u003b\u0a15\u000a" /* Easy characters - new line" so should default!*/ 804 "\u003b\u0a15\u000a", /* Back to Gurmukhi*/ 805 :intvector { 0, 1, 2, 3, 3, 3, 5, 7, 8, 9, 13, 15, 16, 17, 18, 19, 20, 21, 24, 25, 26 }, 806 :int{1}, :int{0}, "", "", :bin{ "" } 807 } 808 // Test Gurmukhi (Bindi Tippi and Consonant Cluster) 809 { 810 "iscii-gur", 811 :bin{EF4BB3A2A2B3E8B3C0E9BFE9E8D8}, 812 "\u0a15\u0a70\u0a02\u0a71\u0a15\u0a5c\u0a4d\u0a39\u0a5c\u0a4d\u0a39", 813 :intvector { 2, 3, 4, 5, 5, 8, 8, 8, 10, 12, 13 }, 814 :int{1}, :int{0}, "", ".", :bin{ "" } 815 } 816 { // Verify Supplementary support 817 "Big5-HKSCS", 818 :bin{ fcfcfcfd }, 819 "\U000233E6\U00026DA0", 820 :intvector{ 0, 0, 2, 2 }, 821 :int{1}, :int{0}, "", "?", :bin{""} 822 } 823 { // Verify non-supplementary support 824 "big5-hkscs:unicode3.0", 825 :bin{ fcfcfcfd }, 826 "\uE1D4\uE1D5", 827 :intvector{ 0, 2 }, 828 :int{1}, :int{0}, "", "?", :bin{""} 829 } 830 { // Verify gb18030 enumeration 831 "gb18030", 832 :bin{ 8130D2398130D3308136A5318136A532 }, 833 "\u0450\u0452\u200F\u2011", 834 :intvector{ 0, 4, 8, 12 }, 835 :int{1}, :int{0}, "", "?", :bin{""} 836 } 837 { // gb18030->U 2005 vs 2022 part 1 (gb18030 2-byte) 838 "gb18030", 839 :bin{ A6D9 A6DA A6DB A6DC A6DF A6EC A6ED A6F3 FE59 FE61 FE66 FE67 FE6D FE7E FE90 FEA0 }, 840 "\uFE10\uFE12\uFE11\uFE13\uFE16\uFE17\uFE18\uFE19\u9FB4\u9FB5\u9FB6\u9FB7\u9FB8\u9FB9\u9FBA\u9FBB", // -2005: "\uE78D\uE78E\uE78F\uE790\uE793\uE794\uE795\uE796\uE81E\uE826\uE82B\uE82C\uE832\uE843\uE854\uE864" 841 :intvector{ 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30 }, 842 :int{1}, :int{0}, "", "?", :bin{""} 843 } 844 { // gb18030->U 2005 vs 2022 part 2 (gb18030 4-byte) 845 "gb18030", 846 :bin{ 82359037 82359038 82359039 82359130 82359131 82359132 82359133 82359134 84318236 84318239 84318332 84318335 }, 847 "\u9FB4\u9FB5\u9FB6\u9FB7\u9FB8\u9FB9\u9FBA\u9FBB\uFE10\uFE13\uFE16\uFE19", // unchanged from 2005 mapping 848 :intvector{ 0, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44 }, 849 :int{1}, :int{0}, "", "?", :bin{""} 850 } 851 { // gb18030->U 2005 vs 2022 part 3 (gb18030 4-byte), non-changing mappings next to or in linear ranges partially overridden by new explicit maps 852 "gb18030", 853 :bin{ 82358F33823590368235913584318235843183368431843684318537 }, 854 "\u9FA6\u9FB3\u9FBC\uFE0F\uFE1A\uFE24\uFE2F", 855 :intvector{ 0, 4, 8, 12, 16, 20, 24 }, 856 :int{1}, :int{0}, "", "?", :bin{""} 857 } 858 { 859 "x11-compound-text", 860 :bin{ 1b242944b5ac1b2d41a5e31b2d43d5f51b2d4dd01b2d41411b2d43bc1b2d42ff1b2d54df1b2d44c0b31b2d46b41b2d47b01b2d48e01b2d4ca1 }, 861 "\u54A1\u00A5\u00E3\u0120\u0121\u011E\u0041\u0135\u02D9\u0E3F\u0100\u0157\u0384\u0660\u05D0\u0401", 862 :intvector{ }, 863 :int{1}, :int{0}, "", "?", :bin{""} 864 } 865 // Improve coverage of ISCII 866 { 867 "iscii-bng", 868 :bin{ f0a0f0b0b8 }, 869 "", 870 :intvector{}, 871 :int{1}, :int{0}, "", "0", :bin{""} 872 } 873 // Test iso-2022-jp-2 miscellaneous symbols 874 { 875 "iso-2022-jp-2", 876 :bin{ 1b242843224f224e1b2842 }, 877 "\u260E\u260F", 878 :intvector{ 4, 6 }, 879 :int{1}, :int{0}, "", ".", :bin{""} 880 } 881 882 // Improve Code Coverage for BOCU-1 883 { 884 "BOCU-1", 885 :bin{ 91fbc555fd6349 }, 886 "\u0041\ud841\ud888\udc81", 887 :intvector{ 0, 1, 4, 4 }, 888 :int{1}, :int{0}, "", ".", :bin{""} 889 } 890 { 891 "BOCU-1", 892 :bin{ fbeda44ff0fe189bb821f05926 }, 893 "\ufe88\ufe70\udbff\udfff\u0061", 894 :intvector{}, 895 :int{1}, :int{0}, "", ".", :bin{""} 896 } 897 { 898 "BOCU-1", 899 :bin{ 5b4bccf9 }, 900 "\u000b", 901 :intvector{}, 902 :int{1}, :int{0}, "", "0", :bin{""} 903 } 904 { 905 "BOCU-1", 906 :bin{ fe0053c6 }, 907 "\u0003\u0076", 908 :intvector{}, 909 :int{1}, :int{0}, "", "0", :bin{""} 910 } 911 912 //Improve code coverage for SCSU 913 { 914 "SCSU", 915 :bin{ 0fd899dc7fd888dc99e041424361 }, 916 "\ud899\udc7f\ud888\udc99\u0041\u0042\u0043\u0061", 917 :intvector{}, 918 :int{1}, :int{0}, "", "0", :bin{""} 919 } 920 { 921 "SCSU", 922 :bin{ 41df1281035f10df1b03df1c88800bbfffff }, 923 "\u0041\u00df\u0401\u015f\u00df\u01df\uf000\udbff\udfff", 924 :intvector{}, 925 :int{1}, :int{0}, "", ".", :bin{""} 926 } 927 { 928 "SCSU", 929 :bin{ 1b9a1b541bb2411bfd1b0041 }, 930 "", 931 :intvector{}, 932 :int{1}, :int{0}, "", ".", :bin{""} 933 } 934 { 935 "SCSU", 936 :bin{ 0f6441b413a733f2 }, 937 "\u6441\ub413\ua733", 938 :intvector{}, 939 :int{1}, :int{0}, "illegal", ".", :bin{ f2 } 940 } 941 942 //Improve code coverage for MBCS 943 { 944 "*test5", 945 :bin{ 0506 }, 946 "\ufffd\x06", 947 :intvector{}, 948 :int{1}, :int{0}, "", "?", :bin{""} 949 } 950 { 951 "ibm-1390,swaplfnl", 952 :bin{ 430e4395ecc140400fc1e115 }, 953 "\uff63\u30C8\u30C8\u309A\u3000\x41\u20ac\x0a", 954 :intvector{ 0, 2, 4, 4, 6, 9, 10, 11 }, 955 :int{1}, :int{0}, "", "?", :bin{""} 956 } 957 /* 958 * ICU 4.4 (ticket #7314) removes mappings for CNS 11643 planes 3..7 959 { 960 "ISO-2022-CN-EXT", 961 :bin{ 1b242b4d1b4f66791b242b4d1b4f216a }, 962 "\u3667", 963 :intvector{ 14 }, 964 :int{1}, :int{0}, "", "0", :bin{""} 965 } 966 */ 967 { 968 "*test1bmp", 969 :bin{ 060a05 }, 970 "\u0066\ufffd\u0065", 971 :intvector{ 0, 2, 2 }, 972 :int{0}, :int{1}, "", "?", :bin{""} 973 } 974 { 975 "*test1bmp", 976 :bin{ 060708 }, 977 "\u0066", 978 :intvector{}, 979 :int{1}, :int{0}, "invalid", ".", :bin{ 07 } 980 } 981 { 982 "*test5", 983 :bin{ 010304 }, 984 "\u0034\ufffd", 985 :intvector{}, 986 :int{1}, :int{0}, "", "?", :bin{""} 987 } 988 { 989 "*test1", 990 :bin{ 0a0b }, 991 "", 992 :intvector{}, 993 :int{1}, :int{0}, "", "0", :bin{""} 994 } 995 { 996 "*test1bmp", 997 :bin{ 0c06 }, 998 "\u0066", 999 :intvector{}, 1000 :int{0}, :int{0}, "", ".", :bin{""} 1001 } 1002 { 1003 "*test5", 1004 :bin{ 0906 }, 1005 "\udbc8\udf45\u0006", 1006 :intvector{}, 1007 :int{0}, :int{0}, "", ".", :bin{""} 1008 } 1009 { 1010 "ibm-16684", 1011 :bin{ 0e }, 1012 "", 1013 :intvector{}, 1014 :int{0}, :int{0}, "illegal", ".", :bin{ 0e } 1015 } 1016 { 1017 "UTF-7", 1018 :bin{ 2b414b4d2d492b414b4d4170412d }, 1019 "\u00a3I\u00a3\u00a4", 1020 :intvector{ 1,5,7,9 }, 1021 :int{0}, :int{0}, "", ".", :bin{""} 1022 } 1023 { 1024 "x11-compound-text", 1025 :bin{ 1e6ddc9b26bc10801bbcad50a040fc }, 1026 "\u001e\u006d\u00dc\u009b\u0026\u00bc\u0010\u0080\ufffd\u00bc\u00ad\u0050\u00a0\u0040\u00fc", 1027 :intvector{ }, 1028 :int{1}, :int{0}, "", "?", :bin{""} 1029 } 1030 } 1031 } 1032 1033 // --------------------------------------------------------------------- *** 1034 1035 fromUnicode { 1036 Headers { "charset", "unicode", "bytes", "offsets", "flush", "fallbacks", "errorCode", "callback", "invalidUChars" } 1037 Cases { 1038 // Test ticket 9602: Add "good one-way" mapping type (|4). 1039 // Such mappings are used regardless of the fallback flag. 1040 { 1041 "+*test3", "##\uFE0E#\uFE0F", 1042 :bin{ 010204010204010204 }, :intvector{ 0,0,0,1,1,1,3,3,3 }, 1043 :int{1}, :int{0}, // no fallbacks 1044 "", "?", "" 1045 } 1046 { 1047 "+*test3", "##\uFE0E#\uFE0F", 1048 :bin{ 010204010204010204 }, :intvector{ 0,0,0,1,1,1,3,3,3 }, 1049 :int{1}, :int{1}, // with fallbacks 1050 "", "?", "" 1051 } 1052 // Test ticket 6789: implement Java-compatible Unicode, UnicodeBig and UnicodeLittle converters 1053 // For details about these encodings see convrtrs.txt. 1054 // Standard UTF-16BE 1055 { "UTF-16BE", "a", :bin{ 0061 }, :intvector{ 0,0 }, :int{1}, :int{0}, "", "?", "" } 1056 // Java "UnicodeBig" writes a BOM 1057 { "UnicodeBig", "a", :bin{ feff0061 }, :intvector{ -1,-1,0,0 }, :int{1}, :int{0}, "", "?", "" } 1058 // Standard UTF-16LE 1059 { "UTF-16LE", "a", :bin{ 6100 }, :intvector{ 0,0 }, :int{1}, :int{0}, "", "?", "" } 1060 // Java "UnicodeLittle" writes a BOM 1061 { "UnicodeLittle", "a", :bin{ fffe6100 }, :intvector{ -1,-1,0,0 }, :int{1}, :int{0}, "", "?", "" } 1062 1063 // Test ticket 7704: implement Java-compatible "UTF-16" converter. 1064 // Same as standard UTF-16 but fromUnicode always writes big-endian byte stream. 1065 { "+UTF-16,version=2", "a", :bin{ feff0061 }, :intvector{ -1,-1,0,0 }, :int{1}, :int{0}, "", "?", "" } 1066 1067 // Test bug 6071 (1:2 Unicode:charset SBCS mapping). 1068 { 1069 "*test1bmp", 1070 "e@t", 1071 :bin{ 05000709 }, 1072 :intvector{ 0,1,2,2 }, 1073 :int{1}, :int{0}, "", "?", "" 1074 } 1075 // improve coverage of ISO-2022-JP converter with hardcoded JIS X 0201 and 1076 // using the Shift-JIS table for JIS X 0208 (ticket #5797) 1077 { 1078 "ISO-2022-JP", 1079 "\u203e\xa5\u4e00\ufa10\u6f3e\u0391", 1080 :bin{ 1b284a7e5c1b2442306c222e5f2126211b2842 }, 1081 :intvector{ 0,0,0,0,1,2,2,2,2,2,3,3,4,4,5,5,5,5,5 }, 1082 :int{1}, :int{0}, "", "?=\u3013", "" // U+3013 Geta Mark converts to 222e 1083 } 1084 // Verify that mappings that would result in byte values outside 20..7F (for SBCS) 1085 // or 21..7E (for DBCS) are not used. 1086 // ibm-9005_X110-2007.ucm (ISO 8859-7, <ESC>.F=1b2e46): 1087 // <U009F> \x9F |0 (also in ISO 8859-1) 1088 // <U0387> \xB7 |1 1089 // windows-949-2000 (KSC_5601, <ESC>$(C=1b242843): 1090 // <UC829> \xA0\xA1 |0 1091 // <UD4FE> \xC0\x41 |0 1092 // <UD79D> \xC8\xFE |0 1093 { 1094 "JIS8", // =ISO_2022,locale=ja,version=4 1095 "\u009f\u0387\uc829\ud4fe\ud79d", 1096 :bin{ 1a1b2e461b4e371a1a1b242843487e1b2842 }, 1097 :intvector{ 0,1,1,1,1,1,1,2,3,4,4,4,4,4,4,4,4,4 }, 1098 :int{1}, :int{1}, "", "?", "" 1099 } 1100 // Ticket 5483: ISO 2022 converter incorrectly using fallback mapping 1101 // Verify that a roundtrip mapping is used even when a fallback mapping is 1102 // available in the current state. 1103 // U+FF61 is handled in code 1104 // jisx-208.ucm (<ESC>$B=1b2442): 1105 // <U30FE> \x21\x34 |0 1106 // <UFF5D> \x21\x51 |0 and 1107 // ibm-897_P100-1995.ucm (JIS X 0201, <ESC>(J=1b284a): 1108 // <UFF5D> \x7D |1 1109 // ibm-9005_X110-2007.ucm (ISO 8859-7, <ESC>.F=1b2e46): 1110 // <U03D5> \xF6 |1 1111 // <U2015> \xAF |0 1112 // <UFF5D> \x7D |1 (not legal for ISO 2022) 1113 // windows-949-2000 (KSC_5601, <ESC>$(C=1b242843): 1114 // <UAC00> \xB0\xA1 |0 1115 // <UFF5D> \xA3\xFD |0 1116 // <U223C> \xA1\xAD |0 (in extension table) 1117 { 1118 "JIS8", // =ISO_2022,locale=ja,version=4 1119 "a\uff61\u03d5\uff5d\uac00\u223c\uff5d\u30fe\uff5d", // Make it switch to ISO-8859-7, KSC 5601 and JIS X 0208. 1120 :bin{ 61a11b2e461b4e761b244221511b2428433021212d237d1b2442213421511b2842 }, 1121 :intvector{ 0,1,2,2,2,2,2,2,3,3,3,3,3,4,4,4,4,4,4,5,5,6,6,7,7,7,7,7,8,8,8,8,8 }, 1122 :int{1}, :int{1}, "", "?", "" 1123 } 1124 // Code coverage for UTF-8->SBCS conversion (ucnv_convertEx()). 1125 // Test code path for non-roundtripping ASCII characters 1126 // (try EBCDIC SBCS, and IBM PC SBCS with control code rotation). 1127 { 1128 "ibm-37", 1129 "a\x85c", 1130 :bin{ 811583 }, 1131 :intvector{ 0,1,2 }, 1132 :int{1}, :int{0}, "", "?", "" 1133 } 1134 { 1135 "ibm-850", 1136 "a\x1ac", 1137 :bin{ 617f63 }, 1138 :intvector{ 0,1,2 }, 1139 :int{1}, :int{0}, "", "?", "" 1140 } 1141 // Code coverage for UTF-8->DBCS conversion (ucnv_convertEx()). 1142 // Test code path for non-roundtripping ASCII characters 1143 // (try IBM PC DBCS with control code rotation). 1144 { 1145 "ibm-943", 1146 "a\x1ac\u30a1\x7ff", 1147 :bin{ 617f6383401c66 }, 1148 :intvector{ 0,1,2,3,3,4,5 }, 1149 :int{1}, :int{0}, "", "?", "" 1150 } 1151 // SCSU regression test. 1152 { 1153 "SCSU", 1154 "1\U00010001\u0085\U000500022\ud8003\udc014\ue001", 1155 :bin{ 310be0008102050fd900dc02e7320efffd330efffd34186881 }, 1156 :intvector{ 0,1,1,1,1,3,3,4,4,4,4,4,6,6,7,7,7,8,9,9,9,10,11,11,11 }, 1157 :int{1}, :int{0}, "", "?", "" 1158 } 1159 // LMBCS 1160 { 1161 "lmbcs", 1162 "\u0061\u2013\u00fc\u00e1\u0007\u0091\u0a74\u0200\ud84d\udc56\u5516", 1163 :bin { 61039681a00f270f91140a7414f60214d84d14dc561088a0 }, 1164 :intvector{ 0, 1, 1, 2, 3, 4, 4, 5, 5, 6, 6, 6, 7, 7, 7, 8, 8, 8, 9, 9, 9, 10, 10, 10 }, 1165 :int{1}, :int{0}, "", ".", "" 1166 } 1167 // Test substitution strings. 1168 { 1169 "windows-1252", // stateless MBCS with WriteSub(), should internally set char * 1170 "a\ufdd0c", 1171 :bin{ 61402421402463 }, 1172 :intvector{ 0,1,1,1,1,1,2 }, 1173 :int{1}, :int{0}, "", "?=@$!@$", "" 1174 } 1175 { 1176 "windows-1252", 1177 "1\U00010001\u0085\U000500022\ud8003\udc014\ue001", 1178 :bin{ 311a1a1a321a331a341a }, 1179 :intvector{ 0,1,3,4,6,7,8,9,10,11 }, 1180 :int{1}, :int{0}, "", "?", "" 1181 } 1182 { 1183 "windows-1252", 1184 "\uD87E", // lone surrogate can cause an offset overflow 1185 :bin{ 1a }, 1186 :intvector{ 0 }, 1187 :int{1}, :int{0}, "", "?", "" 1188 } 1189 { 1190 "windows-1252", 1191 "\uD87E", // lone surrogate can cause an offset overflow 1192 :bin{ 6875683f }, 1193 :intvector{ 0,0,0,0 }, 1194 :int{1}, :int{0}, "", "?=huh?", "" // Use a long substitution character 1195 } 1196 { 1197 "*test4", 1198 "\u30ab", // An incomplete multi-codepoint character 1199 :bin{ ff }, 1200 :intvector{ 0 }, 1201 :int{1}, :int{0}, "", "?", "" 1202 } 1203 { 1204 "ibm-930", // stateful MBCS 1205 "a\ufdd0\u4e00\ufdd0e", 1206 :bin{ 620e4bce0f400e45414bce0f4066 }, 1207 :intvector{ 0,1,1,1,1,1,2,2,2,3,3,3,3,4 }, 1208 :int{1}, :int{0}, "", "?=\u4e01 ", "" 1209 } 1210 { 1211 "iso-2022-jp", 1212 "a\x1bc", // Unicode ESC must not occur as a character 1213 :bin{ 6163 }, 1214 :intvector{ 0,2 }, 1215 :int{1}, :int{0}, "", "?=", "" // empty substitution string 1216 } 1217 { 1218 "iso-2022-cn", 1219 "a\x1bc", // Unicode ESC must not occur as a character 1220 :bin{ 61202063 }, 1221 :intvector{ 0,1,1,2 }, 1222 :int{1}, :int{0}, "", "?= ", "" 1223 } 1224 { 1225 "iso-2022-cn", 1226 "a\x1bc", // Unicode ESC must not occur as a character 1227 :bin{ 611b2429410e523b0f2063 }, 1228 :intvector{ 0,1,1,1,1,1,1,1,1,1,2 }, 1229 :int{1}, :int{0}, "", "?=\u4e00 ", "" 1230 } 1231 { 1232 "us-ascii", 1233 "a\x85c", 1234 :bin{ 61402421402463 }, 1235 :intvector{ 0,1,1,1,1,1,2 }, 1236 :int{1}, :int{0}, "", "?=@$!@$", "" 1237 } 1238 // ISO 2022-CN: test a single-byte subchar, j5171 1239 { 1240 "iso-2022-cn", 1241 "a\x1bc", // Unicode ESC must not occur as a character 1242 :bin{ 612163 }, 1243 :intvector{ 0,1,2 }, 1244 :int{1}, :int{0}, "", "?\x00\x21", "" 1245 } 1246 // UTF-16/32: do not output a BOM if there is no data at all 1247 { 1248 "UTF-16", 1249 "", 1250 :bin{ "" }, 1251 :intvector{ }, 1252 :int{1}, :int{1}, "", "?", "" 1253 } 1254 { 1255 "UTF-32", 1256 "", 1257 :bin{ "" }, 1258 :intvector{ }, 1259 :int{1}, :int{1}, "", "?", "" 1260 } 1261 1262 // do not convert SO/SI/ESC 1263 { 1264 "iso-2022-jp", 1265 "A\x0eB\x0f\x09\x1bC", 1266 :bin{ 411a421a091a43 }, 1267 :intvector{ 0,1,2,3,4,5,6 }, 1268 :int{1}, :int{1}, "", "?", "" 1269 } 1270 { 1271 "iso-2022-cn", 1272 "A\x0eB\x0f\x09\x1bC", 1273 :bin{ 411a421a091a43 }, 1274 :intvector{ 0,1,2,3,4,5,6 }, 1275 :int{1}, :int{1}, "", "?", "" 1276 } 1277 { 1278 "iso-2022-kr", 1279 "A\x0eB\x0f\x09\x1bC", 1280 :bin{ 1b242943411a421a091a43 }, 1281 :intvector{ -1,-1,-1,-1,0,1,2,3,4,5,6 }, 1282 :int{1}, :int{1}, "", "?", "" 1283 } 1284 { 1285 "ibm-25546", 1286 "A\x0eB\x0f\x09\x1bC", 1287 :bin{ 1b242943411a421a091a43 }, 1288 :intvector{ -1,-1,-1,-1,0,1,2,3,4,5,6 }, 1289 :int{1}, :int{1}, "", "?", "" 1290 } 1291 1292 // test ISO 8859-1/7 vs. JIS X 0201 1293 { 1294 "ISO-2022-JP-2", 1295 "?@A\u00e4\u03b1\u203EB", 1296 :bin{ 3f40411B2E411B4E641b244226411b284a7e421b2842 }, 1297 :intvector{ 0,1,2,3,3,3,3,3,3,4,4,4,4,4,5,5,5,5,6,6,6,6 }, 1298 :int{1}, :int{1}, "", "?", "" 1299 } 1300 1301 // Improve ucnv_ext.c code coverage: 1302 // There will be a partial match up to the lead surrogate of U+603ff 1303 // which then results in one more unit in the prefetch buffer 1304 // than the match length when converting one code unit at a time. 1305 // See ucnv_extContinueMatchFromU() comment 1306 // "the match did not use all of preFromU[] - keep the rest for replay" 1307 { 1308 "*test3", 1309 "\U00101234\U00101234\U00050005\U000603ff", 1310 :bin{ 07070001020e05ff }, 1311 :intvector{ 0, 2, 2, 2, 2, 2, 2, 6 }, 1312 :int{1}, :int{0}, "", "?", "" 1313 } 1314 1315 // test that ISO-2022-JP encodes ASCII as itself 1316 { 1317 "ISO-2022-JP", 1318 "?@ABCDEFG", 1319 :bin{ 3f4041424344454647 }, 1320 :intvector{ 0,1,2,3,4,5,6,7,8 }, 1321 :int{1}, :int{1}, "", "?", "" 1322 } 1323 // test that ISO-2022-CN encodes ASCII as itself 1324 { 1325 "ISO-2022-CN", 1326 "?@ABCDEFG", 1327 :bin{ 3f4041424344454647 }, 1328 :intvector{ 0,1,2,3,4,5,6,7,8 }, 1329 :int{1}, :int{1}, "", "?", "" 1330 } 1331 1332 // moved from cintltst /tsconv/nccbtst/TestSkipCallBack 1333 { 1334 "iso-2022-jp", 1335 "\u3000\xe9\u3001", 1336 :bin{ 1b2442212121221b2842 }, 1337 :intvector{ 0,0,0,0,0,2,2,2,2,2 }, 1338 :int{1}, :int{1}, "", "0", "" 1339 } 1340 // moved from cintltst /tsconv/nccbtst/TestSubCallBack 1341 { 1342 "iso-2022-jp", 1343 "A\xe9B\xe9\u3000", 1344 :bin{ 411a421a1b244221211b2842 }, 1345 :intvector{ 0,1,2,3,4,4,4,4,4,4,4,4 }, 1346 :int{1}, :int{1}, "", "?", "" 1347 } 1348 // moved from cintltst /tsconv/nccbtst/TestSubWithValueCallBack 1349 { 1350 "iso-2022-jp", 1351 "A\xe9B\xe9\u3000", 1352 :bin{ 41255530304539422555303045391b244221211b2842 }, 1353 :intvector{ 0,1,1,1,1,1,1,2,3,3,3,3,3,3,4,4,4,4,4,4,4,4 }, 1354 :int{1}, :int{1}, "", "&", "" 1355 } 1356 { 1357 "iso-2022-cn", 1358 "\u4e00\u3712\u4e01", 1359 :bin{ 1b2429410e523b0f2555333731320e36210f }, 1360 :intvector{ 0,0,0,0,0,0,0,1,1,1,1,1,1,1,2,2,2,2 }, 1361 :int{1}, :int{1}, "", "&", "" 1362 } 1363 { 1364 "iso-2022-cn", 1365 "A\u3712\u4e00", 1366 :bin{ 412555333731321b2429410e523b0f }, 1367 :intvector{ 0,1,1,1,1,1,1,2,2,2,2,2,2,2,2 }, 1368 :int{1}, :int{1}, "", "&", "" 1369 } 1370 { 1371 "iso-2022-cn", 1372 "\u3000\u3712\u3001", 1373 :bin{ 1b2429410e21210f2555333731320e21220f }, 1374 :intvector{ 0,0,0,0,0,0,0,1,1,1,1,1,1,1,2,2,2,2 }, 1375 :int{1}, :int{1}, "", "&", "" 1376 } 1377 1378 // moved from cintltst /tsconv/nucnvtst/TestJIS 1379 { 1380 "JIS", 1381 "\uFF81\uFF82\u30EC\u30ED\u30EE\u30EF\uFF93\uFF94\uFF95\uFF96\uFF97\uFF98", 1382 :bin{ 1b244225412544256c256d256e256F25622564256625682569256a1b2842 }, 1383 :intvector{ 0,0,0,0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,8,8,9,9,10,10,11,11,11,11,11 }, 1384 :int{1}, :int{1}, "", "?", "" 1385 } 1386 { 1387 "JIS7", 1388 "\uFF81\uFF82\u30EC\u30ED\u30EE\u30EF\uFF93\uFF94\uFF95\uFF96\uFF97\uFF98", 1389 :bin{ 0e41420f1b2442256c256d256e256F0e5354555657580f1b2842 }, 1390 :intvector{ 0,0,1,2,2,2,2,2,2,3,3,4,4,5,5,6,6,7,8,9,10,11,11,11,11,11 }, 1391 :int{1}, :int{1}, "", "?", "" 1392 } 1393 { 1394 "JIS8", 1395 "\uFF81\uFF82\u30EC\u30ED\u30EE\u30EF\uFF93\uFF94\uFF95\uFF96\uFF97\uFF98", 1396 :bin{ C1C21b2442256c256d256e256F1b284AD3D4D5D6D7D81b2842 }, 1397 :intvector{ 0,1,2,2,2,2,2,3,3,4,4,5,5,6,6,6,6,7,8,9,10,11,11,11,11 }, 1398 :int{1}, :int{1}, "", "?", "" 1399 } 1400 { 1401 "JIS8", 1402 "\u2019Aaa10\u4ED5\u5165\u5148\u30B3", 1403 :bin{ 1b244221471b284241616131301b24423b45467e406825331b2842 }, 1404 :intvector{ 0,0,0,0,0,1,1,1,1,2,3,4,5,6,6,6,6,6,7,7,8,8,9,9,9,9,9 }, 1405 :int{1}, :int{1}, "", "?", "" 1406 } 1407 1408 // moved from cintltst /tsconv/ncnvtst/TestErrorBehaviour 1409 { 1410 "iso-2022-jp", 1411 "\u3000\x50\udc01\u3001", 1412 :bin{ 1B244221211B2842501A1B24422122 }, 1413 :intvector{ 0,0,0,0,0,1,1,1,1,2,3,3,3,3,3 }, 1414 :int{0}, :int{1}, "", "?", "\udc01" 1415 } 1416 { 1417 "iso-2022-jp", 1418 "\u3000\x50\udc01\u3001", 1419 :bin{ 1B244221211B2842501A1B244221221b2842 }, 1420 :intvector{ 0,0,0,0,0,1,1,1,1,2,3,3,3,3,3,3,3,3 }, 1421 :int{1}, :int{1}, "", "?", "" 1422 } 1423 { 1424 "iso-2022-kr", 1425 "\x61\u4e00\udc01\u4e00", 1426 :bin{ 1b242943610e6c690f1a0e6c69 }, 1427 :intvector{ -1,-1,-1,-1,0,1,1,1,2,2,3,3,3 }, 1428 :int{0}, :int{1}, "", "?", "\udc01" 1429 } 1430 { 1431 "iso-2022-kr", 1432 "\x61\u4e00\udc01\u4e00", 1433 :bin{ 1b242943610e6c690f1a0e6c690f }, 1434 :intvector{ -1,-1,-1,-1,0,1,1,1,2,2,3,3,3,3 }, 1435 :int{1}, :int{1}, "", "?", "" 1436 } 1437 1438 // ISO-2022-KR 1439 { 1440 "ibm-25546", 1441 "AB\uc88b\U00050005\uacccC", 1442 :bin{ 1b24294341420e41410f7b552b35303030357d0e306a0f43 }, 1443 :intvector{ -1,-1,-1,-1,0,1,2,2,2,3,3,3,3,3,3,3,3,3,3,5,5,5,6,6 }, 1444 :int{1}, :int{1}, "", "&U", "" 1445 } 1446 { 1447 "ibm-25546", 1448 "AB\uc88b\U00050005\uacccC", 1449 :bin{ 1b24294341420e41410f1a0e306a0f43 }, 1450 :intvector{ -1,-1,-1,-1,0,1,2,2,2,3,3,5,5,5,6,6 }, 1451 :int{1}, :int{1}, "", "?\x00\x1a", "" 1452 } 1453 { 1454 "ibm-25546", 1455 "AB\uc88b\U00050005\uacccC", 1456 :bin{ 1b24294341420e41412f7e306a0f43 }, 1457 :intvector{ -1,-1,-1,-1,0,1,2,2,2,3,3,5,5,6,6 }, 1458 :int{1}, :int{1}, "", "?", "" 1459 } 1460 { 1461 "ibm-25546", 1462 "AB\uc88b\U00050005\uaccc", 1463 :bin{ 1b24294341420e41412f7e306a0f }, 1464 :intvector{ -1,-1,-1,-1,0,1,2,2,2,3,3,5,5,5 }, 1465 :int{1}, :int{1}, "", "?", "" 1466 } 1467 { 1468 "ISO-2022-KR", 1469 "AB\uc88b\U00050005\uacccC", 1470 :bin{ 1b24294341420e41410f7b552b35303030357d0e306a0f43 }, 1471 :intvector{ -1,-1,-1,-1,0,1,2,2,2,3,3,3,3,3,3,3,3,3,3,5,5,5,6,6 }, 1472 :int{1}, :int{1}, "", "&U", "" 1473 } 1474 { 1475 "ISO-2022-KR", 1476 "AB\uc88b\U00050005\uacccC", 1477 :bin{ 1b24294341420e41410f1a0e306a0f43 }, 1478 :intvector{ -1,-1,-1,-1,0,1,2,2,2,3,3,5,5,5,6,6 }, 1479 :int{1}, :int{1}, "", "?", "" 1480 } 1481 { 1482 "ISO-2022-KR", 1483 "AB\uc88b\U00050005\uacccC", 1484 :bin{ 1b24294341420e41412f7e306a0f43 }, 1485 :intvector{ -1,-1,-1,-1,0,1,2,2,2,3,3,5,5,6,6 }, 1486 :int{1}, :int{1}, "", "?\x00\x2f\x7e", "" 1487 } 1488 { 1489 "ISO-2022-KR", 1490 "AB\uc88b\U00050005\uaccc", 1491 :bin{ 1b24294341420e41412f7e306a0f }, 1492 :intvector{ -1,-1,-1,-1,0,1,2,2,2,3,3,5,5,5 }, 1493 :int{1}, :int{1}, "", "?\x00\x2f\x7e", "" 1494 } 1495 // ISO-2022-KR 1496 { 1497 "ibm-25546", 1498 "AB\uc88b\U00050005\uacccC", 1499 :bin{ 1b24294341420e41410f5c3530303035200e306a0f43 }, 1500 :intvector{ -1,-1,-1,-1,0,1,2,2,2,3,3,3,3,3,3,3,3,5,5,5,6,6 }, 1501 :int{1}, :int{1}, "", "&S", "" 1502 } 1503 1504 // ISO-2022-JP-2 with G2 designator & SS2 shift 1505 { 1506 "ISO-2022-JP-2", 1507 "CF\u758f\u038f\u7591", 1508 :bin{ 43461b244241411b2e461b4e3f353f1b2842 }, 1509 :intvector{ 0,1,2,2,2,2,2,3,3,3,3,3,3,4,4,4,4,4 }, 1510 :int{1}, :int{1}, "", ".", "" 1511 } 1512 // JIS7 with Katakana 1513 { 1514 "JIS7", 1515 "AB\uff81\uff82AB", 1516 :bin{ 41420e41420f4142 }, 1517 :intvector{ 0,1,2,2,3,4,4,5 }, 1518 :int{1}, :int{1}, "", ".", "" 1519 } 1520 // JIS7 with shift to ASCII at the very end 1521 { 1522 "JIS7", 1523 "AB\uff81\uff82", 1524 :bin{ 41420e41420f }, 1525 :intvector{ 0,1,2,2,3,3 }, 1526 :int{1}, :int{1}, "", ".", "" 1527 } 1528 // JIS8 with Katakana 1529 { 1530 "JIS8", 1531 "A\uff81\\\xa5\uff82B", 1532 :bin{ 41c15c1b284a5cc2421b2842 }, 1533 :intvector{ 0,1,2,3,3,3,3,4,5,5,5,5 }, 1534 :int{1}, :int{1}, "", ".", "" 1535 } 1536 1537 /* 1538 * ICU 4.4 (ticket #7314) removes mappings for CNS 11643 planes 3..7 1539 // ISO-2022-CN-EXT with all subcharsets and shifts and with supplementary code points 1540 { 1541 "ISO-2022-CN-EXT", 1542 "\u4eae\u9f82\u56cd\u56cc\U0002a6d6\x30", 1543 :bin{ 1b2429410e41411b2429457e7c1b242a481b4e70341b2429477c341b242b4d1b4f664c0f30 }, 1544 :intvector{ 0,0,0,0,0,0,0,1,1,1,1,1,1,2,2,2,2,2,2,2,2,3,3,3,3,3,3,4,4,4,4,4,4,4,4,6,6 }, 1545 :int{1}, :int{1}, "", ".", "" 1546 } 1547 // ISO-2022-CN-EXT with shift to ASCII at the very end 1548 { 1549 "ISO-2022-CN-EXT", 1550 "\u4eae\u9f82\u56cd\u56cc\U0002a6d6", 1551 :bin{ 1b2429410e41411b2429457e7c1b242a481b4e70341b2429477c341b242b4d1b4f664c0f }, 1552 :intvector{ 0,0,0,0,0,0,0,1,1,1,1,1,1,2,2,2,2,2,2,2,2,3,3,3,3,3,3,4,4,4,4,4,4,4,4,4 }, 1553 :int{1}, :int{1}, "", ".", "" 1554 } 1555 // ISO-2022-CN-EXT without flush so do not shift to ASCII at the very end 1556 { 1557 "ISO-2022-CN-EXT", 1558 "\u4eae\u9f82\u56cd\u56cc\U0002a6d6", 1559 :bin{ 1b2429410e41411b2429457e7c1b242a481b4e70341b2429477c341b242b4d1b4f664c }, 1560 :intvector{ 0,0,0,0,0,0,0,1,1,1,1,1,1,2,2,2,2,2,2,2,2,3,3,3,3,3,3,4,4,4,4,4,4,4,4 }, 1561 :int{0}, :int{1}, "", ".", "" 1562 } 1563 */ 1564 1565 // windows-936 vs. ibm-1386 1566 { 1567 "ibm-1386", 1568 "\x1a\u20ac\u5555\x80\x81\U00055555", 1569 :bin{ 7fa2e3dffb7f7fa1a1 }, 1570 :intvector{ 0, 1, 1, 2, 2, 3, 4, 5, 5 }, 1571 :int{1}, :int{1}, "", "?", "" 1572 } 1573 { 1574 "windows-936", 1575 "\x1a\u20ac\u5555\x80\x81\U00055555", 1576 :bin{ 1a80dffb3f3f3f }, 1577 :intvector{ 0, 1, 2, 2, 3, 4, 5 }, 1578 :int{1}, :int{1}, "", "?", "" 1579 } 1580 1581 // verify that if a conversion table does not have any mapping for U+0000, 1582 // then there will not even be a phantom fallback to 00 1583 { 1584 "ibm-971", 1585 "\x00", 1586 :bin{ affe }, 1587 :intvector{ 0, 0 }, 1588 :int{1}, :int{1}, "", "?", "" 1589 } 1590 1591 { 1592 "*test4", 1593 "\x00", 1594 :bin{ ff }, 1595 :intvector{ 0 }, 1596 :int{1}, :int{1}, "", "?", "" 1597 } 1598 1599 // extension in testdata 1600 { 1601 "*test4x", 1602 "\u20ac\x09", 1603 :bin{ 0009 }, 1604 :intvector{ 0, 1 }, 1605 :int{1}, :int{1}, "", "?", "" 1606 } 1607 1608 // DBCS-only extensions 1609 { 1610 "ibm-970", 1611 "\x61\uffa1\u2015\ub000", 1612 :bin{ 611aa1aab2eb }, 1613 :intvector{ 0, 1, 2, 2, 3, 3 }, 1614 :int{1}, :int{1}, "", "?", "" 1615 } 1616 1617 { 1618 "ibm-971", 1619 "\x61\uffa1\u2015\ub000", 1620 :bin{ affeaffeaffeb2eb }, 1621 :intvector{ 0, 0, 1, 1, 2, 2, 3, 3 }, 1622 :int{1}, :int{1}, "", "?", "" 1623 } 1624 1625 { 1626 "ibm-1390,swaplfnl", 1627 "\uff63\u30C8\u30C8\u309A\u3000\x41\u20ac\x0a", 1628 :bin{ 430e4395ecc140400fc1e115 }, 1629 :intvector{ 0, 1, 1, 1, 2, 2, 4, 4, 5, 5, 6, 7 }, 1630 :int{1}, :int{0}, "", "?", "" 1631 } 1632 1633 { 1634 "ibm-16684", 1635 "\uff63\u30C8\u30C8\u309A\u3000\x41\u20ac\x0a", 1636 :bin{ fefe4395ecc14040fefe42e1fefe }, 1637 :intvector{ 0, 0, 1, 1, 2, 2, 4, 4, 5, 5, 6, 6, 7, 7 }, 1638 :int{1}, :int{0}, "", "?", "" 1639 } 1640 1641 { 1642 "ibm-1399", 1643 "\uff63\u30C8\u30C8\u309A\u3000\x41\u20ac\x0a", 1644 :bin{ 440e4395ecc140400fc1e125 }, 1645 :intvector{ 0, 1, 1, 1, 2, 2, 4, 4, 5, 5, 6, 7 }, 1646 :int{1}, :int{0}, "", "?", "" 1647 } 1648 1649 // <subchar1> from |2 mappings 1650 { 1651 "ibm-1390", 1652 "\x0e\x0f\u0901\U00050000\uffe8\uffee", 1653 :bin{ 3f3f0efefefefe0f3f3f }, 1654 :intvector{ 0, 1, 2, 2, 2, 3, 3, 5, 5, 6 }, 1655 :int{1}, :int{1}, "", "?", "" 1656 } 1657 1658 // <subchar1> from |2 mappings, and also contains a fallback to 00 1659 { 1660 "*test4", 1661 "\u20ac\u20ad\U00050005\U00023456\U0010ffff\x30", 1662 :bin{ 0000e10102030affff }, 1663 :intvector{ 0, 1, 2, 4, 4, 4, 4, 6, 8 }, 1664 :int{1}, :int{1}, "", "?", "" 1665 } 1666 1667 // setting a <subchar> resets the <subchar1> 1668 { 1669 "*test4", 1670 "\u20ac\u20ad\U00050005\U00023456\U0010ffff\x30", 1671 :bin{ 00000102030f0102030a0102030f0102030f }, 1672 :intvector{ 0, 1, 2, 2, 2, 2, 4, 4, 4, 4, 6, 6, 6, 6, 8, 8, 8, 8 }, 1673 :int{1}, :int{1}, "", "?\x00\x01\x02\x03\x0f", "" 1674 } 1675 1676 // fallback to 00 with old single-byte data structure 1677 { 1678 "*test1", 1679 "\u20ac\u20ad\U00101234\U00050000", 1680 :bin{ 000007ff }, 1681 :intvector{ 0, 1, 2, 4 }, 1682 :int{1}, :int{1}, "", "?", "" 1683 } 1684 1685 // extensions 1686 { 1687 "ibm-1390", 1688 "\u025a\u025a\u0300\u025a\u0301\u025a\u0302\uffe8\U0002a0f9", 1689 :bin{ 0ed896eccaeccbd896ea530f3f0eb7c20f }, 1690 :intvector{ 0, 0, 0, 1, 1, 3, 3, 5, 5, 6, 6, 7, 7, 8, 8, 8, 8 }, 1691 :int{1}, :int{0}, "", "?", "" 1692 } 1693 1694 { 1695 "*test3", 1696 "\xc4\xc4\xc4\U00101234\xc4\xc4\U00101234\x05", 1697 :bin{ ffffff070501020c }, 1698 :intvector{ 0, 1, 2, 3, 5, 5, 5, 5 }, 1699 :int{1}, :int{0}, "", "?", "" 1700 } 1701 1702 { 1703 "*test3", 1704 "\U00101234\U00101234\U00050005\U00101234\U00050005\U00060006", 1705 :bin{ 07070001020e05070001020f09 }, 1706 :intvector{ 0, 2, 2, 2, 2, 2, 2, 6, 6, 6, 6, 6, 6 }, 1707 :int{1}, :int{0}, "", "?", "" 1708 } 1709 1710 // normal conversions 1711 { 1712 "UTF-16LE", 1713 "1\U00010001\U000500022\ud8003\udc014", 1714 :bin{ 310000d801dc00d902dc3200fdff3300fdff3400 }, 1715 :intvector{ 0, 0, 1, 1, 1, 1, 3, 3, 3, 3, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9 }, 1716 :int{1}, :int{0}, "", "?", "" 1717 } 1718 { "UTF-16LE", "\ud800", :bin{""}, :intvector{}, :int{1}, :int{0}, "truncated", ".", "\ud800" } 1719 1720 { 1721 "UTF-16BE", 1722 "1\U00010001\U000500022\ud8003\udc014", 1723 :bin{ 0031d800dc01d900dc020032fffd0033fffd0034 }, 1724 :intvector{ 0, 0, 1, 1, 1, 1, 3, 3, 3, 3, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9 }, 1725 :int{1}, :int{0}, "", "?", "" 1726 } 1727 { "UTF-16BE", "\ud800", :bin{""}, :intvector{}, :int{1}, :int{0}, "truncated", ".", "\ud800" } 1728 1729 { 1730 "SCSU", 1731 "1\U00010001\u00082\ud8003\udc014\ue001", 1732 :bin{ 310be000810108320efffd330efffd34186881 }, 1733 :intvector{ 0, 1, 1, 1, 1, 3, 3, 4, 5, 5, 5, 6, 7, 7, 7, 8, 9, 9, 9 }, 1734 :int{1}, :int{0}, "", "?", "" 1735 } 1736 { 1737 "x11-compound-text", 1738 "\u54A1\u00A5\u00E3\u0120\u0121\u011E\u0041\u0135\u02D9\u0E3F\u0100\u0157\u0384\u0660\u05D0\u0401", 1739 :bin{ 1b242944b5ac1b2d41a5e31b2d43d5f51b2d4dd01b2d41411b2d43bc1b2d42ff1b2d54df1b2d44c0b31b2d46b41b2d47b01b2d48e01b2d4ca1 }, 1740 :intvector{ }, 1741 :int{1}, :int{0}, "", "?", "" 1742 } 1743 // Test Gurmukhi (Bindi Tippi and Consonant clusters) 1744 { 1745 "iscii-gur", 1746 "\u0a15\u0a70\u0a02\u0a71\u0a15\u0a5c\u0a4d\u0a39\u000a\u0043\u0041\u000a", 1747 :bin { ef4bb3a2a2b3e8b3bfe9e8d80aef4b43410a }, 1748 :intvector{ 0, 0, 0, 1, 2, 3, 3, 3, 5, 5, 6, 7, 8, 8, 8, 9, 10, 11 }, 1749 :int{1}, :int{0}, "", ".", "" 1750 } 1751 // escape callback 1752 { 1753 "iscii-dev", 1754 "A\u0901\U00023456\u0902B\U00023456C", 1755 :bin{ 41ef42a1255544383444255544433536a24225554438344425554443353643 }, 1756 :intvector{ 1757 0, 1758 1,1,1, 1759 2,2,2,2,2,2, 1760 2,2,2,2,2,2, 1761 4, 1762 5, 1763 6,6,6,6,6,6, 1764 6,6,6,6,6,6, 1765 8 1766 }, 1767 :int{1}, :int{0}, "", "&", "" 1768 } 1769 1770 // escape callback (hex) 1771 { 1772 "iso-2022-jp", 1773 "\u3000\U00023456\u3001\U00023456B\u901c", 1774 :bin{ 1b244221211b284226237832333435363b1b244221221b284226237832333435363b42262378393031433b }, 1775 :intvector{ 1776 0,0,0,0,0, 1777 1,1,1,1,1,1,1,1,1,1,1,1, 1778 3,3,3,3,3, 1779 4,4,4,4,4,4,4,4,4,4,4,4, 1780 6, 1781 7,7,7,7,7,7,7,7 1782 }, 1783 :int{1}, :int{0}, "", "&X", "" 1784 } 1785 1786 // sub callback 1787 { 1788 "gb18030", 1789 "$\x7f\x80\u01f9\u20ac\u4e00\u9fa6\uffff\U00010000\U0010ffff", 1790 :bin{ 247f81308130a8bfa2e3d2bb82358f338431a43990308130e3329a35 }, 1791 :intvector{ 0, 1, 2, 2, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 6, 6, 7, 7, 7, 7, 8, 8, 8, 8, 10, 10, 10, 10 }, 1792 :int{1}, :int{0}, "", "?", "" 1793 } 1794 1795 // skip callback 1796 { "ibm-930", "\u6D63\u6D64\u6D65\u6D66", :bin{ 0e5d5f5d63466b0f }, :intvector{ 0, 0, 0, 1, 1, 3, 3, 3 }, :int{1}, :int{0}, "", "0", "" } 1797 { "ibm-930", "\u6D63\u6D64\ud89a\u6D66", :bin{ 0e5d5f5d63466b0f }, :intvector{ 0, 0, 0, 1, 1, 3, 3, 3 }, :int{1}, :int{0}, "", "0", "" } 1798 { "ibm-930", "\u6D63\u6D64\ud89a\u6D66", :bin{ 0e5d5f5d63 }, :intvector{ 0, 0, 0, 1, 1 }, :int{1}, :int{0}, "illegal", "0i", "\ud89a" } 1799 1800 // sub callback for supplementary code point 1801 { "LATIN1", "1\U000104012", :bin{ 311a32 }, :intvector{ 0, 1, 3 }, :int{1}, :int{0}, "", "", "" } 1802 { "ibm-920", "1\U000104012", :bin{ 311a32 }, :intvector{ 0, 1, 3 }, :int{1}, :int{0}, "", "", "" } 1803 1804 // sub callback with AA as subchar 1805 { "ibm-920", "1\U000104012", :bin{ 31AA32 }, :intvector{ 0, 1, 3 }, :int{1}, :int{0}, "", "?\x00\xAA", "" } 1806 1807 // same but not flushing 1808 { "LATIN1", "1\U000104012", :bin{ 311a32 }, :intvector{ 0, 1, 3 }, :int{0}, :int{0}, "", "", "\U00010401" } 1809 { "ibm-920", "1\U000104012", :bin{ 311a32 }, :intvector{ 0, 1, 3 }, :int{0}, :int{0}, "", "", "\U00010401" } 1810 1811 // simple sample, no error handling 1812 { "UTF-8", "a\U0010FFFF", :bin{ 61F48FBFBF }, :intvector{ 0, 1, 1, 1, 1 }, :int{1}, :int{0}, "", "", "" } 1813 1814 // Verify that incomplete surrogates are handled as an error 1815 { "UTF-8", "a\udc00", :bin{ 61efbfbd }, :intvector{ 0, 1, 1, 1 }, :int{1}, :int{0}, "", "", "" } 1816 { "UTF-8", "a\ud800", :bin{ 61efbfbd }, :intvector{ 0, 1, 1, 1 }, :int{1}, :int{0}, "", "", "" } 1817 { "UTF-8", "a\udc00b", :bin{ 61efbfbd62 }, :intvector{ 0, 1, 1, 1, 2 }, :int{1}, :int{0}, "", "", "" } 1818 { "UTF-8", "a\ud800b", :bin{ 61efbfbd62 }, :intvector{ 0, 1, 1, 1, 2 }, :int{1}, :int{0}, "", "", "" } 1819 1820 // Code coverage for the EUC variants. 1821 { "IBM-eucJP", "\u0061\u4edd\u5bec\ud801\udc01\ud801\u0061\u00a2", :bin{ 61a1b88ff4ae618ee0 }, :intvector{ 0, 1, 1, 2, 2, 2, 6, 7, 7 }, :int{1}, :int{0}, "", "0", "" } 1822 { "IBM-eucJP", "\u0061\u4edd\u5bec\ud801\udc01\ud801\u0061\u00a2", :bin{ 61a1b88ff4aef4fef4fe618ee0 }, :intvector{ 0, 1, 1, 2, 2, 2, 3, 3, 5, 5, 6, 7, 7 }, :int{1}, :int{0}, "", "", "" } 1823 { "EUC-TW", "\u0061\u2295\u5BF2\ud801\udc01\ud801\u0061\u8706\u008a", :bin{ 61a2d38ea2dce561e6ca8a }, :intvector{ 0, 1, 1, 2, 2, 2, 2, 6, 7, 7, 8 }, :int{1}, :int{0}, "", "0", "" } 1824 { "EUC-TW", "\u0061\u2295\u5BF2\ud801\udc01\ud801\u0061\u8706\u008a", :bin{ 61a2d38ea2dce5fdfefdfe61e6ca8a }, :intvector{ 0, 1, 1, 2, 2, 2, 2, 3, 3, 5, 5, 6, 7, 7, 8 }, :int{1}, :int{0}, "", "", "" } 1825 1826 // Code Coverage for BOCU-1 1827 { 1828 "BOCU-1", 1829 "\u0041\ud841\ud888\udc81", 1830 :bin{ 91fbc555fd6349 }, 1831 :intvector{ 0, 1, 1, 1, 2, 2, 2 }, 1832 :int{1}, :int{0}, "", ".", "" 1833 } 1834 { 1835 "BOCU-1", 1836 "\ufe88\ufe70\udbff\udfff\u0061", 1837 :bin{ fbeda44ff0fe189bb821f05926 }, 1838 :intvector{}, 1839 :int{1}, :int{0}, "", ".", "" 1840 } 1841 // Improve code coverage for SCSU 1842 { 1843 "SCSU", 1844 "\ud899\udc7f\ud977\ud888\udc99\ud888\u0041", 1845 :bin{ 0fd899dc7fd888dc99e041 }, 1846 :intvector{}, 1847 :int{1}, :int{0}, "", "0", "" 1848 } 1849 { // gb18030-2022 changes mappings for 0xA6DC,0xA6DB 1850 "gb18030", 1851 "\U00020087\ufe13\ufe11\u1e3f", 1852 :bin{ 95329031a6dca6dba8bc }, 1853 :intvector{ 0,0,0,0,2,2,3,3,4,4 }, 1854 :int{1}, :int{0}, "", "0", "" 1855 } 1856 { // U->gb18030 2005 vs 2022 part 1 (gb18030 2-byte) 1857 "gb18030", 1858 "\uFE10\uFE12\uFE11\uFE13\uFE16\uFE17\uFE18\uFE19\u9FB4\u9FB5\u9FB6\u9FB7\u9FB8\u9FB9\u9FBA\u9FBB", // -2005: "\uE78D\uE78E\uE78F\uE790\uE793\uE794\uE795\uE796\uE81E\uE826\uE82B\uE82C\uE832\uE843\uE854\uE864" 1859 :bin{ A6D9A6DAA6DBA6DCA6DFA6ECA6EDA6F3FE59FE61FE66FE67FE6DFE7EFE90FEA0 }, 1860 :intvector{ 0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15 }, 1861 :int{1}, :int{0}, "", "0", "" 1862 } 1863 { // U->gb18030 2005 vs 2022 part 2 (gb18030 fallback mappings from Unicode PUA) 1864 "gb18030", 1865 "\uE78D\uE793\uE794\uE795\uE796\uE81E\uE826\uE82B\uE82C\uE832\uE843\uE854\uE864", 1866 :bin{ A6D9A6DFA6ECA6EDA6F3FE59FE61FE66FE67FE6DFE7EFE90FEA0 }, 1867 :intvector{ 0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,8,8,9,9,10,10,11,11,12,12 }, 1868 :int{1}, :int{0}, "", "0", "" 1869 } 1870 { // U->gb18030 2005 vs 2022 part 3 (gb18030 4-byte), non-changing mappings next to or in linear ranges partially overridden by new explicit maps 1871 "gb18030", 1872 "\u9FA6\u9FB3\u9FBC\uFE0F\uFE1A\uFE24\uFE2F", 1873 :bin{ 82358F33823590368235913584318235843183368431843684318537 }, 1874 :intvector{ 0,0,0,0,1,1,1,1,2,2,2,2,3,3,3,3,4,4,4,4,5,5,5,5,6,6,6,6 }, 1875 :int{1}, :int{0}, "", "0", "" 1876 } 1877 { 1878 "UTF-7", 1879 "\u00a3I\u00a3\u00a4", 1880 :bin{ 2b414b4d2d492b414b4d4170412d }, 1881 :intvector{ 0,0,0,0,0,1,2,2,2,3,3,3,3,3 }, 1882 :int{1}, :int{0}, "", "0", "" 1883 } 1884 // Bug #9601 direct-from-UTF-8 m:n Unicode:charset conversion. 1885 { 1886 "*test1bmp", 1887 "uv", 1888 :bin{ 08 }, 1889 :intvector{ 0 }, 1890 :int{1}, :int{0}, "", "?", "" 1891 } 1892 { 1893 "*test2", 1894 "\U00101234\U00050005", 1895 :bin{ 0700010e05 }, 1896 :intvector{ 0,0,0,0,0 }, 1897 :int{1}, :int{0}, "", "?", "" 1898 } 1899 } 1900 } 1901 1902 getUnicodeSet { 1903 // charset - will be opened, and ucnv_getUnicodeSet() called on it 1904 // map - set of code points and strings that must be in the returned set 1905 // mapnot - set of code points and strings that must *not* be in the returned set 1906 // which - numeric UConverterUnicodeSet value 1907 Headers { "charset", "map", "mapnot", "which" } 1908 Cases { 1909 // Test ticket 9602: Add "good one-way" mapping type (|4). 1910 // Excluded from roundtrip set, included in the set with fallbacks. 1911 { 1912 "+*test3", 1913 "[{#\uFE0F}]", 1914 "[#{#\uFE0E}]", 1915 :int{0} 1916 } 1917 { 1918 "+*test3", 1919 "[#{#\uFE0E}{#\uFE0F}]", 1920 "[]", 1921 :int{1} 1922 } 1923 // Unicode charsets that do not map surrogate code points 1924 { 1925 "UTF-8", 1926 "[\x00-\ud7ff\ue000-\U0010ffff]", 1927 "[\ud800-\udfff]", 1928 :int{0} 1929 } 1930 { 1931 "UTF-16", 1932 "[\x00-\ud7ff\ue000-\U0010ffff]", 1933 "[\ud800-\udfff]", 1934 :int{0} 1935 } 1936 { 1937 "UTF-16BE", 1938 "[\x00-\ud7ff\ue000-\U0010ffff]", 1939 "[\ud800-\udfff]", 1940 :int{0} 1941 } 1942 { 1943 "UTF-16LE", 1944 "[\x00-\ud7ff\ue000-\U0010ffff]", 1945 "[\ud800-\udfff]", 1946 :int{0} 1947 } 1948 { 1949 "UTF-32", 1950 "[\x00-\ud7ff\ue000-\U0010ffff]", 1951 "[\ud800-\udfff]", 1952 :int{0} 1953 } 1954 { 1955 "UTF-32BE", 1956 "[\x00-\ud7ff\ue000-\U0010ffff]", 1957 "[\ud800-\udfff]", 1958 :int{0} 1959 } 1960 { 1961 "UTF-32LE", 1962 "[\x00-\ud7ff\ue000-\U0010ffff]", 1963 "[\ud800-\udfff]", 1964 :int{0} 1965 } 1966 1967 // Unicode charsets that do map surrogate code points 1968 { 1969 "UTF-7", 1970 "[\x00-\U0010ffff]", 1971 "[]", 1972 :int{0} 1973 } 1974 { 1975 "SCSU", 1976 "[\x00-\U0010ffff]", 1977 "[]", 1978 :int{0} 1979 } 1980 { 1981 "BOCU-1", 1982 "[\x00-\U0010ffff]", 1983 "[]", 1984 :int{0} 1985 } 1986 { 1987 "CESU-8", 1988 "[\x00-\U0010ffff]", 1989 "[]", 1990 :int{0} 1991 } 1992 1993 // versions of ISO-2022-KR 1994 { 1995 "ISO-2022-KR", 1996 "[\x00-\x0d\x10-\x1a\x1c-\x7f\xa1\xa4\xfe\u0111\u4e00\u4e01\uac00-\uac02\uffe6]", 1997 "[\x0e\x0f\x1b\x80-\xa0\xa3\xa5\xff-\u0110\uac03\uffe7-\U0010ffff]", 1998 :int{0} 1999 } 2000 { 2001 "ibm-25546", 2002 "[\x00-\x0d\x10-\x1a\x1c-\x7f\xa1\xa4\xfe\u0111\u4e00\u4e01\uac00-\uac01\uffe6]", 2003 "[\x0e\x0f\x1b\x80-\xa0\xa3\xa5\xff-\u0110\uac02\uffe7-\U0010ffff]", 2004 :int{0} 2005 } 2006 2007 // versions of ISO-2022-JP 2008 { 2009 "ISO-2022-JP", 2010 "[\x00-\x0d\x10-\x1a\x1c-\x7f\xa5\u0391-\u03a1\u2015\u203e\u4e00\u4e01\uffe5]", 2011 "[\x0e\x0f\x1b\u0100-\u0113\u0385-\u038a\u2014\u301c\u4e02\u4e27-\u4e29\u4fe0\u663b\u9eb5\ufa0e-\ufa2d\uff61-\uff9f\uffe4\uffe6-\U0010ffff]", 2012 :int{0} 2013 } 2014 { 2015 "ISO-2022-JP-2", 2016 "[\x00-\x0d\x10-\x1a\x1c-\x7f\xa0-\u0113\u0384-\u0386\u0388-\u038a\u0390-\u03a1\u203e\u4e00-\u4e05\u4fe0\u663b\uffe6]", 2017 "[\x0e\x0f\x1b\uff61-\uff9f\uffe4\uffe7-\U0010ffff]", 2018 :int{0} 2019 } 2020 { 2021 "JIS7", 2022 "[\x00-\x0d\x10-\x1a\x1c-\x7f\xa0-\u0113\u0384-\u0386\u0388-\u038a\u0390-\u03a1\u203e\u4e00-\u4e05\u4fe0\u663b\uff61-\uff9f\uffe6]", 2023 "[\x0e\x0f\x1b\uffe4\uffe7-\U0010ffff]", 2024 :int{0} 2025 } 2026 // with fallbacks 2027 { 2028 "ISO-2022-JP", 2029 "[\x00-\x0d\x10-\x1a\x1c-\x7f\xa5\u0391-\u03a1\u2014\u2015\u203e\u301c\u4e00\u4e01\u4fe0\u9eb5\uff61-\uff9f\uffe5]", 2030 "[\x0e\x0f\x1b\xa6\u0100-\u0113\u0385-\u038a\u4e02\u4e27-\u4e29\u663b\ufa0e-\ufa2d\uffe4\uffe6-\U0010ffff]", 2031 :int{1} 2032 } 2033 2034 // versions of ISO-2022-CN 2035 { 2036 "ISO-2022-CN", 2037 "[\x00-\x0d\x10-\x1a\x1c-\x7f\u4e00\u4e01\u9f98\ufe6b]", 2038 "[\x0e\x0f\x1b\u4e29\uffe6-\U0010ffff]", 2039 :int{0} 2040 } 2041 /* 2042 * ICU 4.4 (ticket #7314) removes mappings for CNS 11643 planes 3..7 2043 { 2044 "ISO-2022-CN-EXT", 2045 "[\x00-\x0d\x10-\x1a\x1c-\x7f\u4e00-\u4e05\u9f98\ufe6b\u4e28-\u4e2b\U00020000\U00020003-\U00020005\U00029664]", 2046 "[\x0e\x0f\x1b\U00020001\U00020002\U0002a6d7-\U0010ffff]", 2047 :int{0} 2048 } 2049 */ 2050 2051 // HZ 2052 { 2053 "HZ", 2054 "[\u0410-\u044f\u4e00\u4e01\u4e03]", 2055 "[\u4e02\u4e04-\u4e06\uac00-\ud7ff]", 2056 :int{0} 2057 } 2058 2059 // LMBCS 2060 { 2061 "LMBCS", 2062 "[\x00-\U0010ffff]", 2063 "[]", 2064 :int{0} 2065 } 2066 2067 // ISCII,version=0 (note: all versions of ISCII generates the same Unicodeset) 2068 { 2069 "iscii-dev", 2070 "[\x00-\xa0\u0909-\u0939\u0993-\u09a8\u0a13-\u0a28\u0a93-\u0aa8\u0ae6-\u0aef\u0b05-\u0b0c\u0b13-\u0b28\u0bae-\u0bb5\u0c12-\u0c28\u0c92-\u0ca8\u0d12-\u0d28]", 2071 "[\u0971-\u0975\u09e4\u0a4e-\u0a58\u0a80\u0b72-\u0b7a\u0bfb-\u0bff\u0c70-\u0c7f\u0c80\u0d00]", 2072 :int{0} 2073 } 2074 2075 { 2076 "iso-8859-1", 2077 "[\x00-\xff]", 2078 "[\u0100-\u01ff]", 2079 :int{0} 2080 } 2081 2082 { 2083 "us-ascii", 2084 "[\x00-\x7f]", 2085 "[\u0100-\u01ff]", 2086 :int{0} 2087 } 2088 // DBCS-only 2089 { 2090 "ibm-971", 2091 "[\xa1\xa4\uac01\ub000]", 2092 "[\x00-\x9f\u2015]", 2093 :int{0} 2094 } 2095 2096 { 2097 "ibm-16684", 2098 "[\xa0\xa1\xa4\xa6-\xab\xad-\u017f\u0254\u309b-\u30ff\u4e00-\u4e05\U00023d00\U000243bc\U0002a6b2" 2099 "{\u0254\u0300}{\u0254\u0301}{\u304b\u309a}{\u30ad\u309a}{\u30af\u309a}]", 2100 "[\x00-0x9f\xa2\xa3\xa5\xac\u0200-\u024f\U00010000-\U0001ffff\U0002a61b-\U0002a6b1]", 2101 :int{0} 2102 } 2103 2104 // extensions 2105 { 2106 "ibm-1390", 2107 "[\x00-\x0d\x10-\u017f\u0254\u309b-\u30ff\u4e00-\u4e05\U00023d00\U000243bc\U0002a6b2" 2108 "{\u0254\u0300}{\u0254\u0301}{\u304b\u309a}{\u30ad\u309a}{\u30af\u309a}]", 2109 "[\x0e\x0f\u0200-\u024f\U00010000-\U0001ffff\U0002a61b-\U0002a6b1]", 2110 :int{0} 2111 } 2112 2113 { 2114 "*test3", 2115 "[\x05\x0b\xc0\u20ac\U00023456\U00101234" 2116 "{\U00101234\U00050005\U00060006}{\U00101234\U00050005}{\U00101234\U00060006}{\xc4\xc4\U00101234\x05}]", 2117 "[\x06\x0e\U00034567\U000febcd{\U00101234\U00070007}]", 2118 :int{0} 2119 } 2120 } 2121 } 2122 } 2123 }