bocsu.cpp (5140B)
1 // © 2016 and later: Unicode, Inc. and others. 2 // License & terms of use: http://www.unicode.org/copyright.html 3 /* 4 ******************************************************************************* 5 * Copyright (C) 2001-2014, International Business Machines 6 * Corporation and others. All Rights Reserved. 7 ******************************************************************************* 8 * file name: bocsu.cpp 9 * encoding: UTF-8 10 * tab size: 8 (not used) 11 * indentation:4 12 * 13 * Author: Markus W. Scherer 14 * 15 * Modification history: 16 * 05/18/2001 weiv Made into separate module 17 */ 18 19 20 #include "unicode/utypes.h" 21 22 #if !UCONFIG_NO_COLLATION 23 24 #include "unicode/bytestream.h" 25 #include "unicode/utf16.h" 26 #include "bocsu.h" 27 28 /* 29 * encode one difference value -0x10ffff..+0x10ffff in 1..4 bytes, 30 * preserving lexical order 31 */ 32 static uint8_t * 33 u_writeDiff(int32_t diff, uint8_t *p) { 34 if(diff>=SLOPE_REACH_NEG_1) { 35 if(diff<=SLOPE_REACH_POS_1) { 36 *p++ = static_cast<uint8_t>(SLOPE_MIDDLE + diff); 37 } else if(diff<=SLOPE_REACH_POS_2) { 38 *p++ = static_cast<uint8_t>(SLOPE_START_POS_2 + (diff / SLOPE_TAIL_COUNT)); 39 *p++ = static_cast<uint8_t>(SLOPE_MIN + diff % SLOPE_TAIL_COUNT); 40 } else if(diff<=SLOPE_REACH_POS_3) { 41 p[2] = static_cast<uint8_t>(SLOPE_MIN + diff % SLOPE_TAIL_COUNT); 42 diff/=SLOPE_TAIL_COUNT; 43 p[1] = static_cast<uint8_t>(SLOPE_MIN + diff % SLOPE_TAIL_COUNT); 44 *p = static_cast<uint8_t>(SLOPE_START_POS_3 + (diff / SLOPE_TAIL_COUNT)); 45 p+=3; 46 } else { 47 p[3] = static_cast<uint8_t>(SLOPE_MIN + diff % SLOPE_TAIL_COUNT); 48 diff/=SLOPE_TAIL_COUNT; 49 p[2] = static_cast<uint8_t>(SLOPE_MIN + diff % SLOPE_TAIL_COUNT); 50 diff/=SLOPE_TAIL_COUNT; 51 p[1] = static_cast<uint8_t>(SLOPE_MIN + diff % SLOPE_TAIL_COUNT); 52 *p=SLOPE_MAX; 53 p+=4; 54 } 55 } else { 56 int32_t m; 57 58 if(diff>=SLOPE_REACH_NEG_2) { 59 NEGDIVMOD(diff, SLOPE_TAIL_COUNT, m); 60 *p++ = static_cast<uint8_t>(SLOPE_START_NEG_2 + diff); 61 *p++ = static_cast<uint8_t>(SLOPE_MIN + m); 62 } else if(diff>=SLOPE_REACH_NEG_3) { 63 NEGDIVMOD(diff, SLOPE_TAIL_COUNT, m); 64 p[2] = static_cast<uint8_t>(SLOPE_MIN + m); 65 NEGDIVMOD(diff, SLOPE_TAIL_COUNT, m); 66 p[1] = static_cast<uint8_t>(SLOPE_MIN + m); 67 *p = static_cast<uint8_t>(SLOPE_START_NEG_3 + diff); 68 p+=3; 69 } else { 70 NEGDIVMOD(diff, SLOPE_TAIL_COUNT, m); 71 p[3] = static_cast<uint8_t>(SLOPE_MIN + m); 72 NEGDIVMOD(diff, SLOPE_TAIL_COUNT, m); 73 p[2] = static_cast<uint8_t>(SLOPE_MIN + m); 74 NEGDIVMOD(diff, SLOPE_TAIL_COUNT, m); 75 p[1] = static_cast<uint8_t>(SLOPE_MIN + m); 76 *p=SLOPE_MIN; 77 p+=4; 78 } 79 } 80 return p; 81 } 82 83 /* 84 * Encode the code points of a string as 85 * a sequence of byte-encoded differences (slope detection), 86 * preserving lexical order. 87 * 88 * Optimize the difference-taking for runs of Unicode text within 89 * small scripts: 90 * 91 * Most small scripts are allocated within aligned 128-blocks of Unicode 92 * code points. Lexical order is preserved if "prev" is always moved 93 * into the middle of such a block. 94 * 95 * Additionally, "prev" is moved from anywhere in the Unihan 96 * area into the middle of that area. 97 * Note that the identical-level run in a sort key is generated from 98 * NFD text - there are never Hangul characters included. 99 */ 100 U_CFUNC UChar32 101 u_writeIdenticalLevelRun(UChar32 prev, const char16_t *s, int32_t length, icu::ByteSink &sink) { 102 char scratch[64]; 103 int32_t capacity; 104 105 int32_t i=0; 106 while(i<length) { 107 char *buffer=sink.GetAppendBuffer(1, length*2, scratch, (int32_t)sizeof(scratch), &capacity); 108 uint8_t *p; 109 // We must have capacity>=SLOPE_MAX_BYTES in case u_writeDiff() writes that much, 110 // but we do not want to force the sink.GetAppendBuffer() to allocate 111 // for a large min_capacity because we might actually only write one byte. 112 if(capacity<16) { 113 buffer=scratch; 114 capacity=(int32_t)sizeof(scratch); 115 } 116 p=reinterpret_cast<uint8_t *>(buffer); 117 uint8_t *lastSafe=p+capacity-SLOPE_MAX_BYTES; 118 while(i<length && p<=lastSafe) { 119 if(prev<0x4e00 || prev>=0xa000) { 120 prev=(prev&~0x7f)-SLOPE_REACH_NEG_1; 121 } else { 122 /* 123 * Unihan U+4e00..U+9fa5: 124 * double-bytes down from the upper end 125 */ 126 prev=0x9fff-SLOPE_REACH_POS_2; 127 } 128 129 UChar32 c; 130 U16_NEXT(s, i, length, c); 131 if(c==0xfffe) { 132 *p++=2; // merge separator 133 prev=0; 134 } else { 135 p=u_writeDiff(c-prev, p); 136 prev=c; 137 } 138 } 139 sink.Append(buffer, (int32_t)(p-reinterpret_cast<uint8_t *>(buffer))); 140 } 141 return prev; 142 } 143 144 #endif /* #if !UCONFIG_NO_COLLATION */