diff options
Diffstat (limited to 'vendor/golang.org/x/text/unicode/norm/composition.go')
| -rw-r--r-- | vendor/golang.org/x/text/unicode/norm/composition.go | 512 |
1 files changed, 0 insertions, 512 deletions
diff --git a/vendor/golang.org/x/text/unicode/norm/composition.go b/vendor/golang.org/x/text/unicode/norm/composition.go deleted file mode 100644 index e2087bc..0000000 --- a/vendor/golang.org/x/text/unicode/norm/composition.go +++ /dev/null @@ -1,512 +0,0 @@ -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package norm - -import "unicode/utf8" - -const ( - maxNonStarters = 30 - // The maximum number of characters needed for a buffer is - // maxNonStarters + 1 for the starter + 1 for the GCJ - maxBufferSize = maxNonStarters + 2 - maxNFCExpansion = 3 // NFC(0x1D160) - maxNFKCExpansion = 18 // NFKC(0xFDFA) - - maxByteBufferSize = utf8.UTFMax * maxBufferSize // 128 -) - -// ssState is used for reporting the segment state after inserting a rune. -// It is returned by streamSafe.next. -type ssState int - -const ( - // Indicates a rune was successfully added to the segment. - ssSuccess ssState = iota - // Indicates a rune starts a new segment and should not be added. - ssStarter - // Indicates a rune caused a segment overflow and a CGJ should be inserted. - ssOverflow -) - -// streamSafe implements the policy of when a CGJ should be inserted. -type streamSafe uint8 - -// first inserts the first rune of a segment. It is a faster version of next if -// it is known p represents the first rune in a segment. -func (ss *streamSafe) first(p Properties) { - *ss = streamSafe(p.nTrailingNonStarters()) -} - -// insert returns a ssState value to indicate whether a rune represented by p -// can be inserted. -func (ss *streamSafe) next(p Properties) ssState { - if *ss > maxNonStarters { - panic("streamSafe was not reset") - } - n := p.nLeadingNonStarters() - if *ss += streamSafe(n); *ss > maxNonStarters { - *ss = 0 - return ssOverflow - } - // The Stream-Safe Text Processing prescribes that the counting can stop - // as soon as a starter is encountered. However, there are some starters, - // like Jamo V and T, that can combine with other runes, leaving their - // successive non-starters appended to the previous, possibly causing an - // overflow. We will therefore consider any rune with a non-zero nLead to - // be a non-starter. Note that it always hold that if nLead > 0 then - // nLead == nTrail. - if n == 0 { - *ss = streamSafe(p.nTrailingNonStarters()) - return ssStarter - } - return ssSuccess -} - -// backwards is used for checking for overflow and segment starts -// when traversing a string backwards. Users do not need to call first -// for the first rune. The state of the streamSafe retains the count of -// the non-starters loaded. -func (ss *streamSafe) backwards(p Properties) ssState { - if *ss > maxNonStarters { - panic("streamSafe was not reset") - } - c := *ss + streamSafe(p.nTrailingNonStarters()) - if c > maxNonStarters { - return ssOverflow - } - *ss = c - if p.nLeadingNonStarters() == 0 { - return ssStarter - } - return ssSuccess -} - -func (ss streamSafe) isMax() bool { - return ss == maxNonStarters -} - -// GraphemeJoiner is inserted after maxNonStarters non-starter runes. -const GraphemeJoiner = "\u034F" - -// reorderBuffer is used to normalize a single segment. Characters inserted with -// insert are decomposed and reordered based on CCC. The compose method can -// be used to recombine characters. Note that the byte buffer does not hold -// the UTF-8 characters in order. Only the rune array is maintained in sorted -// order. flush writes the resulting segment to a byte array. -type reorderBuffer struct { - rune [maxBufferSize]Properties // Per character info. - byte [maxByteBufferSize]byte // UTF-8 buffer. Referenced by runeInfo.pos. - nbyte uint8 // Number or bytes. - ss streamSafe // For limiting length of non-starter sequence. - nrune int // Number of runeInfos. - f formInfo - - src input - nsrc int - tmpBytes input - - out []byte - flushF func(*reorderBuffer) bool -} - -func (rb *reorderBuffer) init(f Form, src []byte) { - rb.f = *formTable[f] - rb.src.setBytes(src) - rb.nsrc = len(src) - rb.ss = 0 -} - -func (rb *reorderBuffer) initString(f Form, src string) { - rb.f = *formTable[f] - rb.src.setString(src) - rb.nsrc = len(src) - rb.ss = 0 -} - -func (rb *reorderBuffer) setFlusher(out []byte, f func(*reorderBuffer) bool) { - rb.out = out - rb.flushF = f -} - -// reset discards all characters from the buffer. -func (rb *reorderBuffer) reset() { - rb.nrune = 0 - rb.nbyte = 0 -} - -func (rb *reorderBuffer) doFlush() bool { - if rb.f.composing { - rb.compose() - } - res := rb.flushF(rb) - rb.reset() - return res -} - -// appendFlush appends the normalized segment to rb.out. -func appendFlush(rb *reorderBuffer) bool { - for i := 0; i < rb.nrune; i++ { - start := rb.rune[i].pos - end := start + rb.rune[i].size - rb.out = append(rb.out, rb.byte[start:end]...) - } - return true -} - -// flush appends the normalized segment to out and resets rb. -func (rb *reorderBuffer) flush(out []byte) []byte { - for i := 0; i < rb.nrune; i++ { - start := rb.rune[i].pos - end := start + rb.rune[i].size - out = append(out, rb.byte[start:end]...) - } - rb.reset() - return out -} - -// flushCopy copies the normalized segment to buf and resets rb. -// It returns the number of bytes written to buf. -func (rb *reorderBuffer) flushCopy(buf []byte) int { - p := 0 - for i := 0; i < rb.nrune; i++ { - runep := rb.rune[i] - p += copy(buf[p:], rb.byte[runep.pos:runep.pos+runep.size]) - } - rb.reset() - return p -} - -// insertOrdered inserts a rune in the buffer, ordered by Canonical Combining Class. -// It returns false if the buffer is not large enough to hold the rune. -// It is used internally by insert and insertString only. -func (rb *reorderBuffer) insertOrdered(info Properties) { - n := rb.nrune - b := rb.rune[:] - cc := info.ccc - if cc > 0 { - // Find insertion position + move elements to make room. - for ; n > 0; n-- { - if b[n-1].ccc <= cc { - break - } - b[n] = b[n-1] - } - } - rb.nrune += 1 - pos := uint8(rb.nbyte) - rb.nbyte += utf8.UTFMax - info.pos = pos - b[n] = info -} - -// insertErr is an error code returned by insert. Using this type instead -// of error improves performance up to 20% for many of the benchmarks. -type insertErr int - -const ( - iSuccess insertErr = -iota - iShortDst - iShortSrc -) - -// insertFlush inserts the given rune in the buffer ordered by CCC. -// If a decomposition with multiple segments are encountered, they leading -// ones are flushed. -// It returns a non-zero error code if the rune was not inserted. -func (rb *reorderBuffer) insertFlush(src input, i int, info Properties) insertErr { - if rune := src.hangul(i); rune != 0 { - rb.decomposeHangul(rune) - return iSuccess - } - if info.hasDecomposition() { - return rb.insertDecomposed(info.Decomposition()) - } - rb.insertSingle(src, i, info) - return iSuccess -} - -// insertUnsafe inserts the given rune in the buffer ordered by CCC. -// It is assumed there is sufficient space to hold the runes. It is the -// responsibility of the caller to ensure this. This can be done by checking -// the state returned by the streamSafe type. -func (rb *reorderBuffer) insertUnsafe(src input, i int, info Properties) { - if rune := src.hangul(i); rune != 0 { - rb.decomposeHangul(rune) - } - if info.hasDecomposition() { - // TODO: inline. - rb.insertDecomposed(info.Decomposition()) - } else { - rb.insertSingle(src, i, info) - } -} - -// insertDecomposed inserts an entry in to the reorderBuffer for each rune -// in dcomp. dcomp must be a sequence of decomposed UTF-8-encoded runes. -// It flushes the buffer on each new segment start. -func (rb *reorderBuffer) insertDecomposed(dcomp []byte) insertErr { - rb.tmpBytes.setBytes(dcomp) - // As the streamSafe accounting already handles the counting for modifiers, - // we don't have to call next. However, we do need to keep the accounting - // intact when flushing the buffer. - for i := 0; i < len(dcomp); { - info := rb.f.info(rb.tmpBytes, i) - if info.BoundaryBefore() && rb.nrune > 0 && !rb.doFlush() { - return iShortDst - } - i += copy(rb.byte[rb.nbyte:], dcomp[i:i+int(info.size)]) - rb.insertOrdered(info) - } - return iSuccess -} - -// insertSingle inserts an entry in the reorderBuffer for the rune at -// position i. info is the runeInfo for the rune at position i. -func (rb *reorderBuffer) insertSingle(src input, i int, info Properties) { - src.copySlice(rb.byte[rb.nbyte:], i, i+int(info.size)) - rb.insertOrdered(info) -} - -// insertCGJ inserts a Combining Grapheme Joiner (0x034f) into rb. -func (rb *reorderBuffer) insertCGJ() { - rb.insertSingle(input{str: GraphemeJoiner}, 0, Properties{size: uint8(len(GraphemeJoiner))}) -} - -// appendRune inserts a rune at the end of the buffer. It is used for Hangul. -func (rb *reorderBuffer) appendRune(r rune) { - bn := rb.nbyte - sz := utf8.EncodeRune(rb.byte[bn:], rune(r)) - rb.nbyte += utf8.UTFMax - rb.rune[rb.nrune] = Properties{pos: bn, size: uint8(sz)} - rb.nrune++ -} - -// assignRune sets a rune at position pos. It is used for Hangul and recomposition. -func (rb *reorderBuffer) assignRune(pos int, r rune) { - bn := rb.rune[pos].pos - sz := utf8.EncodeRune(rb.byte[bn:], rune(r)) - rb.rune[pos] = Properties{pos: bn, size: uint8(sz)} -} - -// runeAt returns the rune at position n. It is used for Hangul and recomposition. -func (rb *reorderBuffer) runeAt(n int) rune { - inf := rb.rune[n] - r, _ := utf8.DecodeRune(rb.byte[inf.pos : inf.pos+inf.size]) - return r -} - -// bytesAt returns the UTF-8 encoding of the rune at position n. -// It is used for Hangul and recomposition. -func (rb *reorderBuffer) bytesAt(n int) []byte { - inf := rb.rune[n] - return rb.byte[inf.pos : int(inf.pos)+int(inf.size)] -} - -// For Hangul we combine algorithmically, instead of using tables. -const ( - hangulBase = 0xAC00 // UTF-8(hangulBase) -> EA B0 80 - hangulBase0 = 0xEA - hangulBase1 = 0xB0 - hangulBase2 = 0x80 - - hangulEnd = hangulBase + jamoLVTCount // UTF-8(0xD7A4) -> ED 9E A4 - hangulEnd0 = 0xED - hangulEnd1 = 0x9E - hangulEnd2 = 0xA4 - - jamoLBase = 0x1100 // UTF-8(jamoLBase) -> E1 84 00 - jamoLBase0 = 0xE1 - jamoLBase1 = 0x84 - jamoLEnd = 0x1113 - jamoVBase = 0x1161 - jamoVEnd = 0x1176 - jamoTBase = 0x11A7 - jamoTEnd = 0x11C3 - - jamoTCount = 28 - jamoVCount = 21 - jamoVTCount = 21 * 28 - jamoLVTCount = 19 * 21 * 28 -) - -const hangulUTF8Size = 3 - -func isHangul(b []byte) bool { - if len(b) < hangulUTF8Size { - return false - } - b0 := b[0] - if b0 < hangulBase0 { - return false - } - b1 := b[1] - switch { - case b0 == hangulBase0: - return b1 >= hangulBase1 - case b0 < hangulEnd0: - return true - case b0 > hangulEnd0: - return false - case b1 < hangulEnd1: - return true - } - return b1 == hangulEnd1 && b[2] < hangulEnd2 -} - -func isHangulString(b string) bool { - if len(b) < hangulUTF8Size { - return false - } - b0 := b[0] - if b0 < hangulBase0 { - return false - } - b1 := b[1] - switch { - case b0 == hangulBase0: - return b1 >= hangulBase1 - case b0 < hangulEnd0: - return true - case b0 > hangulEnd0: - return false - case b1 < hangulEnd1: - return true - } - return b1 == hangulEnd1 && b[2] < hangulEnd2 -} - -// Caller must ensure len(b) >= 2. -func isJamoVT(b []byte) bool { - // True if (rune & 0xff00) == jamoLBase - return b[0] == jamoLBase0 && (b[1]&0xFC) == jamoLBase1 -} - -func isHangulWithoutJamoT(b []byte) bool { - c, _ := utf8.DecodeRune(b) - c -= hangulBase - return c < jamoLVTCount && c%jamoTCount == 0 -} - -// decomposeHangul writes the decomposed Hangul to buf and returns the number -// of bytes written. len(buf) should be at least 9. -func decomposeHangul(buf []byte, r rune) int { - const JamoUTF8Len = 3 - r -= hangulBase - x := r % jamoTCount - r /= jamoTCount - utf8.EncodeRune(buf, jamoLBase+r/jamoVCount) - utf8.EncodeRune(buf[JamoUTF8Len:], jamoVBase+r%jamoVCount) - if x != 0 { - utf8.EncodeRune(buf[2*JamoUTF8Len:], jamoTBase+x) - return 3 * JamoUTF8Len - } - return 2 * JamoUTF8Len -} - -// decomposeHangul algorithmically decomposes a Hangul rune into -// its Jamo components. -// See https://unicode.org/reports/tr15/#Hangul for details on decomposing Hangul. -func (rb *reorderBuffer) decomposeHangul(r rune) { - r -= hangulBase - x := r % jamoTCount - r /= jamoTCount - rb.appendRune(jamoLBase + r/jamoVCount) - rb.appendRune(jamoVBase + r%jamoVCount) - if x != 0 { - rb.appendRune(jamoTBase + x) - } -} - -// combineHangul algorithmically combines Jamo character components into Hangul. -// See https://unicode.org/reports/tr15/#Hangul for details on combining Hangul. -func (rb *reorderBuffer) combineHangul(s, i, k int) { - b := rb.rune[:] - bn := rb.nrune - for ; i < bn; i++ { - cccB := b[k-1].ccc - cccC := b[i].ccc - if cccB == 0 { - s = k - 1 - } - if s != k-1 && cccB >= cccC { - // b[i] is blocked by greater-equal cccX below it - b[k] = b[i] - k++ - } else { - l := rb.runeAt(s) // also used to compare to hangulBase - v := rb.runeAt(i) // also used to compare to jamoT - switch { - case jamoLBase <= l && l < jamoLEnd && - jamoVBase <= v && v < jamoVEnd: - // 11xx plus 116x to LV - rb.assignRune(s, hangulBase+ - (l-jamoLBase)*jamoVTCount+(v-jamoVBase)*jamoTCount) - case hangulBase <= l && l < hangulEnd && - jamoTBase < v && v < jamoTEnd && - ((l-hangulBase)%jamoTCount) == 0: - // ACxx plus 11Ax to LVT - rb.assignRune(s, l+v-jamoTBase) - default: - b[k] = b[i] - k++ - } - } - } - rb.nrune = k -} - -// compose recombines the runes in the buffer. -// It should only be used to recompose a single segment, as it will not -// handle alternations between Hangul and non-Hangul characters correctly. -func (rb *reorderBuffer) compose() { - // Lazily load the map used by the combine func below, but do - // it outside of the loop. - recompMapOnce.Do(buildRecompMap) - - // UAX #15, section X5 , including Corrigendum #5 - // "In any character sequence beginning with starter S, a character C is - // blocked from S if and only if there is some character B between S - // and C, and either B is a starter or it has the same or higher - // combining class as C." - bn := rb.nrune - if bn == 0 { - return - } - k := 1 - b := rb.rune[:] - for s, i := 0, 1; i < bn; i++ { - if isJamoVT(rb.bytesAt(i)) { - // Redo from start in Hangul mode. Necessary to support - // U+320E..U+321E in NFKC mode. - rb.combineHangul(s, i, k) - return - } - ii := b[i] - // We can only use combineForward as a filter if we later - // get the info for the combined character. This is more - // expensive than using the filter. Using combinesBackward() - // is safe. - if ii.combinesBackward() { - cccB := b[k-1].ccc - cccC := ii.ccc - blocked := false // b[i] blocked by starter or greater or equal CCC? - if cccB == 0 { - s = k - 1 - } else { - blocked = s != k-1 && cccB >= cccC - } - if !blocked { - combined := combine(rb.runeAt(s), rb.runeAt(i)) - if combined != 0 { - rb.assignRune(s, combined) - continue - } - } - } - b[k] = b[i] - k++ - } - rb.nrune = k -} |