// Copyright 2012 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package build import ( "fmt" "log" "sort" "strings" "unicode" "golang.org/x/text/internal/colltab" "golang.org/x/text/unicode/norm" ) type logicalAnchor int const ( firstAnchor logicalAnchor = -1 noAnchor = 0 lastAnchor = 1 ) // entry is used to keep track of a single entry in the collation element table // during building. Examples of entries can be found in the Default Unicode // Collation Element Table. // See http://www.unicode.org/Public/UCA/6.0.0/allkeys.txt. type entry struct { str string // same as string(runes) runes []rune elems []rawCE // the collation elements extend string // weights of extend to be appended to elems before bool // weights relative to next instead of previous. lock bool // entry is used in extension and can no longer be moved. // prev, next, and level are used to keep track of tailorings. prev, next *entry level colltab.Level // next differs at this level skipRemove bool // do not unlink when removed decompose bool // can use NFKD decomposition to generate elems exclude bool // do not include in table implicit bool // derived, is not included in the list modified bool // entry was modified in tailoring logical logicalAnchor expansionIndex int // used to store index into expansion table contractionHandle ctHandle contractionIndex int // index into contraction elements } func (e *entry) String() string { return fmt.Sprintf("%X (%q) -> %X (ch:%x; ci:%d, ei:%d)", e.runes, e.str, e.elems, e.contractionHandle, e.contractionIndex, e.expansionIndex) } func (e *entry) skip() bool { return e.contraction() } func (e *entry) expansion() bool { return !e.decompose && len(e.elems) > 1 } func (e *entry) contraction() bool { return len(e.runes) > 1 } func (e *entry) contractionStarter() bool { return e.contractionHandle.n != 0 } // nextIndexed gets the next entry that needs to be stored in the table. // It returns the entry and the collation level at which the next entry differs // from the current entry. // Entries that can be explicitly derived and logical reset positions are // examples of entries that will not be indexed. func (e *entry) nextIndexed() (*entry, colltab.Level) { level := e.level for e = e.next; e != nil && (e.exclude || len(e.elems) == 0); e = e.next { if e.level < level { level = e.level } } return e, level } // remove unlinks entry e from the sorted chain and clears the collation // elements. e may not be at the front or end of the list. This should always // be the case, as the front and end of the list are always logical anchors, // which may not be removed. func (e *entry) remove() { if e.logical != noAnchor { log.Fatalf("may not remove anchor %q", e.str) } // TODO: need to set e.prev.level to e.level if e.level is smaller? e.elems = nil if !e.skipRemove { if e.prev != nil { e.prev.next = e.next } if e.next != nil { e.next.prev = e.prev } } e.skipRemove = false } // insertAfter inserts n after e. func (e *entry) insertAfter(n *entry) { if e == n { panic("e == anchor") } if e == nil { panic("unexpected nil anchor") } n.remove() n.decompose = false // redo decomposition test n.next = e.next n.prev = e if e.next != nil { e.next.prev = n } e.next = n } // insertBefore inserts n before e. func (e *entry) insertBefore(n *entry) { if e == n { panic("e == anchor") } if e == nil { panic("unexpected nil anchor") } n.remove() n.decompose = false // redo decomposition test n.prev = e.prev n.next = e if e.prev != nil { e.prev.next = n } e.prev = n } func (e *entry) encodeBase() (ce uint32, err error) { switch { case e.expansion(): ce, err = makeExpandIndex(e.expansionIndex) default: if e.decompose { log.Fatal("decompose should be handled elsewhere") } ce, err = makeCE(e.elems[0]) } return } func (e *entry) encode() (ce uint32, err error) { if e.skip() { log.Fatal("cannot build colElem for entry that should be skipped") } switch { case e.decompose: t1 := e.elems[0].w[2] t2 := 0 if len(e.elems) > 1 { t2 = e.elems[1].w[2] } ce, err = makeDecompose(t1, t2) case e.contractionStarter(): ce, err = makeContractIndex(e.contractionHandle, e.contractionIndex) default: if len(e.runes) > 1 { log.Fatal("colElem: contractions are handled in contraction trie") } ce, err = e.encodeBase() } return } // entryLess returns true if a sorts before b and false otherwise. func entryLess(a, b *entry) bool { if res, _ := compareWeights(a.elems, b.elems); res != 0 { return res == -1 } if a.logical != noAnchor { return a.logical == firstAnchor } if b.logical != noAnchor { return b.logical == lastAnchor } return a.str < b.str } type sortedEntries []*entry func (s sortedEntries) Len() int { return len(s) } func (s sortedEntries) Swap(i, j int) { s[i], s[j] = s[j], s[i] } func (s sortedEntries) Less(i, j int) bool { return entryLess(s[i], s[j]) } type ordering struct { id string entryMap map[string]*entry ordered []*entry handle *trieHandle } // insert inserts e into both entryMap and ordered. // Note that insert simply appends e to ordered. To reattain a sorted // order, o.sort() should be called. func (o *ordering) insert(e *entry) { if e.logical == noAnchor { o.entryMap[e.str] = e } else { // Use key format as used in UCA rules. o.entryMap[fmt.Sprintf("[%s]", e.str)] = e // Also add index entry for XML format. o.entryMap[fmt.Sprintf("<%s/>", strings.Replace(e.str, " ", "_", -1))] = e } o.ordered = append(o.ordered, e) } // newEntry creates a new entry for the given info and inserts it into // the index. func (o *ordering) newEntry(s string, ces []rawCE) *entry { e := &entry{ runes: []rune(s), elems: ces, str: s, } o.insert(e) return e } // find looks up and returns the entry for the given string. // It returns nil if str is not in the index and if an implicit value // cannot be derived, that is, if str represents more than one rune. func (o *ordering) find(str string) *entry { e := o.entryMap[str] if e == nil { r := []rune(str) if len(r) == 1 { const ( firstHangul = 0xAC00 lastHangul = 0xD7A3 ) if r[0] >= firstHangul && r[0] <= lastHangul { ce := []rawCE{} nfd := norm.NFD.String(str) for _, r := range nfd { ce = append(ce, o.find(string(r)).elems...) } e = o.newEntry(nfd, ce) } else { e = o.newEntry(string(r[0]), []rawCE{ {w: []int{ implicitPrimary(r[0]), defaultSecondary, defaultTertiary, int(r[0]), }, }, }) e.modified = true } e.exclude = true // do not index implicits } } return e } // makeRootOrdering returns a newly initialized ordering value and populates // it with a set of logical reset points that can be used as anchors. // The anchors first_tertiary_ignorable and __END__ will always sort at // the beginning and end, respectively. This means that prev and next are non-nil // for any indexed entry. func makeRootOrdering() ordering { const max = unicode.MaxRune o := ordering{ entryMap: make(map[string]*entry), } insert := func(typ logicalAnchor, s string, ce []int) { e := &entry{ elems: []rawCE{{w: ce}}, str: s, exclude: true, logical: typ, } o.insert(e) } insert(firstAnchor, "first tertiary ignorable", []int{0, 0, 0, 0}) insert(lastAnchor, "last tertiary ignorable", []int{0, 0, 0, max}) insert(lastAnchor, "last primary ignorable", []int{0, defaultSecondary, defaultTertiary, max}) insert(lastAnchor, "last non ignorable", []int{maxPrimary, defaultSecondary, defaultTertiary, max}) insert(lastAnchor, "__END__", []int{1 << maxPrimaryBits, defaultSecondary, defaultTertiary, max}) return o } // patchForInsert eleminates entries from the list with more than one collation element. // The next and prev fields of the eliminated entries still point to appropriate // values in the newly created list. // It requires that sort has been called. func (o *ordering) patchForInsert() { for i := 0; i < len(o.ordered)-1; { e := o.ordered[i] lev := e.level n := e.next for ; n != nil && len(n.elems) > 1; n = n.next { if n.level < lev { lev = n.level } n.skipRemove = true } for ; o.ordered[i] != n; i++ { o.ordered[i].level = lev o.ordered[i].next = n o.ordered[i+1].prev = e } } } // clone copies all ordering of es into a new ordering value. func (o *ordering) clone() *ordering { o.sort() oo := ordering{ entryMap: make(map[string]*entry), } for _, e := range o.ordered { ne := &entry{ runes: e.runes, elems: e.elems, str: e.str, decompose: e.decompose, exclude: e.exclude, logical: e.logical, } oo.insert(ne) } oo.sort() // link all ordering. oo.patchForInsert() return &oo } // front returns the first entry to be indexed. // It assumes that sort() has been called. func (o *ordering) front() *entry { e := o.ordered[0] if e.prev != nil { log.Panicf("unexpected first entry: %v", e) } // The first entry is always a logical position, which should not be indexed. e, _ = e.nextIndexed() return e } // sort sorts all ordering based on their collation elements and initializes // the prev, next, and level fields accordingly. func (o *ordering) sort() { sort.Sort(sortedEntries(o.ordered)) l := o.ordered for i := 1; i < len(l); i++ { k := i - 1 l[k].next = l[i] _, l[k].level = compareWeights(l[k].elems, l[i].elems) l[i].prev = l[k] } } // genColElems generates a collation element array from the runes in str. This // assumes that all collation elements have already been added to the Builder. func (o *ordering) genColElems(str string) []rawCE { elems := []rawCE{} for _, r := range []rune(str) { for _, ce := range o.find(string(r)).elems { if ce.w[0] != 0 || ce.w[1] != 0 || ce.w[2] != 0 { elems = append(elems, ce) } } } return elems }