// Copyright 2015 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. // +build ignore // This program generates the trie for width operations. The generated table // includes width category information as well as the normalization mappings. package main import ( "bytes" "fmt" "io" "log" "math" "unicode/utf8" "golang.org/x/text/internal/gen" "golang.org/x/text/internal/triegen" ) // See gen_common.go for flags. func main() { gen.Init() genTables() genTests() gen.Repackage("gen_trieval.go", "trieval.go", "width") gen.Repackage("gen_common.go", "common_test.go", "width") } func genTables() { t := triegen.NewTrie("width") // fold and inverse mappings. See mapComment for a description of the format // of each entry. Add dummy value to make an index of 0 mean no mapping. inverse := [][4]byte{{}} mapping := map[[4]byte]int{[4]byte{}: 0} getWidthData(func(r rune, tag elem, alt rune) { idx := 0 if alt != 0 { var buf [4]byte buf[0] = byte(utf8.EncodeRune(buf[1:], alt)) s := string(r) buf[buf[0]] ^= s[len(s)-1] var ok bool if idx, ok = mapping[buf]; !ok { idx = len(mapping) if idx > math.MaxUint8 { log.Fatalf("Index %d does not fit in a byte.", idx) } mapping[buf] = idx inverse = append(inverse, buf) } } t.Insert(r, uint64(tag|elem(idx))) }) w := &bytes.Buffer{} gen.WriteUnicodeVersion(w) sz, err := t.Gen(w) if err != nil { log.Fatal(err) } sz += writeMappings(w, inverse) fmt.Fprintf(w, "// Total table size %d bytes (%dKiB)\n", sz, sz/1024) gen.WriteVersionedGoFile(*outputFile, "width", w.Bytes()) } const inverseDataComment = ` // inverseData contains 4-byte entries of the following format: // <0 padding> // The last byte of the UTF-8-encoded rune is xor-ed with the last byte of the // UTF-8 encoding of the original rune. Mappings often have the following // pattern: // A -> A (U+FF21 -> U+0041) // B -> B (U+FF22 -> U+0042) // ... // By xor-ing the last byte the same entry can be shared by many mappings. This // reduces the total number of distinct entries by about two thirds. // The resulting entry for the aforementioned mappings is // { 0x01, 0xE0, 0x00, 0x00 } // Using this entry to map U+FF21 (UTF-8 [EF BC A1]), we get // E0 ^ A1 = 41. // Similarly, for U+FF22 (UTF-8 [EF BC A2]), we get // E0 ^ A2 = 42. // Note that because of the xor-ing, the byte sequence stored in the entry is // not valid UTF-8.` func writeMappings(w io.Writer, data [][4]byte) int { fmt.Fprintln(w, inverseDataComment) fmt.Fprintf(w, "var inverseData = [%d][4]byte{\n", len(data)) for _, x := range data { fmt.Fprintf(w, "{ 0x%02x, 0x%02x, 0x%02x, 0x%02x },\n", x[0], x[1], x[2], x[3]) } fmt.Fprintln(w, "}") return len(data) * 4 } func genTests() { w := &bytes.Buffer{} fmt.Fprintf(w, "\nvar mapRunes = map[rune]struct{r rune; e elem}{\n") getWidthData(func(r rune, tag elem, alt rune) { if alt != 0 { fmt.Fprintf(w, "\t0x%X: {0x%X, 0x%X},\n", r, alt, tag) } }) fmt.Fprintln(w, "}") gen.WriteGoFile("runes_test.go", "width", w.Bytes()) }