mirror of
https://codeberg.org/forgejo/forgejo.git
synced 2025-06-23 09:30:50 +00:00
Update to last common x/text (#3994)
This commit is contained in:
Antoine GIRARD
Lunny Xiao
parent
000b2d33a9
commit
edc78b9633
29 changed files with 4131 additions and 7612 deletions
60
vendor/golang.org/x/text/encoding/simplifiedchinese/gbk.go
generated
vendored
60
vendor/golang.org/x/text/encoding/simplifiedchinese/gbk.go
generated
vendored
|
|
@ -5,7 +5,6 @@
|
|||
package simplifiedchinese
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"unicode/utf8"
|
||||
|
||||
"golang.org/x/text/encoding"
|
||||
|
|
@ -40,11 +39,6 @@ var gbk18030 = internal.Encoding{
|
|||
identifier.GB18030,
|
||||
}
|
||||
|
||||
var (
|
||||
errInvalidGB18030 = errors.New("simplifiedchinese: invalid GB18030 encoding")
|
||||
errInvalidGBK = errors.New("simplifiedchinese: invalid GBK encoding")
|
||||
)
|
||||
|
||||
type gbkDecoder struct {
|
||||
transform.NopResetter
|
||||
gb18030 bool
|
||||
|
|
@ -66,8 +60,12 @@ loop:
|
|||
|
||||
case c0 < 0xff:
|
||||
if nSrc+1 >= len(src) {
|
||||
err = transform.ErrShortSrc
|
||||
break loop
|
||||
if !atEOF {
|
||||
err = transform.ErrShortSrc
|
||||
break loop
|
||||
}
|
||||
r, size = utf8.RuneError, 1
|
||||
goto write
|
||||
}
|
||||
c1 := src[nSrc+1]
|
||||
switch {
|
||||
|
|
@ -77,18 +75,24 @@ loop:
|
|||
c1 -= 0x41
|
||||
case d.gb18030 && 0x30 <= c1 && c1 < 0x40:
|
||||
if nSrc+3 >= len(src) {
|
||||
err = transform.ErrShortSrc
|
||||
break loop
|
||||
if !atEOF {
|
||||
err = transform.ErrShortSrc
|
||||
break loop
|
||||
}
|
||||
// The second byte here is always ASCII, so we can set size
|
||||
// to 1 in all cases.
|
||||
r, size = utf8.RuneError, 1
|
||||
goto write
|
||||
}
|
||||
c2 := src[nSrc+2]
|
||||
if c2 < 0x81 || 0xff <= c2 {
|
||||
err = errInvalidGB18030
|
||||
break loop
|
||||
r, size = utf8.RuneError, 1
|
||||
goto write
|
||||
}
|
||||
c3 := src[nSrc+3]
|
||||
if c3 < 0x30 || 0x3a <= c3 {
|
||||
err = errInvalidGB18030
|
||||
break loop
|
||||
r, size = utf8.RuneError, 1
|
||||
goto write
|
||||
}
|
||||
size = 4
|
||||
r = ((rune(c0-0x81)*10+rune(c1-0x30))*126+rune(c2-0x81))*10 + rune(c3-0x30)
|
||||
|
|
@ -109,17 +113,13 @@ loop:
|
|||
r -= 189000
|
||||
if 0 <= r && r < 0x100000 {
|
||||
r += 0x10000
|
||||
goto write
|
||||
}
|
||||
err = errInvalidGB18030
|
||||
break loop
|
||||
default:
|
||||
if d.gb18030 {
|
||||
err = errInvalidGB18030
|
||||
} else {
|
||||
err = errInvalidGBK
|
||||
r, size = utf8.RuneError, 1
|
||||
}
|
||||
break loop
|
||||
goto write
|
||||
default:
|
||||
r, size = utf8.RuneError, 1
|
||||
goto write
|
||||
}
|
||||
r, size = '\ufffd', 2
|
||||
if i := int(c0-0x81)*190 + int(c1); i < len(decode) {
|
||||
|
|
@ -130,12 +130,7 @@ loop:
|
|||
}
|
||||
|
||||
default:
|
||||
if d.gb18030 {
|
||||
err = errInvalidGB18030
|
||||
} else {
|
||||
err = errInvalidGBK
|
||||
}
|
||||
break loop
|
||||
r, size = utf8.RuneError, 1
|
||||
}
|
||||
|
||||
write:
|
||||
|
|
@ -145,13 +140,6 @@ loop:
|
|||
}
|
||||
nDst += utf8.EncodeRune(dst[nDst:], r)
|
||||
}
|
||||
if atEOF && err == transform.ErrShortSrc {
|
||||
if d.gb18030 {
|
||||
err = errInvalidGB18030
|
||||
} else {
|
||||
err = errInvalidGBK
|
||||
}
|
||||
}
|
||||
return nDst, nSrc, err
|
||||
}
|
||||
|
||||
|
|
|
|||
49
vendor/golang.org/x/text/encoding/simplifiedchinese/hzgb2312.go
generated
vendored
49
vendor/golang.org/x/text/encoding/simplifiedchinese/hzgb2312.go
generated
vendored
|
|
@ -5,7 +5,6 @@
|
|||
package simplifiedchinese
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"unicode/utf8"
|
||||
|
||||
"golang.org/x/text/encoding"
|
||||
|
|
@ -31,8 +30,6 @@ func hzGB2312NewEncoder() transform.Transformer {
|
|||
return new(hzGB2312Encoder)
|
||||
}
|
||||
|
||||
var errInvalidHZGB2312 = errors.New("simplifiedchinese: invalid HZ-GB2312 encoding")
|
||||
|
||||
const (
|
||||
asciiState = iota
|
||||
gbState
|
||||
|
|
@ -50,14 +47,18 @@ loop:
|
|||
for ; nSrc < len(src); nSrc += size {
|
||||
c0 := src[nSrc]
|
||||
if c0 >= utf8.RuneSelf {
|
||||
err = errInvalidHZGB2312
|
||||
break loop
|
||||
r, size = utf8.RuneError, 1
|
||||
goto write
|
||||
}
|
||||
|
||||
if c0 == '~' {
|
||||
if nSrc+1 >= len(src) {
|
||||
err = transform.ErrShortSrc
|
||||
break loop
|
||||
if !atEOF {
|
||||
err = transform.ErrShortSrc
|
||||
break loop
|
||||
}
|
||||
r = utf8.RuneError
|
||||
goto write
|
||||
}
|
||||
size = 2
|
||||
switch src[nSrc+1] {
|
||||
|
|
@ -78,8 +79,8 @@ loop:
|
|||
case '\n':
|
||||
continue
|
||||
default:
|
||||
err = errInvalidHZGB2312
|
||||
break loop
|
||||
r = utf8.RuneError
|
||||
goto write
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -87,33 +88,37 @@ loop:
|
|||
r, size = rune(c0), 1
|
||||
} else {
|
||||
if nSrc+1 >= len(src) {
|
||||
err = transform.ErrShortSrc
|
||||
break loop
|
||||
if !atEOF {
|
||||
err = transform.ErrShortSrc
|
||||
break loop
|
||||
}
|
||||
r, size = utf8.RuneError, 1
|
||||
goto write
|
||||
}
|
||||
size = 2
|
||||
c1 := src[nSrc+1]
|
||||
if c0 < 0x21 || 0x7e <= c0 || c1 < 0x21 || 0x7f <= c1 {
|
||||
err = errInvalidHZGB2312
|
||||
break loop
|
||||
}
|
||||
|
||||
r, size = '\ufffd', 2
|
||||
if i := int(c0-0x01)*190 + int(c1+0x3f); i < len(decode) {
|
||||
// error
|
||||
} else if i := int(c0-0x01)*190 + int(c1+0x3f); i < len(decode) {
|
||||
r = rune(decode[i])
|
||||
if r == 0 {
|
||||
r = '\ufffd'
|
||||
if r != 0 {
|
||||
goto write
|
||||
}
|
||||
}
|
||||
if c1 > utf8.RuneSelf {
|
||||
// Be consistent and always treat non-ASCII as a single error.
|
||||
size = 1
|
||||
}
|
||||
r = utf8.RuneError
|
||||
}
|
||||
|
||||
write:
|
||||
if nDst+utf8.RuneLen(r) > len(dst) {
|
||||
err = transform.ErrShortDst
|
||||
break loop
|
||||
}
|
||||
nDst += utf8.EncodeRune(dst[nDst:], r)
|
||||
}
|
||||
if atEOF && err == transform.ErrShortSrc {
|
||||
err = errInvalidHZGB2312
|
||||
}
|
||||
return nDst, nSrc, err
|
||||
}
|
||||
|
||||
|
|
|
|||
161
vendor/golang.org/x/text/encoding/simplifiedchinese/maketables.go
generated
vendored
161
vendor/golang.org/x/text/encoding/simplifiedchinese/maketables.go
generated
vendored
|
|
@ -1,161 +0,0 @@
|
|||
// Copyright 2013 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
// +build ignore
|
||||
|
||||
package main
|
||||
|
||||
// This program generates tables.go:
|
||||
// go run maketables.go | gofmt > tables.go
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func main() {
|
||||
fmt.Printf("// generated by go run maketables.go; DO NOT EDIT\n\n")
|
||||
fmt.Printf("// Package simplifiedchinese provides Simplified Chinese encodings such as GBK.\n")
|
||||
fmt.Printf(`package simplifiedchinese // import "golang.org/x/text/encoding/simplifiedchinese"` + "\n\n")
|
||||
|
||||
printGB18030()
|
||||
printGBK()
|
||||
}
|
||||
|
||||
func printGB18030() {
|
||||
res, err := http.Get("http://encoding.spec.whatwg.org/index-gb18030.txt")
|
||||
if err != nil {
|
||||
log.Fatalf("Get: %v", err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
|
||||
fmt.Printf("// gb18030 is the table from http://encoding.spec.whatwg.org/index-gb18030.txt\n")
|
||||
fmt.Printf("var gb18030 = [...][2]uint16{\n")
|
||||
scanner := bufio.NewScanner(res.Body)
|
||||
for scanner.Scan() {
|
||||
s := strings.TrimSpace(scanner.Text())
|
||||
if s == "" || s[0] == '#' {
|
||||
continue
|
||||
}
|
||||
x, y := uint32(0), uint32(0)
|
||||
if _, err := fmt.Sscanf(s, "%d 0x%x", &x, &y); err != nil {
|
||||
log.Fatalf("could not parse %q", s)
|
||||
}
|
||||
if x < 0x10000 && y < 0x10000 {
|
||||
fmt.Printf("\t{0x%04x, 0x%04x},\n", x, y)
|
||||
}
|
||||
}
|
||||
fmt.Printf("}\n\n")
|
||||
}
|
||||
|
||||
func printGBK() {
|
||||
res, err := http.Get("http://encoding.spec.whatwg.org/index-gbk.txt")
|
||||
if err != nil {
|
||||
log.Fatalf("Get: %v", err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
|
||||
mapping := [65536]uint16{}
|
||||
reverse := [65536]uint16{}
|
||||
|
||||
scanner := bufio.NewScanner(res.Body)
|
||||
for scanner.Scan() {
|
||||
s := strings.TrimSpace(scanner.Text())
|
||||
if s == "" || s[0] == '#' {
|
||||
continue
|
||||
}
|
||||
x, y := uint16(0), uint16(0)
|
||||
if _, err := fmt.Sscanf(s, "%d 0x%x", &x, &y); err != nil {
|
||||
log.Fatalf("could not parse %q", s)
|
||||
}
|
||||
if x < 0 || 126*190 <= x {
|
||||
log.Fatalf("GBK code %d is out of range", x)
|
||||
}
|
||||
mapping[x] = y
|
||||
if reverse[y] == 0 {
|
||||
c0, c1 := x/190, x%190
|
||||
if c1 >= 0x3f {
|
||||
c1++
|
||||
}
|
||||
reverse[y] = (0x81+c0)<<8 | (0x40 + c1)
|
||||
}
|
||||
}
|
||||
if err := scanner.Err(); err != nil {
|
||||
log.Fatalf("scanner error: %v", err)
|
||||
}
|
||||
|
||||
fmt.Printf("// decode is the decoding table from GBK code to Unicode.\n")
|
||||
fmt.Printf("// It is defined at http://encoding.spec.whatwg.org/index-gbk.txt\n")
|
||||
fmt.Printf("var decode = [...]uint16{\n")
|
||||
for i, v := range mapping {
|
||||
if v != 0 {
|
||||
fmt.Printf("\t%d: 0x%04X,\n", i, v)
|
||||
}
|
||||
}
|
||||
fmt.Printf("}\n\n")
|
||||
|
||||
// Any run of at least separation continuous zero entries in the reverse map will
|
||||
// be a separate encode table.
|
||||
const separation = 1024
|
||||
|
||||
intervals := []interval(nil)
|
||||
low, high := -1, -1
|
||||
for i, v := range reverse {
|
||||
if v == 0 {
|
||||
continue
|
||||
}
|
||||
if low < 0 {
|
||||
low = i
|
||||
} else if i-high >= separation {
|
||||
if high >= 0 {
|
||||
intervals = append(intervals, interval{low, high})
|
||||
}
|
||||
low = i
|
||||
}
|
||||
high = i + 1
|
||||
}
|
||||
if high >= 0 {
|
||||
intervals = append(intervals, interval{low, high})
|
||||
}
|
||||
sort.Sort(byDecreasingLength(intervals))
|
||||
|
||||
fmt.Printf("const numEncodeTables = %d\n\n", len(intervals))
|
||||
fmt.Printf("// encodeX are the encoding tables from Unicode to GBK code,\n")
|
||||
fmt.Printf("// sorted by decreasing length.\n")
|
||||
for i, v := range intervals {
|
||||
fmt.Printf("// encode%d: %5d entries for runes in [%5d, %5d).\n", i, v.len(), v.low, v.high)
|
||||
}
|
||||
fmt.Printf("\n")
|
||||
|
||||
for i, v := range intervals {
|
||||
fmt.Printf("const encode%dLow, encode%dHigh = %d, %d\n\n", i, i, v.low, v.high)
|
||||
fmt.Printf("var encode%d = [...]uint16{\n", i)
|
||||
for j := v.low; j < v.high; j++ {
|
||||
x := reverse[j]
|
||||
if x == 0 {
|
||||
continue
|
||||
}
|
||||
fmt.Printf("\t%d-%d: 0x%04X,\n", j, v.low, x)
|
||||
}
|
||||
fmt.Printf("}\n\n")
|
||||
}
|
||||
}
|
||||
|
||||
// interval is a half-open interval [low, high).
|
||||
type interval struct {
|
||||
low, high int
|
||||
}
|
||||
|
||||
func (i interval) len() int { return i.high - i.low }
|
||||
|
||||
// byDecreasingLength sorts intervals by decreasing length.
|
||||
type byDecreasingLength []interval
|
||||
|
||||
func (b byDecreasingLength) Len() int { return len(b) }
|
||||
func (b byDecreasingLength) Less(i, j int) bool { return b[i].len() > b[j].len() }
|
||||
func (b byDecreasingLength) Swap(i, j int) { b[i], b[j] = b[j], b[i] }
|
||||
Loading…
Add table
Add a link
Reference in a new issue