#2185 fall back to use custom chardet lib

This commit is contained in:
Unknwon 2015-12-31 22:13:47 -05:00
parent a62290de52
commit 4993ab1a76
8 changed files with 22 additions and 17 deletions

View File

@ -16,6 +16,7 @@ github.com/go-macaron/toolbox = commit:ab30a81
github.com/go-sql-driver/mysql = commit:d512f20 github.com/go-sql-driver/mysql = commit:d512f20
github.com/go-xorm/core = commit:acb6f00 github.com/go-xorm/core = commit:acb6f00
github.com/go-xorm/xorm = commit:a8fba4d github.com/go-xorm/xorm = commit:a8fba4d
github.com/gogits/chardet = commit:2404f77725
github.com/gogits/git-module = commit:5cd57b9 github.com/gogits/git-module = commit:5cd57b9
github.com/gogits/go-gogs-client = commit:78460e9 github.com/gogits/go-gogs-client = commit:78460e9
github.com/issue9/identicon = commit:f8c0d2c github.com/issue9/identicon = commit:f8c0d2c

View File

@ -3,7 +3,7 @@ Gogs - Go Git Service [![Build Status](https://travis-ci.org/gogits/gogs.svg?bra
![](https://github.com/gogits/gogs/blob/master/public/img/gogs-large-resize.png?raw=true) ![](https://github.com/gogits/gogs/blob/master/public/img/gogs-large-resize.png?raw=true)
##### Current version: 0.8.14 ##### Current version: 0.8.15
| Web | UI | Preview | | Web | UI | Preview |
|:-------------:|:-------:|:-------:| |:-------------:|:-------:|:-------:|

View File

@ -17,7 +17,7 @@ import (
"github.com/gogits/gogs/modules/setting" "github.com/gogits/gogs/modules/setting"
) )
const APP_VER = "0.8.14.1230" const APP_VER = "0.8.15.1231"
func init() { func init() {
runtime.GOMAXPROCS(runtime.NumCPU()) runtime.GOMAXPROCS(runtime.NumCPU())

View File

@ -246,8 +246,8 @@ func ParsePatch(maxlines int, reader io.Reader) (*Diff, error) {
buf.WriteString("\n") buf.WriteString("\n")
} }
} }
charsetLabel := base.DetectEncoding(buf.Bytes()) charsetLabel, err := base.DetectEncoding(buf.Bytes())
if charsetLabel != "UTF-8" { if charsetLabel != "UTF-8" && err == nil {
encoding, _ := charset.Lookup(charsetLabel) encoding, _ := charset.Lookup(charsetLabel)
if encoding != nil { if encoding != nil {
d := encoding.NewDecoder() d := encoding.NewDecoder()

View File

@ -23,7 +23,8 @@ import (
"github.com/Unknwon/com" "github.com/Unknwon/com"
"github.com/Unknwon/i18n" "github.com/Unknwon/i18n"
"github.com/microcosm-cc/bluemonday" "github.com/microcosm-cc/bluemonday"
"golang.org/x/net/html/charset"
"github.com/gogits/chardet"
"github.com/gogits/gogs/modules/avatar" "github.com/gogits/gogs/modules/avatar"
"github.com/gogits/gogs/modules/log" "github.com/gogits/gogs/modules/log"
@ -53,19 +54,20 @@ func ShortSha(sha1 string) string {
return sha1 return sha1
} }
func DetectEncoding(content []byte) string { func DetectEncoding(content []byte) (string, error) {
if utf8.Valid(content[:1024]) { if utf8.Valid(content) {
log.Debug("Detected encoding: utf-8 (fast)") log.Debug("Detected encoding: utf-8 (fast)")
return "utf-8" return "UTF-8", nil
} }
_, name, certain := charset.DetermineEncoding(content, "") result, err := chardet.NewTextDetector().DetectBest(content)
if name != "utf-8" && len(setting.Repository.AnsiCharset) > 0 { if result.Charset != "UTF-8" && len(setting.Repository.AnsiCharset) > 0 {
log.Debug("Using default AnsiCharset: %s", setting.Repository.AnsiCharset) log.Debug("Using default AnsiCharset: %s", setting.Repository.AnsiCharset)
return setting.Repository.AnsiCharset return setting.Repository.AnsiCharset, err
} }
log.Debug("Detected encoding: %s (%v)", name, certain)
return name log.Debug("Detected encoding: %s", result.Charset)
return result.Charset, err
} }
func BasicAuthDecode(encoded string) (string, string, error) { func BasicAuthDecode(encoded string) (string, string, error) {

View File

@ -130,8 +130,10 @@ func Sha1(str string) string {
} }
func ToUtf8WithErr(content []byte) (error, string) { func ToUtf8WithErr(content []byte) (error, string) {
charsetLabel := base.DetectEncoding(content) charsetLabel, err := base.DetectEncoding(content)
if charsetLabel == "utf-8" { if err != nil {
return err, ""
} else if charsetLabel == "UTF-8" {
return nil, string(content) return nil, string(content)
} }

View File

@ -1 +1 @@
0.8.14.1230 0.8.15.1231

View File

@ -7,7 +7,7 @@
<footer> <footer>
<div class="ui container"> <div class="ui container">
<div class="ui left"> <div class="ui left">
© 2015 Gogs {{if (or .ShowFooterVersion .PageIsAdmin)}}{{.i18n.Tr "version"}}: {{AppVer}}{{end}} {{.i18n.Tr "page"}}: <strong>{{LoadTimes .PageStartTime}}</strong> {{.i18n.Tr "template"}}: <strong>{{call .TmplLoadTimes}}</strong> © 2016 Gogs {{if (or .ShowFooterVersion .PageIsAdmin)}}{{.i18n.Tr "version"}}: {{AppVer}}{{end}} {{.i18n.Tr "page"}}: <strong>{{LoadTimes .PageStartTime}}</strong> {{.i18n.Tr "template"}}: <strong>{{call .TmplLoadTimes}}</strong>
</div> </div>
<div class="ui right links"> <div class="ui right links">
{{if .ShowFooterBranding}} {{if .ShowFooterBranding}}