toutf8 improved & add max git diff lines

conf/app.ini

@@ -255,3 +255,6 @@ CONN =
 [i18n]
 LANGS = en-US,zh-CN,de-DE
 NAMES = English,简体中文,Deutsch
+
+[git]
+MAX_GITDIFF_LINES = 10000

models/git_diff.go

@@ -70,7 +70,7 @@ func (diff *Diff) NumFiles() int {
 
 const DIFF_HEAD = "diff --git "
 
-func ParsePatch(pid int64, cmd *exec.Cmd, reader io.Reader) (*Diff, error) {
+func ParsePatch(pid int64, maxlines int, cmd *exec.Cmd, reader io.Reader) (*Diff, error) {
 	scanner := bufio.NewScanner(reader)
 	var (
 		curFile    *DiffFile
@@ -79,6 +79,7 @@ func ParsePatch(pid int64, cmd *exec.Cmd, reader io.Reader) (*Diff, error) {
 		}
 
 		leftLine, rightLine int
+		isTooLong           bool
 	)
 
 	diff := &Diff{Files: make([]*DiffFile, 0)}
@@ -90,16 +91,17 @@ func ParsePatch(pid int64, cmd *exec.Cmd, reader io.Reader) (*Diff, error) {
 			continue
 		}
 
+		if line == "" {
+			continue
+		}
+
 		i = i + 1
 
-		// Diff data too large.
+		// Diff data too large, we only show the first about maxlines lines
-		if i == 5000 {
+		if i == maxlines {
+			isTooLong = true
 			log.Warn("Diff data too large")
-			return &Diff{}, nil
+			//return &Diff{}, nil
-		}
-
-		if line == "" {
-			continue
 		}
 
 		switch {
@@ -110,6 +112,10 @@ func ParsePatch(pid int64, cmd *exec.Cmd, reader io.Reader) (*Diff, error) {
 			curSection.Lines = append(curSection.Lines, diffLine)
 			continue
 		case line[0] == '@':
+			if isTooLong {
+				return diff, nil
+			}
+
 			curSection = &DiffSection{}
 			curFile.Sections = append(curFile.Sections, curSection)
 			ss := strings.Split(line, "@@")
@@ -143,6 +149,10 @@ func ParsePatch(pid int64, cmd *exec.Cmd, reader io.Reader) (*Diff, error) {
 
 		// Get new file.
 		if strings.HasPrefix(line, DIFF_HEAD) {
+			if isTooLong {
+				return diff, nil
+			}
+
 			fs := strings.Split(line[len(DIFF_HEAD):], " ")
 			a := fs[0]
 
@@ -174,7 +184,7 @@ func ParsePatch(pid int64, cmd *exec.Cmd, reader io.Reader) (*Diff, error) {
 	return diff, nil
 }
 
-func GetDiffRange(repoPath, beforeCommitId string, afterCommitId string) (*Diff, error) {
+func GetDiffRange(repoPath, beforeCommitId string, afterCommitId string, maxlines int) (*Diff, error) {
 	repo, err := git.OpenRepository(repoPath)
 	if err != nil {
 		return nil, err
@@ -228,9 +238,9 @@ func GetDiffRange(repoPath, beforeCommitId string, afterCommitId string) (*Diff,
 		}
 	}()
 
-	return ParsePatch(pid, cmd, rd)
+	return ParsePatch(pid, maxlines, cmd, rd)
 }
 
-func GetDiffCommit(repoPath, commitId string) (*Diff, error) {
+func GetDiffCommit(repoPath, commitId string, maxlines int) (*Diff, error) {
-	return GetDiffRange(repoPath, "", commitId)
+	return GetDiffRange(repoPath, "", commitId, maxlines)
 }

modules/base/template.go

@@ -8,13 +8,16 @@ import (
 	"bytes"
 	"container/list"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"html/template"
 	"runtime"
 	"strings"
 	"time"
+	"code.google.com/p/mahonia"
 
 	"github.com/gogits/gogs/modules/setting"
+	"github.com/saintfish/chardet"
 )
 
 func Str2html(raw string) template.HTML {
@@ -45,6 +48,29 @@ func ShortSha(sha1 string) string {
 	return sha1
 }
 
+func ToUtf8WithErr(content []byte) (error, string) {
+	detector := chardet.NewTextDetector()
+	result, err := detector.DetectBest(content)
+	if err != nil {
+		return err, ""
+	}
+
+	if result.Charset == "utf8" {
+		return nil, string(content)
+	}
+
+	decoder := mahonia.NewDecoder(result.Charset)
+	if decoder != nil {
+		return nil, decoder.ConvertString(string(content))
+	}
+	return errors.New("unknow char decoder"), string(content)
+}
+
+func ToUtf8(content string) string {
+	_, res := ToUtf8WithErr([]byte(content))
+	return res
+}
+
 var mailDomains = map[string]string{
 	"gmail.com": "gmail.com",
 }
@@ -103,6 +129,7 @@ var TemplateFuncs template.FuncMap = map[string]interface{}{
 	"ActionContent2Commits": ActionContent2Commits,
 	"Oauth2Icon":            Oauth2Icon,
 	"Oauth2Name":            Oauth2Name,
+	"ToUtf8":                ToUtf8,
 }
 
 type Actioner interface {

modules/setting/setting.go

@@ -64,6 +64,7 @@ var (
 	// Picture settings.
 	PictureService  string
 	DisableGravatar bool
+	MaxGitDiffLines int
 
 	// Log settings.
 	LogRootPath string
@@ -241,6 +242,8 @@ func NewConfigContext() {
 		[]string{"server"})
 	DisableGravatar = Cfg.MustBool("picture", "DISABLE_GRAVATAR")
 
+	MaxGitDiffLines = Cfg.MustInt("git", "MAX_GITDIFF_LINES", 5000)
+
 	Langs = Cfg.MustValueArray("i18n", "LANGS", ",")
 	Names = Cfg.MustValueArray("i18n", "NAMES", ",")
 }

routers/repo/commit.go

@@ -12,6 +12,7 @@ import (
 	"github.com/gogits/gogs/models"
 	"github.com/gogits/gogs/modules/base"
 	"github.com/gogits/gogs/modules/middleware"
+	"github.com/gogits/gogs/modules/setting"
 )
 
 const (
@@ -114,7 +115,8 @@ func Diff(ctx *middleware.Context) {
 
 	commit := ctx.Repo.Commit
 
-	diff, err := models.GetDiffCommit(models.RepoPath(userName, repoName), commitId)
+	diff, err := models.GetDiffCommit(models.RepoPath(userName, repoName),
+		commitId, setting.MaxGitDiffLines)
 	if err != nil {
 		ctx.Handle(404, "GetDiffCommit", err)
 		return
@@ -176,7 +178,8 @@ func CompareDiff(ctx *middleware.Context) {
 		return
 	}
 
-	diff, err := models.GetDiffRange(models.RepoPath(userName, repoName), beforeCommitId, afterCommitId)
+	diff, err := models.GetDiffRange(models.RepoPath(userName, repoName), beforeCommitId,
+		afterCommitId, setting.MaxGitDiffLines)
 	if err != nil {
 		ctx.Handle(404, "GetDiffRange", err)
 		return

routers/repo/view.go

@@ -11,12 +11,9 @@ import (
 	"path/filepath"
 	"strings"
 
-	"github.com/saintfish/chardet"
-
 	"github.com/gogits/gogs/modules/base"
 	"github.com/gogits/gogs/modules/git"
 	"github.com/gogits/gogs/modules/log"
-	"github.com/gogits/gogs/modules/mahonia"
 	"github.com/gogits/gogs/modules/middleware"
 )
 
@@ -24,21 +21,6 @@ const (
 	HOME base.TplName = "repo/home"
 )
 
-func toUtf8(content []byte) (error, string) {
-	detector := chardet.NewTextDetector()
-	result, err := detector.DetectBest(content)
-	if err != nil {
-		return err, ""
-	}
-
-	if result.Charset == "utf8" {
-		return nil, string(content)
-	}
-
-	decoder := mahonia.NewDecoder(result.Charset)
-	return nil, decoder.ConvertString(string(content))
-}
-
 func Home(ctx *middleware.Context) {
 	ctx.Data["Title"] = ctx.Repo.Repository.Name
 
@@ -117,7 +99,7 @@ func Home(ctx *middleware.Context) {
 				if readmeExist {
 					ctx.Data["FileContent"] = string(base.RenderMarkdown(buf, ""))
 				} else {
-					if err, content := toUtf8(buf); err != nil {
+					if err, content := base.ToUtf8WithErr(buf); err != nil {
 						if err != nil {
 							log.Error(4, "Convert content encoding: %s", err)
 						}

templates/repo/diff.tmpl

@@ -105,7 +105,7 @@
                                 <span rel="L1">{{if .RightIdx}}{{.RightIdx}}{{end}}</span>
                             </td>
                             <td class="lines-code">
-                                <pre>{{.Content}}</pre>
+                                <pre>{{ToUtf8 .Content}}</pre>
                             </td>
                         </tr>
                         {{end}}