Read expected buffer size (#17409)

* Read expected buffer size.

* Changed name.
tokarchuk/v1.17
KN4CK3R 3 years ago committed by GitHub
parent 932780c2bb
commit f99d50fc9f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 5
      modules/charset/charset.go
  2. 5
      modules/csv/csv.go
  3. 3
      modules/git/blob.go
  4. 5
      modules/repofiles/update.go
  5. 6
      modules/typesniffer/typesniffer.go
  6. 20
      modules/util/io.go
  7. 5
      routers/common/repo.go
  8. 4
      routers/web/repo/editor.go
  9. 7
      routers/web/repo/lfs.go
  10. 13
      routers/web/repo/view.go
  11. 5
      services/attachment/attachment.go

@ -13,6 +13,7 @@ import (
"code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/util"
"github.com/gogs/chardet" "github.com/gogs/chardet"
"golang.org/x/net/html/charset" "golang.org/x/net/html/charset"
@ -25,9 +26,9 @@ var UTF8BOM = []byte{'\xef', '\xbb', '\xbf'}
// ToUTF8WithFallbackReader detects the encoding of content and coverts to UTF-8 reader if possible // ToUTF8WithFallbackReader detects the encoding of content and coverts to UTF-8 reader if possible
func ToUTF8WithFallbackReader(rd io.Reader) io.Reader { func ToUTF8WithFallbackReader(rd io.Reader) io.Reader {
var buf = make([]byte, 2048) var buf = make([]byte, 2048)
n, err := rd.Read(buf) n, err := util.ReadAtMost(rd, buf)
if err != nil { if err != nil {
return rd return io.MultiReader(bytes.NewReader(RemoveBOMIfPresent(buf[:n])), rd)
} }
charsetLabel, err := DetectEncoding(buf[:n]) charsetLabel, err := DetectEncoding(buf[:n])

@ -29,11 +29,8 @@ func CreateReader(input io.Reader, delimiter rune) *stdcsv.Reader {
// CreateReaderAndGuessDelimiter tries to guess the field delimiter from the content and creates a csv.Reader. // CreateReaderAndGuessDelimiter tries to guess the field delimiter from the content and creates a csv.Reader.
func CreateReaderAndGuessDelimiter(rd io.Reader) (*stdcsv.Reader, error) { func CreateReaderAndGuessDelimiter(rd io.Reader) (*stdcsv.Reader, error) {
var data = make([]byte, 1e4) var data = make([]byte, 1e4)
size, err := rd.Read(data) size, err := util.ReadAtMost(rd, data)
if err != nil { if err != nil {
if err == io.EOF {
return CreateReader(bytes.NewReader([]byte{}), rune(',')), nil
}
return nil, err return nil, err
} }

@ -11,6 +11,7 @@ import (
"io" "io"
"code.gitea.io/gitea/modules/typesniffer" "code.gitea.io/gitea/modules/typesniffer"
"code.gitea.io/gitea/modules/util"
) )
// This file contains common functions between the gogit and !gogit variants for git Blobs // This file contains common functions between the gogit and !gogit variants for git Blobs
@ -28,7 +29,7 @@ func (b *Blob) GetBlobContent() (string, error) {
} }
defer dataRc.Close() defer dataRc.Close()
buf := make([]byte, 1024) buf := make([]byte, 1024)
n, _ := dataRc.Read(buf) n, _ := util.ReadAtMost(dataRc, buf)
buf = buf[:n] buf = buf[:n]
return string(buf), nil return string(buf), nil
} }

@ -19,6 +19,7 @@ import (
repo_module "code.gitea.io/gitea/modules/repository" repo_module "code.gitea.io/gitea/modules/repository"
"code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/structs"
"code.gitea.io/gitea/modules/util"
stdcharset "golang.org/x/net/html/charset" stdcharset "golang.org/x/net/html/charset"
"golang.org/x/text/transform" "golang.org/x/text/transform"
@ -61,7 +62,7 @@ func detectEncodingAndBOM(entry *git.TreeEntry, repo *models.Repository) (string
} }
defer reader.Close() defer reader.Close()
buf := make([]byte, 1024) buf := make([]byte, 1024)
n, err := reader.Read(buf) n, err := util.ReadAtMost(reader, buf)
if err != nil { if err != nil {
// return default // return default
return "UTF-8", false return "UTF-8", false
@ -84,7 +85,7 @@ func detectEncodingAndBOM(entry *git.TreeEntry, repo *models.Repository) (string
} }
defer dataRc.Close() defer dataRc.Close()
buf = make([]byte, 1024) buf = make([]byte, 1024)
n, err = dataRc.Read(buf) n, err = util.ReadAtMost(dataRc, buf)
if err != nil { if err != nil {
// return default // return default
return "UTF-8", false return "UTF-8", false

@ -10,6 +10,8 @@ import (
"net/http" "net/http"
"regexp" "regexp"
"strings" "strings"
"code.gitea.io/gitea/modules/util"
) )
// Use at most this many bytes to determine Content Type. // Use at most this many bytes to determine Content Type.
@ -86,8 +88,8 @@ func DetectContentType(data []byte) SniffedType {
// DetectContentTypeFromReader guesses the content type contained in the reader. // DetectContentTypeFromReader guesses the content type contained in the reader.
func DetectContentTypeFromReader(r io.Reader) (SniffedType, error) { func DetectContentTypeFromReader(r io.Reader) (SniffedType, error) {
buf := make([]byte, sniffLen) buf := make([]byte, sniffLen)
n, err := r.Read(buf) n, err := util.ReadAtMost(r, buf)
if err != nil && err != io.EOF { if err != nil {
return SniffedType{}, fmt.Errorf("DetectContentTypeFromReader io error: %w", err) return SniffedType{}, fmt.Errorf("DetectContentTypeFromReader io error: %w", err)
} }
buf = buf[:n] buf = buf[:n]

@ -0,0 +1,20 @@
// Copyright 2021 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package util
import (
"io"
)
// ReadAtMost reads at most len(buf) bytes from r into buf.
// It returns the number of bytes copied. n is only less then len(buf) if r provides fewer bytes.
// If EOF occurs while reading, err will be nil.
func ReadAtMost(r io.Reader, buf []byte) (n int, err error) {
n, err = io.ReadFull(r, buf)
if err == io.EOF || err == io.ErrUnexpectedEOF {
err = nil
}
return
}

@ -18,6 +18,7 @@ import (
"code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/typesniffer" "code.gitea.io/gitea/modules/typesniffer"
"code.gitea.io/gitea/modules/util"
) )
// ServeBlob download a git.Blob // ServeBlob download a git.Blob
@ -42,8 +43,8 @@ func ServeBlob(ctx *context.Context, blob *git.Blob) error {
// ServeData download file from io.Reader // ServeData download file from io.Reader
func ServeData(ctx *context.Context, name string, size int64, reader io.Reader) error { func ServeData(ctx *context.Context, name string, size int64, reader io.Reader) error {
buf := make([]byte, 1024) buf := make([]byte, 1024)
n, err := reader.Read(buf) n, err := util.ReadAtMost(reader, buf)
if err != nil && err != io.EOF { if err != nil {
return err return err
} }
if n >= 0 { if n >= 0 {

@ -118,7 +118,7 @@ func editFile(ctx *context.Context, isNewFile bool) {
ctx.Data["FileName"] = blob.Name() ctx.Data["FileName"] = blob.Name()
buf := make([]byte, 1024) buf := make([]byte, 1024)
n, _ := dataRc.Read(buf) n, _ := util.ReadAtMost(dataRc, buf)
buf = buf[:n] buf = buf[:n]
// Only some file types are editable online as text. // Only some file types are editable online as text.
@ -751,7 +751,7 @@ func UploadFileToServer(ctx *context.Context) {
defer file.Close() defer file.Close()
buf := make([]byte, 1024) buf := make([]byte, 1024)
n, _ := file.Read(buf) n, _ := util.ReadAtMost(file, buf)
if n > 0 { if n > 0 {
buf = buf[:n] buf = buf[:n]
} }

@ -25,6 +25,7 @@ import (
"code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/storage" "code.gitea.io/gitea/modules/storage"
"code.gitea.io/gitea/modules/typesniffer" "code.gitea.io/gitea/modules/typesniffer"
"code.gitea.io/gitea/modules/util"
) )
const ( const (
@ -271,7 +272,7 @@ func LFSFileGet(ctx *context.Context) {
} }
defer dataRc.Close() defer dataRc.Close()
buf := make([]byte, 1024) buf := make([]byte, 1024)
n, err := dataRc.Read(buf) n, err := util.ReadAtMost(dataRc, buf)
if err != nil { if err != nil {
ctx.ServerError("Data", err) ctx.ServerError("Data", err)
return return
@ -296,10 +297,10 @@ func LFSFileGet(ctx *context.Context) {
break break
} }
buf := charset.ToUTF8WithFallbackReader(io.MultiReader(bytes.NewReader(buf), dataRc)) rd := charset.ToUTF8WithFallbackReader(io.MultiReader(bytes.NewReader(buf), dataRc))
// Building code view blocks with line number on server side. // Building code view blocks with line number on server side.
fileContent, _ := io.ReadAll(buf) fileContent, _ := io.ReadAll(rd)
var output bytes.Buffer var output bytes.Buffer
lines := strings.Split(string(fileContent), "\n") lines := strings.Split(string(fileContent), "\n")

@ -33,6 +33,7 @@ import (
"code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/structs"
"code.gitea.io/gitea/modules/typesniffer" "code.gitea.io/gitea/modules/typesniffer"
"code.gitea.io/gitea/modules/util"
) )
const ( const (
@ -250,7 +251,7 @@ func renderDirectory(ctx *context.Context, treeLink string) {
defer dataRc.Close() defer dataRc.Close()
buf := make([]byte, 1024) buf := make([]byte, 1024)
n, _ := dataRc.Read(buf) n, _ := util.ReadAtMost(dataRc, buf)
buf = buf[:n] buf = buf[:n]
st := typesniffer.DetectContentType(buf) st := typesniffer.DetectContentType(buf)
@ -285,7 +286,7 @@ func renderDirectory(ctx *context.Context, treeLink string) {
defer dataRc.Close() defer dataRc.Close()
buf = make([]byte, 1024) buf = make([]byte, 1024)
n, err = dataRc.Read(buf) n, err = util.ReadAtMost(dataRc, buf)
if err != nil { if err != nil {
ctx.ServerError("Data", err) ctx.ServerError("Data", err)
return return
@ -377,7 +378,7 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st
ctx.Data["RawFileLink"] = rawLink + "/" + ctx.Repo.TreePath ctx.Data["RawFileLink"] = rawLink + "/" + ctx.Repo.TreePath
buf := make([]byte, 1024) buf := make([]byte, 1024)
n, _ := dataRc.Read(buf) n, _ := util.ReadAtMost(dataRc, buf)
buf = buf[:n] buf = buf[:n]
st := typesniffer.DetectContentType(buf) st := typesniffer.DetectContentType(buf)
@ -409,10 +410,8 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st
defer dataRc.Close() defer dataRc.Close()
buf = make([]byte, 1024) buf = make([]byte, 1024)
n, err = dataRc.Read(buf) n, err = util.ReadAtMost(dataRc, buf)
// Error EOF don't mean there is an error, it just means we read to if err != nil {
// the end
if err != nil && err != io.EOF {
ctx.ServerError("Data", err) ctx.ServerError("Data", err)
return return
} }

@ -14,6 +14,7 @@ import (
"code.gitea.io/gitea/models/db" "code.gitea.io/gitea/models/db"
"code.gitea.io/gitea/modules/storage" "code.gitea.io/gitea/modules/storage"
"code.gitea.io/gitea/modules/upload" "code.gitea.io/gitea/modules/upload"
"code.gitea.io/gitea/modules/util"
"github.com/google/uuid" "github.com/google/uuid"
) )
@ -41,10 +42,8 @@ func NewAttachment(attach *models.Attachment, file io.Reader) (*models.Attachmen
// UploadAttachment upload new attachment into storage and update database // UploadAttachment upload new attachment into storage and update database
func UploadAttachment(file io.Reader, actorID, repoID, releaseID int64, fileName string, allowedTypes string) (*models.Attachment, error) { func UploadAttachment(file io.Reader, actorID, repoID, releaseID int64, fileName string, allowedTypes string) (*models.Attachment, error) {
buf := make([]byte, 1024) buf := make([]byte, 1024)
n, _ := file.Read(buf) n, _ := util.ReadAtMost(file, buf)
if n > 0 {
buf = buf[:n] buf = buf[:n]
}
if err := upload.Verify(buf, fileName, allowedTypes); err != nil { if err := upload.Verify(buf, fileName, allowedTypes); err != nil {
return nil, err return nil, err

Loading…
Cancel
Save