gitea/routers/repo/download.go

216 lines
5.8 KiB
Go
Raw Normal View History

2014-04-16 00:27:29 +08:00
// Copyright 2014 The Gogs Authors. All rights reserved.
// Copyright 2018 The Gitea Authors. All rights reserved.
2014-04-16 00:27:29 +08:00
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package repo
import (
"fmt"
"io"
"path"
"path/filepath"
2016-11-26 21:26:03 +08:00
"strings"
2014-04-16 00:27:29 +08:00
"code.gitea.io/gitea/modules/base"
"code.gitea.io/gitea/modules/charset"
"code.gitea.io/gitea/modules/context"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/httpcache"
"code.gitea.io/gitea/modules/lfs"
2019-06-13 03:41:28 +08:00
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
2014-04-16 00:27:29 +08:00
)
2016-11-24 15:04:31 +08:00
// ServeData download file from io.Reader
func ServeData(ctx *context.Context, name string, size int64, reader io.Reader) error {
buf := make([]byte, 1024)
Add a storage layer for attachments (#11387) * Add a storage layer for attachments * Fix some bug * fix test * Fix copyright head and lint * Fix bug * Add setting for minio and flags for migrate-storage * Add documents * fix lint * Add test for minio store type on attachments * fix test * fix test * Apply suggestions from code review Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com> * Add warning when storage migrated successfully * Fix drone * fix test * rebase * Fix test * display the error on console * Move minio test to amd64 since minio docker don't support arm64 * refactor the codes * add trace * Fix test * remove log on xorm * Fi download bug * Add a storage layer for attachments * Add setting for minio and flags for migrate-storage * fix lint * Add test for minio store type on attachments * Apply suggestions from code review Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com> * Fix drone * fix test * Fix test * display the error on console * Move minio test to amd64 since minio docker don't support arm64 * refactor the codes * add trace * Fix test * Add URL function to serve attachments directly from S3/Minio * Add ability to enable/disable redirection in attachment configuration * Fix typo * Add a storage layer for attachments * Add setting for minio and flags for migrate-storage * fix lint * Add test for minio store type on attachments * Apply suggestions from code review Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com> * Fix drone * fix test * Fix test * display the error on console * Move minio test to amd64 since minio docker don't support arm64 * don't change unrelated files * Fix lint * Fix build * update go.mod and go.sum * Use github.com/minio/minio-go/v6 * Remove unused function * Upgrade minio to v7 and some other improvements * fix lint * Fix go mod Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com> Co-authored-by: Tyler <tystuyfzand@gmail.com>
2020-08-18 12:23:45 +08:00
n, err := reader.Read(buf)
if err != nil && err != io.EOF {
return err
}
2017-04-20 10:38:56 +08:00
if n >= 0 {
buf = buf[:n]
}
ctx.Resp.Header().Set("Cache-Control", "public,max-age=86400")
if size >= 0 {
ctx.Resp.Header().Set("Content-Length", fmt.Sprintf("%d", size))
} else {
log.Error("ServeData called to serve data: %s with size < 0: %d", name, size)
}
name = path.Base(name)
2016-11-26 21:26:03 +08:00
// Google Chrome dislike commas in filenames, so let's change it to a space
name = strings.ReplaceAll(name, ",", " ")
2016-11-26 21:26:03 +08:00
if base.IsTextFile(buf) || ctx.QueryBool("render") {
cs, err := charset.DetectEncoding(buf)
if err != nil {
log.Error("Detect raw file %s charset failed: %v, using by default utf-8", name, err)
cs = "utf-8"
}
ctx.Resp.Header().Set("Content-Type", "text/plain; charset="+strings.ToLower(cs))
} else if base.IsImageFile(buf) || base.IsPDFFile(buf) {
ctx.Resp.Header().Set("Content-Disposition", fmt.Sprintf(`inline; filename="%s"`, name))
ctx.Resp.Header().Set("Access-Control-Expose-Headers", "Content-Disposition")
if base.IsSVGImageFile(buf) {
ctx.Resp.Header().Set("Content-Security-Policy", "default-src 'none'; style-src 'unsafe-inline'; sandbox")
ctx.Resp.Header().Set("X-Content-Type-Options", "nosniff")
ctx.Resp.Header().Set("Content-Type", base.SVGMimeType)
}
} else {
ctx.Resp.Header().Set("Content-Disposition", fmt.Sprintf(`attachment; filename="%s"`, name))
ctx.Resp.Header().Set("Access-Control-Expose-Headers", "Content-Disposition")
if setting.MimeTypeMap.Enabled {
fileExtension := strings.ToLower(filepath.Ext(name))
if mimetype, ok := setting.MimeTypeMap.Map[fileExtension]; ok {
ctx.Resp.Header().Set("Content-Type", mimetype)
}
}
}
Add a storage layer for attachments (#11387) * Add a storage layer for attachments * Fix some bug * fix test * Fix copyright head and lint * Fix bug * Add setting for minio and flags for migrate-storage * Add documents * fix lint * Add test for minio store type on attachments * fix test * fix test * Apply suggestions from code review Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com> * Add warning when storage migrated successfully * Fix drone * fix test * rebase * Fix test * display the error on console * Move minio test to amd64 since minio docker don't support arm64 * refactor the codes * add trace * Fix test * remove log on xorm * Fi download bug * Add a storage layer for attachments * Add setting for minio and flags for migrate-storage * fix lint * Add test for minio store type on attachments * Apply suggestions from code review Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com> * Fix drone * fix test * Fix test * display the error on console * Move minio test to amd64 since minio docker don't support arm64 * refactor the codes * add trace * Fix test * Add URL function to serve attachments directly from S3/Minio * Add ability to enable/disable redirection in attachment configuration * Fix typo * Add a storage layer for attachments * Add setting for minio and flags for migrate-storage * fix lint * Add test for minio store type on attachments * Apply suggestions from code review Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com> * Fix drone * fix test * Fix test * display the error on console * Move minio test to amd64 since minio docker don't support arm64 * don't change unrelated files * Fix lint * Fix build * update go.mod and go.sum * Use github.com/minio/minio-go/v6 * Remove unused function * Upgrade minio to v7 and some other improvements * fix lint * Fix go mod Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com> Co-authored-by: Tyler <tystuyfzand@gmail.com>
2020-08-18 12:23:45 +08:00
_, err = ctx.Resp.Write(buf)
2019-06-13 03:41:28 +08:00
if err != nil {
return err
}
_, err = io.Copy(ctx.Resp, reader)
return err
2014-11-17 10:32:26 +08:00
}
2016-11-24 15:04:31 +08:00
// ServeBlob download a git.Blob
2016-03-12 00:56:52 +08:00
func ServeBlob(ctx *context.Context, blob *git.Blob) error {
if httpcache.HandleGenericETagCache(ctx.Req, ctx.Resp, `"`+blob.ID.String()+`"`) {
return nil
}
dataRc, err := blob.DataAsync()
2015-08-12 04:49:51 +08:00
if err != nil {
return err
}
2019-06-13 03:41:28 +08:00
defer func() {
if err = dataRc.Close(); err != nil {
log.Error("ServeBlob: Close: %v", err)
}
}()
2015-08-12 04:49:51 +08:00
return ServeData(ctx, ctx.Repo.TreePath, blob.Size(), dataRc)
2015-08-12 04:49:51 +08:00
}
// ServeBlobOrLFS download a git.Blob redirecting to LFS if necessary
func ServeBlobOrLFS(ctx *context.Context, blob *git.Blob) error {
if httpcache.HandleGenericETagCache(ctx.Req, ctx.Resp, `"`+blob.ID.String()+`"`) {
return nil
}
dataRc, err := blob.DataAsync()
if err != nil {
return err
}
closed := false
2019-06-13 03:41:28 +08:00
defer func() {
if closed {
return
}
2019-06-13 03:41:28 +08:00
if err = dataRc.Close(); err != nil {
log.Error("ServeBlobOrLFS: Close: %v", err)
}
}()
Add LFS Migration and Mirror (#14726) * Implemented LFS client. * Implemented scanning for pointer files. * Implemented downloading of lfs files. * Moved model-dependent code into services. * Removed models dependency. Added TryReadPointerFromBuffer. * Migrated code from service to module. * Centralised storage creation. * Removed dependency from models. * Moved ContentStore into modules. * Share structs between server and client. * Moved method to services. * Implemented lfs download on clone. * Implemented LFS sync on clone and mirror update. * Added form fields. * Updated templates. * Fixed condition. * Use alternate endpoint. * Added missing methods. * Fixed typo and make linter happy. * Detached pointer parser from gogit dependency. * Fixed TestGetLFSRange test. * Added context to support cancellation. * Use ReadFull to probably read more data. * Removed duplicated code from models. * Moved scan implementation into pointer_scanner_nogogit. * Changed method name. * Added comments. * Added more/specific log/error messages. * Embedded lfs.Pointer into models.LFSMetaObject. * Moved code from models to module. * Moved code from models to module. * Moved code from models to module. * Reduced pointer usage. * Embedded type. * Use promoted fields. * Fixed unexpected eof. * Added unit tests. * Implemented migration of local file paths. * Show an error on invalid LFS endpoints. * Hide settings if not used. * Added LFS info to mirror struct. * Fixed comment. * Check LFS endpoint. * Manage LFS settings from mirror page. * Fixed selector. * Adjusted selector. * Added more tests. * Added local filesystem migration test. * Fixed typo. * Reset settings. * Added special windows path handling. * Added unit test for HTTPClient. * Added unit test for BasicTransferAdapter. * Moved into util package. * Test if LFS endpoint is allowed. * Added support for git:// * Just use a static placeholder as the displayed url may be invalid. * Reverted to original code. * Added "Advanced Settings". * Updated wording. * Added discovery info link. * Implemented suggestion. * Fixed missing format parameter. * Added Pointer.IsValid(). * Always remove model on error. * Added suggestions. * Use channel instead of array. * Update routers/repo/migrate.go * fmt Signed-off-by: Andrew Thornton <art27@cantab.net> Co-authored-by: zeripath <art27@cantab.net>
2021-04-09 06:25:57 +08:00
pointer, _ := lfs.ReadPointer(dataRc)
if pointer.IsValid() {
meta, _ := ctx.Repo.Repository.GetLFSMetaObjectByOid(pointer.Oid)
if meta == nil {
if err = dataRc.Close(); err != nil {
log.Error("ServeBlobOrLFS: Close: %v", err)
}
closed = true
return ServeBlob(ctx, blob)
}
if httpcache.HandleGenericETagCache(ctx.Req, ctx.Resp, `"`+pointer.Oid+`"`) {
return nil
}
Add LFS Migration and Mirror (#14726) * Implemented LFS client. * Implemented scanning for pointer files. * Implemented downloading of lfs files. * Moved model-dependent code into services. * Removed models dependency. Added TryReadPointerFromBuffer. * Migrated code from service to module. * Centralised storage creation. * Removed dependency from models. * Moved ContentStore into modules. * Share structs between server and client. * Moved method to services. * Implemented lfs download on clone. * Implemented LFS sync on clone and mirror update. * Added form fields. * Updated templates. * Fixed condition. * Use alternate endpoint. * Added missing methods. * Fixed typo and make linter happy. * Detached pointer parser from gogit dependency. * Fixed TestGetLFSRange test. * Added context to support cancellation. * Use ReadFull to probably read more data. * Removed duplicated code from models. * Moved scan implementation into pointer_scanner_nogogit. * Changed method name. * Added comments. * Added more/specific log/error messages. * Embedded lfs.Pointer into models.LFSMetaObject. * Moved code from models to module. * Moved code from models to module. * Moved code from models to module. * Reduced pointer usage. * Embedded type. * Use promoted fields. * Fixed unexpected eof. * Added unit tests. * Implemented migration of local file paths. * Show an error on invalid LFS endpoints. * Hide settings if not used. * Added LFS info to mirror struct. * Fixed comment. * Check LFS endpoint. * Manage LFS settings from mirror page. * Fixed selector. * Adjusted selector. * Added more tests. * Added local filesystem migration test. * Fixed typo. * Reset settings. * Added special windows path handling. * Added unit test for HTTPClient. * Added unit test for BasicTransferAdapter. * Moved into util package. * Test if LFS endpoint is allowed. * Added support for git:// * Just use a static placeholder as the displayed url may be invalid. * Reverted to original code. * Added "Advanced Settings". * Updated wording. * Added discovery info link. * Implemented suggestion. * Fixed missing format parameter. * Added Pointer.IsValid(). * Always remove model on error. * Added suggestions. * Use channel instead of array. * Update routers/repo/migrate.go * fmt Signed-off-by: Andrew Thornton <art27@cantab.net> Co-authored-by: zeripath <art27@cantab.net>
2021-04-09 06:25:57 +08:00
lfsDataRc, err := lfs.ReadMetaObject(meta.Pointer)
if err != nil {
return err
}
defer func() {
if err = lfsDataRc.Close(); err != nil {
log.Error("ServeBlobOrLFS: Close: %v", err)
}
}()
return ServeData(ctx, ctx.Repo.TreePath, meta.Size, lfsDataRc)
}
if err = dataRc.Close(); err != nil {
log.Error("ServeBlobOrLFS: Close: %v", err)
}
closed = true
return ServeBlob(ctx, blob)
}
2016-11-24 15:04:31 +08:00
// SingleDownload download a file by repos path
2016-03-12 00:56:52 +08:00
func SingleDownload(ctx *context.Context) {
2016-08-25 12:35:03 +08:00
blob, err := ctx.Repo.Commit.GetBlobByPath(ctx.Repo.TreePath)
2014-11-17 10:32:26 +08:00
if err != nil {
if git.IsErrNotExist(err) {
ctx.NotFound("GetBlobByPath", nil)
2014-11-17 10:32:26 +08:00
} else {
ctx.ServerError("GetBlobByPath", err)
2014-11-17 10:32:26 +08:00
}
return
}
if err = ServeBlob(ctx, blob); err != nil {
ctx.ServerError("ServeBlob", err)
2014-11-17 10:32:26 +08:00
}
2014-04-16 00:27:29 +08:00
}
// SingleDownloadOrLFS download a file by repos path redirecting to LFS if necessary
func SingleDownloadOrLFS(ctx *context.Context) {
blob, err := ctx.Repo.Commit.GetBlobByPath(ctx.Repo.TreePath)
if err != nil {
if git.IsErrNotExist(err) {
ctx.NotFound("GetBlobByPath", nil)
} else {
ctx.ServerError("GetBlobByPath", err)
}
return
}
if err = ServeBlobOrLFS(ctx, blob); err != nil {
ctx.ServerError("ServeBlobOrLFS", err)
}
}
// DownloadByID download a file by sha1 ID
func DownloadByID(ctx *context.Context) {
blob, err := ctx.Repo.GitRepo.GetBlob(ctx.Params("sha"))
if err != nil {
if git.IsErrNotExist(err) {
ctx.NotFound("GetBlob", nil)
} else {
ctx.ServerError("GetBlob", err)
}
return
}
if err = ServeBlob(ctx, blob); err != nil {
ctx.ServerError("ServeBlob", err)
}
}
// DownloadByIDOrLFS download a file by sha1 ID taking account of LFS
func DownloadByIDOrLFS(ctx *context.Context) {
blob, err := ctx.Repo.GitRepo.GetBlob(ctx.Params("sha"))
if err != nil {
if git.IsErrNotExist(err) {
ctx.NotFound("GetBlob", nil)
} else {
ctx.ServerError("GetBlob", err)
}
return
}
if err = ServeBlobOrLFS(ctx, blob); err != nil {
ctx.ServerError("ServeBlob", err)
}
}