Retrieve remote file size without the Content-Size header (#1537)

* Get downloaded file size regardless of the Content-Size header

Signed-off-by: Florian Le Minoux <darkgallium+git@deuxfleurs.fr>

* fixing lint issues

* Don't exhaust memory for large files, don't limit more than necessary

* Don't use errors.Wrap in download.go

Co-authored-by: Neil Alexander <neilalexander@users.noreply.github.com>
This commit is contained in:
darkgallium 2021-02-17 14:54:53 +01:00 committed by GitHub
parent 2386e0c7af
commit da797c7998
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 62 additions and 29 deletions

View file

@ -147,7 +147,7 @@ func (r *uploadRequest) doUpload(
// r.storeFileAndMetadata(ctx, tmpDir, ...)
// before you return from doUpload else we will leak a temp file. We could make this nicer with a `WithTransaction` style of
// nested function to guarantee either storage or cleanup.
hash, bytesWritten, tmpDir, err := fileutils.WriteTempFile(ctx, reqReader, *cfg.MaxFileSizeBytes, cfg.AbsBasePath)
hash, bytesWritten, tmpDir, err := fileutils.WriteTempFile(ctx, reqReader, cfg.AbsBasePath)
if err != nil {
r.Logger.WithError(err).WithFields(log.Fields{
"MaxFileSizeBytes": *cfg.MaxFileSizeBytes,