Ark/openmaxio-object-browser
1
0
Fork 0
mirror of https://github.com/OpenMaxIO/openmaxio-object-browser synced 2026-07-01 15:51:18 -07:00
Code Issues Packages Projects Releases Wiki Activity

change server side uploader to fully streaming based (#1066)

Browse Source 
this PR fixes the behavior to avoid `/tmp` folder as
staging directory for large uploads, instead rely on
the client upload stream itself to upload the object
entirely.
This commit is contained in:
Harshavardhana
2021-09-21 17:43:43 -07:00
committed by GitHub
parent 2b6d5e5128
commit 27689a17f3
13 changed files with 50 additions and 90 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
node_modules/.yarn-integrity generated vendored
View File

@@ -1,10 +1,12 @@
{
"systemParams": "darwin-x64-93",
"systemParams": "linux-x64-93",
"modulesFolders": [
"node_modules"
],
"flags": [],
"linkedModules": [],
"linkedModules": [
"one-ui-common"
],
"topLevelPatterns": [],
"lockfileEntries": {},
"files": [],

14
portal-ui/build/asset-manifest.json
View File

@@ -1,23 +1,23 @@
{
"files": {
"main.css": "./static/css/main.8cfac526.chunk.css",
"main.js": "./static/js/main.7ad4b880.chunk.js",
"main.js.map": "./static/js/main.7ad4b880.chunk.js.map",
"main.js": "./static/js/main.625561fd.chunk.js",
"main.js.map": "./static/js/main.625561fd.chunk.js.map",
"runtime-main.js": "./static/js/runtime-main.3fe0c1ac.js",
"runtime-main.js.map": "./static/js/runtime-main.3fe0c1ac.js.map",
"static/css/2.60e04a19.chunk.css": "./static/css/2.60e04a19.chunk.css",
"static/js/2.b63e7857.chunk.js": "./static/js/2.b63e7857.chunk.js",
"static/js/2.b63e7857.chunk.js.map": "./static/js/2.b63e7857.chunk.js.map",
"static/js/2.f169c574.chunk.js": "./static/js/2.f169c574.chunk.js",
"static/js/2.f169c574.chunk.js.map": "./static/js/2.f169c574.chunk.js.map",
"index.html": "./index.html",
"static/css/2.60e04a19.chunk.css.map": "./static/css/2.60e04a19.chunk.css.map",
"static/css/main.8cfac526.chunk.css.map": "./static/css/main.8cfac526.chunk.css.map",
"static/js/2.b63e7857.chunk.js.LICENSE.txt": "./static/js/2.b63e7857.chunk.js.LICENSE.txt"
"static/js/2.f169c574.chunk.js.LICENSE.txt": "./static/js/2.f169c574.chunk.js.LICENSE.txt"
},
"entrypoints": [
"static/js/runtime-main.3fe0c1ac.js",
"static/css/2.60e04a19.chunk.css",
"static/js/2.b63e7857.chunk.js",
"static/js/2.f169c574.chunk.js",
"static/css/main.8cfac526.chunk.css",
"static/js/main.7ad4b880.chunk.js"
"static/js/main.625561fd.chunk.js"
]
}

2
portal-ui/build/index.html
View File

@@ -1 +1 @@
<!doctype html><html lang="en"><head><meta charset="utf-8"/><meta name="viewport" content="width=device-width,initial-scale=1"/><meta name="theme-color" content="#081C42" media="(prefers-color-scheme: light)"/><meta name="theme-color" content="#081C42" media="(prefers-color-scheme: dark)"/><meta name="description" content="MinIO Console"/><link href="https://fonts.googleapis.com/css2?family=Lato:wght@400;500;700;900&display=swap" rel="stylesheet"/><link href="./styles/root-styles.css" rel="stylesheet"/><link rel="apple-touch-icon" sizes="180x180" href="./apple-icon-180x180.png"/><link rel="icon" type="image/png" sizes="32x32" href="./favicon-32x32.png"/><link rel="icon" type="image/png" sizes="96x96" href="./favicon-96x96.png"/><link rel="icon" type="image/png" sizes="16x16" href="./favicon-16x16.png"/><link rel="manifest" href="./manifest.json"/><link rel="mask-icon" href="./safari-pinned-tab.svg" color="#3a4e54"/><title>MinIO Console</title><link href="./static/css/2.60e04a19.chunk.css" rel="stylesheet"><link href="./static/css/main.8cfac526.chunk.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"><div id="loader-block"><svg class="loader-svg-container" viewBox="22 22 44 44"><circle class="loader-style MuiCircularProgress-circle MuiCircularProgress-circleIndeterminate" cx="44" cy="44" r="20.2" fill="none" stroke-width="3.6"></circle></svg></div></div><script>!function(e){function r(r){for(var n,l,i=r[0],a=r[1],p=r[2],c=0,s=[];c<i.length;c++)l=i[c],Object.prototype.hasOwnProperty.call(o,l)&&o[l]&&s.push(o[l][0]),o[l]=0;for(n in a)Object.prototype.hasOwnProperty.call(a,n)&&(e[n]=a[n]);for(f&&f(r);s.length;)s.shift()();return u.push.apply(u,p||[]),t()}function t(){for(var e,r=0;r<u.length;r++){for(var t=u[r],n=!0,i=1;i<t.length;i++){var a=t[i];0!==o[a]&&(n=!1)}n&&(u.splice(r--,1),e=l(l.s=t[0]))}return e}var n={},o={1:0},u=[];function l(r){if(n[r])return n[r].exports;var t=n[r]={i:r,l:!1,exports:{}};return e[r].call(t.exports,t,t.exports,l),t.l=!0,t.exports}l.m=e,l.c=n,l.d=function(e,r,t){l.o(e,r)||Object.defineProperty(e,r,{enumerable:!0,get:t})},l.r=function(e){"undefined"!=typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(e,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(e,"__esModule",{value:!0})},l.t=function(e,r){if(1&r&&(e=l(e)),8&r)return e;if(4&r&&"object"==typeof e&&e&&e.__esModule)return e;var t=Object.create(null);if(l.r(t),Object.defineProperty(t,"default",{enumerable:!0,value:e}),2&r&&"string"!=typeof e)for(var n in e)l.d(t,n,function(r){return e[r]}.bind(null,n));return t},l.n=function(e){var r=e&&e.__esModule?function(){return e.default}:function(){return e};return l.d(r,"a",r),r},l.o=function(e,r){return Object.prototype.hasOwnProperty.call(e,r)},l.p="./";var i=this["webpackJsonpportal-ui"]=this["webpackJsonpportal-ui"]||[],a=i.push.bind(i);i.push=r,i=i.slice();for(var p=0;p<i.length;p++)r(i[p]);var f=a;t()}([])</script><script src="./static/js/2.b63e7857.chunk.js"></script><script src="./static/js/main.7ad4b880.chunk.js"></script></body></html>
<!doctype html><html lang="en"><head><meta charset="utf-8"/><meta name="viewport" content="width=device-width,initial-scale=1"/><meta name="theme-color" content="#081C42" media="(prefers-color-scheme: light)"/><meta name="theme-color" content="#081C42" media="(prefers-color-scheme: dark)"/><meta name="description" content="MinIO Console"/><link href="https://fonts.googleapis.com/css2?family=Lato:wght@400;500;700;900&display=swap" rel="stylesheet"/><link href="./styles/root-styles.css" rel="stylesheet"/><link rel="apple-touch-icon" sizes="180x180" href="./apple-icon-180x180.png"/><link rel="icon" type="image/png" sizes="32x32" href="./favicon-32x32.png"/><link rel="icon" type="image/png" sizes="96x96" href="./favicon-96x96.png"/><link rel="icon" type="image/png" sizes="16x16" href="./favicon-16x16.png"/><link rel="manifest" href="./manifest.json"/><link rel="mask-icon" href="./safari-pinned-tab.svg" color="#3a4e54"/><title>MinIO Console</title><link href="./static/css/2.60e04a19.chunk.css" rel="stylesheet"><link href="./static/css/main.8cfac526.chunk.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"><div id="loader-block"><svg class="loader-svg-container" viewBox="22 22 44 44"><circle class="loader-style MuiCircularProgress-circle MuiCircularProgress-circleIndeterminate" cx="44" cy="44" r="20.2" fill="none" stroke-width="3.6"></circle></svg></div></div><script>!function(e){function r(r){for(var n,l,i=r[0],a=r[1],p=r[2],c=0,s=[];c<i.length;c++)l=i[c],Object.prototype.hasOwnProperty.call(o,l)&&o[l]&&s.push(o[l][0]),o[l]=0;for(n in a)Object.prototype.hasOwnProperty.call(a,n)&&(e[n]=a[n]);for(f&&f(r);s.length;)s.shift()();return u.push.apply(u,p||[]),t()}function t(){for(var e,r=0;r<u.length;r++){for(var t=u[r],n=!0,i=1;i<t.length;i++){var a=t[i];0!==o[a]&&(n=!1)}n&&(u.splice(r--,1),e=l(l.s=t[0]))}return e}var n={},o={1:0},u=[];function l(r){if(n[r])return n[r].exports;var t=n[r]={i:r,l:!1,exports:{}};return e[r].call(t.exports,t,t.exports,l),t.l=!0,t.exports}l.m=e,l.c=n,l.d=function(e,r,t){l.o(e,r)||Object.defineProperty(e,r,{enumerable:!0,get:t})},l.r=function(e){"undefined"!=typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(e,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(e,"__esModule",{value:!0})},l.t=function(e,r){if(1&r&&(e=l(e)),8&r)return e;if(4&r&&"object"==typeof e&&e&&e.__esModule)return e;var t=Object.create(null);if(l.r(t),Object.defineProperty(t,"default",{enumerable:!0,value:e}),2&r&&"string"!=typeof e)for(var n in e)l.d(t,n,function(r){return e[r]}.bind(null,n));return t},l.n=function(e){var r=e&&e.__esModule?function(){return e.default}:function(){return e};return l.d(r,"a",r),r},l.o=function(e,r){return Object.prototype.hasOwnProperty.call(e,r)},l.p="./";var i=this["webpackJsonpportal-ui"]=this["webpackJsonpportal-ui"]||[],a=i.push.bind(i);i.push=r,i=i.slice();for(var p=0;p<i.length;p++)r(i[p]);var f=a;t()}([])</script><script src="./static/js/2.f169c574.chunk.js"></script><script src="./static/js/main.625561fd.chunk.js"></script></body></html>

4
portal-ui/build/static/js/2.b63e7857.chunk.js → portal-ui/build/static/js/2.f169c574.chunk.js
View File

File diff suppressed because one or more lines are too long

0
portal-ui/build/static/js/2.b63e7857.chunk.js.LICENSE.txt → portal-ui/build/static/js/2.f169c574.chunk.js.LICENSE.txt
View File

2
portal-ui/build/static/js/2.b63e7857.chunk.js.map → portal-ui/build/static/js/2.f169c574.chunk.js.map
View File

File diff suppressed because one or more lines are too long

2
portal-ui/build/static/js/main.625561fd.chunk.js Normal file
View File

File diff suppressed because one or more lines are too long

1
portal-ui/build/static/js/main.625561fd.chunk.js.map Normal file
View File

File diff suppressed because one or more lines are too long

2
portal-ui/build/static/js/main.7ad4b880.chunk.js
View File

File diff suppressed because one or more lines are too long

1
portal-ui/build/static/js/main.7ad4b880.chunk.js.map
View File

File diff suppressed because one or more lines are too long

4
portal-ui/src/screens/Console/Buckets/ListBuckets/Objects/ListObjects/ListObjects.tsx
View File

@@ -559,8 +559,8 @@ const ListObjects = ({
for (let file of files) {
const fileName = file.name;
const blobFile = new Blob([file]);
formData.append(fileName, blobFile);
const blobFile = new Blob([file], { type: file.type });
formData.append(file.size, blobFile, fileName);
}
xhr.send(formData);

98
restapi/user_objects.go
View File

@@ -21,11 +21,11 @@ import (
"fmt"
"io"
"log"
"mime/multipart"
"net/http"
"path"
"path/filepath"
"regexp"
"strconv"
"strings"
"time"
@@ -468,16 +468,35 @@ func uploadFiles(ctx context.Context, client MinioClient, params user_api.PostBu
prefix = *params.Prefix
}
// get object files from request
o, err := getFormFiles(params.HTTPRequest)
// parse a request body as multipart/form-data.
// 32 << 20 is default max memory
mr, err := params.HTTPRequest.MultipartReader()
if err != nil {
return err
}
defer o.Close()
// upload files one by one
for _, obj := range o.Files() {
if err := uploadObject(ctx, client, params.BucketName, path.Join(prefix, obj.name), obj.size, obj.file); err != nil {
for {
p, err := mr.NextPart()
if err == io.EOF {
break
}
size, err := strconv.ParseInt(p.FormName(), 10, 64)
if err != nil {
return err
}
contentType := p.Header.Get("content-type")
if contentType == "" {
contentType = mimedb.TypeByExtension(filepath.Ext(p.FileName()))
}
_, err = client.putObject(ctx, params.BucketName, path.Join(prefix, p.FileName()), p, size, minio.PutObjectOptions{
ContentType: contentType,
DisableMultipart: true, // Do not upload as multipart stream for console uploader.
})
if err != nil {
return err
}
}
@@ -485,71 +504,6 @@ func uploadFiles(ctx context.Context, client MinioClient, params user_api.PostBu
return nil
}
type objectFile struct {
name string
size int64
file multipart.File
}
type objectFiles struct {
files []objectFile
form *multipart.Form
}
func (o objectFiles) Files() []objectFile {
return o.files
}
func (o objectFiles) Close() error {
for _, fl := range o.files {
fl.file.Close()
}
if o.form != nil {
return o.form.RemoveAll()
}
return nil
}
// getFormFiles parses the request body and gets all the files from the Request
// it includes name, size and file content
func getFormFiles(r *http.Request) (o objectFiles, err error) {
if r == nil {
return o, errors.New("http.Request is nil")
}
// parse a request body as multipart/form-data.
// 32 << 20 is default max memory
if err := r.ParseMultipartForm(32 << 20); err != nil {
return o, err
}
for fileName, files := range r.MultipartForm.File {
if len(files) > 0 {
f, err := files[0].Open()
if err != nil {
return o, err
}
o.files = append(o.files, objectFile{
name: fileName,
size: files[0].Size,
file: f,
})
}
}
o.form = r.MultipartForm
return o, nil
}
func uploadObject(ctx context.Context, client MinioClient, bucket, object string, size int64, reader io.ReadCloser) error {
contentType := mimedb.TypeByExtension(filepath.Ext(object))
_, err := client.putObject(ctx, bucket, object, reader, size, minio.PutObjectOptions{
ContentType: contentType,
DisableMultipart: true, // Do not upload as multipart stream for console uploader.
})
return err
}
// getShareObjectResponse returns a share object url
func getShareObjectResponse(session *models.Principal, params user_api.ShareObjectParams) (*string, *models.Error) {
ctx := context.Background()

4
yarn.lock Normal file
View File

@@ -0,0 +1,4 @@
# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
# yarn lockfile v1