diff --git a/app/assets/javascripts/admin/addon/controllers/admin-backups-index.js b/app/assets/javascripts/admin/addon/controllers/admin-backups-index.js index 7b53b2170d7..c53adeecf9a 100644 --- a/app/assets/javascripts/admin/addon/controllers/admin-backups-index.js +++ b/app/assets/javascripts/admin/addon/controllers/admin-backups-index.js @@ -12,9 +12,6 @@ export default Controller.extend({ uploadLabel: i18n("admin.backups.upload.label"), backupLocation: setting("backup_location"), localBackupStorage: equal("backupLocation", "local"), - enableExperimentalBackupUploader: setting( - "enable_experimental_backup_uploader" - ), @discourseComputed("status.allowRestore", "status.isOperationRunning") restoreTitle(allowRestore, isOperationRunning) { diff --git a/app/assets/javascripts/admin/addon/templates/backups-index.hbs b/app/assets/javascripts/admin/addon/templates/backups-index.hbs index 1a3ae7181b9..da2e488425b 100644 --- a/app/assets/javascripts/admin/addon/templates/backups-index.hbs +++ b/app/assets/javascripts/admin/addon/templates/backups-index.hbs @@ -1,14 +1,18 @@
{{#if localBackupStorage}} - {{resumable-upload - target="/admin/backups/upload" - success=(route-action "uploadSuccess") - error=(route-action "uploadError") - uploadText=uploadLabel - title="admin.backups.upload.title" - class="btn-default"}} + {{#if siteSettings.enable_experimental_backup_uploader}} + {{uppy-backup-uploader done=(route-action "uploadSuccess") localBackupStorage=localBackupStorage}} + {{else}} + {{resumable-upload + target="/admin/backups/upload" + success=(route-action "uploadSuccess") + error=(route-action "uploadError") + uploadText=uploadLabel + title="admin.backups.upload.title" + class="btn-default"}} + {{/if}} {{else}} - {{#if enableExperimentalBackupUploader}} + {{#if (and siteSettings.enable_direct_s3_uploads siteSettings.enable_experimental_backup_uploader)}} {{uppy-backup-uploader done=(route-action "remoteUploadSuccess")}} {{else}} {{backup-uploader done=(route-action "remoteUploadSuccess")}} diff --git a/app/assets/javascripts/discourse-shims.js b/app/assets/javascripts/discourse-shims.js index 006a98ed696..5870456b45f 100644 --- a/app/assets/javascripts/discourse-shims.js +++ b/app/assets/javascripts/discourse-shims.js @@ -45,3 +45,19 @@ define("@uppy/xhr-upload", ["exports"], function (__exports__) { define("@uppy/drop-target", ["exports"], function (__exports__) { __exports__.default = window.Uppy.DropTarget; }); + +define("@uppy/utils/lib/delay", ["exports"], function (__exports__) { + __exports__.default = window.Uppy.Utils.delay; +}); + +define("@uppy/utils/lib/EventTracker", ["exports"], function (__exports__) { + __exports__.default = window.Uppy.Utils.EventTracker; +}); + +define("@uppy/utils/lib/AbortController", ["exports"], function (__exports__) { + __exports__.AbortController = + window.Uppy.Utils.AbortControllerLib.AbortController; + __exports__.AbortSignal = window.Uppy.Utils.AbortControllerLib.AbortSignal; + __exports__.createAbortError = + window.Uppy.Utils.AbortControllerLib.createAbortError; +}); diff --git a/app/assets/javascripts/discourse/app/components/uppy-backup-uploader.js b/app/assets/javascripts/discourse/app/components/uppy-backup-uploader.js index 55be0926b74..ac07d28a400 100644 --- a/app/assets/javascripts/discourse/app/components/uppy-backup-uploader.js +++ b/app/assets/javascripts/discourse/app/components/uppy-backup-uploader.js @@ -1,13 +1,29 @@ import Component from "@ember/component"; +import { alias, not } from "@ember/object/computed"; import I18n from "I18n"; import UppyUploadMixin from "discourse/mixins/uppy-upload"; import discourseComputed from "discourse-common/utils/decorators"; export default Component.extend(UppyUploadMixin, { + id: "uppy-backup-uploader", tagName: "span", type: "backup", - useMultipartUploadsIfAvailable: true, + uploadRootPath: "/admin/backups", + uploadUrl: "/admin/backups/upload", + + // TODO (martin) Add functionality to make this usable _without_ multipart + // uploads, direct to S3, which needs to call get-presigned-put on the + // BackupsController (which extends ExternalUploadHelpers) rather than + // the old create_upload_url route. The two are functionally equivalent; + // they both generate a presigned PUT url for the upload to S3, and do + // the whole thing in one request rather than multipart. + + // direct s3 backups + useMultipartUploadsIfAvailable: not("localBackupStorage"), + + // local backups + useChunkedUploads: alias("localBackupStorage"), @discourseComputed("uploading", "uploadProgress") uploadButtonText(uploading, progress) { @@ -20,7 +36,7 @@ export default Component.extend(UppyUploadMixin, { return { skipValidation: true }; }, - uploadDone() { - this.done(); + uploadDone(responseData) { + this.done(responseData.file_name); }, }); diff --git a/app/assets/javascripts/discourse/app/lib/uppy-chunked-upload.js b/app/assets/javascripts/discourse/app/lib/uppy-chunked-upload.js new file mode 100644 index 00000000000..73edc96f0e2 --- /dev/null +++ b/app/assets/javascripts/discourse/app/lib/uppy-chunked-upload.js @@ -0,0 +1,339 @@ +import { Promise } from "rsvp"; +import delay from "@uppy/utils/lib/delay"; +import { + AbortController, + createAbortError, +} from "@uppy/utils/lib/AbortController"; + +const MB = 1024 * 1024; + +const defaultOptions = { + limit: 5, + retryDelays: [0, 1000, 3000, 5000], + getChunkSize() { + return 5 * MB; + }, + onStart() {}, + onProgress() {}, + onChunkComplete() {}, + onSuccess() {}, + onError(err) { + throw err; + }, +}; + +/** + * Used mainly as a replacement for Resumable.js, using code cribbed from + * uppy's S3 Multipart class, which we mainly use the chunking algorithm + * and retry/abort functions of. The _buildFormData function is the one + * which shapes the data into the same parameters as Resumable.js used. + * + * See the UppyChunkedUploader class for the uppy uploader plugin which + * uses UppyChunkedUpload. + */ +export default class UppyChunkedUpload { + constructor(file, options) { + this.options = { + ...defaultOptions, + ...options, + }; + this.file = file; + + if (!this.options.getChunkSize) { + this.options.getChunkSize = defaultOptions.getChunkSize; + this.chunkSize = this.options.getChunkSize(this.file); + } + + this.abortController = new AbortController(); + this._initChunks(); + } + + _aborted() { + return this.abortController.signal.aborted; + } + + _initChunks() { + this.chunksInProgress = 0; + this.chunks = null; + this.chunkState = null; + + const chunks = []; + + if (this.file.size === 0) { + chunks.push(this.file.data); + } else { + for (let i = 0; i < this.file.data.size; i += this.chunkSize) { + const end = Math.min(this.file.data.size, i + this.chunkSize); + chunks.push(this.file.data.slice(i, end)); + } + } + + this.chunks = chunks; + this.chunkState = chunks.map(() => ({ + bytesUploaded: 0, + busy: false, + done: false, + })); + } + + _createUpload() { + if (this._aborted()) { + throw createAbortError(); + } + this.options.onStart(); + this._uploadChunks(); + } + + _uploadChunks() { + if (this.chunkState.every((state) => state.done)) { + this._completeUpload(); + return; + } + + // For a 100MB file, with the default min chunk size of 5MB and a limit of 10: + // + // Total 20 chunks + // --------- + // Need 1 is 10 + // Need 2 is 5 + // Need 3 is 5 + const need = this.options.limit - this.chunksInProgress; + const completeChunks = this.chunkState.filter((state) => state.done).length; + const remainingChunks = this.chunks.length - completeChunks; + let minNeeded = Math.ceil(this.options.limit / 2); + if (minNeeded > remainingChunks) { + minNeeded = remainingChunks; + } + if (need < minNeeded) { + return; + } + + const candidates = []; + for (let i = 0; i < this.chunkState.length; i++) { + const state = this.chunkState[i]; + if (!state.done && !state.busy) { + candidates.push(i); + if (candidates.length >= need) { + break; + } + } + } + + if (candidates.length === 0) { + return; + } + + candidates.forEach((index) => { + this._uploadChunkRetryable(index).then( + () => { + this._uploadChunks(); + }, + (err) => { + this._onError(err); + } + ); + }); + } + + _shouldRetry(err) { + if (err.source && typeof err.source.status === "number") { + const { status } = err.source; + // 0 probably indicates network failure + return ( + status === 0 || + status === 409 || + status === 423 || + (status >= 500 && status < 600) + ); + } + return false; + } + + _retryable({ before, attempt, after }) { + const { retryDelays } = this.options; + const { signal } = this.abortController; + + if (before) { + before(); + } + + const doAttempt = (retryAttempt) => + attempt().catch((err) => { + if (this._aborted()) { + throw createAbortError(); + } + + if (this._shouldRetry(err) && retryAttempt < retryDelays.length) { + return delay(retryDelays[retryAttempt], { signal }).then(() => + doAttempt(retryAttempt + 1) + ); + } + throw err; + }); + + return doAttempt(0).then( + (result) => { + if (after) { + after(); + } + return result; + }, + (err) => { + if (after) { + after(); + } + throw err; + } + ); + } + + _uploadChunkRetryable(index) { + return this._retryable({ + before: () => { + this.chunksInProgress += 1; + }, + attempt: () => this._uploadChunk(index), + after: () => { + this.chunksInProgress -= 1; + }, + }); + } + + _uploadChunk(index) { + this.chunkState[index].busy = true; + + if (this._aborted()) { + this.chunkState[index].busy = false; + throw createAbortError(); + } + + return this._uploadChunkBytes( + index, + this.options.url, + this.options.headers + ); + } + + _onChunkProgress(index, sent) { + this.chunkState[index].bytesUploaded = parseInt(sent, 10); + + const totalUploaded = this.chunkState.reduce( + (total, chunk) => total + chunk.bytesUploaded, + 0 + ); + this.options.onProgress(totalUploaded, this.file.data.size); + } + + _onChunkComplete(index) { + this.chunkState[index].done = true; + this.options.onChunkComplete(index); + } + + _uploadChunkBytes(index, url, headers) { + const body = this.chunks[index]; + const { signal } = this.abortController; + + return new Promise((resolve, reject) => { + const xhr = new XMLHttpRequest(); + function cleanup() { + signal.removeEventListener("abort", () => xhr.abort()); + } + signal.addEventListener("abort", xhr.abort()); + + xhr.open(this.options.method || "POST", url, true); + if (headers) { + Object.keys(headers).forEach((key) => { + xhr.setRequestHeader(key, headers[key]); + }); + } + xhr.responseType = "text"; + xhr.upload.addEventListener("progress", (ev) => { + if (!ev.lengthComputable) { + return; + } + + this._onChunkProgress(index, ev.loaded, ev.total); + }); + + xhr.addEventListener("abort", () => { + cleanup(); + this.chunkState[index].busy = false; + + reject(createAbortError()); + }); + + xhr.addEventListener("load", (ev) => { + cleanup(); + this.chunkState[index].busy = false; + + if (ev.target.status < 200 || ev.target.status >= 300) { + const error = new Error("Non 2xx"); + error.source = ev.target; + reject(error); + return; + } + + // This avoids the net::ERR_OUT_OF_MEMORY in Chromium Browsers. + this.chunks[index] = null; + + this._onChunkProgress(index, body.size, body.size); + + this._onChunkComplete(index); + resolve(); + }); + + xhr.addEventListener("error", (ev) => { + cleanup(); + this.chunkState[index].busy = false; + + const error = new Error("Unknown error"); + error.source = ev.target; + reject(error); + }); + + xhr.send(this._buildFormData(index + 1, body)); + }); + } + + async _completeUpload() { + this.options.onSuccess(); + } + + _buildFormData(currentChunkNumber, body) { + const uniqueIdentifier = + this.file.data.size + + "-" + + this.file.data.name.replace(/[^0-9a-zA-Z_-]/gim, ""); + const formData = new FormData(); + formData.append("file", body); + formData.append("resumableChunkNumber", currentChunkNumber); + formData.append("resumableCurrentChunkSize", body.size); + formData.append("resumableChunkSize", this.chunkSize); + formData.append("resumableTotalSize", this.file.data.size); + formData.append("resumableFilename", this.file.data.name); + formData.append("resumableIdentifier", uniqueIdentifier); + return formData; + } + + _abortUpload() { + this.abortController.abort(); + } + + _onError(err) { + if (err && err.name === "AbortError") { + return; + } + + this.options.onError(err); + } + + start() { + this._createUpload(); + } + + abort(opts = undefined) { + if (opts?.really) { + this._abortUpload(); + } + } +} diff --git a/app/assets/javascripts/discourse/app/lib/uppy-chunked-uploader-plugin.js b/app/assets/javascripts/discourse/app/lib/uppy-chunked-uploader-plugin.js new file mode 100644 index 00000000000..44ec6358560 --- /dev/null +++ b/app/assets/javascripts/discourse/app/lib/uppy-chunked-uploader-plugin.js @@ -0,0 +1,211 @@ +import { UploaderPlugin } from "discourse/lib/uppy-plugin-base"; +import { next } from "@ember/runloop"; +import getURL from "discourse-common/lib/get-url"; +import { Promise } from "rsvp"; +import UppyChunkedUpload from "discourse/lib/uppy-chunked-upload"; +import EventTracker from "@uppy/utils/lib/EventTracker"; + +// Limited use uppy uploader function to replace Resumable.js, which +// is only used by the local backup uploader at this point in time, +// and has been that way for many years. Uses the skeleton of uppy's +// AwsS3Multipart uploader plugin to provide a similar API, with unnecessary +// code removed. +// +// See also UppyChunkedUpload class for more detail. +export default class UppyChunkedUploader extends UploaderPlugin { + static pluginId = "uppy-chunked-uploader"; + + constructor(uppy, opts) { + super(uppy, opts); + const defaultOptions = { + limit: 0, + retryDelays: [0, 1000, 3000, 5000], + }; + + this.opts = { ...defaultOptions, ...opts }; + this.url = getURL(opts.url); + this.method = opts.method || "POST"; + + this.uploaders = Object.create(null); + this.uploaderEvents = Object.create(null); + } + + _resetUploaderReferences(fileID, opts = {}) { + if (this.uploaders[fileID]) { + this.uploaders[fileID].abort({ really: opts.abort || false }); + this.uploaders[fileID] = null; + } + if (this.uploaderEvents[fileID]) { + this.uploaderEvents[fileID].remove(); + this.uploaderEvents[fileID] = null; + } + } + + _uploadFile(file) { + return new Promise((resolve, reject) => { + const onStart = () => { + this.uppy.emit("upload-started", file); + }; + + const onProgress = (bytesUploaded, bytesTotal) => { + this.uppy.emit("upload-progress", file, { + uploader: this, + bytesUploaded, + bytesTotal, + }); + }; + + const onError = (err) => { + this.uppy.log(err); + this.uppy.emit("upload-error", file, err); + + this._resetUploaderReferences(file.id); + reject(err); + }; + + const onSuccess = () => { + this._resetUploaderReferences(file.id); + + const cFile = this.uppy.getFile(file.id); + const uploadResponse = {}; + this.uppy.emit("upload-success", cFile || file, uploadResponse); + + resolve(upload); + }; + + const onChunkComplete = (chunk) => { + const cFile = this.uppy.getFile(file.id); + if (!cFile) { + return; + } + + this.uppy.emit("chunk-uploaded", cFile, chunk); + }; + + const upload = new UppyChunkedUpload(file, { + getChunkSize: this.opts.getChunkSize + ? this.opts.getChunkSize.bind(this) + : null, + + onStart, + onProgress, + onChunkComplete, + onSuccess, + onError, + + limit: this.opts.limit || 5, + retryDelays: this.opts.retryDelays || [], + method: this.method, + url: this.url, + headers: this.opts.headers, + }); + + this.uploaders[file.id] = upload; + this.uploaderEvents[file.id] = new EventTracker(this.uppy); + + next(() => { + if (!file.isPaused) { + upload.start(); + } + }); + + this._onFileRemove(file.id, (removed) => { + this._resetUploaderReferences(file.id, { abort: true }); + resolve(`upload ${removed.id} was removed`); + }); + + this._onCancelAll(file.id, () => { + this._resetUploaderReferences(file.id, { abort: true }); + resolve(`upload ${file.id} was canceled`); + }); + + this._onFilePause(file.id, (isPaused) => { + if (isPaused) { + upload.pause(); + } else { + next(() => { + upload.start(); + }); + } + }); + + this._onPauseAll(file.id, () => { + upload.pause(); + }); + + this._onResumeAll(file.id, () => { + if (file.error) { + upload.abort(); + } + next(() => { + upload.start(); + }); + }); + + // Don't double-emit upload-started for restored files that were already started + if (!file.progress.uploadStarted || !file.isRestored) { + this.uppy.emit("upload-started", file); + } + }); + } + + _onFileRemove(fileID, cb) { + this.uploaderEvents[fileID].on("file-removed", (file) => { + if (fileID === file.id) { + cb(file.id); + } + }); + } + + _onFilePause(fileID, cb) { + this.uploaderEvents[fileID].on("upload-pause", (targetFileID, isPaused) => { + if (fileID === targetFileID) { + cb(isPaused); + } + }); + } + + _onPauseAll(fileID, cb) { + this.uploaderEvents[fileID].on("pause-all", () => { + if (!this.uppy.getFile(fileID)) { + return; + } + cb(); + }); + } + + _onCancelAll(fileID, cb) { + this.uploaderEvents[fileID].on("cancel-all", () => { + if (!this.uppy.getFile(fileID)) { + return; + } + cb(); + }); + } + + _onResumeAll(fileID, cb) { + this.uploaderEvents[fileID].on("resume-all", () => { + if (!this.uppy.getFile(fileID)) { + return; + } + cb(); + }); + } + + _upload(fileIDs) { + const promises = fileIDs.map((id) => { + const file = this.uppy.getFile(id); + return this._uploadFile(file); + }); + + return Promise.all(promises); + } + + install() { + this._install(this._upload.bind(this)); + } + + uninstall() { + this._uninstall(this._upload.bind(this)); + } +} diff --git a/app/assets/javascripts/discourse/app/lib/uppy-plugin-base.js b/app/assets/javascripts/discourse/app/lib/uppy-plugin-base.js index f7a0f871460..51e5907ebdd 100644 --- a/app/assets/javascripts/discourse/app/lib/uppy-plugin-base.js +++ b/app/assets/javascripts/discourse/app/lib/uppy-plugin-base.js @@ -30,32 +30,6 @@ export class UppyPluginBase extends BasePlugin { _setFileState(fileId, state) { this.uppy.setFileState(fileId, state); } -} - -export class UploadPreProcessorPlugin extends UppyPluginBase { - static pluginType = "preprocessor"; - - constructor(uppy, opts) { - super(uppy, opts); - this.type = this.constructor.pluginType; - } - - _install(fn) { - this.uppy.addPreProcessor(fn); - } - - _uninstall(fn) { - this.uppy.removePreProcessor(fn); - } - - _emitProgress(file) { - this.uppy.emit("preprocess-progress", file, null, this.id); - } - - _emitComplete(file, skipped = false) { - this.uppy.emit("preprocess-complete", file, skipped, this.id); - return Promise.resolve(); - } _emitAllComplete(fileIds, skipped = false) { fileIds.forEach((fileId) => { @@ -82,3 +56,55 @@ export class UploadPreProcessorPlugin extends UppyPluginBase { return this._emitAllComplete(file, true); } } + +export class UploadPreProcessorPlugin extends UppyPluginBase { + static pluginType = "preprocessor"; + + constructor(uppy, opts) { + super(uppy, opts); + this.type = this.constructor.pluginType; + } + + _install(fn) { + this.uppy.addPreProcessor(fn); + } + + _uninstall(fn) { + this.uppy.removePreProcessor(fn); + } + + _emitProgress(file) { + this.uppy.emit("preprocess-progress", file, null, this.id); + } + + _emitComplete(file, skipped = false) { + this.uppy.emit("preprocess-complete", file, skipped, this.id); + return Promise.resolve(); + } +} + +export class UploaderPlugin extends UppyPluginBase { + static pluginType = "uploader"; + + constructor(uppy, opts) { + super(uppy, opts); + this.type = this.constructor.pluginType; + } + + _install(fn) { + this.uppy.addUploader(fn); + } + + _uninstall(fn) { + this.uppy.removeUploader(fn); + } + + _emitProgress(file) { + this.uppy.emit("upload-progress", file, null, this.id); + } + + _emitComplete(file, skipped = false) { + this.uppy.emit("upload-complete", file, skipped, this.id); + return Promise.resolve(); + } +} diff --git a/app/assets/javascripts/discourse/app/mixins/uppy-upload.js b/app/assets/javascripts/discourse/app/mixins/uppy-upload.js index f94ca0b40e9..d2de36205cf 100644 --- a/app/assets/javascripts/discourse/app/mixins/uppy-upload.js +++ b/app/assets/javascripts/discourse/app/mixins/uppy-upload.js @@ -14,6 +14,7 @@ import XHRUpload from "@uppy/xhr-upload"; import AwsS3 from "@uppy/aws-s3"; import UppyChecksum from "discourse/lib/uppy-checksum-plugin"; import UppyS3Multipart from "discourse/mixins/uppy-s3-multipart"; +import UppyChunkedUploader from "discourse/lib/uppy-chunked-uploader-plugin"; import { on } from "discourse-common/utils/decorators"; import { warn } from "@ember/debug"; import bootbox from "bootbox"; @@ -152,7 +153,9 @@ export default Mixin.create(UppyS3Multipart, { this.setProperties({ uploading: false, processing: true }); this._completeExternalUpload(file) .then((completeResponse) => { - this.uploadDone(completeResponse); + this.uploadDone( + deepMerge(completeResponse, { file_name: file.name }) + ); if (this._inProgressUploads === 0) { this._reset(); @@ -165,7 +168,9 @@ export default Mixin.create(UppyS3Multipart, { } }); } else { - this.uploadDone(response.body); + this.uploadDone( + deepMerge(response?.body || {}, { file_name: file.name }) + ); if (this._inProgressUploads === 0) { this._reset(); } @@ -185,7 +190,8 @@ export default Mixin.create(UppyS3Multipart, { // allow these other uploaders to go direct to S3. if ( this.siteSettings.enable_direct_s3_uploads && - !this.preventDirectS3Uploads + !this.preventDirectS3Uploads && + !this.useChunkedUploads ) { if (this.useMultipartUploadsIfAvailable) { this._useS3MultipartUploads(); @@ -193,7 +199,11 @@ export default Mixin.create(UppyS3Multipart, { this._useS3Uploads(); } } else { - this._useXHRUploads(); + if (this.useChunkedUploads) { + this._useChunkedUploads(); + } else { + this._useXHRUploads(); + } } }, @@ -206,6 +216,16 @@ export default Mixin.create(UppyS3Multipart, { }); }, + _useChunkedUploads() { + this.set("usingChunkedUploads", true); + this._uppyInstance.use(UppyChunkedUploader, { + url: this._xhrUploadUrl(), + headers: { + "X-CSRF-Token": this.session.csrfToken, + }, + }); + }, + _useS3Uploads() { this.set("usingS3Uploads", true); this._uppyInstance.use(AwsS3, { @@ -251,7 +271,7 @@ export default Mixin.create(UppyS3Multipart, { _xhrUploadUrl() { return ( - getUrl(this.getWithDefault("uploadUrl", "/uploads")) + + getUrl(this.getWithDefault("uploadUrl", this.uploadRootPath)) + ".json?client_id=" + this.messageBus?.clientId ); diff --git a/app/assets/javascripts/discourse/package.json b/app/assets/javascripts/discourse/package.json index bf35016ac41..9bfd19569ba 100644 --- a/app/assets/javascripts/discourse/package.json +++ b/app/assets/javascripts/discourse/package.json @@ -22,6 +22,7 @@ "@ember/test-helpers": "^2.2.0", "@glimmer/component": "^1.0.0", "@popperjs/core": "2.10.2", + "@uppy/utils": "^4.0.3", "@uppy/aws-s3": "^2.0.4", "@uppy/aws-s3-multipart": "^2.1.0", "@uppy/core": "^2.1.0", diff --git a/app/controllers/admin/backups_controller.rb b/app/controllers/admin/backups_controller.rb index 188db1fdcc9..64dfeb0ee48 100644 --- a/app/controllers/admin/backups_controller.rb +++ b/app/controllers/admin/backups_controller.rb @@ -180,13 +180,11 @@ class Admin::BackupsController < Admin::AdminController current_chunk_size = params.fetch(:resumableCurrentChunkSize).to_i previous_chunk_number = chunk_number - 1 - # path to chunk file chunk = BackupRestore::LocalBackupStore.chunk_path(identifier, filename, chunk_number) - # upload chunk HandleChunkUpload.upload_chunk(chunk, file: file) - uploaded_file_size = previous_chunk_number * chunk_size # when all chunks are uploaded + uploaded_file_size = previous_chunk_number * chunk_size if uploaded_file_size + current_chunk_size >= total_size # merge all the chunks in a background thread Jobs.enqueue_in(5.seconds, :backup_chunks_merger, filename: filename, identifier: identifier, chunks: chunk_number) diff --git a/package.json b/package.json index 10a3879b45d..316c1f65653 100644 --- a/package.json +++ b/package.json @@ -12,6 +12,7 @@ "@json-editor/json-editor": "^2.5.2", "@popperjs/core": "v2.10.2", "@uppy/aws-s3": "^2.0.4", + "@uppy/utils": "^4.0.3", "@uppy/aws-s3-multipart": "^2.1.0", "@uppy/core": "^2.1.0", "@uppy/drop-target": "^1.1.0", diff --git a/vendor/assets/javascripts/custom-uppy.js b/vendor/assets/javascripts/custom-uppy.js index e146b136628..964c7bc3262 100644 --- a/vendor/assets/javascripts/custom-uppy.js +++ b/vendor/assets/javascripts/custom-uppy.js @@ -9,3 +9,8 @@ Uppy.XHRUpload = require('@uppy/xhr-upload') Uppy.AwsS3 = require('@uppy/aws-s3') Uppy.AwsS3Multipart = require('@uppy/aws-s3-multipart') Uppy.DropTarget = require('@uppy/drop-target') +Uppy.Utils = { + delay: require('@uppy/utils/lib/delay'), + EventTracker: require('@uppy/utils/lib/EventTracker'), + AbortControllerLib: require('@uppy/utils/lib/AbortController') +} diff --git a/vendor/assets/javascripts/uppy.js b/vendor/assets/javascripts/uppy.js index 7545f505ad1..32bfb01026f 100644 --- a/vendor/assets/javascripts/uppy.js +++ b/vendor/assets/javascripts/uppy.js @@ -7578,5 +7578,10 @@ Uppy.XHRUpload = require('@uppy/xhr-upload') Uppy.AwsS3 = require('@uppy/aws-s3') Uppy.AwsS3Multipart = require('@uppy/aws-s3-multipart') Uppy.DropTarget = require('@uppy/drop-target') +Uppy.Utils = { + delay: require('@uppy/utils/lib/delay'), + EventTracker: require('@uppy/utils/lib/EventTracker'), + AbortControllerLib: require('@uppy/utils/lib/AbortController') +} -},{"@uppy/aws-s3":5,"@uppy/aws-s3-multipart":3,"@uppy/core":18,"@uppy/drop-target":21,"@uppy/xhr-upload":49}]},{},[58]); +},{"@uppy/aws-s3":5,"@uppy/aws-s3-multipart":3,"@uppy/core":18,"@uppy/drop-target":21,"@uppy/utils/lib/AbortController":23,"@uppy/utils/lib/EventTracker":24,"@uppy/utils/lib/delay":29,"@uppy/xhr-upload":49}]},{},[58]);