mirror of
https://github.com/discourse/discourse.git
synced 2025-01-31 06:09:30 +08:00
5eaf214594
This commit introduces two new APIs for handling unused uploads, one can be used to exclude uploads in bulk when the data model allow and the other one excludes uploads one by one.
386 lines
12 KiB
Ruby
386 lines
12 KiB
Ruby
# frozen_string_literal: true
|
|
|
|
require 'rails_helper'
|
|
|
|
describe Jobs::CleanUpUploads do
|
|
|
|
def fabricate_upload(attributes = {})
|
|
Fabricate(:upload, { created_at: 2.hours.ago }.merge(attributes))
|
|
end
|
|
|
|
fab! :expired_upload do
|
|
fabricate_upload
|
|
end
|
|
|
|
before do
|
|
SiteSetting.clean_up_uploads = true
|
|
SiteSetting.clean_orphan_uploads_grace_period_hours = 1
|
|
|
|
# pre-fabrication resets created_at, so re-expire the upload
|
|
expired_upload
|
|
freeze_time 2.hours.from_now
|
|
|
|
Jobs::CleanUpUploads.new.reset_last_cleanup!
|
|
end
|
|
|
|
it "only runs upload cleanup every grace period / 2 time" do
|
|
|
|
SiteSetting.clean_orphan_uploads_grace_period_hours = 48
|
|
expired = fabricate_upload(created_at: 49.hours.ago)
|
|
Jobs::CleanUpUploads.new.execute(nil)
|
|
|
|
expect(Upload.exists?(id: expired.id)).to eq(false)
|
|
|
|
upload = fabricate_upload(created_at: 72.hours.ago)
|
|
Jobs::CleanUpUploads.new.execute(nil)
|
|
|
|
expect(Upload.exists?(id: upload.id)).to eq(true)
|
|
|
|
freeze_time 25.hours.from_now
|
|
|
|
Jobs::CleanUpUploads.new.execute(nil)
|
|
expect(Upload.exists?(id: upload.id)).to eq(false)
|
|
|
|
end
|
|
|
|
it "deletes orphan uploads" do
|
|
expect do
|
|
Jobs::CleanUpUploads.new.execute(nil)
|
|
end.to change { Upload.count }.by(-1)
|
|
|
|
expect(Upload.exists?(id: expired_upload.id)).to eq(false)
|
|
end
|
|
|
|
describe 'unused callbacks' do
|
|
before do
|
|
Upload.add_unused_callback do |uploads|
|
|
uploads.where.not(id: expired_upload.id)
|
|
end
|
|
end
|
|
|
|
after do
|
|
Upload.reset_unused_callbacks
|
|
end
|
|
|
|
it 'does not delete uploads skipped by an unused callback' do
|
|
expect do
|
|
Jobs::CleanUpUploads.new.execute(nil)
|
|
end.to change { Upload.count }.by(0)
|
|
|
|
expect(Upload.exists?(id: expired_upload.id)).to eq(true)
|
|
end
|
|
|
|
it 'deletes other uploads not skipped by an unused callback' do
|
|
expired_upload2 = fabricate_upload
|
|
upload = fabricate_upload
|
|
PostUpload.create(post: Fabricate(:post), upload: upload)
|
|
|
|
expect do
|
|
Jobs::CleanUpUploads.new.execute(nil)
|
|
end.to change { Upload.count }.by(-1)
|
|
|
|
expect(Upload.exists?(id: expired_upload.id)).to eq(true)
|
|
expect(Upload.exists?(id: expired_upload2.id)).to eq(false)
|
|
expect(Upload.exists?(id: upload.id)).to eq(true)
|
|
end
|
|
end
|
|
|
|
describe 'in use callbacks' do
|
|
before do
|
|
Upload.add_in_use_callback do |upload|
|
|
expired_upload.id == upload.id
|
|
end
|
|
end
|
|
|
|
after do
|
|
Upload.reset_in_use_callbacks
|
|
end
|
|
|
|
it 'does not delete uploads that are in use by callback' do
|
|
expect do
|
|
Jobs::CleanUpUploads.new.execute(nil)
|
|
end.to change { Upload.count }.by(0)
|
|
|
|
expect(Upload.exists?(id: expired_upload.id)).to eq(true)
|
|
end
|
|
|
|
it 'deletes other uploads that are not in use by callback' do
|
|
expired_upload2 = fabricate_upload
|
|
upload = fabricate_upload
|
|
PostUpload.create(post: Fabricate(:post), upload: upload)
|
|
|
|
expect do
|
|
Jobs::CleanUpUploads.new.execute(nil)
|
|
end.to change { Upload.count }.by(-1)
|
|
|
|
expect(Upload.exists?(id: expired_upload.id)).to eq(true)
|
|
expect(Upload.exists?(id: expired_upload2.id)).to eq(false)
|
|
expect(Upload.exists?(id: upload.id)).to eq(true)
|
|
end
|
|
end
|
|
|
|
describe 'when clean_up_uploads is disabled' do
|
|
before do
|
|
SiteSetting.clean_up_uploads = false
|
|
end
|
|
|
|
it 'should still delete invalid upload records' do
|
|
upload2 = fabricate_upload(
|
|
url: "",
|
|
retain_hours: nil
|
|
)
|
|
|
|
expect do
|
|
Jobs::CleanUpUploads.new.execute(nil)
|
|
end.to change { Upload.count }.by(-1)
|
|
|
|
expect(Upload.exists?(id: expired_upload.id)).to eq(true)
|
|
expect(Upload.exists?(id: upload2.id)).to eq(false)
|
|
end
|
|
end
|
|
|
|
it 'does not clean up upload site settings' do
|
|
begin
|
|
original_provider = SiteSetting.provider
|
|
SiteSetting.provider = SiteSettings::DbProvider.new(SiteSetting)
|
|
SiteSetting.clean_orphan_uploads_grace_period_hours = 1
|
|
|
|
system_upload = fabricate_upload(id: -999)
|
|
logo_upload = fabricate_upload
|
|
logo_small_upload = fabricate_upload
|
|
digest_logo_upload = fabricate_upload
|
|
mobile_logo_upload = fabricate_upload
|
|
large_icon_upload = fabricate_upload
|
|
opengraph_image_upload = fabricate_upload
|
|
twitter_summary_large_image_upload = fabricate_upload
|
|
favicon_upload = fabricate_upload
|
|
apple_touch_icon_upload = fabricate_upload
|
|
|
|
SiteSetting.logo = logo_upload
|
|
SiteSetting.logo_small = logo_small_upload
|
|
SiteSetting.digest_logo = digest_logo_upload
|
|
SiteSetting.mobile_logo = mobile_logo_upload
|
|
SiteSetting.large_icon = large_icon_upload
|
|
SiteSetting.opengraph_image = opengraph_image_upload
|
|
|
|
SiteSetting.twitter_summary_large_image =
|
|
twitter_summary_large_image_upload
|
|
|
|
SiteSetting.favicon = favicon_upload
|
|
SiteSetting.apple_touch_icon = apple_touch_icon_upload
|
|
|
|
Jobs::CleanUpUploads.new.execute(nil)
|
|
|
|
[
|
|
logo_upload,
|
|
logo_small_upload,
|
|
digest_logo_upload,
|
|
mobile_logo_upload,
|
|
large_icon_upload,
|
|
opengraph_image_upload,
|
|
twitter_summary_large_image_upload,
|
|
favicon_upload,
|
|
apple_touch_icon_upload,
|
|
system_upload
|
|
].each { |record| expect(Upload.exists?(id: record.id)).to eq(true) }
|
|
|
|
fabricate_upload
|
|
SiteSetting.opengraph_image = ''
|
|
|
|
Jobs::CleanUpUploads.new.execute(nil)
|
|
ensure
|
|
SiteSetting.delete_all
|
|
SiteSetting.provider = original_provider
|
|
end
|
|
end
|
|
|
|
it "does not clean up selectable avatars" do
|
|
avatar1_upload = fabricate_upload
|
|
avatar2_upload = fabricate_upload
|
|
|
|
SiteSetting.selectable_avatars = [avatar1_upload, avatar2_upload]
|
|
|
|
Jobs::CleanUpUploads.new.execute(nil)
|
|
|
|
expect(Upload.exists?(id: expired_upload.id)).to eq(false)
|
|
expect(Upload.exists?(id: avatar1_upload.id)).to eq(true)
|
|
expect(Upload.exists?(id: avatar2_upload.id)).to eq(true)
|
|
end
|
|
|
|
it "does not delete profile background uploads" do
|
|
profile_background_upload = fabricate_upload
|
|
UserProfile.last.upload_profile_background(profile_background_upload)
|
|
|
|
Jobs::CleanUpUploads.new.execute(nil)
|
|
|
|
expect(Upload.exists?(id: expired_upload.id)).to eq(false)
|
|
expect(Upload.exists?(id: profile_background_upload.id)).to eq(true)
|
|
end
|
|
|
|
it "does not delete card background uploads" do
|
|
card_background_upload = fabricate_upload
|
|
UserProfile.last.upload_card_background(card_background_upload)
|
|
|
|
Jobs::CleanUpUploads.new.execute(nil)
|
|
|
|
expect(Upload.exists?(id: expired_upload.id)).to eq(false)
|
|
expect(Upload.exists?(id: card_background_upload.id)).to eq(true)
|
|
end
|
|
|
|
it "does not delete category logo uploads" do
|
|
category_logo_upload = fabricate_upload
|
|
Fabricate(:category, uploaded_logo: category_logo_upload)
|
|
|
|
Jobs::CleanUpUploads.new.execute(nil)
|
|
|
|
expect(Upload.exists?(id: expired_upload.id)).to eq(false)
|
|
expect(Upload.exists?(id: category_logo_upload.id)).to eq(true)
|
|
end
|
|
|
|
it "does not delete category background url uploads" do
|
|
category_logo_upload = fabricate_upload
|
|
Fabricate(:category, uploaded_background: category_logo_upload)
|
|
|
|
Jobs::CleanUpUploads.new.execute(nil)
|
|
|
|
expect(Upload.exists?(id: expired_upload.id)).to eq(false)
|
|
expect(Upload.exists?(id: category_logo_upload.id)).to eq(true)
|
|
end
|
|
|
|
it "does not delete post uploads" do
|
|
upload = fabricate_upload
|
|
Fabricate(:post, uploads: [upload])
|
|
|
|
Jobs::CleanUpUploads.new.execute(nil)
|
|
|
|
expect(Upload.exists?(id: expired_upload.id)).to eq(false)
|
|
expect(Upload.exists?(id: upload.id)).to eq(true)
|
|
end
|
|
|
|
it "does not delete user uploaded avatar" do
|
|
upload = fabricate_upload
|
|
Fabricate(:user, uploaded_avatar: upload)
|
|
|
|
Jobs::CleanUpUploads.new.execute(nil)
|
|
|
|
expect(Upload.exists?(id: expired_upload.id)).to eq(false)
|
|
expect(Upload.exists?(id: upload.id)).to eq(true)
|
|
end
|
|
|
|
it "does not delete user gravatar" do
|
|
upload = fabricate_upload
|
|
Fabricate(:user, user_avatar: Fabricate(:user_avatar, gravatar_upload: upload))
|
|
|
|
Jobs::CleanUpUploads.new.execute(nil)
|
|
|
|
expect(Upload.exists?(id: expired_upload.id)).to eq(false)
|
|
expect(Upload.exists?(id: upload.id)).to eq(true)
|
|
end
|
|
|
|
it "does not delete user custom upload" do
|
|
upload = fabricate_upload
|
|
Fabricate(:user, user_avatar: Fabricate(:user_avatar, custom_upload: upload))
|
|
|
|
Jobs::CleanUpUploads.new.execute(nil)
|
|
|
|
expect(Upload.exists?(id: expired_upload.id)).to eq(false)
|
|
expect(Upload.exists?(id: upload.id)).to eq(true)
|
|
end
|
|
|
|
it "does not delete uploads in a queued post" do
|
|
upload = fabricate_upload
|
|
upload2 = fabricate_upload
|
|
upload3 = fabricate_upload
|
|
|
|
Fabricate(:reviewable_queued_post_topic, payload: {
|
|
raw: "#{upload.sha1}\n#{upload2.short_url}"
|
|
})
|
|
|
|
Fabricate(:reviewable_queued_post_topic,
|
|
payload: {
|
|
raw: "#{upload3.sha1}"
|
|
},
|
|
status: Reviewable.statuses[:rejected]
|
|
)
|
|
|
|
Jobs::CleanUpUploads.new.execute(nil)
|
|
|
|
expect(Upload.exists?(id: expired_upload.id)).to eq(false)
|
|
expect(Upload.exists?(id: upload.id)).to eq(true)
|
|
expect(Upload.exists?(id: upload2.id)).to eq(true)
|
|
expect(Upload.exists?(id: upload3.id)).to eq(false)
|
|
end
|
|
|
|
it "does not delete uploads in a draft" do
|
|
upload = fabricate_upload
|
|
upload2 = fabricate_upload
|
|
|
|
Draft.set(Fabricate(:user), "test", 0, "#{upload.sha1}\n#{upload2.short_url}")
|
|
|
|
Jobs::CleanUpUploads.new.execute(nil)
|
|
|
|
expect(Upload.exists?(id: expired_upload.id)).to eq(false)
|
|
expect(Upload.exists?(id: upload.id)).to eq(true)
|
|
expect(Upload.exists?(id: upload2.id)).to eq(true)
|
|
end
|
|
|
|
it "does not delete uploads with an access control post ID (secure uploads)" do
|
|
upload = fabricate_upload(access_control_post_id: Fabricate(:post).id, secure: true)
|
|
|
|
Jobs::CleanUpUploads.new.execute(nil)
|
|
|
|
expect(Upload.exists?(id: expired_upload.id)).to eq(false)
|
|
expect(Upload.exists?(id: upload.id)).to eq(true)
|
|
end
|
|
|
|
it "does not delete custom emojis" do
|
|
upload = fabricate_upload
|
|
CustomEmoji.create!(name: 'test', upload: upload)
|
|
|
|
Jobs::CleanUpUploads.new.execute(nil)
|
|
|
|
expect(Upload.exists?(id: expired_upload.id)).to eq(false)
|
|
expect(Upload.exists?(id: upload.id)).to eq(true)
|
|
end
|
|
|
|
it "does not delete user exported csv uploads" do
|
|
csv_file = fabricate_upload
|
|
UserExport.create(file_name: "export.csv", user_id: Fabricate(:user).id, upload_id: csv_file.id)
|
|
|
|
Jobs::CleanUpUploads.new.execute(nil)
|
|
|
|
expect(Upload.exists?(id: expired_upload.id)).to eq(false)
|
|
expect(Upload.exists?(id: csv_file.id)).to eq(true)
|
|
end
|
|
|
|
it "does not delete theme setting uploads" do
|
|
theme = Fabricate(:theme)
|
|
theme_upload = fabricate_upload
|
|
ThemeSetting.create!(theme: theme, data_type: ThemeSetting.types[:upload], value: theme_upload.id.to_s, name: "my_setting_name")
|
|
|
|
Jobs::CleanUpUploads.new.execute(nil)
|
|
|
|
expect(Upload.exists?(id: expired_upload.id)).to eq(false)
|
|
expect(Upload.exists?(id: theme_upload.id)).to eq(true)
|
|
end
|
|
|
|
it "does not delete badges uploads" do
|
|
badge_image = fabricate_upload
|
|
badge = Fabricate(:badge, image_upload_id: badge_image.id)
|
|
|
|
Jobs::CleanUpUploads.new.execute(nil)
|
|
|
|
expect(Upload.exists?(id: expired_upload.id)).to eq(false)
|
|
expect(Upload.exists?(id: badge_image.id)).to eq(true)
|
|
end
|
|
|
|
it "deletes external upload stubs that have expired" do
|
|
external_stub1 = Fabricate(:external_upload_stub, status: ExternalUploadStub.statuses[:created], created_at: 10.minutes.ago)
|
|
external_stub2 = Fabricate(:external_upload_stub, status: ExternalUploadStub.statuses[:created], created_at: (ExternalUploadStub::CREATED_EXPIRY_HOURS.hours + 10.minutes).ago)
|
|
external_stub3 = Fabricate(:external_upload_stub, status: ExternalUploadStub.statuses[:uploaded], created_at: 10.minutes.ago)
|
|
external_stub4 = Fabricate(:external_upload_stub, status: ExternalUploadStub.statuses[:uploaded], created_at: (ExternalUploadStub::UPLOADED_EXPIRY_HOURS.hours + 10.minutes).ago)
|
|
Jobs::CleanUpUploads.new.execute(nil)
|
|
expect(ExternalUploadStub.pluck(:id)).to contain_exactly(external_stub1.id, external_stub3.id)
|
|
end
|
|
end
|