discourse/spec/models/upload_spec.rb
Sam cedcdb0057
FEATURE: allow for local theme js assets (#16374)
Due to default CSP web workers instantiated from CDN based assets are still
treated as "same-origin" meaning that we had no way of safely instansiating
a web worker from a theme.

This limits the theme system and adds the arbitrary restriction that WASM
based components can not be safely used.

To resolve this limitation all js assets in about.json are also cached on
local domain.

{
  "name": "Header Icons",
  "assets" : {
    "worker" : "assets/worker.js"
  }
}

This can then be referenced in JS via:

settings.theme_uploads_local.worker

local_js_assets are unconditionally served from the site directly and
bypass the entire CDN, using the pre-existing JavascriptCache

Previous to this change this code was completely dormant on sites which
used s3 based uploads, this reuses the very well tested and cached asset
system on s3 based sites.

Note, when creating local_js_assets it is highly recommended to keep the
assets lean and keep all the heavy working in CDN based assets. For example
wasm files can still live on the CDN but the lean worker that loads it can
live on local.

This change unlocks wasm in theme components, so wasm is now also allowed
in `theme_authorized_extensions`

* more usages of upload.content

* add a specific test for upload.content

* Adjust logic to ensure that after upgrades we still get a cached local js
on save
2022-04-07 07:58:10 +10:00

594 lines
20 KiB
Ruby

# frozen_string_literal: true
describe Upload do
let(:upload) { build(:upload) }
let(:user_id) { 1 }
let(:image_filename) { "logo.png" }
let(:image) { file_from_fixtures(image_filename) }
let(:image_svg_filename) { "image.svg" }
let(:image_svg) { file_from_fixtures(image_svg_filename) }
let(:huge_image_filename) { "huge.jpg" }
let(:huge_image) { file_from_fixtures(huge_image_filename) }
let(:attachment_path) { __FILE__ }
let(:attachment) { File.new(attachment_path) }
describe '.with_no_non_post_relations' do
it "does not find non-post related uploads" do
post_upload = Fabricate(:upload)
post = Fabricate(:post, raw: "<img src='#{post_upload.url}'>")
post.link_post_uploads
badge_upload = Fabricate(:upload)
Fabricate(:badge, image_upload: badge_upload)
avatar_upload = Fabricate(:upload)
Fabricate(:user, uploaded_avatar: avatar_upload)
site_setting_upload = Fabricate(:upload)
SiteSetting.create!(
name: "logo",
data_type: SiteSettings::TypeSupervisor.types[:upload],
value: site_setting_upload.id
)
upload_ids = Upload
.by_users
.with_no_non_post_relations
.pluck(:id)
expect(upload_ids).to eq([post_upload.id])
end
end
context ".create_thumbnail!" do
it "does not create a thumbnail when disabled" do
SiteSetting.create_thumbnails = false
OptimizedImage.expects(:create_for).never
upload.create_thumbnail!(100, 100)
end
it "creates a thumbnail" do
upload = Fabricate(:upload)
thumbnail = Fabricate(:optimized_image, upload: upload)
SiteSetting.expects(:create_thumbnails?).returns(true)
OptimizedImage.expects(:create_for).returns(thumbnail)
upload.create_thumbnail!(100, 100)
upload.reload
expect(upload.optimized_images.count).to eq(1)
end
end
it "supports <style> element in SVG" do
SiteSetting.authorized_extensions = "svg"
upload = UploadCreator.new(image_svg, image_svg_filename).create_for(user_id)
expect(upload.valid?).to eq(true)
path = Discourse.store.path_for(upload)
expect(File.read(path)).to match(/<style>/)
end
it "can reconstruct dimensions on demand" do
upload = UploadCreator.new(huge_image, "image.png").create_for(user_id)
upload.update_columns(width: nil, height: nil, thumbnail_width: nil, thumbnail_height: nil)
upload = Upload.find(upload.id)
expect(upload.width).to eq(8900)
expect(upload.height).to eq(8900)
upload.reload
expect(upload.read_attribute(:width)).to eq(8900)
upload.update_columns(width: nil, height: nil, thumbnail_width: nil, thumbnail_height: nil)
expect(upload.thumbnail_width).to eq(500)
expect(upload.thumbnail_height).to eq(500)
end
it "dimension calculation returns nil on missing image" do
upload = UploadCreator.new(huge_image, "image.png").create_for(user_id)
upload.update_columns(width: nil, height: nil, thumbnail_width: nil, thumbnail_height: nil)
missing_url = "wrong_folder#{upload.url}"
upload.update_columns(url: missing_url)
expect(upload.thumbnail_height).to eq(nil)
expect(upload.thumbnail_width).to eq(nil)
end
it 'returns error when image resolution is to big' do
SiteSetting.max_image_megapixels = 10
upload = UploadCreator.new(huge_image, "image.png").create_for(user_id)
expect(upload.persisted?).to eq(false)
expect(upload.errors.messages[:base].first).to eq(I18n.t("upload.images.larger_than_x_megapixels", max_image_megapixels: 20))
end
it "extracts file extension" do
created_upload = UploadCreator.new(image, image_filename).create_for(user_id)
expect(created_upload.extension).to eq("png")
end
it "should create an invalid upload when the filename is blank" do
SiteSetting.authorized_extensions = "*"
created_upload = UploadCreator.new(attachment, nil).create_for(user_id)
expect(created_upload.valid?).to eq(false)
end
context ".extract_url" do
let(:url) { 'https://example.com/uploads/default/original/1X/d1c2d40ab994e8410c.png' }
it 'should return the right part of url' do
expect(Upload.extract_url(url).to_s).to eq('/original/1X/d1c2d40ab994e8410c.png')
end
end
context ".get_from_url" do
let(:sha1) { "10f73034616a796dfd70177dc54b6def44c4ba6f" }
let(:upload) { Fabricate(:upload, sha1: sha1) }
it "works when the file has been uploaded" do
expect(Upload.get_from_url(upload.url)).to eq(upload)
end
describe 'for an extensionless url' do
before do
upload.update!(url: upload.url.sub('.png', ''))
upload.reload
end
it 'should return the right upload' do
expect(Upload.get_from_url(upload.url)).to eq(upload)
end
end
it "should return the right upload as long as the upload's URL matches" do
upload.update!(url: "/uploads/default/12345/971308e535305c51.png")
expect(Upload.get_from_url(upload.url)).to eq(upload)
expect(Upload.get_from_url("/uploads/default/123131/971308e535305c51.png"))
.to eq(nil)
end
describe 'for a url a tree' do
before do
upload.update!(url:
Discourse.store.get_path_for(
"original",
16001,
upload.sha1,
".#{upload.extension}"
)
)
end
it 'should return the right upload' do
expect(Upload.get_from_url(upload.url)).to eq(upload)
end
end
it "works when using a cdn" do
begin
original_asset_host = Rails.configuration.action_controller.asset_host
Rails.configuration.action_controller.asset_host = 'http://my.cdn.com'
expect(Upload.get_from_url(
URI.join("http://my.cdn.com", upload.url).to_s
)).to eq(upload)
ensure
Rails.configuration.action_controller.asset_host = original_asset_host
end
end
it "should return the right upload when using the full URL" do
expect(Upload.get_from_url(
URI.join("http://discourse.some.com:3000/", upload.url).to_s
)).to eq(upload)
end
it "doesn't blow up with an invalid URI" do
expect { Upload.get_from_url("http://ip:port/index.html") }.not_to raise_error
expect { Upload.get_from_url("mailto:admin%40example.com") }.not_to raise_error
expect { Upload.get_from_url("mailto:example") }.not_to raise_error
end
describe "s3 store" do
let(:upload) { Fabricate(:upload_s3) }
let(:path) { upload.url.sub(SiteSetting.Upload.s3_base_url, '') }
before do
setup_s3
end
it "can download an s3 upload" do
stub_request(:get, upload.url)
.to_return(status: 200, body: "hello", headers: {})
expect(upload.content).to eq("hello")
end
it "should return the right upload when using base url (not CDN) for s3" do
upload
expect(Upload.get_from_url(upload.url)).to eq(upload)
end
describe 'when using a cdn' do
let(:s3_cdn_url) { 'https://mycdn.slowly.net' }
before do
SiteSetting.s3_cdn_url = s3_cdn_url
end
it "should return the right upload" do
upload
expect(Upload.get_from_url(URI.join(s3_cdn_url, path).to_s)).to eq(upload)
end
describe 'when upload bucket contains subfolder' do
before do
SiteSetting.s3_upload_bucket = "s3-upload-bucket/path/path2"
end
it "should return the right upload" do
upload
expect(Upload.get_from_url(URI.join(s3_cdn_url, path).to_s)).to eq(upload)
end
end
end
it "should return the right upload when using one CDN for both s3 and assets" do
begin
original_asset_host = Rails.configuration.action_controller.asset_host
cdn_url = 'http://my.cdn.com'
Rails.configuration.action_controller.asset_host = cdn_url
SiteSetting.s3_cdn_url = cdn_url
upload
expect(Upload.get_from_url(
URI.join(cdn_url, path).to_s
)).to eq(upload)
ensure
Rails.configuration.action_controller.asset_host = original_asset_host
end
end
end
end
context ".get_from_urls" do
let(:upload) { Fabricate(:upload, sha1: "10f73034616a796dfd70177dc54b6def44c4ba6f") }
let(:upload2) { Fabricate(:upload, sha1: "2a7081e615f9075befd87a9a6d273935c0262cd5") }
it "works with multiple uploads" do
expect(Upload.get_from_urls([upload.url, upload2.url])).to contain_exactly(upload, upload2)
end
it "works for an extensionless URL" do
url = upload.url.sub('.png', '')
upload.update!(url: url)
expect(Upload.get_from_urls([url])).to contain_exactly(upload)
end
it "works with uploads with mismatched URLs" do
upload.update!(url: "/uploads/default/12345/971308e535305c51.png")
expect(Upload.get_from_urls([upload.url])).to contain_exactly(upload)
expect(Upload.get_from_urls(["/uploads/default/123131/971308e535305c51.png"])).to be_empty
end
it "works with an upload with a URL containing a deep tree" do
upload.update!(url: Discourse.store.get_path_for("original", 16001, upload.sha1, ".#{upload.extension}"))
expect(Upload.get_from_urls([upload.url])).to contain_exactly(upload)
end
it "works when using a CDN" do
begin
original_asset_host = Rails.configuration.action_controller.asset_host
Rails.configuration.action_controller.asset_host = 'http://my.cdn.com'
expect(Upload.get_from_urls([
URI.join("http://my.cdn.com", upload.url).to_s
])).to contain_exactly(upload)
ensure
Rails.configuration.action_controller.asset_host = original_asset_host
end
end
it "works with full URLs" do
expect(Upload.get_from_urls([
URI.join("http://discourse.some.com:3000/", upload.url).to_s
])).to contain_exactly(upload)
end
it "handles invalid URIs" do
urls = ["http://ip:port/index.html", "mailto:admin%40example.com", "mailto:example"]
expect { Upload.get_from_urls(urls) }.not_to raise_error
end
end
describe '.generate_digest' do
it "should return the right digest" do
expect(Upload.generate_digest(image.path)).to eq('bc975735dfc6409c1c2aa5ebf2239949bcbdbd65')
end
end
describe '.short_url' do
it "should generate a correct short url" do
upload = Upload.new(sha1: 'bda2c513e1da04f7b4e99230851ea2aafeb8cc4e', extension: 'png')
expect(upload.short_url).to eq('upload://r3AYqESanERjladb4vBB7VsMBm6.png')
upload.extension = nil
expect(upload.short_url).to eq('upload://r3AYqESanERjladb4vBB7VsMBm6')
end
end
describe '.sha1_from_short_url' do
it "should be able to look up sha1" do
sha1 = 'bda2c513e1da04f7b4e99230851ea2aafeb8cc4e'
expect(Upload.sha1_from_short_url('upload://r3AYqESanERjladb4vBB7VsMBm6.png')).to eq(sha1)
expect(Upload.sha1_from_short_url('upload://r3AYqESanERjladb4vBB7VsMBm6')).to eq(sha1)
expect(Upload.sha1_from_short_url('r3AYqESanERjladb4vBB7VsMBm6')).to eq(sha1)
end
it "should be able to look up sha1 even with leading zeros" do
sha1 = '0000c513e1da04f7b4e99230851ea2aafeb8cc4e'
expect(Upload.sha1_from_short_url('upload://1Eg9p8rrCURq4T3a6iJUk0ri6.png')).to eq(sha1)
end
end
describe '#base62_sha1' do
it 'should return the right value' do
upload.update!(sha1: "0000c513e1da04f7b4e99230851ea2aafeb8cc4e")
expect(upload.base62_sha1).to eq("1Eg9p8rrCURq4T3a6iJUk0ri6")
end
end
describe '.sha1_from_short_path' do
it "should be able to lookup sha1" do
path = "/uploads/short-url/3UjQ4jHoyeoQndk5y3qHzm3QVTQ.png"
sha1 = "1b6453892473a467d07372d45eb05abc2031647a"
expect(Upload.sha1_from_short_path(path)).to eq(sha1)
expect(Upload.sha1_from_short_path(path.sub(".png", ""))).to eq(sha1)
end
end
describe '#to_s' do
it 'should return the right value' do
expect(upload.to_s).to eq(upload.url)
end
end
describe '.migrate_to_new_scheme' do
it 'should not migrate system uploads' do
SiteSetting.migrate_to_new_scheme = true
expect { Upload.migrate_to_new_scheme }
.to_not change { Upload.pluck(:url) }
end
end
describe '.update_secure_status' do
it "respects the override parameter if provided" do
upload.update!(secure: true)
upload.update_secure_status(override: true)
expect(upload.secure).to eq(true)
upload.update_secure_status(override: false)
expect(upload.secure).to eq(false)
end
it 'marks a local upload as not secure with default settings' do
upload.update!(secure: true)
expect { upload.update_secure_status }
.to change { upload.secure }
expect(upload.secure).to eq(false)
end
context "local attachment" do
before do
SiteSetting.authorized_extensions = "pdf"
end
let(:upload) { Fabricate(:upload, original_filename: "small.pdf", extension: "pdf", secure: true) }
it 'marks a local attachment as secure if secure media enabled' do
upload.update!(secure: false, access_control_post: Fabricate(:private_message_post))
enable_secure_media
expect { upload.update_secure_status }
.to change { upload.secure }
expect(upload.secure).to eq(true)
end
it 'marks a local attachment as not secure if secure media enabled' do
expect { upload.update_secure_status }
.to change { upload.secure }
expect(upload.secure).to eq(false)
end
end
it 'does not change secure status of a non-attachment when prevent_anons_from_downloading_files is enabled by itself' do
SiteSetting.prevent_anons_from_downloading_files = true
SiteSetting.authorized_extensions = "mp4"
upload.update!(original_filename: "small.mp4", extension: "mp4")
expect { upload.update_secure_status }
.not_to change { upload.secure }
expect(upload.secure).to eq(false)
end
context "secure media enabled" do
before do
enable_secure_media
end
it 'does not mark an image upload as not secure when there is no access control post id, to avoid unintentional exposure' do
upload.update!(secure: true)
upload.update_secure_status
expect(upload.secure).to eq(true)
end
it 'marks the upload as not secure if its access control post is a public post' do
upload.update!(secure: true, access_control_post: Fabricate(:post))
upload.update_secure_status
expect(upload.secure).to eq(false)
end
it 'leaves the upload as secure if its access control post is a PM post' do
upload.update!(secure: true, access_control_post: Fabricate(:private_message_post))
upload.update_secure_status
expect(upload.secure).to eq(true)
end
it 'marks an image upload as secure if login_required is enabled' do
SiteSetting.login_required = true
upload.update!(secure: false)
expect { upload.update_secure_status }
.to change { upload.secure }
expect(upload.reload.secure).to eq(true)
end
it 'does not mark an upload used for a custom emoji as secure' do
SiteSetting.login_required = true
upload.update!(secure: false)
CustomEmoji.create(name: 'meme', upload: upload)
upload.update_secure_status
expect(upload.reload.secure).to eq(false)
end
it 'does not mark an upload whose origin matches a regular emoji as secure (sometimes emojis are downloaded in pull_hotlinked_images)' do
SiteSetting.login_required = true
falafel = Emoji.all.find { |e| e.url == "/images/emoji/twitter/falafel.png?v=#{Emoji::EMOJI_VERSION}" }
upload.update!(secure: false, origin: "http://localhost:3000#{falafel.url}")
upload.update_secure_status
expect(upload.reload.secure).to eq(false)
end
it 'does not mark any upload with origin containing images/emoji in the URL' do
SiteSetting.login_required = true
upload.update!(secure: false, origin: "http://localhost:3000/images/emoji/test.png")
upload.update_secure_status
expect(upload.reload.secure).to eq(false)
end
it "does not throw an error if the object storage provider does not support ACLs" do
FileStore::S3Store.any_instance.stubs(:update_upload_ACL).raises(
Aws::S3::Errors::NotImplemented.new("A header you provided implies functionality that is not implemented", "")
)
upload.update!(secure: true, access_control_post: Fabricate(:private_message_post))
expect { upload.update_secure_status }.not_to raise_error
end
end
end
def enable_secure_media
setup_s3
SiteSetting.secure_media = true
stub_upload(upload)
end
context '.destroy' do
it "can correctly clear information when destroying an upload" do
upload = Fabricate(:upload)
user = Fabricate(:user)
user.user_profile.update!(
card_background_upload_id: upload.id,
profile_background_upload_id: upload.id
)
upload.destroy
user.user_profile.reload
expect(user.user_profile.card_background_upload_id).to eq(nil)
expect(user.user_profile.profile_background_upload_id).to eq(nil)
end
end
context ".signed_url_from_secure_media_url" do
before do
# must be done so signed_url_for_path exists
enable_secure_media
end
it "correctly gives back a signed url from a path only" do
secure_url = "/secure-media-uploads/original/1X/c5a2c4ba0fa390f5aac5c2c1a12416791ebdd9e9.png"
signed_url = Upload.signed_url_from_secure_media_url(secure_url)
expect(signed_url).not_to include("secure-media-uploads")
expect(UrlHelper.s3_presigned_url?(signed_url)).to eq(true)
end
it "correctly gives back a signed url from a full url" do
secure_url = "http://localhost:3000/secure-media-uploads/original/1X/c5a2c4ba0fa390f5aac5c2c1a12416791ebdd9e9.png"
signed_url = Upload.signed_url_from_secure_media_url(secure_url)
expect(signed_url).not_to include(Discourse.base_url)
expect(UrlHelper.s3_presigned_url?(signed_url)).to eq(true)
end
end
context ".secure_media_url_from_upload_url" do
before do
# must be done so signed_url_for_path exists
enable_secure_media
end
it "gets the secure media url from an S3 upload url" do
upload = Fabricate(:upload_s3, secure: true)
url = upload.url
secure_url = Upload.secure_media_url_from_upload_url(url)
expect(secure_url).not_to include(SiteSetting.Upload.absolute_base_url)
end
end
context ".secure_media_url?" do
it "works for a secure media url with or without schema + host" do
url = "//localhost:3000/secure-media-uploads/original/2X/f/f62055931bb702c7fd8f552fb901f977e0289a18.png"
expect(Upload.secure_media_url?(url)).to eq(true)
url = "/secure-media-uploads/original/2X/f/f62055931bb702c7fd8f552fb901f977e0289a18.png"
expect(Upload.secure_media_url?(url)).to eq(true)
url = "http://localhost:3000/secure-media-uploads/original/2X/f/f62055931bb702c7fd8f552fb901f977e0289a18.png"
expect(Upload.secure_media_url?(url)).to eq(true)
end
it "does not get false positives on a topic url" do
url = "/t/secure-media-uploads-are-cool/42839"
expect(Upload.secure_media_url?(url)).to eq(false)
end
it "returns true only for secure media URL for actual media (images/video/audio)" do
url = "/secure-media-uploads/original/2X/f/f62055931bb702c7fd8f552fb901f977e0289a18.mp4"
expect(Upload.secure_media_url?(url)).to eq(true)
url = "/secure-media-uploads/original/2X/f/f62055931bb702c7fd8f552fb901f977e0289a18.png"
expect(Upload.secure_media_url?(url)).to eq(true)
url = "/secure-media-uploads/original/2X/f/f62055931bb702c7fd8f552fb901f977e0289a18.mp3"
expect(Upload.secure_media_url?(url)).to eq(true)
url = "/secure-media-uploads/original/2X/f/f62055931bb702c7fd8f552fb901f977e0289a18.pdf"
expect(Upload.secure_media_url?(url)).to eq(false)
end
it "does not work for regular upload urls" do
url = "/uploads/default/test_0/original/1X/e1864389d8252958586c76d747b069e9f68827e3.png"
expect(Upload.secure_media_url?(url)).to eq(false)
end
it "does not raise for invalid URLs" do
url = "http://URL:%20https://google.com"
expect(Upload.secure_media_url?(url)).to eq(false)
end
end
end