DEV: move discourse_dev gem to the core. (#13360)

And get avatar images from `discourse_dev_assets` gem.
This commit is contained in:
Vinoth Kannan 2021-06-14 20:34:44 +05:30 committed by GitHub
parent c44650eec5
commit 6abc45e57b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
19 changed files with 913 additions and 10 deletions

View File

@ -174,7 +174,8 @@ group :development do
gem 'binding_of_caller' gem 'binding_of_caller'
gem 'yaml-lint' gem 'yaml-lint'
gem 'annotate' gem 'annotate'
gem 'discourse_dev' gem 'discourse_dev_assets'
gem 'faker', "~> 2.16"
end end
# this is an optional gem, it provides a high performance replacement # this is an optional gem, it provides a high performance replacement

View File

@ -115,7 +115,7 @@ GEM
railties (>= 3.1) railties (>= 3.1)
discourse-ember-source (3.12.2.3) discourse-ember-source (3.12.2.3)
discourse-fonts (0.0.8) discourse-fonts (0.0.8)
discourse_dev (0.2.1) discourse_dev_assets (0.0.2)
faker (~> 2.16) faker (~> 2.16)
docile (1.4.0) docile (1.4.0)
ecma-re-validator (0.3.0) ecma-re-validator (0.3.0)
@ -504,12 +504,13 @@ DEPENDENCIES
discourse-ember-rails (= 0.18.6) discourse-ember-rails (= 0.18.6)
discourse-ember-source (~> 3.12.2) discourse-ember-source (~> 3.12.2)
discourse-fonts discourse-fonts
discourse_dev discourse_dev_assets
email_reply_trimmer email_reply_trimmer
ember-handlebars-template (= 0.8.0) ember-handlebars-template (= 0.8.0)
excon excon
execjs execjs
fabrication fabrication
faker (~> 2.16)
fakeweb fakeweb
fast_blank fast_blank
fast_xs fast_xs

38
config/dev_defaults.yml Normal file
View File

@ -0,0 +1,38 @@
site_settings:
tagging_enabled: true
verbose_discourse_connect_logging: true
seed: 1
start_date: 2020-01-01
auth_plugin_enabled: true
allow_anonymous_to_impersonate: false
category:
count: 30
group:
count: 15
post:
include_images: false
max_likes_count: 10
tag:
count: 30
topic:
count: 30
replies:
# number of replies per topic between min and max
min: 0
max: 12
overrides:
# topic titles can be found in config/locales/faker.en.yml
- title: "Coolest thing you have seen today"
count: 99
tags:
# number of tags per topic between min and max
min: 0
max: 3
user:
count: 30
new_user:
username: new_user
email: new_user@example.com

View File

@ -977,10 +977,6 @@ Discourse::Application.routes.draw do
post "/do-not-disturb" => "do_not_disturb#create" post "/do-not-disturb" => "do_not_disturb#create"
delete "/do-not-disturb" => "do_not_disturb#destroy" delete "/do-not-disturb" => "do_not_disturb#destroy"
if Rails.env.development?
mount DiscourseDev::Engine => "/dev/"
end
get "*url", to: 'permalinks#show', constraints: PermalinkConstraint.new get "*url", to: 'permalinks#show', constraints: PermalinkConstraint.new
end end
end end

15
lib/discourse_dev.rb Normal file
View File

@ -0,0 +1,15 @@
# frozen_string_literal: true
module DiscourseDev
def self.config
@config ||= Config.new
end
def self.settings_file
File.join(root, "config", "settings.yml")
end
def self.root
File.expand_path("..", __dir__)
end
end

View File

@ -0,0 +1,60 @@
# frozen_string_literal: true
require 'discourse_dev/record'
require 'rails'
require 'faker'
module DiscourseDev
class Category < Record
def initialize
super(::Category, DiscourseDev.config.category[:count])
@parent_category_ids = ::Category.where(parent_category_id: nil).pluck(:id)
end
def data
name = Faker::Discourse.unique.category
parent_category_id = nil
if Faker::Boolean.boolean(true_ratio: 0.6)
offset = Faker::Number.between(from: 0, to: @parent_category_ids.count - 1)
parent_category_id = @parent_category_ids[offset]
@permissions = ::Category.find(parent_category_id).permissions_params.presence
else
@permissions = nil
end
{
name: name,
description: Faker::Lorem.paragraph,
user_id: ::Discourse::SYSTEM_USER_ID,
color: Faker::Color.hex_color.last(6),
parent_category_id: parent_category_id
}
end
def permissions
return @permissions if @permissions.present?
return { everyone: :full } if Faker::Boolean.boolean(true_ratio: 0.75)
permission = {}
group = Group.random
permission[group.id] = Faker::Number.between(from: 1, to: 3)
permission
end
def create!
super do |category|
category.set_permissions(permissions)
category.save!
@parent_category_ids << category.id if category.parent_category_id.blank?
end
end
def self.random
super(::Category)
end
end
end

149
lib/discourse_dev/config.rb Normal file
View File

@ -0,0 +1,149 @@
# frozen_string_literal: true
require 'rails'
require 'highline/import'
module DiscourseDev
class Config
attr_reader :config, :file_path
def initialize
default_file_path = File.join(Rails.root, "config", "dev_defaults.yml")
@file_path = File.join(Rails.root, "config", "dev.yml")
default_config = YAML.load_file(default_file_path)
if File.exists?(file_path)
user_config = YAML.load_file(file_path)
else
puts "I did no detect a custom `config/dev.yml` file, creating one for you where you can amend defaults."
FileUtils.cp(default_file_path, file_path)
user_config = {}
end
@config = default_config.deep_merge(user_config).deep_symbolize_keys
end
def update!
update_site_settings
create_admin_user
create_new_user
set_seed
end
private
def update_site_settings
puts "Updating site settings..."
site_settings = config[:site_settings] || {}
site_settings.each do |key, value|
puts "#{key} = #{value}"
SiteSetting.set(key, value)
end
SiteSetting.refresh!
end
def create_admin_user
puts "Creating default admin user account..."
settings = config[:admin]
if settings.present?
create_admin_user_from_settings(settings)
else
create_admin_user_from_input
end
end
def create_new_user
settings = config[:new_user]
if settings.present?
email = settings[:email] || "new_user@example.com"
new_user = ::User.create!(
email: email,
username: settings[:username] || UserNameSuggester.suggest(email)
)
new_user.email_tokens.update_all confirmed: true
new_user.activate
end
end
def set_seed
seed = self.seed || 1
Faker::Config.random = Random.new(seed)
end
def start_date
DateTime.parse(config[:start_date])
end
def method_missing(name)
config[name.to_sym]
end
def create_admin_user_from_settings(settings)
email = settings[:email]
admin = ::User.with_email(email).first_or_create!(
email: email,
username: settings[:username] || UserNameSuggester.suggest(email),
password: settings[:password]
)
admin.grant_admin!
if admin.trust_level < 1
admin.change_trust_level!(1)
end
admin.email_tokens.update_all confirmed: true
admin.activate
end
def create_admin_user_from_input
begin
email = ask("Email: ")
password = ask("Password (optional, press ENTER to skip): ")
username = UserNameSuggester.suggest(email)
admin = ::User.new(
email: email,
username: username
)
if password.present?
admin.password = password
else
puts "Once site is running use https://localhost:9292/user/#{username}/become to access the account in development"
end
admin.name = ask("Full name: ") if SiteSetting.full_name_required
saved = admin.save
if saved
File.open(file_path, 'a') do | file|
file.puts("admin:")
file.puts(" username: #{admin.username}")
file.puts(" email: #{admin.email}")
file.puts(" password: #{password}") if password.present?
end
else
say(admin.errors.full_messages.join("\n"))
end
end while !saved
admin.active = true
admin.save
admin.grant_admin!
if admin.trust_level < 1
admin.change_trust_level!(1)
end
admin.email_tokens.update_all confirmed: true
admin.activate
say("\nAdmin account created successfully with username #{admin.username}")
end
end
end

View File

@ -0,0 +1,38 @@
# frozen_string_literal: true
require 'discourse_dev/record'
require 'rails'
require 'faker'
module DiscourseDev
class Group < Record
def initialize
super(::Group, DiscourseDev.config.group[:count])
end
def data
{
name: Faker::Discourse.unique.group,
public_exit: Faker::Boolean.boolean,
public_admission: Faker::Boolean.boolean,
primary_group: Faker::Boolean.boolean,
created_at: Faker::Time.between(from: DiscourseDev.config.start_date, to: DateTime.now),
}
end
def create!
super do |group|
if Faker::Boolean.boolean
group.add_owner(::Discourse.system_user)
group.allow_membership_requests = true
group.save!
end
end
end
def self.random
super(::Group)
end
end
end

107
lib/discourse_dev/post.rb Normal file
View File

@ -0,0 +1,107 @@
# frozen_string_literal: true
require 'discourse_dev/record'
require 'faker'
module DiscourseDev
class Post < Record
attr_reader :topic
def initialize(topic, count)
super(::Post, count)
@topic = topic
category = topic.category
@max_likes_count = DiscourseDev.config.post[:max_likes_count]
unless category.groups.blank?
group_ids = category.groups.pluck(:id)
@user_ids = ::GroupUser.where(group_id: group_ids).pluck(:user_id)
@user_count = @user_ids.count
@max_likes_count = @user_count - 1
end
end
def data
{
topic_id: topic.id,
raw: Faker::DiscourseMarkdown.sandwich(sentences: 5),
created_at: Faker::Time.between(from: topic.last_posted_at, to: DateTime.now),
skip_validations: true,
skip_guardian: true
}
end
def create!
user = self.user
data = Faker::DiscourseMarkdown.with_user(user.id) { self.data }
post = PostCreator.new(user, data).create!
topic.reload
generate_likes(post)
end
def generate_likes(post)
user_ids = [post.user_id]
Faker::Number.between(from: 0, to: @max_likes_count).times do
user = self.user
next if user_ids.include?(user.id)
PostActionCreator.new(user, post, PostActionType.types[:like], created_at: Faker::Time.between(from: post.created_at, to: DateTime.now)).perform
user_ids << user.id
end
end
def user
return User.random if topic.category.groups.blank?
return Discourse.system_user if @user_ids.blank?
position = Faker::Number.between(from: 0, to: @user_count - 1)
::User.find(@user_ids[position])
end
def populate!
generate_likes(topic.first_post)
@count.times do
create!
end
end
def self.add_replies!(args)
if !args[:topic_id]
puts "Topic ID is required. Aborting."
return
end
if !::Topic.find_by_id(args[:topic_id])
puts "Topic ID does not match topic in DB, aborting."
return
end
topic = ::Topic.find_by_id(args[:topic_id])
count = args[:count] ? args[:count].to_i : 50
puts "Creating #{count} replies in '#{topic.title}'"
count.times do |i|
begin
user = User.random
reply = Faker::DiscourseMarkdown.with_user(user.id) do
{
topic_id: topic.id,
raw: Faker::DiscourseMarkdown.sandwich(sentences: 5),
skip_validations: true
}
end
PostCreator.new(user, reply).create!
rescue ActiveRecord::RecordNotSaved => e
puts e
end
end
puts "Done!"
end
end
end

View File

@ -0,0 +1,68 @@
# frozen_string_literal: true
require 'discourse_dev'
require 'rails'
require 'faker'
module DiscourseDev
class Record
DEFAULT_COUNT = 30.freeze
attr_reader :model, :type
def initialize(model, count = DEFAULT_COUNT)
@@initialized ||= begin
Faker::Discourse.unique.clear
RateLimiter.disable
true
end
@model = model
@type = model.to_s
@count = count
end
def create!
record = model.create!(data)
yield(record) if block_given?
end
def populate!
if current_count >= @count
puts "Already have #{current_count} #{type.downcase} records"
Rake.application.top_level_tasks.each do |task_name|
Rake::Task[task_name].reenable
end
Rake::Task['dev:repopulate'].invoke
return
elsif current_count > 0
@count -= current_count
puts "There are #{current_count} #{type.downcase} records. Creating #{@count} more."
else
puts "Creating #{@count} sample #{type.downcase} records"
end
@count.times do
create!
putc "."
end
puts
end
def current_count
model.count
end
def self.populate!
self.new.populate!
end
def self.random(model)
offset = Faker::Number.between(from: 0, to: model.count - 1)
model.offset(offset).first
end
end
end

32
lib/discourse_dev/tag.rb Normal file
View File

@ -0,0 +1,32 @@
# frozen_string_literal: true
require 'discourse_dev/record'
require 'rails'
require 'faker'
module DiscourseDev
class Tag < Record
def initialize
super(::Tag, DiscourseDev.config.tag[:count])
end
def create!
super
rescue ActiveRecord::RecordInvalid => e
# If the name is taken, try again
retry
end
def populate!
return unless SiteSetting.tagging_enabled
super
end
def data
{
name: Faker::Discourse.unique.tag,
}
end
end
end

108
lib/discourse_dev/topic.rb Normal file
View File

@ -0,0 +1,108 @@
# frozen_string_literal: true
require 'discourse_dev/record'
require 'faker'
module DiscourseDev
class Topic < Record
def initialize
@settings = DiscourseDev.config.topic
super(::Topic, @settings[:count])
end
def data
max_views = 0
case Faker::Number.between(from: 0, to: 5)
when 0
max_views = 10
when 1
max_views = 100
when 2
max_views = SiteSetting.topic_views_heat_low
when 3
max_views = SiteSetting.topic_views_heat_medium
when 4
max_views = SiteSetting.topic_views_heat_high
when 5
max_views = SiteSetting.topic_views_heat_high + SiteSetting.topic_views_heat_medium
end
{
title: title[0, SiteSetting.max_topic_title_length],
raw: Faker::DiscourseMarkdown.sandwich(sentences: 5),
category: @category.id,
created_at: Faker::Time.between(from: DiscourseDev.config.start_date, to: DateTime.now),
tags: tags,
topic_opts: {
import_mode: true,
views: Faker::Number.between(from: 1, to: max_views),
custom_fields: { dev_sample: true }
},
skip_validations: true
}
end
def title
if current_count < I18n.t("faker.discourse.topics").count
Faker::Discourse.unique.topic
else
Faker::Lorem.unique.sentence(word_count: 5, supplemental: true, random_words_to_add: 4).chomp(".")
end
end
def tags
return unless SiteSetting.tagging_enabled
@tags = []
Faker::Number.between(from: @settings.dig(:tags, :min), to: @settings.dig(:tags, :max)).times do
@tags << Faker::Discourse.tag
end
@tags.uniq
end
def create!
@category = Category.random
user = self.user
topic = Faker::DiscourseMarkdown.with_user(user.id) { data }
post = PostCreator.new(user, topic).create!
if override = @settings.dig(:replies, :overrides).find { |o| o[:title] == topic[:title] }
reply_count = override[:count]
else
reply_count = Faker::Number.between(from: @settings.dig(:replies, :min), to: @settings.dig(:replies, :max))
end
Post.new(post.topic, reply_count).populate!
end
def populate!
super
delete_unwanted_sidekiq_jobs
end
def user
return User.random if @category.groups.blank?
group_ids = @category.groups.pluck(:id)
user_ids = ::GroupUser.where(group_id: group_ids).pluck(:user_id)
user_count = user_ids.count
position = Faker::Number.between(from: 0, to: user_count - 1)
::User.find(user_ids[position] || Discourse::SYSTEM_USER_ID)
end
def current_count
category_definition_topic_ids = ::Category.pluck(:topic_id)
::Topic.where(archetype: :regular).where.not(id: category_definition_topic_ids).count
end
def delete_unwanted_sidekiq_jobs
Sidekiq::ScheduledSet.new.each do |job|
job.delete if job.item["class"] == "Jobs::UserEmail"
end
end
end
end

93
lib/discourse_dev/user.rb Normal file
View File

@ -0,0 +1,93 @@
# frozen_string_literal: true
require 'discourse_dev/record'
require 'faker'
module DiscourseDev
class User < Record
attr_reader :images
def initialize
super(::User, DiscourseDev.config.user[:count])
@images = DiscourseDevAssets.avatars
end
def data
name = Faker::Name.unique.name
email = Faker::Internet.unique.email(name: name, domain: "faker.invalid")
username = Faker::Internet.unique.username(specifier: ::User.username_length)
username = UserNameSuggester.suggest(username)
username_lower = ::User.normalize_username(username)
{
name: name,
email: email,
username: username,
username_lower: username_lower,
moderator: Faker::Boolean.boolean(true_ratio: 0.1),
trust_level: Faker::Number.between(from: 1, to: 4),
created_at: Faker::Time.between(from: DiscourseDev.config.start_date, to: DateTime.now),
}
end
def create!
super do |user|
user.activate
set_random_avatar(user)
Faker::Number.between(from: 0, to: 2).times do
group = Group.random
group.add(user)
end
end
end
def self.random
super(::User)
end
def set_random_avatar(user)
return if images.blank?
return unless Faker::Boolean.boolean
avatar_index = Faker::Number.between(from: 0, to: images.count - 1)
avatar_path = images[avatar_index]
create_avatar(user, avatar_path)
@images.delete_at(avatar_index)
end
def create_avatar(user, avatar_path)
tempfile = copy_to_tempfile(avatar_path)
filename = "avatar#{File.extname(avatar_path)}"
upload = UploadCreator.new(tempfile, filename, type: "avatar").create_for(user.id)
if upload.present? && upload.persisted?
user.create_user_avatar
user.user_avatar.update(custom_upload_id: upload.id)
user.update(uploaded_avatar_id: upload.id)
else
STDERR.puts "Failed to upload avatar for user #{user.username}: #{avatar_path}"
STDERR.puts upload.errors.inspect if upload
end
rescue
STDERR.puts "Failed to create avatar for user #{user.username}: #{avatar_path}"
ensure
tempfile.close! if tempfile
end
private
def copy_to_tempfile(source_path)
extension = File.extname(source_path)
tmp = Tempfile.new(['discourse-upload', extension])
File.open(source_path) do |source_stream|
IO.copy_stream(source_stream, tmp)
end
tmp.rewind
tmp
end
end
end

26
lib/faker/discourse.rb Normal file
View File

@ -0,0 +1,26 @@
# frozen_string_literal: true
require 'faker'
module Faker
class Discourse < Base
class << self
def tag
fetch('discourse.tags')
end
def category
fetch('discourse.categories')
end
def group
fetch('discourse.groups')
end
def topic
fetch('discourse.topics')
end
end
end
end

View File

@ -0,0 +1,96 @@
# frozen_string_literal: true
require 'faker'
require 'net/http'
require 'json'
module Faker
class DiscourseMarkdown < Markdown
class << self
attr_writer(:user_id)
def user_id
@user_id || ::Discourse::SYSTEM_USER_ID
end
def with_user(user_id)
current_user_id = self.user_id
self.user_id = user_id
begin
yield
ensure
self.user_id = current_user_id
end
end
def image
image = next_image
image_file = load_image(image)
upload = ::UploadCreator.new(
image_file,
image[:filename],
origin: image[:url]
).create_for(user_id)
::UploadMarkdown.new(upload).to_markdown if upload.present? && upload.persisted?
rescue => e
STDERR.puts e
STDERR.puts e.backtrace.join("\n")
end
private
def next_image
if @images.blank?
if @stop_loading_images
@images = @all_images.dup
else
@next_page = (@next_page || 0) + 1
url = URI("https://picsum.photos/v2/list?page=#{@next_page}&limit=50")
response = Net::HTTP.get(url)
json = JSON.parse(response)
if json.blank?
@stop_loading_images = true
@images = @all_images.dup
else
@images = json.sort_by { |image| image["id"] }
@all_images = (@all_images || []).concat(@images)
end
end
end
image = @images.pop
{ filename: "#{image['id']}.jpg", url: "#{image['download_url']}.jpg" }
end
def image_cache_dir
@image_cache_dir ||= ::File.join(Rails.root, "tmp", "discourse_dev", "images")
end
def load_image(image)
cache_path = ::File.join(image_cache_dir, image[:filename])
if !::File.exists?(cache_path)
FileUtils.mkdir_p(image_cache_dir)
temp_file = ::FileHelper.download(
image[:url],
max_file_size: [SiteSetting.max_image_size_kb.kilobytes, 10.megabytes].max,
tmp_file_name: "image",
follow_redirect: true
)
FileUtils.cp(temp_file, cache_path)
end
::File.open(cache_path)
end
def available_methods
methods = super
methods << :image if ::DiscourseDev.config.post[:include_images]
methods
end
end
end
end

View File

@ -4,7 +4,6 @@ module JsLocaleHelper
def self.plugin_client_files(locale_str) def self.plugin_client_files(locale_str)
files = Dir["#{Rails.root}/plugins/*/config/locales/client*.#{locale_str}.yml"] files = Dir["#{Rails.root}/plugins/*/config/locales/client*.#{locale_str}.yml"]
files += DiscourseDev.client_locale_files(locale_str) if Rails.env.development?
I18n::Backend::DiscourseI18n.sort_locale_files(files) I18n::Backend::DiscourseI18n.sort_locale_files(files)
end end

View File

@ -90,8 +90,6 @@ class Plugin::Instance
metadata = Plugin::Metadata.parse(source) metadata = Plugin::Metadata.parse(source)
plugins << self.new(metadata, path) plugins << self.new(metadata, path)
end end
plugins << DiscourseDev.auth_plugin if Rails.env.development? && DiscourseDev.auth_plugin_enabled?
} }
end end

47
lib/tasks/dev.rake Normal file
View File

@ -0,0 +1,47 @@
# frozen_string_literal: true
def check_environment!
if !Rails.env.development?
raise "Database commands are only supported in development environment"
end
ENV['SKIP_TEST_DATABASE'] = "1"
ENV['SKIP_MULTISITE'] = "1"
end
desc 'Run db:migrate:reset task and populate sample content for development environment'
task 'dev:reset' => ['db:load_config'] do |_, args|
check_environment!
Rake::Task['db:migrate:reset'].invoke
Rake::Task['dev:config'].invoke
Rake::Task['dev:populate'].invoke
end
desc 'Initialize development environment'
task 'dev:config' => ['db:load_config'] do |_, args|
DiscourseDev.config.update!
end
desc 'Populate sample content for development environment'
task 'dev:populate' => ['db:load_config'] do |_, args|
system("redis-cli flushall")
Rake::Task['groups:populate'].invoke
Rake::Task['users:populate'].invoke
Rake::Task['categories:populate'].invoke
Rake::Task['tags:populate'].invoke
Rake::Task['topics:populate'].invoke
end
desc 'Repopulate sample datas in development environment'
task 'dev:repopulate' => ['db:load_config'] do |_, args|
require 'highline/import'
answer = ask("Do you want to repopulate the database with fresh data? It will recreate DBs and run migration from scratch before generating all the samples. (Y/n) ")
if (answer == "" || answer.downcase == 'y')
Rake::Task['dev:reset'].invoke
else
puts "You can run `bin/rails dev:reset` to repopulate anytime."
end
end

31
lib/tasks/populate.rake Normal file
View File

@ -0,0 +1,31 @@
# frozen_string_literal: true
desc 'Creates sample categories'
task 'groups:populate' => ['db:load_config'] do |_, args|
DiscourseDev::Group.populate!
end
desc 'Creates sample user accounts'
task 'users:populate' => ['db:load_config'] do |_, args|
DiscourseDev::User.populate!
end
desc 'Creates sample categories'
task 'categories:populate' => ['db:load_config'] do |_, args|
DiscourseDev::Category.populate!
end
desc 'Creates sample tags'
task 'tags:populate' => ['db:load_config'] do |_, args|
DiscourseDev::Tag.populate!
end
desc 'Creates sample topics'
task 'topics:populate' => ['db:load_config'] do |_, args|
DiscourseDev::Topic.populate!
end
desc 'Add replies to a topic'
task 'replies:populate', [:topic_id, :count] => ['db:load_config'] do |_, args|
DiscourseDev::Post.add_replies!(args)
end