mirror of
https://github.com/discourse/discourse.git
synced 2024-12-14 17:53:42 +08:00
30990006a9
This reduces chances of errors where consumers of strings mutate inputs and reduces memory usage of the app. Test suite passes now, but there may be some stuff left, so we will run a few sites on a branch prior to merging
29 lines
748 B
Ruby
29 lines
748 B
Ruby
# frozen_string_literal: true
|
|
|
|
module Jobs
|
|
|
|
class CleanUpCrawlerStats < Jobs::Scheduled
|
|
every 1.day
|
|
|
|
def execute(args)
|
|
WebCrawlerRequest.where('date < ?', WebCrawlerRequest.max_record_age.ago).delete_all
|
|
|
|
# keep count of only the top user agents
|
|
DB.exec <<~SQL
|
|
WITH ranked_requests AS (
|
|
SELECT row_number() OVER (ORDER BY count DESC) as row_number, id
|
|
FROM web_crawler_requests
|
|
WHERE date = '#{1.day.ago.strftime("%Y-%m-%d")}'
|
|
)
|
|
DELETE FROM web_crawler_requests
|
|
WHERE id IN (
|
|
SELECT ranked_requests.id
|
|
FROM ranked_requests
|
|
WHERE row_number > #{WebCrawlerRequest.max_records_per_day}
|
|
)
|
|
SQL
|
|
end
|
|
end
|
|
|
|
end
|