github-mirror/discourse
2
0
Fork 0
mirror of https://github.com/discourse/discourse.git synced 2024-11-24 20:51:50 +08:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

DEV: Remove summarization code (#27373)

Browse Source
This commit is contained in:
Keegan George 2024-07-02 08:51:47 -07:00 committed by GitHub
parent 052550c6e0
commit ea58140032
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
48 changed files with 586 additions and 2317 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
app/assets/javascripts/discourse/app/components/ai-summary-skeleton.hbs
View File

@ -1,28 +0,0 @@
<div class="ai-summary__container">
<ul class="ai-summary__list" {{did-insert this.setupAnimation}}>
{{#each this.blocks as |block|}}
<li
class={{concat-class
"ai-summary__list-item"
(if block.show "show")
(if block.shown "is-shown")
(if block.blinking "blink")
}}
{{did-update (fn this.onBlinking block) block.blinking}}
{{did-update (fn this.onShowing block) block.show}}
{{will-destroy this.teardownAnimation}}
></li>
{{/each}}
</ul>
<span>
<div class="ai-summary__generating-text">
{{i18n "summary.in_progress"}}
</div>
<span class="ai-summary__indicator-wave">
<span class="ai-summary__indicator-dot">.</span>
<span class="ai-summary__indicator-dot">.</span>
<span class="ai-summary__indicator-dot">.</span>
</span>
</span>
</div>

92
app/assets/javascripts/discourse/app/components/ai-summary-skeleton.js
View File

@ -1,92 +0,0 @@
import Component from "@glimmer/component";
import { tracked } from "@glimmer/tracking";
import { action } from "@ember/object";
import { cancel } from "@ember/runloop";
import discourseLater from "discourse-common/lib/later";
class Block {
@tracked show = false;
@tracked shown = false;
@tracked blinking = false;
constructor(args = {}) {
this.show = args.show ?? false;
this.shown = args.shown ?? false;
}
}
const BLOCKS_SIZE = 20; // changing this requires to change css accordingly
export default class AiSummarySkeleton extends Component {
blocks = [...Array.from({ length: BLOCKS_SIZE }, () => new Block())];
#nextBlockBlinkingTimer;
#blockBlinkingTimer;
#blockShownTimer;
@action
setupAnimation() {
this.blocks.firstObject.show = true;
this.blocks.firstObject.shown = true;
}
@action
onBlinking(block) {
if (!block.blinking) {
return;
}
block.show = false;
this.#nextBlockBlinkingTimer = discourseLater(
this,
() => {
this.#nextBlock(block).blinking = true;
},
250
);
this.#blockBlinkingTimer = discourseLater(
this,
() => {
block.blinking = false;
},
500
);
}
@action
onShowing(block) {
if (!block.show) {
return;
}
this.#blockShownTimer = discourseLater(
this,
() => {
this.#nextBlock(block).show = true;
this.#nextBlock(block).shown = true;
if (this.blocks.lastObject === block) {
this.blocks.firstObject.blinking = true;
}
},
250
);
}
@action
teardownAnimation() {
cancel(this.#blockShownTimer);
cancel(this.#nextBlockBlinkingTimer);
cancel(this.#blockBlinkingTimer);
}
#nextBlock(currentBlock) {
if (currentBlock === this.blocks.lastObject) {
return this.blocks.firstObject;
} else {
return this.blocks.objectAt(this.blocks.indexOf(currentBlock) + 1);
}
}
}

55
app/assets/javascripts/discourse/app/components/summary-box.hbs
View File

@ -1,55 +0,0 @@
<div class="summary-box__container old-summary-box-temporary">
{{#if @topic.summarizable}}
{{#if this.summary.showSummaryBox}}
<DButton
@action={{@collapseSummary}}
@title="summary.buttons.hide"
@label="summary.buttons.hide"
@icon="chevron-up"
class="btn-primary topic-strategy-summarization"
/>
{{else}}
<DButton
@action={{@showSummary}}
@translatedLabel={{this.generateSummaryTitle}}
@translatedTitle={{this.generateSummaryTitle}}
@icon={{this.generateSummaryIcon}}
@disabled={{this.summary.loading}}
class="btn-primary topic-strategy-summarization"
/>
{{/if}}
{{/if}}
{{#if this.summary.showSummaryBox}}
<article class="summary-box">
{{#if (and this.summary.loading (not this.summary.text))}}
<AiSummarySkeleton />
{{else}}
<div class="generated-summary">{{this.summary.text}}</div>
{{#if this.summary.summarizedOn}}
<div class="summarized-on">
<p>
{{i18n "summary.summarized_on" date=this.summary.summarizedOn}}
<DTooltip @placements={{array "top-end"}}>
<:trigger>
{{d-icon "info-circle"}}
</:trigger>
<:content>
{{i18n "summary.model_used" model=this.summary.summarizedBy}}
</:content>
</DTooltip>
</p>
{{#if this.summary.outdated}}
<p class="outdated-summary">
{{this.outdatedSummaryWarningText}}
</p>
{{/if}}
</div>
{{/if}}
{{/if}}
</article>
{{/if}}
</div>

43
app/assets/javascripts/discourse/app/components/summary-box.js
View File

@ -1,43 +0,0 @@
import Component from "@glimmer/component";
import { service } from "@ember/service";
import I18n from "discourse-i18n";
export default class SummaryBox extends Component {
@service siteSettings;
get summary() {
return this.args.postStream.topicSummary;
}
get generateSummaryTitle() {
const title = this.summary.canRegenerate
? "summary.buttons.regenerate"
: "summary.buttons.generate";
return I18n.t(title);
}
get generateSummaryIcon() {
return this.summary.canRegenerate ? "sync" : "discourse-sparkles";
}
get outdatedSummaryWarningText() {
let outdatedText = I18n.t("summary.outdated");
if (
!this.topRepliesSummaryEnabled &&
this.summary.newPostsSinceSummary > 0
) {
outdatedText += " ";
outdatedText += I18n.t("summary.outdated_posts", {
count: this.summary.newPostsSinceSummary,
});
}
return outdatedText;
}
get topRepliesSummaryEnabled() {
return this.args.postStream.summary;
}
}

17
app/assets/javascripts/discourse/app/components/topic-map.gjs
View File

@ -6,13 +6,11 @@ import { service } from "@ember/service";
import { htmlSafe } from "@ember/template";
import DButton from "discourse/components/d-button";
import PluginOutlet from "discourse/components/plugin-outlet";
import SummaryBox from "discourse/components/summary-box";
import PrivateMessageMap from "discourse/components/topic-map/private-message-map";
import TopicMapExpanded from "discourse/components/topic-map/topic-map-expanded";
import TopicMapSummary from "discourse/components/topic-map/topic-map-summary";
import concatClass from "discourse/helpers/concat-class";
import I18n from "discourse-i18n";
import or from "truth-helpers/helpers/or";
const MIN_POST_READ_TIME = 4;
@ -98,7 +96,6 @@ export default class TopicMap extends Component {
</section>
{{/unless}}
{{#if (or @model.has_summary @model.summarizable)}}
<section class="information toggle-summary">
{{#if @model.has_summary}}
<p>{{htmlSafe this.topRepliesSummaryInfo}}</p>
@ -108,7 +105,6 @@ export default class TopicMap extends Component {
@defaultGlimmer={{true}}
@outletArgs={{hash topic=@model postStream=@postStream}}
>
<div class="summarization-buttons">
{{#if @model.has_summary}}
<DButton
@action={{if @postStream.summary @cancelFilter @showTopReplies}}
@ -118,21 +114,8 @@ export default class TopicMap extends Component {
class="top-replies"
/>
{{/if}}
</div>
{{#if @model.summarizable}}
<SummaryBox
@topic={{@model}}
@postStream={{@postStream}}
@cancelFilter={{@cancelFilter}}
@showTopReplies={{@showTopReplies}}
@collapseSummary={{@collapseSummary}}
@showSummary={{@showSummary}}
/>
{{/if}}
</PluginOutlet>
</section>
{{/if}}
{{#if @showPMMap}}
<section class="information private-message-map">

18
app/assets/javascripts/discourse/app/controllers/topic.js
View File

@ -568,14 +568,6 @@ export default Controller.extend(bufferedProperty("model"), {
});
},
collapseSummary() {
this.get("model.postStream").collapseSummary();
},
showSummary() {
this.get("model.postStream").showSummary(this.currentUser);
},
removeAllowedUser(user) {
return this.get("model.details")
.removeAllowedUser(user)
@ -1649,9 +1641,6 @@ export default Controller.extend(bufferedProperty("model"), {
this.onMessage,
this.get("model.message_bus_last_id")
);
const summariesChannel = `/summaries/topic/${this.get("model.id")}`;
this.messageBus.subscribe(summariesChannel, this._updateSummary);
},
unsubscribe() {
@ -1661,13 +1650,6 @@ export default Controller.extend(bufferedProperty("model"), {
}
this.messageBus.unsubscribe("/topic/*", this.onMessage);
this.messageBus.unsubscribe("/summaries/topic/*", this._updateSummary);
},
@bind
_updateSummary(update) {
const postStream = this.get("model.postStream");
postStream.processSummaryUpdate(update);
},
@bind

72
app/assets/javascripts/discourse/app/lib/topic-summary.js
View File

@ -1,72 +0,0 @@
import { tracked } from "@glimmer/tracking";
import { ajax } from "discourse/lib/ajax";
import { shortDateNoYear } from "discourse/lib/formatter";
import { cook } from "discourse/lib/text";
export default class TopicSummary {
@tracked text = "";
@tracked summarizedOn = null;
@tracked summarizedBy = null;
@tracked newPostsSinceSummary = null;
@tracked outdated = false;
@tracked canRegenerate = false;
@tracked regenerated = false;
@tracked showSummaryBox = false;
@tracked canCollapseSummary = false;
@tracked loadingSummary = false;
processUpdate(update) {
const topicSummary = update.topic_summary;
return cook(topicSummary.summarized_text)
.then((cooked) => {
this.text = cooked;
this.loading = false;
})
.then(() => {
if (update.done) {
this.summarizedOn = shortDateNoYear(topicSummary.summarized_on);
this.summarizedBy = topicSummary.algorithm;
this.newPostsSinceSummary = topicSummary.new_posts_since_summary;
this.outdated = topicSummary.outdated;
this.newPostsSinceSummary = topicSummary.new_posts_since_summary;
this.canRegenerate =
topicSummary.outdated && topicSummary.can_regenerate;
}
});
}
collapse() {
this.showSummaryBox = false;
this.canCollapseSummary = false;
}
generateSummary(currentUser, topicId) {
this.showSummaryBox = true;
if (this.text && !this.canRegenerate) {
this.canCollapseSummary = false;
return;
}
let fetchURL = `/t/${topicId}/strategy-summary?`;
if (currentUser) {
fetchURL += `stream=true`;
if (this.canRegenerate) {
fetchURL += "&skip_age_check=true";
}
}
this.loading = true;
return ajax(fetchURL).then((data) => {
if (!currentUser) {
data.done = true;
this.processUpdate(data);
}
});
}
}

15
app/assets/javascripts/discourse/app/models/post-stream.js
View File

@ -6,7 +6,6 @@ import { isEmpty } from "@ember/utils";
import { Promise } from "rsvp";
import { ajax } from "discourse/lib/ajax";
import PostsWithPlaceholders from "discourse/lib/posts-with-placeholders";
import TopicSummary from "discourse/lib/topic-summary";
import DiscourseURL from "discourse/lib/url";
import { highlightPost } from "discourse/lib/utilities";
import RestModel from "discourse/models/rest";
@ -51,7 +50,6 @@ export default class PostStream extends RestModel {
filterRepliesToPostNumber = null;
filterUpwardsPostID = null;
filter = null;
topicSummary = null;
lastId = null;
@or("loadingAbove", "loadingBelow", "loadingFilter", "stagingPost") loading;
@ -86,7 +84,6 @@ export default class PostStream extends RestModel {
loadingFilter: false,
stagingPost: false,
timelineLookup: [],
topicSummary: new TopicSummary(),
});
}
@ -1259,18 +1256,6 @@ export default class PostStream extends RestModel {
}
}
collapseSummary() {
this.topicSummary.collapse();
}
showSummary(currentUser) {
this.topicSummary.generateSummary(currentUser, this.get("topic.id"));
}
processSummaryUpdate(update) {
this.topicSummary.processUpdate(update);
}
_initUserModels(post) {
post.user = this.store.createRecord("user", {
id: post.user_id,

2
app/assets/javascripts/discourse/app/templates/topic.hbs
View File

@ -361,8 +361,6 @@
@toggleWiki={{action "toggleWiki"}}
@showTopReplies={{action "showTopReplies"}}
@cancelFilter={{action "cancelFilter"}}
@collapseSummary={{action "collapseSummary"}}
@showSummary={{action "showSummary"}}
@removeAllowedUser={{action "removeAllowedUser"}}
@removeAllowedGroup={{action "removeAllowedGroup"}}
@topVisibleChanged={{action "topVisibleChanged"}}

4
app/assets/javascripts/discourse/app/widgets/post.js
View File

@ -761,8 +761,6 @@ createWidget("post-body", {
@showPMMap={{@data.showPMMap}}
@cancelFilter={{@data.cancelFilter}}
@showTopReplies={{@data.showTopReplies}}
@collapseSummary={{@data.collapseSummary}}
@showSummary={{@data.showSummary}}
@showInvite={{@data.showInvite}}
@removeAllowedGroup={{@data.removeAllowedGroup}}
@removeAllowedUser={{@data.removeAllowedUser}}
@ -774,8 +772,6 @@ createWidget("post-body", {
showPMMap: attrs.showPMMap,
cancelFilter: () => this.sendWidgetAction("cancelFilter"),
showTopReplies: () => this.sendWidgetAction("showTopReplies"),
collapseSummary: () => this.sendWidgetAction("collapseSummary"),
showSummary: () => this.sendWidgetAction("showSummary"),
showInvite: () => this.sendWidgetAction("showInvite"),
removeAllowedGroup: (group) =>
this.sendWidgetAction("removeAllowedGroup", group),

106
app/assets/javascripts/discourse/tests/acceptance/topic-summary-test.js
View File

@ -1,106 +0,0 @@
import { click, visit } from "@ember/test-helpers";
import { test } from "qunit";
import topicFixtures from "discourse/tests/fixtures/topic";
import {
acceptance,
publishToMessageBus,
updateCurrentUser,
} from "discourse/tests/helpers/qunit-helpers";
import { cloneJSON } from "discourse-common/lib/object";
acceptance("Topic - Summary", function (needs) {
const currentUserId = 5;
needs.user();
needs.pretender((server, helper) => {
server.get("/t/1.json", () => {
const json = cloneJSON(topicFixtures["/t/130.json"]);
json.id = 1;
json.summarizable = true;
return helper.response(json);
});
server.get("/t/1/strategy-summary", () => {
return helper.response({});
});
});
needs.hooks.beforeEach(() => {
updateCurrentUser({ id: currentUserId });
});
test("displays streamed summary", async function (assert) {
await visit("/t/-/1");
const partialSummary = "This a";
await publishToMessageBus("/summaries/topic/1", {
done: false,
topic_summary: { summarized_text: partialSummary },
});
await click(".topic-strategy-summarization");
assert
.dom(".summary-box .generated-summary p")
.hasText(partialSummary, "Updates the summary with a partial result");
const finalSummary = "This is a completed summary";
await publishToMessageBus("/summaries/topic/1", {
done: true,
topic_summary: {
summarized_text: finalSummary,
summarized_on: "2023-01-01T04:00:00.000Z",
algorithm: "OpenAI GPT-4",
outdated: false,
new_posts_since_summary: false,
can_regenerate: true,
},
});
assert
.dom(".summary-box .generated-summary p")
.hasText(finalSummary, "Updates the summary with a final result");
assert.dom(".summary-box .summarized-on").exists("summary metadata exists");
});
});
acceptance("Topic - Summary - Anon", function (needs) {
const finalSummary = "This is a completed summary";
needs.pretender((server, helper) => {
server.get("/t/1.json", () => {
const json = cloneJSON(topicFixtures["/t/280/1.json"]);
json.id = 1;
json.summarizable = true;
return helper.response(json);
});
server.get("/t/1/strategy-summary", () => {
return helper.response({
topic_summary: {
summarized_text: finalSummary,
summarized_on: "2023-01-01T04:00:00.000Z",
algorithm: "OpenAI GPT-4",
outdated: false,
new_posts_since_summary: false,
can_regenerate: false,
},
});
});
});
test("displays cached summary inmediately", async function (assert) {
await visit("/t/-/1");
await click(".topic-strategy-summarization");
assert
.dom(".summary-box .generated-summary p")
.hasText(finalSummary, "Updates the summary with the result");
assert.dom(".summary-box .summarized-on").exists("summary metadata exists");
});
});

4
app/assets/javascripts/discourse/tests/integration/components/widgets/post-test.js
View File

@ -853,14 +853,14 @@ module("Integration | Component | Widget | post", function (hooks) {
assert.dom(".topic-map-expanded .topic-link").exists({ count: 6 });
});
test("topic map - no summary", async function (assert) {
test("topic map - no top reply summary", async function (assert) {
const store = getOwner(this).lookup("service:store");
const topic = store.createRecord("topic", { id: 123 });
this.set("args", { topic, showTopicMap: true });
await render(hbs`<MountWidget @widget="post" @args={{this.args}} />`);
assert.dom(".toggle-summary").doesNotExist();
assert.dom(".toggle-summary .top-replies").doesNotExist();
});
test("topic map - has top replies summary", async function (assert) {

1
app/assets/stylesheets/common/base/_index.scss
View File

@ -58,7 +58,6 @@
@import "tooltip";
@import "topic-admin-menu";
@import "topic-post";
@import "topic-summary";
@import "topic";
@import "upload";
@import "user-badges";

197
app/assets/stylesheets/common/base/topic-summary.scss
View File

@ -1,197 +0,0 @@
.topic-map .toggle-summary {
.summarization-buttons {
display: flex;
gap: 0.5em;
}
.ai-summary {
&__list {
list-style: none;
display: flex;
flex-wrap: wrap;
padding: 0;
margin: 0;
}
&__list-item {
background: var(--primary-300);
border-radius: var(--d-border-radius);
margin-right: 8px;
margin-bottom: 8px;
height: 18px;
opacity: 0;
display: block;
&:nth-child(1) {
width: 10%;
}
&:nth-child(2) {
width: 12%;
}
&:nth-child(3) {
width: 18%;
}
&:nth-child(4) {
width: 14%;
}
&:nth-child(5) {
width: 18%;
}
&:nth-child(6) {
width: 14%;
}
&:nth-child(7) {
width: 22%;
}
&:nth-child(8) {
width: 05%;
}
&:nth-child(9) {
width: 25%;
}
&:nth-child(10) {
width: 14%;
}
&:nth-child(11) {
width: 18%;
}
&:nth-child(12) {
width: 12%;
}
&:nth-child(13) {
width: 22%;
}
&:nth-child(14) {
width: 18%;
}
&:nth-child(15) {
width: 13%;
}
&:nth-child(16) {
width: 22%;
}
&:nth-child(17) {
width: 19%;
}
&:nth-child(18) {
width: 13%;
}
&:nth-child(19) {
width: 22%;
}
&:nth-child(20) {
width: 25%;
}
&.is-shown {
opacity: 1;
}
&.show {
animation: appear 0.5s cubic-bezier(0.445, 0.05, 0.55, 0.95) 0s forwards;
@media (prefers-reduced-motion) {
animation-duration: 0s;
}
}
@media (prefers-reduced-motion: no-preference) {
&.blink {
animation: blink 0.5s cubic-bezier(0.55, 0.085, 0.68, 0.53) both;
}
}
}
&__generating-text {
display: inline-block;
margin-left: 3px;
}
&__indicator-wave {
flex: 0 0 auto;
display: inline-flex;
}
&__indicator-dot {
display: inline-block;
@media (prefers-reduced-motion: no-preference) {
animation: ai-summary__indicator-wave 1.8s linear infinite;
}
&:nth-child(2) {
animation-delay: -1.6s;
}
&:nth-child(3) {
animation-delay: -1.4s;
}
}
}
.placeholder-summary {
padding-top: 0.5em;
}
.placeholder-summary-text {
display: inline-block;
height: 1em;
margin-top: 0.6em;
width: 100%;
}
.summarized-on {
text-align: right;
.info-icon {
margin-left: 3px;
}
}
.outdated-summary {
color: var(--primary-medium);
}
.old-summary-box-temporary {
margin-top: 10px;
}
}
@keyframes ai-summary__indicator-wave {
0%,
60%,
100% {
transform: initial;
}
30% {
transform: translateY(-0.2em);
}
}
@keyframes appear {
0% {
opacity: 0;
}
100% {
opacity: 1;
}
}
@keyframes blink {
0% {
opacity: 1;
}
50% {
opacity: 0.5;
}
100% {
opacity: 1;
}
}

31
app/controllers/topics_controller.rb
View File

@ -1189,37 +1189,6 @@ class TopicsController < ApplicationController
head :ok
end
def summary
topic = Topic.find(params[:topic_id])
guardian.ensure_can_see!(topic)
strategy = Summarization::Base.selected_strategy
if strategy.nil? || !Summarization::Base.can_see_summary?(topic, current_user)
raise Discourse::NotFound
end
RateLimiter.new(current_user, "summary", 6, 5.minutes).performed! if current_user
opts = params.permit(:skip_age_check)
if params[:stream] && current_user
Jobs.enqueue(
:stream_topic_summary,
topic_id: topic.id,
user_id: current_user.id,
opts: opts.as_json,
)
render json: success_json
else
hijack do
summary = TopicSummarization.new(strategy).summarize(topic, current_user, opts)
render_serialized(summary, TopicSummarySerializer)
end
end
end
private
def topic_params

49
app/jobs/regular/stream_topic_summary.rb
View File

@ -1,49 +0,0 @@
# frozen_string_literal: true
module Jobs
class StreamTopicSummary < ::Jobs::Base
sidekiq_options retry: false
def execute(args)
return unless topic = Topic.find_by(id: args[:topic_id])
return unless user = User.find_by(id: args[:user_id])
strategy = Summarization::Base.selected_strategy
return if strategy.nil? || !Summarization::Base.can_see_summary?(topic, user)
guardian = Guardian.new(user)
return unless guardian.can_see?(topic)
opts = args[:opts] || {}
streamed_summary = +""
start = Time.now
summary =
TopicSummarization
.new(strategy)
.summarize(topic, user, opts) do |partial_summary|
streamed_summary << partial_summary
# Throttle updates.
if (Time.now - start > 0.5) || Rails.env.test?
payload = { done: false, topic_summary: { summarized_text: streamed_summary } }
publish_update(topic, user, payload)
start = Time.now
end
end
publish_update(
topic,
user,
TopicSummarySerializer.new(summary, { scope: guardian }).as_json.merge(done: true),
)
end
private
def publish_update(topic, user, payload)
MessageBus.publish("/summaries/topic/#{topic.id}", payload, user_ids: [user.id])
end
end
end

2
app/models/summarization_strategy.rb
View File

@ -1,5 +1,7 @@
# frozen_string_literal: true
# TODO(@keegan): Remove after removing SiteSetting.summarization_strategy
require "enum_site_setting"
class SummarizationStrategy < EnumSiteSetting

29
app/models/summary_section.rb
View File

@ -1,29 +0,0 @@
# frozen_string_literal: true
class SummarySection < ActiveRecord::Base
belongs_to :target, polymorphic: true
def mark_as_outdated
@outdated = true
end
def outdated
@outdated || false
end
end
# == Schema Information
#
# Table name: summary_sections
#
# id :bigint not null, primary key
# target_id :integer not null
# target_type :string not null
# content_range :int4range
# summarized_text :string not null
# meta_section_id :integer
# original_content_sha :string not null
# algorithm :string not null
# created_at :datetime not null
# updated_at :datetime not null
#

5
app/serializers/current_user_serializer.rb
View File

@ -28,7 +28,6 @@ class CurrentUserSerializer < BasicUserSerializer
:can_post_anonymously,
:can_ignore_users,
:can_delete_all_posts_and_topics,
:can_summarize,
:custom_fields,
:muted_category_ids,
:indirectly_muted_category_ids,
@ -155,10 +154,6 @@ class CurrentUserSerializer < BasicUserSerializer
object.in_any_groups?(SiteSetting.delete_all_posts_and_topics_allowed_groups_map)
end
def can_summarize
object.in_any_groups?(SiteSetting.custom_summarization_allowed_groups_map)
end
def can_upload_avatar
!is_anonymous && object.in_any_groups?(SiteSetting.uploaded_avatars_allowed_groups_map)
end

22
app/serializers/topic_summary_serializer.rb
View File

@ -1,22 +0,0 @@
# frozen_string_literal: true
class TopicSummarySerializer < ApplicationSerializer
attributes :summarized_text, :algorithm, :outdated, :can_regenerate, :new_posts_since_summary
def can_regenerate
Summarization::Base.can_request_summary_for?(scope.current_user)
end
def new_posts_since_summary
# Postgres uses discrete range types for int4range, which means
# (1..2) is stored as (1...3).
#
# We use Range#max to work around this, which in the case above always returns 2.
# Be careful with using Range#end here, it could lead to unexpected results as:
#
# (1..2).end => 2
# (1...3).end => 3
object.target.highest_post_number.to_i - object.content_range&.max.to_i
end
end

5
app/serializers/topic_view_serializer.rb
View File

@ -78,7 +78,6 @@ class TopicViewSerializer < ApplicationSerializer
:user_last_posted_at,
:is_shared_draft,
:slow_mode_enabled_until,
:summarizable,
)
has_one :details, serializer: TopicViewDetailsSerializer, root: false, embed: :objects
@ -311,10 +310,6 @@ class TopicViewSerializer < ApplicationSerializer
object.topic.slow_mode_topic_timer&.execute_at
end
def summarizable
object.summarizable?
end
def include_categories?
scope.can_lazy_load_categories?
end

1
app/serializers/web_hook_topic_view_serializer.rb
View File

@ -22,7 +22,6 @@ class WebHookTopicViewSerializer < TopicViewSerializer
slow_mode_seconds
slow_mode_enabled_until
bookmarks
summarizable
].each { |attr| define_method("include_#{attr}?") { false } }
def include_show_read_indicator?

129
app/services/topic_summarization.rb
View File

@ -1,129 +0,0 @@
# frozen_string_literal: true
class TopicSummarization
def initialize(strategy)
@strategy = strategy
end
def summarize(topic, user, opts = {}, &on_partial_blk)
existing_summary = SummarySection.find_by(target: topic, meta_section_id: nil)
# Existing summary shouldn't be nil in this scenario because the controller checks its existence.
return if !user && !existing_summary
targets_data = summary_targets(topic).pluck(:post_number, :raw, :username)
current_topic_sha = build_sha(targets_data.map(&:first))
can_summarize = Summarization::Base.can_request_summary_for?(user)
if use_cached?(existing_summary, can_summarize, current_topic_sha, !!opts[:skip_age_check])
# It's important that we signal a cached summary is outdated
existing_summary.mark_as_outdated if new_targets?(existing_summary, current_topic_sha)
return existing_summary
end
delete_cached_summaries_of(topic) if existing_summary
content = {
resource_path: "#{Discourse.base_path}/t/-/#{topic.id}",
content_title: topic.title,
contents: [],
}
targets_data.map do |(pn, raw, username)|
raw_text = raw
if pn == 1 && topic.topic_embed&.embed_content_cache.present?
raw_text = topic.topic_embed&.embed_content_cache
end
content[:contents] << { poster: username, id: pn, text: raw_text }
end
summarization_result = strategy.summarize(content, user, &on_partial_blk)
cache_summary(summarization_result, targets_data.map(&:first), topic)
end
def summary_targets(topic)
topic.has_summary? ? best_replies(topic) : pick_selection(topic)
end
private
attr_reader :strategy
def best_replies(topic)
Post
.summary(topic.id)
.where("post_type = ?", Post.types[:regular])
.where("NOT hidden")
.joins(:user)
.order(:post_number)
end
def pick_selection(topic)
posts =
Post
.where(topic_id: topic.id)
.where("post_type = ?", Post.types[:regular])
.where("NOT hidden")
.order(:post_number)
post_numbers = posts.limit(5).pluck(:post_number)
post_numbers += posts.reorder("posts.score desc").limit(50).pluck(:post_number)
post_numbers += posts.reorder("post_number desc").limit(5).pluck(:post_number)
Post
.where(topic_id: topic.id)
.joins(:user)
.where("post_number in (?)", post_numbers)
.order(:post_number)
end
def delete_cached_summaries_of(topic)
SummarySection.where(target: topic).destroy_all
end
# For users without permissions to generate a summary or fresh summaries, we return what we have cached.
def use_cached?(existing_summary, can_summarize, current_sha, skip_age_check)
existing_summary &&
!(
can_summarize && new_targets?(existing_summary, current_sha) &&
(skip_age_check || existing_summary.created_at < 1.hour.ago)
)
end
def new_targets?(summary, current_sha)
summary.original_content_sha != current_sha
end
def cache_summary(result, post_numbers, topic)
main_summary =
SummarySection.create!(
target: topic,
algorithm: strategy.display_name,
content_range: (post_numbers.first..post_numbers.last),
summarized_text: result[:summary],
original_content_sha: build_sha(post_numbers),
)
result[:chunks].each do |chunk|
SummarySection.create!(
target: topic,
algorithm: strategy.display_name,
content_range: chunk[:ids].min..chunk[:ids].max,
summarized_text: chunk[:summary],
original_content_sha: build_sha(chunk[:ids]),
meta_section_id: main_summary.id,
)
end
main_summary
end
def build_sha(ids)
Digest::SHA256.hexdigest(ids.join)
end
end

5
config/routes.rb
View File

@ -1358,11 +1358,6 @@ Discourse::Application.routes.draw do
topic_id: /\d+/,
}
get "t/:topic_id/summary" => "topics#show", :constraints => { topic_id: /\d+/ }
get "t/:topic_id/strategy-summary" => "topics#summary",
:constraints => {
topic_id: /\d+/,
},
:format => :json
put "t/:slug/:topic_id" => "topics#update", :constraints => { topic_id: /\d+/ }
put "t/:slug/:topic_id/status" => "topics#status", :constraints => { topic_id: /\d+/ }
put "t/:topic_id/status" => "topics#status", :constraints => { topic_id: /\d+/ }

2
config/site_settings.yml
View File

@ -2675,10 +2675,12 @@ uncategorized:
summarization_strategy:
client: true
hidden: true
default: ""
enum: "SummarizationStrategy"
validator: "SummarizationValidator"
custom_summarization_allowed_groups:
hidden: true
type: group_list
list_type: compact
default: "3|13" # 3: @staff, 13: @trust_level_3

3
lib/plugin/instance.rb
View File

@ -1182,6 +1182,9 @@ class Plugin::Instance
# to summarize content. Staff can select which strategy to use
# through the `summarization_strategy` setting.
def register_summarization_strategy(strategy)
Discourse.deprecate(
"register_summarization_straegy is deprecated. Summarization code is now moved to Discourse AI",
)
if !strategy.class.ancestors.include?(Summarization::Base)
raise ArgumentError.new("Not a valid summarization strategy")
end

4
lib/summarization/base.rb
View File

@ -1,5 +1,9 @@
# frozen_string_literal: true
# TODO(@keegan): Remove after removing SiteSetting.summarization_strategy
# Keeping because its still needed for SiteSetting to function.
# Remove after settings are migrated to AI
# Base class that defines the interface that every summarization
# strategy must implement.
# Above each method, you'll find an explanation of what

4
lib/topic_view.rb
View File

@ -742,10 +742,6 @@ class TopicView
end
end
def summarizable?
Summarization::Base.can_see_summary?(@topic, @user)
end
def categories
@categories ||= [category&.parent_category, category, suggested_topics&.categories].flatten
.uniq

2
lib/validators/summarization_validator.rb
View File

@ -1,5 +1,7 @@
# frozen_string_literal: true
# TODO(@keegan): Remove after removing SiteSetting.summarization_strategy
class SummarizationValidator
def initialize(opts = {})
@opts = opts

40
plugins/chat/app/controllers/chat/api/summaries_controller.rb
View File

@ -1,40 +0,0 @@
# frozen_string_literal: true
class Chat::Api::SummariesController < Chat::ApiController
VALID_SINCE_VALUES = [1, 3, 6, 12, 24, 72, 168]
def get_summary
since = params[:since].to_i
raise Discourse::InvalidParameters.new(:since) if !VALID_SINCE_VALUES.include?(since)
channel = Chat::Channel.find(params[:channel_id])
guardian.ensure_can_join_chat_channel!(channel)
strategy = Summarization::Base.selected_strategy
raise Discourse::NotFound.new unless strategy
raise Discourse::InvalidAccess unless Summarization::Base.can_request_summary_for?(current_user)
RateLimiter.new(current_user, "channel_summary", 6, 5.minutes).performed!
hijack do
content = { content_title: channel.name }
content[:contents] = channel
.chat_messages
.where("chat_messages.created_at > ?", since.hours.ago)
.includes(:user)
.order(created_at: :asc)
.pluck(:id, :username_lower, :message)
.map { { id: _1, poster: _2, text: _3 } }
summarized_text =
if content[:contents].empty?
I18n.t("chat.summaries.no_targets")
else
strategy.summarize(content, current_user).dig(:summary)
end
render json: { summary: summarized_text }
end
end
end

8
plugins/chat/assets/javascripts/discourse/components/chat-composer.js
View File

@ -26,7 +26,6 @@ import {
import { cloneJSON } from "discourse-common/lib/object";
import { findRawTemplate } from "discourse-common/lib/raw-templates";
import I18n from "discourse-i18n";
import ChatModalChannelSummary from "discourse/plugins/chat/discourse/components/chat/modal/channel-summary";
import { chatComposerButtons } from "discourse/plugins/chat/discourse/lib/chat-composer-buttons";
import ChatMessageInteractor from "discourse/plugins/chat/discourse/lib/chat-message-interactor";
import TextareaInteractor from "discourse/plugins/chat/discourse/lib/textarea-interactor";
@ -396,13 +395,6 @@ export default class ChatComposer extends Component {
}
}
@action
showChannelSummaryModal() {
this.modal.show(ChatModalChannelSummary, {
model: { channelId: this.args.channel.id },
});
}
#addMentionedUser(userData) {
const user = this.store.createRecord("user", userData);
this.draft.mentionedUsers.set(user.id, user);

78
plugins/chat/assets/javascripts/discourse/components/chat/modal/channel-summary.gjs
View File

@ -1,78 +0,0 @@
import Component from "@glimmer/component";
import { tracked } from "@glimmer/tracking";
import { action } from "@ember/object";
import { service } from "@ember/service";
import ConditionalLoadingSection from "discourse/components/conditional-loading-section";
import DModal from "discourse/components/d-modal";
import DModalCancel from "discourse/components/d-modal-cancel";
import { popupAjaxError } from "discourse/lib/ajax-error";
import i18n from "discourse-common/helpers/i18n";
import I18n from "discourse-i18n";
import ComboBox from "select-kit/components/combo-box";
export default class ChatModalChannelSummary extends Component {
@service chatApi;
@tracked sinceHours = null;
@tracked loading = false;
@tracked summary = null;
availableSummaries = {};
sinceOptions = [1, 3, 6, 12, 24, 72, 168].map((hours) => {
return {
name: I18n.t("chat.summarization.since", { count: hours }),
value: hours,
};
});
get channelId() {
return this.args.model.channelId;
}
@action
summarize(since) {
this.sinceHours = since;
this.loading = true;
if (this.availableSummaries[since]) {
this.summary = this.availableSummaries[since];
this.loading = false;
return;
}
return this.chatApi
.summarize(this.channelId, { since })
.then((data) => {
this.availableSummaries[this.sinceHours] = data.summary;
this.summary = this.availableSummaries[this.sinceHours];
})
.catch(popupAjaxError)
.finally(() => (this.loading = false));
}
<template>
<DModal
@closeModal={{@closeModal}}
class="chat-modal-channel-summary"
@title={{i18n "chat.summarization.title"}}
>
<:body>
<span>{{i18n "chat.summarization.description"}}</span>
<ComboBox
@value={{this.sinceHours}}
@content={{this.sinceOptions}}
@onChange={{this.summarize}}
@valueProperty="value"
class="summarization-since"
/>
<ConditionalLoadingSection @isLoading={{this.loading}}>
<p class="summary-area">{{this.summary}}</p>
</ConditionalLoadingSection>
</:body>
<:footer>
<DModalCancel @close={{@closeModal}} />
</:footer>
</DModal>
</template>
}

5
plugins/chat/assets/javascripts/discourse/components/styleguide/chat-modal-channel-summary.hbs
View File

@ -1,5 +0,0 @@
<StyleguideExample @title="<Chat::Modal::ChannelSummary>">
<Styleguide::Controls::Row>
<DButton @translatedLabel="Open modal" @action={{this.openModal}} />
</Styleguide::Controls::Row>
</StyleguideExample>

17
plugins/chat/assets/javascripts/discourse/components/styleguide/chat-modal-channel-summary.js
View File

@ -1,17 +0,0 @@
import Component from "@glimmer/component";
import { getOwner } from "@ember/application";
import { action } from "@ember/object";
import { service } from "@ember/service";
import ChatModalChannelSummary from "discourse/plugins/chat/discourse/components/chat/modal/channel-summary";
import ChatFabricators from "discourse/plugins/chat/discourse/lib/fabricators";
export default class ChatStyleguideChatModalChannelSummary extends Component {
@service modal;
@action
openModal() {
return this.modal.show(ChatModalChannelSummary, {
model: { channelId: new ChatFabricators(getOwner(this)).channel().id },
});
}
}

2
plugins/chat/assets/javascripts/discourse/components/styleguide/organisms/chat.gjs
View File

@ -3,7 +3,6 @@ import ChatComposerMessageDetails from "../chat-composer-message-details";
import ChatHeaderIcon from "../chat-header-icon";
import ChatMessage from "../chat-message";
import ChatModalArchiveChannel from "../chat-modal-archive-channel";
import ChatModalChannelSummary from "../chat-modal-channel-summary";
import ChatModalCreateChannel from "../chat-modal-create-channel";
import ChatModalDeleteChannel from "../chat-modal-delete-channel";
import ChatModalEditChannelDescription from "../chat-modal-edit-channel-description";
@ -32,7 +31,6 @@ const ChatOrganism = <template>
<ChatModalCreateChannel />
<ChatModalToggleChannelStatus />
<ChatModalNewMessage />
<ChatModalChannelSummary />
</template>;
export default ChatOrganism;

15
plugins/chat/assets/javascripts/discourse/initializers/chat-setup.js
View File

@ -97,21 +97,6 @@ export default {
},
});
const canSummarize =
this.siteSettings.summarization_strategy &&
this.currentUser &&
this.currentUser.can_summarize;
if (canSummarize) {
api.registerChatComposerButton({
translatedLabel: "chat.summarization.title",
id: "channel-summary",
icon: "discourse-sparkles",
position: "dropdown",
action: "showChannelSummaryModal",
});
}
// we want to decorate the chat quote dates regardless
// of whether the current user has chat enabled
api.decorateCookedElement((elem) => {

11
plugins/chat/assets/javascripts/discourse/services/chat-api.js
View File

@ -578,17 +578,6 @@ export default class ChatApi extends Service {
});
}
/**
* Summarize a channel.
*
* @param {number} channelId - The ID of the channel to summarize.
* @param {object} options
* @param {number} options.since - Number of hours ago the summary should start (1, 3, 6, 12, 24, 72, 168).
*/
summarize(channelId, options = {}) {
return this.#getRequest(`/channels/${channelId}/summarize`, options);
}
/**
* Add members to a channel.
*

7
plugins/chat/config/locales/client.en.yml
View File

@ -144,13 +144,6 @@ en:
join: "Join"
last_visit: "last visit"
summarization:
title: "Summarize messages"
description: "Select an option below to summarize the conversation sent during the desired timeframe."
summarize: "Summarize"
since:
one: "Last hour"
other: "Last %{count} hours"
mention_warning:
invitations_sent:
one: "Invitation sent"

3
plugins/chat/config/locales/server.en.yml
View File

@ -196,9 +196,6 @@ en:
one: "and %{count} other"
other: "and %{count} others"
summaries:
no_targets: "There were no messages during the selected period."
transcript:
default_thread_title: "Thread"
split_thread_range: "messages %{start} to %{end} of %{total}"

2
plugins/chat/config/routes.rb
View File

@ -56,8 +56,6 @@ Chat::Engine.routes.draw do
put "/channels/:channel_id/messages/:message_id/restore" => "channel_messages#restore"
delete "/channels/:channel_id/messages/:message_id" => "channel_messages#destroy"
delete "/channels/:channel_id/messages" => "channel_messages#bulk_destroy"
get "/channels/:channel_id/summarize" => "summaries#get_summary"
end
namespace :admin, defaults: { format: :json, constraints: StaffConstraint.new } do

34
plugins/chat/spec/requests/chat/api/summaries_controller_spec.rb
View File

@ -1,34 +0,0 @@
# frozen_string_literal: true
RSpec.describe Chat::Api::SummariesController do
fab!(:current_user) { Fabricate(:user) }
fab!(:group)
let(:plugin) { Plugin::Instance.new }
before do
group.add(current_user)
strategy = DummyCustomSummarization.new({ summary: "dummy", chunks: [] })
plugin.register_summarization_strategy(strategy)
SiteSetting.summarization_strategy = strategy.model
SiteSetting.custom_summarization_allowed_groups = group.id
SiteSetting.chat_enabled = true
SiteSetting.chat_allowed_groups = group.id
sign_in(current_user)
end
after { DiscoursePluginRegistry.reset_register!(:summarization_strategies) }
describe "#get_summary" do
context "when the user is not allowed to join the channel" do
fab!(:channel) { Fabricate(:private_category_channel) }
it "returns a 403" do
get "/chat/api/channels/#{channel.id}/summarize", params: { since: 6 }
expect(response.status).to eq(403)
end
end
end
end

39
plugins/chat/spec/system/chat_summarization_spec.rb
View File

@ -1,39 +0,0 @@
# frozen_string_literal: true
RSpec.describe "Summarize a channel since your last visit", type: :system do
fab!(:current_user) { Fabricate(:user) }
fab!(:group)
fab!(:channel) { Fabricate(:chat_channel) }
fab!(:message_1) { Fabricate(:chat_message, chat_channel: channel) }
let(:chat) { PageObjects::Pages::Chat.new }
let(:plugin) { Plugin::Instance.new }
let(:summarization_result) { { summary: "This is a summary", chunks: [] } }
before do
group.add(current_user)
strategy = DummyCustomSummarization.new(summarization_result)
plugin.register_summarization_strategy(strategy)
SiteSetting.summarization_strategy = strategy.model
SiteSetting.custom_summarization_allowed_groups = group.id.to_s
SiteSetting.chat_enabled = true
SiteSetting.chat_allowed_groups = group.id.to_s
sign_in(current_user)
chat_system_bootstrap(current_user, [channel])
end
it "displays a summary of the messages since the selected timeframe" do
chat.visit_channel(channel)
find(".chat-composer-dropdown__trigger-btn").click
find(".chat-composer-dropdown__action-btn.channel-summary").click
expect(page.has_css?(".chat-modal-channel-summary")).to eq(true)
find(".summarization-since").click
find(".select-kit-row[data-value=\"3\"]").click
expect(find(".summary-area").text).to eq(summarization_result[:summary])
end
end

80
spec/jobs/regular/stream_topic_summary_spec.rb
View File

@ -1,80 +0,0 @@
# frozen_string_literal: true
RSpec.describe Jobs::StreamTopicSummary do
subject(:job) { described_class.new }
describe "#execute" do
fab!(:topic) { Fabricate(:topic, highest_post_number: 2) }
fab!(:post_1) { Fabricate(:post, topic: topic, post_number: 1) }
fab!(:post_2) { Fabricate(:post, topic: topic, post_number: 2) }
let(:plugin) { Plugin::Instance.new }
let(:strategy) { DummyCustomSummarization.new({ summary: "dummy", chunks: [] }) }
fab!(:user) { Fabricate(:leader) }
before { Group.find(Group::AUTO_GROUPS[:trust_level_3]).add(user) }
before do
plugin.register_summarization_strategy(strategy)
SiteSetting.summarization_strategy = strategy.model
end
after { DiscoursePluginRegistry.reset_register!(:summarization_strategies) }
describe "validates params" do
it "does nothing if there is no topic" do
messages =
MessageBus.track_publish("/summaries/topic/#{topic.id}") do
job.execute(topic_id: nil, user_id: user.id)
end
expect(messages).to be_empty
end
it "does nothing if there is no user" do
messages =
MessageBus.track_publish("/summaries/topic/#{topic.id}") do
job.execute(topic_id: topic.id, user_id: nil)
end
expect(messages).to be_empty
end
it "does nothing if the user is not allowed to see the topic" do
private_topic = Fabricate(:private_message_topic)
messages =
MessageBus.track_publish("/summaries/topic/#{private_topic.id}") do
job.execute(topic_id: private_topic.id, user_id: user.id)
end
expect(messages).to be_empty
end
end
it "publishes updates with a partial summary" do
messages =
MessageBus.track_publish("/summaries/topic/#{topic.id}") do
job.execute(topic_id: topic.id, user_id: user.id)
end
partial_summary_update = messages.first.data
expect(partial_summary_update[:done]).to eq(false)
expect(partial_summary_update.dig(:topic_summary, :summarized_text)).to eq("dummy")
end
it "publishes a final update to signal we're done and provide metadata" do
messages =
MessageBus.track_publish("/summaries/topic/#{topic.id}") do
job.execute(topic_id: topic.id, user_id: user.id)
end
final_update = messages.last.data
expect(final_update[:done]).to eq(true)
expect(final_update.dig(:topic_summary, :algorithm)).to eq(strategy.model)
expect(final_update.dig(:topic_summary, :outdated)).to eq(false)
expect(final_update.dig(:topic_summary, :can_regenerate)).to eq(true)
expect(final_update.dig(:topic_summary, :new_posts_since_summary)).to be_zero
end
end
end

78
spec/lib/summarization/base_spec.rb
View File

@ -1,78 +0,0 @@
# frozen_string_literal: true
describe Summarization::Base do
fab!(:user)
fab!(:group)
fab!(:topic)
let(:plugin) { Plugin::Instance.new }
before do
group.add(user)
strategy = DummyCustomSummarization.new({ summary: "dummy", chunks: [] })
plugin.register_summarization_strategy(strategy)
SiteSetting.summarization_strategy = strategy.model
end
after { DiscoursePluginRegistry.reset_register!(:summarization_strategies) }
describe "#can_see_summary?" do
context "when the user cannot generate a summary" do
before { SiteSetting.custom_summarization_allowed_groups = "" }
it "returns false" do
SiteSetting.custom_summarization_allowed_groups = ""
expect(described_class.can_see_summary?(topic, user)).to eq(false)
end
it "returns true if there is a cached summary" do
SummarySection.create!(
target: topic,
summarized_text: "test",
original_content_sha: "123",
algorithm: "test",
meta_section_id: nil,
)
expect(described_class.can_see_summary?(topic, user)).to eq(true)
end
end
context "when the user can generate a summary" do
before { SiteSetting.custom_summarization_allowed_groups = group.id }
it "returns true if the user group is present in the custom_summarization_allowed_groups_map setting" do
expect(described_class.can_see_summary?(topic, user)).to eq(true)
end
end
context "when there is no user" do
it "returns false for anons" do
expect(described_class.can_see_summary?(topic, nil)).to eq(false)
end
it "returns true for anons when there is a cached summary" do
SummarySection.create!(
target: topic,
summarized_text: "test",
original_content_sha: "123",
algorithm: "test",
meta_section_id: nil,
)
expect(described_class.can_see_summary?(topic, nil)).to eq(true)
end
end
context "when the topic is a PM" do
before { SiteSetting.custom_summarization_allowed_groups = group.id }
let(:pm) { Fabricate(:private_message_topic) }
it "returns false" do
expect(described_class.can_see_summary?(pm, user)).to eq(false)
end
end
end
end

51
spec/requests/api/schemas/json/topic_show_response.json
View File

@ -7,8 +7,7 @@
"properties": {
"posts": {
"type": "array",
"items":
{
"items": {
"type": "object",
"additionalProperties": false,
"properties": {
@ -125,8 +124,7 @@
},
"link_counts": {
"type": "array",
"items":
{
"items": {
"type": "object",
"additionalProperties": false,
"properties": {
@ -169,8 +167,7 @@
},
"actions_summary": {
"type": "array",
"items":
{
"items": {
"type": "object",
"additionalProperties": false,
"properties": {
@ -291,9 +288,7 @@
},
"stream": {
"type": "array",
"items": {
}
"items": {}
}
},
"required": [
@ -303,14 +298,11 @@
},
"timeline_lookup": {
"type": "array",
"items": {
}
"items": {}
},
"suggested_topics": {
"type": "array",
"items":
{
"items": {
"type": "object",
"additionalProperties": false,
"properties": {
@ -397,15 +389,12 @@
},
"tags": {
"type": "array",
"items": {
}
"items": {}
},
"tags_descriptions": {
"type": "object",
"additionalProperties": false,
"properties": {
}
"properties": {}
},
"like_count": {
"type": "integer"
@ -424,8 +413,7 @@
},
"posters": {
"type": "array",
"items":
{
"items": {
"type": "object",
"additionalProperties": false,
"properties": {
@ -503,15 +491,12 @@
},
"tags": {
"type": "array",
"items": {
}
"items": {}
},
"tags_descriptions": {
"type": "object",
"additionalProperties": false,
"properties": {
}
"properties": {}
},
"id": {
"type": "integer"
@ -650,8 +635,7 @@
},
"actions_summary": {
"type": "array",
"items":
{
"items": {
"type": "object",
"additionalProperties": false,
"properties": {
@ -684,9 +668,7 @@
},
"bookmarks": {
"type": "array",
"items": {
}
"items": {}
},
"topic_timer": {
"type": [
@ -715,9 +697,6 @@
"null"
]
},
"summarizable": {
"type": "boolean"
},
"details": {
"type": "object",
"additionalProperties": false,
@ -784,8 +763,7 @@
},
"participants": {
"type": "array",
"items":
{
"items": {
"type": "object",
"additionalProperties": false,
"properties": {
@ -988,7 +966,6 @@
"show_read_indicator",
"thumbnails",
"slow_mode_enabled_until",
"summarizable",
"details"
]
}

122
spec/requests/topics_controller_spec.rb
View File

@ -5703,126 +5703,4 @@ RSpec.describe TopicsController do
end
end
end
describe "#summary" do
fab!(:topic) { Fabricate(:topic, highest_post_number: 2) }
fab!(:post_1) { Fabricate(:post, topic: topic, post_number: 1) }
fab!(:post_2) { Fabricate(:post, topic: topic, post_number: 2) }
let(:plugin) { Plugin::Instance.new }
let(:strategy) { DummyCustomSummarization.new({ summary: "dummy", chunks: [] }) }
before do
plugin.register_summarization_strategy(strategy)
SiteSetting.summarization_strategy = strategy.model
end
after { DiscoursePluginRegistry.reset_register!(:summarization_strategies) }
context "for anons" do
it "returns a 404 if there is no cached summary" do
get "/t/#{topic.id}/strategy-summary.json"
expect(response.status).to eq(404)
end
it "returns a cached summary" do
section =
SummarySection.create!(
target: topic,
summarized_text: "test",
algorithm: "test",
original_content_sha: "test",
)
get "/t/#{topic.id}/strategy-summary.json"
expect(response.status).to eq(200)
summary = response.parsed_body
expect(summary.dig("topic_summary", "summarized_text")).to eq(section.summarized_text)
end
end
context "when the user is a member of an allowlisted group" do
fab!(:user) { Fabricate(:leader) }
before do
sign_in(user)
Group.find(Group::AUTO_GROUPS[:trust_level_3]).add(user)
end
it "returns a 404 if there is no topic" do
invalid_topic_id = 999
get "/t/#{invalid_topic_id}/strategy-summary.json"
expect(response.status).to eq(404)
end
it "returns a 403 if not allowed to see the topic" do
pm = Fabricate(:private_message_topic)
get "/t/#{pm.id}/strategy-summary.json"
expect(response.status).to eq(403)
end
it "returns a summary" do
get "/t/#{topic.id}/strategy-summary.json"
expect(response.status).to eq(200)
summary = response.parsed_body["topic_summary"]
section = SummarySection.last
expect(summary["summarized_text"]).to eq(section.summarized_text)
expect(summary["algorithm"]).to eq(strategy.model)
expect(summary["outdated"]).to eq(false)
expect(summary["can_regenerate"]).to eq(true)
expect(summary["new_posts_since_summary"]).to be_zero
end
it "signals the summary is outdated" do
get "/t/#{topic.id}/strategy-summary.json"
Fabricate(:post, topic: topic, post_number: 3)
topic.update!(highest_post_number: 3)
get "/t/#{topic.id}/strategy-summary.json"
expect(response.status).to eq(200)
summary = response.parsed_body["topic_summary"]
expect(summary["outdated"]).to eq(true)
expect(summary["new_posts_since_summary"]).to eq(1)
end
end
context "when the user is not a member of an allowlisted group" do
fab!(:user)
before { sign_in(user) }
it "return a 404 if there is no cached summary" do
get "/t/#{topic.id}/strategy-summary.json"
expect(response.status).to eq(404)
end
it "returns a cached summary" do
section =
SummarySection.create!(
target: topic,
summarized_text: "test",
algorithm: "test",
original_content_sha: "test",
)
get "/t/#{topic.id}/strategy-summary.json"
expect(response.status).to eq(200)
summary = response.parsed_body
expect(summary.dig("topic_summary", "summarized_text")).to eq(section.summarized_text)
end
end
end
end

220
spec/services/topic_summarization_spec.rb
View File

@ -1,220 +0,0 @@
# frozen_string_literal: true
describe TopicSummarization do
fab!(:user) { Fabricate(:admin) }
fab!(:topic) { Fabricate(:topic, highest_post_number: 2) }
fab!(:post_1) { Fabricate(:post, topic: topic, post_number: 1) }
fab!(:post_2) { Fabricate(:post, topic: topic, post_number: 2) }
shared_examples "includes only public-visible topics" do
subject { described_class.new(DummyCustomSummarization.new({})) }
it "only includes visible posts" do
topic.first_post.update!(hidden: true)
posts = subject.summary_targets(topic)
expect(posts.none?(&:hidden?)).to eq(true)
end
it "doesn't include posts without users" do
topic.first_post.user.destroy!
posts = subject.summary_targets(topic)
expect(posts.detect { |p| p.id == topic.first_post.id }).to be_nil
end
it "doesn't include deleted posts" do
topic.first_post.update!(user_id: nil)
posts = subject.summary_targets(topic)
expect(posts.detect { |p| p.id == topic.first_post.id }).to be_nil
end
end
describe "#summary_targets" do
context "when the topic has a best replies summary" do
before { topic.has_summary = true }
it_behaves_like "includes only public-visible topics"
end
context "when the topic doesn't have a best replies summary" do
before { topic.has_summary = false }
it_behaves_like "includes only public-visible topics"
end
end
describe "#summarize" do
subject(:summarization) { described_class.new(strategy) }
let(:strategy) { DummyCustomSummarization.new(summary) }
def assert_summary_is_cached(topic, summary_response)
cached_summary = SummarySection.find_by(target: topic, meta_section_id: nil)
expect(cached_summary.content_range).to cover(*topic.posts.map(&:post_number))
expect(cached_summary.summarized_text).to eq(summary_response[:summary])
expect(cached_summary.original_content_sha).to be_present
expect(cached_summary.algorithm).to eq(strategy.model)
end
def assert_chunk_is_cached(topic, chunk_response)
cached_chunk =
SummarySection
.where.not(meta_section_id: nil)
.find_by(
target: topic,
content_range: (chunk_response[:ids].min..chunk_response[:ids].max),
)
expect(cached_chunk.summarized_text).to eq(chunk_response[:summary])
expect(cached_chunk.original_content_sha).to be_present
expect(cached_chunk.algorithm).to eq(strategy.model)
end
context "when the content was summarized in a single chunk" do
let(:summary) { { summary: "This is the final summary", chunks: [] } }
it "caches the summary" do
section = summarization.summarize(topic, user)
expect(section.summarized_text).to eq(summary[:summary])
assert_summary_is_cached(topic, summary)
end
it "returns the cached version in subsequent calls" do
summarization.summarize(topic, user)
cached_summary_text = "This is a cached summary"
cached_summary =
SummarySection.find_by(target: topic, meta_section_id: nil).update!(
summarized_text: cached_summary_text,
updated_at: 24.hours.ago,
)
section = summarization.summarize(topic, user)
expect(section.summarized_text).to eq(cached_summary_text)
end
context "when the topic has embed content cached" do
it "embed content is used instead of the raw text" do
topic_embed =
Fabricate(
:topic_embed,
topic: topic,
embed_content_cache: "<p>hello world new post :D</p>",
)
summarization.summarize(topic, user)
first_post_data =
strategy.content[:contents].detect { |c| c[:id] == topic.first_post.post_number }
expect(first_post_data[:text]).to eq(topic_embed.embed_content_cache)
end
end
end
context "when the content was summarized in multiple chunks" do
let(:summary) do
{
summary: "This is the final summary",
chunks: [
{ ids: [topic.first_post.post_number], summary: "this is the first chunk" },
{ ids: [post_1.post_number, post_2.post_number], summary: "this is the second chunk" },
],
}
end
it "caches the summary and each chunk" do
section = summarization.summarize(topic, user)
expect(section.summarized_text).to eq(summary[:summary])
assert_summary_is_cached(topic, summary)
summary[:chunks].each { |c| assert_chunk_is_cached(topic, c) }
end
end
describe "invalidating cached summaries" do
let(:cached_text) { "This is a cached summary" }
let(:summarized_text) { "This is the final summary" }
let(:summary) do
{
summary: summarized_text,
chunks: [
{ ids: [topic.first_post.post_number], summary: "this is the first chunk" },
{ ids: [post_1.post_number, post_2.post_number], summary: "this is the second chunk" },
],
}
end
def cached_summary
SummarySection.find_by(target: topic, meta_section_id: nil)
end
before do
summarization.summarize(topic, user)
cached_summary.update!(summarized_text: cached_text, created_at: 24.hours.ago)
end
context "when the user can requests new summaries" do
context "when there are no new posts" do
it "returns the cached summary" do
section = summarization.summarize(topic, user)
expect(section.summarized_text).to eq(cached_text)
end
end
context "when there are new posts" do
before { cached_summary.update!(original_content_sha: "outdated_sha") }
it "returns a new summary" do
section = summarization.summarize(topic, user)
expect(section.summarized_text).to eq(summarized_text)
end
context "when the cached summary is less than one hour old" do
before { cached_summary.update!(created_at: 30.minutes.ago) }
it "returns the cached summary" do
cached_summary.update!(created_at: 30.minutes.ago)
section = summarization.summarize(topic, user)
expect(section.summarized_text).to eq(cached_text)
expect(section.outdated).to eq(true)
end
it "returns a new summary if the skip_age_check flag is passed" do
section = summarization.summarize(topic, user, skip_age_check: true)
expect(section.summarized_text).to eq(summarized_text)
end
end
end
end
end
describe "stream partial updates" do
let(:summary) { { summary: "This is the final summary", chunks: [] } }
it "receives a blk that is passed to the underlying strategy and called with partial summaries" do
partial_result = nil
summarization.summarize(topic, user) { |partial_summary| partial_result = partial_summary }
expect(partial_result).to eq(summary[:summary])
end
end
end
end

30
spec/support/dummy_custom_summarization.rb
View File

@ -1,30 +0,0 @@
# frozen_string_literal: true
class DummyCustomSummarization < Summarization::Base
def initialize(summarization_result)
@summarization_result = summarization_result
end
def display_name
"dummy"
end
def correctly_configured?
true
end
def configuration_hint
"hint"
end
def model
"dummy"
end
def summarize(content, _user)
@content = content
@summarization_result.tap { |result| yield(result[:summary]) if block_given? }
end
attr_reader :content
end