Kimirock/zammad
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fixes #5839 - Ticket Summary - Improve UX when an unexpected structure is returned by the LLM.

Browse Source 
Co-authored-by: Martin Gruner <mg@zammad.com>
Co-authored-by: Dusan Vuckovic <dv@zammad.com>
This commit is contained in:
Dusan Vuckovic
2025-11-18 14:29:40 +00:00
parent e3cdc9b6e6
commit 2b6824cf40
9 changed files with 151 additions and 128 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
app/assets/javascripts/app/controllers/ticket_zoom/sidebar_ticket_summary.coffee
View File

@@ -172,42 +172,30 @@ class App.SidebarTicketSummary extends App.Controller
return if !@elSidebar
invalidSummary = @invalidSummary()
summarization = $(App.view('ticket_zoom/sidebar_ticket_summary')(
data: @summaryData
invalidSummary: invalidSummary
structure: @getAvailableDisplayStructure()
data: @summaryData
structure: @getAvailableDisplayStructure()
))
summarization
.on('click', '.js-retry', @retrySummarization)
@stopStripeAnimation() if not @isSummarizationLoading(invalidSummary)
@stopStripeAnimation() if not @isSummarizationLoading()
if not invalidSummary
@feedbackWidget = new App.AIFeedbackWidget(
el: summarization.find('.js-aiFeedback')
runId: @summaryData?.analytics?.run_id
hasProvidedFeedback: @summaryData?.analytics?.usage?.user_has_provided_feedback
regenerateCallback: @loadSummarization
)
@feedbackWidget = new App.AIFeedbackWidget(
el: summarization.find('.js-aiFeedback')
runId: @summaryData?.analytics?.run_id
hasProvidedFeedback: @summaryData?.analytics?.usage?.user_has_provided_feedback
regenerateCallback: @loadSummarization
)
@hasUsage = not _.isNull(@summaryData?.analytics?.usage)
@markAsRead() if @sidebarItem()?.name is @parentSidebar.currentTab and not @hasUsage
@hasUsage = not _.isNull(@summaryData?.analytics?.usage)
@markAsRead() if @sidebarItem()?.name is @parentSidebar.currentTab and not @hasUsage
@elSidebar.html(summarization)
invalidSummary: =>
# In case the summary result does not follow expected structure (at least in some part), we consider it as invalid.
@summaryData?.result and not _.some(@getAvailableDisplayStructure(), (item) =>
key = item.value
return _.some(key, (k) => @summaryData?.result[k]?) if _.isArray(key)
@summaryData?.result[key]?
)
isSummarizationLoading: (invalidSummary) =>
not @summaryData?.error and App.Config.get('ai_provider') and not invalidSummary and not @summaryData?.result
isSummarizationLoading: =>
not @summaryData?.error and App.Config.get('ai_provider') and not @summaryData?.result
retrySummarization: (e) =>
@preventDefaultAndStopPropagation(e)

4
app/assets/javascripts/app/views/ticket_zoom/sidebar_ticket_summary.jst.eco
View File

@@ -13,10 +13,6 @@
<div class="alert alert--danger">
<%- @T('No AI provider is currently set up. Please contact your administrator.') %>
</div>
<% else if @invalidSummary: %>
<div class="alert alert--info">
<%- @T('There is not enough content yet to summarize this ticket.') %>
</div>
<% else if @data?.result: %>
<% for item in @structure: %>
<% values = [] %>

23
app/frontend/apps/desktop/pages/ticket/components/TicketSidebar/TicketSidebarSummary/TicketSidebarSummaryContent.vue
View File

@@ -43,27 +43,11 @@ const hasProvidedFeedback = computed(() => !!props.analyticsMeta?.usage?.userHas
const hasRecentlyRated = shallowRef(false)
const noSummaryPossible = computed(() => {
const { summary } = props
if (!summary) return false
return props.summaryHeadings.every((section) => {
if (Array.isArray(section.key)) return section.key.every((key) => !summary[key]?.length)
return !summary[section.key]?.length
})
})
const titleClass = computed(() => {
let titleClass =
'ai-stripe before:-bottom-3 before:absolute relative before:left:0 before:right-0'
if (
props.isProviderConfigured &&
!errorMessage.value &&
!noSummaryPossible.value &&
!props.summary
)
if (props.isProviderConfigured && !errorMessage.value && !props.summary)
titleClass += ' animate-ai-stripe'
return titleClass
@@ -109,11 +93,6 @@ const titleClass = computed(() => {
}}</CommonButton>
</div>
</template>
<template v-else-if="noSummaryPossible">
<CommonAlert variant="info">
{{ $t('There is not enough content yet to summarize this ticket.') }}
</CommonAlert>
</template>
<template v-else-if="summary">
<template v-for="item in summaryHeadings" :key="item.key">
<article

33
app/frontend/apps/desktop/pages/ticket/components/TicketSidebar/TicketSidebarSummary/__tests__/TicketSidebarSummary.spec.ts
View File

@@ -9,7 +9,6 @@ import { mockRouterHooks } from '#tests/support/mock-vue-router.ts'
import type { TicketById } from '#shared/entities/ticket/types.ts'
import { createDummyArticle } from '#shared/entities/ticket-article/__tests__/mocks/ticket-articles.ts'
import { createDummyTicket } from '#shared/entities/ticket-article/__tests__/mocks/ticket.ts'
import { EnumTicketSummaryGeneration } from '#shared/graphql/types.ts'
import { convertToGraphQLId } from '#shared/graphql/utils.ts'
import { pluginFiles } from '#desktop/pages/ticket/components/TicketSidebar/plugins/index.ts'
@@ -274,38 +273,6 @@ describe('TicketSidebarSummary', () => {
expect(await wrapper.findByText('Any feedback on this result?')).toBeInTheDocument()
})
it('shows info that summary is too short to be generated', async () => {
mockTicketAiAssistanceSummarizeMutation({
ticketAIAssistanceSummarize: {
summary: {
customerRequest: null,
conversationSummary: null,
openQuestions: null,
upcomingEvents: null,
customerMood: null,
customerEmotion: null,
},
},
})
mockApplicationConfig({
ai_provider: true,
ai_assistance_ticket_summary: true,
ai_assistance_ticket_summary_config: {
open_questions: true,
upcoming_events: true,
customer_sentiment: true,
generate_on: EnumTicketSummaryGeneration.OnTicketDetailOpening,
},
})
const wrapper = renderRenderTicketSidebarSummary()
expect(
await wrapper.findByText('There is not enough content yet to summarize this ticket.'),
).toBeInTheDocument()
})
it('shows skeleton loader when summary is not ready', async () => {
mockTicketAiAssistanceSummarizeMutation({
ticketAIAssistanceSummarize: {

31
i18n/zammad.pot
View File

@@ -726,6 +726,10 @@ msgstr ""
msgid "AI provider token is not set"
msgstr ""
#: lib/ai/service/ticket_summarize.rb:59
msgid "AI service result is missing expected keys"
msgstr ""
#: lib/excel_sheet/ai/analytics/with_usage.rb:4
msgid "AI usage and feedback log"
msgstr ""
@@ -769,7 +773,7 @@ msgstr ""
#: lib/ai/provider/mistral.rb:95
#: lib/ai/provider/ollama.rb:89
#: lib/ai/provider/open_ai.rb:100
#: lib/ai/provider/zammad_ai.rb:62
#: lib/ai/provider/zammad_ai.rb:70
msgid "API server not accessible"
msgstr ""
@@ -2403,7 +2407,7 @@ msgid "Basic Settings"
msgstr ""
#: app/assets/javascripts/app/views/ai/ai_feedback_widget.jst.eco:2
#: app/frontend/apps/desktop/pages/ticket/components/TicketSidebar/TicketSidebarSummary/TicketSidebarSummaryContent.vue:141
#: app/frontend/apps/desktop/pages/ticket/components/TicketSidebar/TicketSidebarSummary/TicketSidebarSummaryContent.vue:120
msgid "Be sure to check AI-generated content for accuracy."
msgstr ""
@@ -10602,7 +10606,7 @@ msgstr ""
#: app/assets/javascripts/app/controllers/_ai/ai_agent.coffee:90
#: app/assets/javascripts/app/controllers/_channel/whatsapp.coffee:112
#: app/assets/javascripts/app/controllers/webhook.coffee:162
#: app/assets/javascripts/app/controllers/webhook.coffee:165
#: app/assets/javascripts/app/controllers/widget/two_factor_configuration/modal/password_check.coffee:2
#: app/assets/javascripts/app/controllers/widget/two_factor_configuration/modal/security_keys.coffee:120
#: app/assets/javascripts/app/views/clues/clues_content.jst.eco:19
@@ -10652,7 +10656,7 @@ msgid "No AI agent available, please create a new one or activate an existing on
msgstr ""
#: app/assets/javascripts/app/views/ticket_zoom/sidebar_ticket_summary.jst.eco:14
#: app/frontend/apps/desktop/pages/ticket/components/TicketSidebar/TicketSidebarSummary/TicketSidebarSummaryContent.vue:86
#: app/frontend/apps/desktop/pages/ticket/components/TicketSidebar/TicketSidebarSummary/TicketSidebarSummaryContent.vue:70
msgid "No AI provider is currently set up. Please contact your administrator."
msgstr ""
@@ -12179,7 +12183,7 @@ msgid "Please enter a text."
msgstr ""
#: app/assets/javascripts/app/controllers/_ai/ai_agent.coffee:409
#: app/assets/javascripts/app/controllers/webhook.coffee:131
#: app/assets/javascripts/app/controllers/webhook.coffee:134
msgid "Please enter a valid JSON string."
msgstr ""
@@ -12339,7 +12343,7 @@ msgstr ""
msgid "Pre-defined Webhook"
msgstr ""
#: app/assets/javascripts/app/controllers/webhook.coffee:286
#: app/assets/javascripts/app/controllers/webhook.coffee:291
msgid "Pre-defined webhook for %s."
msgstr ""
@@ -13223,7 +13227,7 @@ msgstr ""
#: app/frontend/apps/desktop/components/TwoFactor/TwoFactorConfiguration/TwoFactorConfigurationSecurityKeys.vue:64
#: app/frontend/apps/desktop/pages/authentication/components/LoginTwoFactor.vue:169
#: app/frontend/apps/desktop/pages/authentication/views/AdminPasswordAuth.vue:131
#: app/frontend/apps/desktop/pages/ticket/components/TicketSidebar/TicketSidebarSummary/TicketSidebarSummaryContent.vue:108
#: app/frontend/apps/desktop/pages/ticket/components/TicketSidebar/TicketSidebarSummary/TicketSidebarSummaryContent.vue:92
#: app/frontend/apps/mobile/pages/authentication/components/LoginTwoFactor.vue:172
msgid "Retry"
msgstr ""
@@ -15124,8 +15128,8 @@ msgstr ""
msgid "Summary Services"
msgstr ""
#: app/assets/javascripts/app/views/ticket_zoom/sidebar_ticket_summary.jst.eco:50
#: app/frontend/apps/desktop/pages/ticket/components/TicketSidebar/TicketSidebarSummary/TicketSidebarSummaryContent.vue:160
#: app/assets/javascripts/app/views/ticket_zoom/sidebar_ticket_summary.jst.eco:46
#: app/frontend/apps/desktop/pages/ticket/components/TicketSidebar/TicketSidebarSummary/TicketSidebarSummaryContent.vue:139
msgid "Summary is being generated…"
msgstr ""
@@ -16452,7 +16456,7 @@ msgid "The structure is a JSON object that defines the result format to be retur
msgstr ""
#: app/assets/javascripts/app/views/ticket_zoom/sidebar_ticket_summary.jst.eco:5
#: app/frontend/apps/desktop/pages/ticket/components/TicketSidebar/TicketSidebarSummary/TicketSidebarSummaryContent.vue:98
#: app/frontend/apps/desktop/pages/ticket/components/TicketSidebar/TicketSidebarSummary/TicketSidebarSummaryContent.vue:82
msgid "The summary could not be generated. Please try again later or contact your administrator."
msgstr ""
@@ -16707,11 +16711,6 @@ msgstr ""
msgid "There is no valid PGP email signer."
msgstr ""
#: app/assets/javascripts/app/views/ticket_zoom/sidebar_ticket_summary.jst.eco:18
#: app/frontend/apps/desktop/pages/ticket/components/TicketSidebar/TicketSidebarSummary/TicketSidebarSummaryContent.vue:114
msgid "There is not enough content yet to summarize this ticket."
msgstr ""
#: lib/secure_mailing/pgp/tool/error.rb:160
msgid "There was an unknown PGP error."
msgstr ""
@@ -18050,7 +18049,7 @@ msgstr ""
msgid "Turn chat into ticket"
msgstr ""
#: app/assets/javascripts/app/controllers/webhook.coffee:275
#: app/assets/javascripts/app/controllers/webhook.coffee:280
#: app/assets/javascripts/app/views/channel/email_account_wizard.jst.eco:89
#: app/assets/javascripts/app/views/channel/email_notification_wizard.jst.eco:48
#: app/assets/javascripts/app/views/getting_started/email.jst.eco:71

8
lib/ai/service.rb
View File

@@ -68,7 +68,11 @@ class AI::Service
end
def request_fresh
response = post_transform_result(ask_provider)
result = ask_provider
validate_result!(result)
response = post_transform_result(result)
if response.nil?
save_analytics_run if analytics?
@@ -166,6 +170,8 @@ class AI::Service
self.class.lookup_version(context_data, locale)
end
def validate_result!(_result); end
def post_transform_result(result)
result
end

17
lib/ai/service/ticket_summarize.rb
View File

@@ -24,6 +24,17 @@ class AI::Service::TicketSummarize < AI::Service
true
end
def validate_result!(result)
raise InvalidResultKeysError if !result.key?('language')
optional_keys = %w[
customer_request
conversation_summary
]
raise InvalidResultKeysError if !optional_keys.intersect?(result.keys)
end
private
def options
@@ -31,4 +42,10 @@ class AI::Service::TicketSummarize < AI::Service
temperature: 0.1,
}
end
class InvalidResultKeysError < StandardError
def initialize
super(__('AI service result is missing expected keys'))
end
end
end

68
spec/lib/ai/service/ticket_summarize_spec.rb Normal file
View File

@@ -0,0 +1,68 @@
# Copyright (C) 2012-2025 Zammad Foundation, https://zammad-foundation.org/
require 'rails_helper'
RSpec.describe AI::Service::TicketSummarize do
subject(:ai_service) { described_class.new(current_user:, context_data:) }
let(:group) { create(:group) }
let(:current_user) { create(:user, groups: [group]) }
let(:ticket) { create(:ticket, group:) }
let(:article) { create(:ticket_article, ticket:) }
let(:context_data) do
{
ticket:,
}
end
let(:llm_response) do
{
'customer_request' => 'Customer requested help with an issue.',
'conversation_summary' => 'This is a summary of the conversation.',
'language' => 'en-us',
}
end
before do
setup_ai_provider('zammad_ai')
allow_any_instance_of(AI::Service::EmailRemoveQuote)
.to receive(:ask_provider).and_return(article.body_as_text)
allow_any_instance_of(AI::Provider::ZammadAI)
.to receive(:ask).and_return(llm_response)
end
it 'passes through received ticket summary' do
result = ai_service.execute
expect(result.content).to include(llm_response)
end
context 'when result is missing a required key' do
let(:llm_response) do
{
'summary' => 'This is a summary of the conversation.',
}
end
it 'raises an error' do
expect { ai_service.execute }
.to raise_error(AI::Service::TicketSummarize::InvalidResultKeysError)
end
end
context 'when result is missing all of the optional keys' do
let(:llm_response) do
{
'language' => 'en-us',
'summary' => 'This is a summary of the conversation.',
}
end
it 'raises an error' do
expect { ai_service.execute }
.to raise_error(AI::Service::TicketSummarize::InvalidResultKeysError)
end
end
end

57
spec/system/ticket/summary_spec.rb
View File

@@ -16,12 +16,13 @@ RSpec.describe 'Ticket Summary', authenticated_as: :authenticate, type: :system
let(:initial_content) do
{
'customer_request' => '',
'customer_request' => article.body_as_text,
'conversation_summary' => initial_summary,
'open_questions' => [],
'upcoming_events' => [],
'customer_mood' => 'Neutral',
'customer_emotion' => '😐',
'language' => 'en-us',
}
end
@@ -33,6 +34,7 @@ RSpec.describe 'Ticket Summary', authenticated_as: :authenticate, type: :system
'upcoming_events' => ['Next meeting on Friday', 'Follow-up call next week'],
'customer_mood' => 'Happy',
'customer_emotion' => '🙂',
'language' => 'en-us',
}.compact
end
@@ -91,8 +93,8 @@ RSpec.describe 'Ticket Summary', authenticated_as: :authenticate, type: :system
it 'displays and updates summary in sidebar', performs_jobs: true do
within '.sidebar[data-tab="summary"]' do
expect(page).to have_text 'Conversation Summary'
expect(page).to have_text initial_summary
expect(page).to have_text('Conversation Summary')
.and have_text(initial_summary)
end
create(:ticket_article, ticket:)
@@ -103,14 +105,15 @@ RSpec.describe 'Ticket Summary', authenticated_as: :authenticate, type: :system
perform_enqueued_jobs(only: TicketAIAssistanceSummarizeJob)
within '.sidebar[data-tab="summary"]' do
expect(page).to have_text updated_summary
expect(page).to have_text('Conversation Summary')
.and have_text(updated_summary)
end
end
it 'shows customer intent in the summary sidebar', performs_jobs: true do
within '.sidebar[data-tab="summary"]' do
expect(page).to have_no_text 'Customer Intent'
expect(page).to have_no_text 'Customer is facing an issue with the product.'
expect(page).to have_text('Customer Intent')
.and have_text(article.body_as_text)
end
create(:ticket_article, ticket:)
@@ -120,16 +123,16 @@ RSpec.describe 'Ticket Summary', authenticated_as: :authenticate, type: :system
perform_enqueued_jobs(only: TicketAIAssistanceSummarizeJob)
within '.sidebar[data-tab="summary"]' do
expect(page).to have_text 'Customer Intent'
expect(page).to have_text 'Customer is facing an issue with the product.'
expect(page).to have_text('Customer Intent')
.and have_text('Customer is facing an issue with the product.')
end
end
it 'shows open questions in the summary sidebar', performs_jobs: true do
within '.sidebar[data-tab="summary"]' do
expect(page).to have_no_text 'Open Questions'
expect(page).to have_no_text 'What is the issue?'
expect(page).to have_no_text 'How can we help?'
expect(page).to have_no_text('Open Questions')
.and have_no_text('What is the issue?')
.and have_no_text('How can we help?')
end
create(:ticket_article, ticket:)
@@ -139,17 +142,17 @@ RSpec.describe 'Ticket Summary', authenticated_as: :authenticate, type: :system
perform_enqueued_jobs(only: TicketAIAssistanceSummarizeJob)
within '.sidebar[data-tab="summary"]' do
expect(page).to have_text 'Open Questions'
expect(page).to have_text 'What is the issue?'
expect(page).to have_text 'How can we help?'
expect(page).to have_text('Open Questions')
.and have_text('What is the issue?')
.and have_text('How can we help?')
end
end
it 'shows upcoming events in the summary sidebar', performs_jobs: true do
within '.sidebar[data-tab="summary"]' do
expect(page).to have_no_text 'Upcoming Events'
expect(page).to have_no_text 'Next meeting on Friday'
expect(page).to have_no_text 'Follow-up call next week'
expect(page).to have_no_text('Upcoming Events')
.and have_no_text('Next meeting on Friday')
.and have_no_text('Follow-up call next week')
end
create(:ticket_article, ticket:)
@@ -159,17 +162,17 @@ RSpec.describe 'Ticket Summary', authenticated_as: :authenticate, type: :system
perform_enqueued_jobs(only: TicketAIAssistanceSummarizeJob)
within '.sidebar[data-tab="summary"]' do
expect(page).to have_text 'Upcoming Events'
expect(page).to have_text 'Next meeting on Friday'
expect(page).to have_text 'Follow-up call next week'
expect(page).to have_text('Upcoming Events')
.and have_text('Next meeting on Friday')
.and have_text('Follow-up call next week')
end
end
it 'shows customer sentiment and emotion in the summary sidebar', performs_jobs: true do
within '.sidebar[data-tab="summary"]' do
expect(page).to have_text 'Customer Sentiment'
expect(page).to have_text 'Neutral'
expect(page).to have_text '😐'
expect(page).to have_text('Customer Sentiment')
.and have_text('Neutral')
.and have_text('😐')
end
create(:ticket_article, ticket:)
@@ -179,9 +182,9 @@ RSpec.describe 'Ticket Summary', authenticated_as: :authenticate, type: :system
perform_enqueued_jobs(only: TicketAIAssistanceSummarizeJob)
within '.sidebar[data-tab="summary"]' do
expect(page).to have_text 'Customer Sentiment'
expect(page).to have_text 'Happy'
expect(page).to have_text '🙂'
expect(page).to have_text('Customer Sentiment')
.and have_text('Happy')
.and have_text('🙂')
end
end
@@ -192,7 +195,7 @@ RSpec.describe 'Ticket Summary', authenticated_as: :authenticate, type: :system
it 'does not show sidebar' do
expect(page).to have_text(ticket.title)
expect(page).to have_no_css('.tabsSidebar-tab[data-tab=summary]')
.and have_no_css('.tabsSidebar-tab[data-tab=summary]')
end
end
end