mirror of
https://github.com/vernu/textbee.git
synced 2026-04-20 23:11:35 -04:00
Merge pull request #191 from vernu/webhooks-improvements
auto-disable webhook subscriptions with high failure rate and increase delivery request timeout
This commit is contained in:
Israel Abebe
@@ -24,6 +24,14 @@ MAIL_USER=
|
||||
MAIL_PASS=
|
||||
MAIL_FROM=
|
||||
MAIL_REPLY_TO=
|
||||
ADMIN_EMAIL=
|
||||
|
||||
# Webhook delivery HTTP timeout in milliseconds (default 30000, min 10000, max 60000)
|
||||
WEBHOOK_DELIVERY_TIMEOUT_MS=30000
|
||||
|
||||
# Auto-disable webhook subscriptions with high failure rate (cron runs daily)
|
||||
WEBHOOK_AUTO_DISABLE_FAILURE_THRESHOLD=50
|
||||
WEBHOOK_AUTO_DISABLE_LOOKBACK_DAYS=30
|
||||
|
||||
# SMS Queue Configuration
|
||||
USE_SMS_QUEUE=false
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
<html>
|
||||
<head>
|
||||
<meta charset='utf-8' />
|
||||
<meta name='viewport' content='width=device-width, initial-scale=1.0' />
|
||||
<title>{{title}} – {{runAt}}</title>
|
||||
<style>
|
||||
body { font-family: Arial, sans-serif; line-height: 1.5; color: #333; margin: 0; padding: 0; }
|
||||
.container { max-width: 900px; margin: 0 auto; padding: 20px; }
|
||||
.header { padding: 16px 0; border-bottom: 1px solid #eee; }
|
||||
.title { font-size: 18px; font-weight: bold; }
|
||||
.meta { font-size: 12px; color: #666; margin-top: 4px; }
|
||||
table { width: 100%; border-collapse: collapse; margin: 16px 0; font-size: 13px; }
|
||||
th, td { padding: 10px; text-align: left; border-bottom: 1px solid #ddd; }
|
||||
th { background-color: #f5f5f5; font-weight: 600; }
|
||||
.url { word-break: break-all; max-width: 240px; }
|
||||
.footer { font-size: 12px; color: #777; margin-top: 24px; padding-top: 16px; border-top: 1px solid #eee; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class='container'>
|
||||
<div class='header'>
|
||||
<div class='title'>{{title}}</div>
|
||||
<div class='meta'>Run at {{runAt}} · {{count}} subscription(s) auto-disabled</div>
|
||||
</div>
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Subscription ID</th>
|
||||
<th>Delivery URL</th>
|
||||
<th>Failed count</th>
|
||||
<th>Period (days)</th>
|
||||
<th>User name</th>
|
||||
<th>User email</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{{#each disabledList}}
|
||||
<tr>
|
||||
<td>{{this.subscriptionId}}</td>
|
||||
<td class='url'>{{this.deliveryUrl}}</td>
|
||||
<td>{{this.failureCount}}</td>
|
||||
<td>{{this.lookbackDays}}</td>
|
||||
<td>{{this.userName}}</td>
|
||||
<td>{{this.userEmail}}</td>
|
||||
</tr>
|
||||
{{/each}}
|
||||
</tbody>
|
||||
</table>
|
||||
<div class='footer'>{{brandName}} – Webhook auto-disable cron</div>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
86
api/src/mail/templates/webhook-subscription-disabled.hbs
Normal file
86
api/src/mail/templates/webhook-subscription-disabled.hbs
Normal file
@@ -0,0 +1,86 @@
|
||||
<html lang='en' xmlns:v='urn:schemas-microsoft-com:vml' xmlns:o='urn:schemas-microsoft-com:office:office'>
|
||||
<head>
|
||||
<meta charset='utf-8' />
|
||||
<meta name='viewport' content='width=device-width, initial-scale=1' />
|
||||
<meta http-equiv='x-ua-compatible' content='ie=edge' />
|
||||
<title>{{title}} • {{brandName}}</title>
|
||||
<style>
|
||||
.preheader { display:none !important; visibility:hidden; opacity:0; color:transparent; height:0; width:0; overflow:hidden; mso-hide:all; }
|
||||
@media screen and (max-width: 600px) { .container { width:100% !important; } .stack { display:block !important; width:100% !important; } .p-sm { padding:16px !important; } .text-center-sm { text-align:center !important; } .hide-sm { display:none !important; } }
|
||||
</style>
|
||||
<!--[if mso]>
|
||||
<style type="text/css"> body, table, td, a { font-family: Helvetica, Arial, sans-serif !important; } </style>
|
||||
<![endif]-->
|
||||
</head>
|
||||
<body style='margin:0; padding:0; background:#f7f9fc;'>
|
||||
<div class='preheader'>Webhook subscription disabled – {{brandName}}</div>
|
||||
<table role='presentation' cellpadding='0' cellspacing='0' border='0' width='100%' class='email-bg' style='background:#f7f9fc;'>
|
||||
<tr>
|
||||
<td align='center' style='padding:24px;'>
|
||||
<table role='presentation' cellpadding='0' cellspacing='0' border='0' width='600' class='container' style='width:600px; max-width:600px;'>
|
||||
<tr>
|
||||
<td style='padding:12px 16px 0 16px;'>
|
||||
<table role='presentation' width='100%' cellspacing='0' cellpadding='0' border='0'>
|
||||
<tr>
|
||||
<td class='stack' valign='middle' style='padding:8px 0;'>
|
||||
<table role='presentation' cellspacing='0' cellpadding='0' border='0'>
|
||||
<tr>
|
||||
<td valign='middle' style='padding-right:10px;'>
|
||||
<img src='https://textbee.dev/images/logo.png' alt='{{brandName}}' width='36' height='36' style='display:block; border:0; outline:none; text-decoration:none;' />
|
||||
</td>
|
||||
<td valign='middle' style='font:600 18px Arial, Helvetica, sans-serif; color:#EA580C;'>{{brandName}}</td>
|
||||
</tr>
|
||||
</table>
|
||||
</td>
|
||||
<td class='stack text-center-sm' valign='middle' align='right' style='padding:8px 0;'>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td align='center' style='padding:16px 16px 0 16px;'>
|
||||
<table role='presentation' width='100%' cellspacing='0' cellpadding='0' border='0' class='card' style='background:#ffffff; border-radius:10px;'>
|
||||
<tr>
|
||||
<td align='center' style='background:#F97316; border-radius:10px 10px 0 0; padding:28px 20px;'>
|
||||
<div style='font:700 24px Arial, Helvetica, sans-serif; color:#ffffff;'>{{title}}</div>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td class='p-sm' style='padding:28px; font:16px/1.6 Arial, Helvetica, sans-serif; color:#111827;'>
|
||||
<div style='font:700 18px Arial, Helvetica, sans-serif; color:#111827; margin-bottom:8px;'>Hi {{name}},</div>
|
||||
<p style='margin:0 0 16px 0;'>{{message}}</p>
|
||||
<div style='text-align:center; padding:8px 0 2px;'>
|
||||
<!--[if mso]>
|
||||
<v:roundrect xmlns:v="urn:schemas-microsoft-com:vml" xmlns:w="urn:schemas-microsoft-com:office:word" href="{{ctaUrl}}" style="height:48px;v-text-anchor:middle;width:260px;" arcsize="10%" strokecolor="#EA580C" fillcolor="#F97316">
|
||||
<w:anchorlock/>
|
||||
<center style="color:#ffffff;font-family:Arial, Helvetica, sans-serif;font-size:16px;font-weight:bold;">{{ctaLabel}}</center>
|
||||
</v:roundrect>
|
||||
<![endif]-->
|
||||
<!--[if !mso]><!-- -->
|
||||
<a href='{{ctaUrl}}' style='background:#F97316; border:1px solid #EA580C; border-radius:6px; color:#ffffff; display:inline-block; font:700 16px Arial, Helvetica, sans-serif; line-height:48px; text-align:center; text-decoration:none; width:260px;'>{{ctaLabel}}</a>
|
||||
<!--<![endif]-->
|
||||
</div>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td align='center' style='padding:16px;'>
|
||||
<table role='presentation' width='100%' cellspacing='0' cellpadding='0' border='0'>
|
||||
<tr>
|
||||
<td align='center' style='font:12px/1.6 Arial, Helvetica, sans-serif; color:#6b7280;'>
|
||||
<div>© 2025 {{brandName}}. All rights reserved.</div>
|
||||
<div class='muted' style='margin-top:4px;'>Manage webhooks in <a href='https://app.textbee.dev/dashboard/account/' style='color:#6b7280; text-decoration:underline;'>Account settings</a>.</div>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
</body>
|
||||
</html>
|
||||
@@ -40,6 +40,12 @@ export class WebhookSubscription {
|
||||
|
||||
@Prop({ type: Date })
|
||||
lastDeliveryFailureAt: Date
|
||||
|
||||
@Prop({
|
||||
type: [{ at: { type: Date }, text: { type: String } }],
|
||||
default: [],
|
||||
})
|
||||
notes: { at: Date; text: string }[]
|
||||
}
|
||||
|
||||
export const WebhookSubscriptionSchema =
|
||||
|
||||
@@ -13,6 +13,7 @@ import {
|
||||
} from './schemas/webhook-notification.schema'
|
||||
import { AuthModule } from 'src/auth/auth.module'
|
||||
import { UsersModule } from 'src/users/users.module'
|
||||
import { MailModule } from 'src/mail/mail.module'
|
||||
import { WebhookQueueService } from './queue/webhook-queue.service'
|
||||
import { WebhookQueueProcessor } from './queue/webhook-queue.processor'
|
||||
|
||||
@@ -38,6 +39,7 @@ import { WebhookQueueProcessor } from './queue/webhook-queue.processor'
|
||||
}),
|
||||
AuthModule,
|
||||
UsersModule,
|
||||
MailModule,
|
||||
],
|
||||
controllers: [WebhookController],
|
||||
providers: [WebhookService, WebhookQueueService, WebhookQueueProcessor],
|
||||
|
||||
@@ -13,11 +13,12 @@ import {
|
||||
import axios from 'axios'
|
||||
import { v4 as uuidv4 } from 'uuid'
|
||||
import { Cron } from '@nestjs/schedule'
|
||||
import { CronExpression } from '@nestjs/schedule'
|
||||
import * as crypto from 'crypto'
|
||||
import mongoose from 'mongoose'
|
||||
import { SMS } from 'src/gateway/schemas/sms.schema'
|
||||
import { SMS } from '../gateway/schemas/sms.schema'
|
||||
import { WebhookQueueService } from './queue/webhook-queue.service'
|
||||
import { MailService } from '../mail/mail.service'
|
||||
import { UsersService } from '../users/users.service'
|
||||
|
||||
@Injectable()
|
||||
export class WebhookService {
|
||||
@@ -27,6 +28,8 @@ export class WebhookService {
|
||||
@InjectModel(WebhookNotification.name)
|
||||
private webhookNotificationModel: Model<WebhookNotificationDocument>,
|
||||
private webhookQueueService: WebhookQueueService,
|
||||
private mailService: MailService,
|
||||
private usersService: UsersService,
|
||||
) {}
|
||||
|
||||
async findOne({ user, webhookId }) {
|
||||
@@ -446,12 +449,17 @@ export class WebhookService {
|
||||
let responseBody: string | undefined
|
||||
let errorType: 'retryable' | 'non-retryable' | undefined
|
||||
|
||||
const deliveryTimeoutMs = Math.min(
|
||||
60000,
|
||||
Math.max(10000, parseInt(process.env.WEBHOOK_DELIVERY_TIMEOUT_MS ?? '30000', 10) || 30000),
|
||||
)
|
||||
|
||||
try {
|
||||
const response = await axios.post(deliveryUrl, webhookNotification.payload, {
|
||||
headers: {
|
||||
'X-Signature': signature,
|
||||
},
|
||||
timeout: 10000,
|
||||
timeout: deliveryTimeoutMs,
|
||||
})
|
||||
|
||||
httpStatusCode = response.status
|
||||
@@ -551,10 +559,8 @@ export class WebhookService {
|
||||
return new Date(Date.now() + delayInMinutes * 60 * 1000)
|
||||
}
|
||||
|
||||
// Check for notifications that need to be delivered every 3 minutes
|
||||
@Cron('0 */3 * * * *', {
|
||||
disabled: process.env.NODE_ENV !== 'production',
|
||||
})
|
||||
// Check for notifications that need to be delivered every 5 minutes
|
||||
@Cron('0 */5 * * * *')
|
||||
async checkForNotificationsToDeliver() {
|
||||
const now = new Date()
|
||||
const fiveMinutesAgo = new Date(now.getTime() - 5 * 60 * 1000)
|
||||
@@ -566,7 +572,7 @@ export class WebhookService {
|
||||
deliveryAttemptAbortedAt: null,
|
||||
})
|
||||
.sort({ nextDeliveryAttemptAt: 1 })
|
||||
.limit(30)
|
||||
.limit(200)
|
||||
|
||||
if (notifications.length === 0) {
|
||||
return
|
||||
@@ -580,4 +586,160 @@ export class WebhookService {
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
private getAutoDisableConfig(): {
|
||||
threshold: number
|
||||
lookbackDays: number
|
||||
} {
|
||||
const threshold = Math.max(
|
||||
1,
|
||||
parseInt(process.env.WEBHOOK_AUTO_DISABLE_FAILURE_THRESHOLD ?? '50', 10) || 50,
|
||||
)
|
||||
const lookbackDays = Math.max(
|
||||
1,
|
||||
Math.min(
|
||||
365,
|
||||
parseInt(process.env.WEBHOOK_AUTO_DISABLE_LOOKBACK_DAYS ?? '30', 10) || 30,
|
||||
),
|
||||
)
|
||||
return { threshold, lookbackDays }
|
||||
}
|
||||
|
||||
@Cron('0 6 * * *')
|
||||
async autoDisableSubscriptionsWithHighFailureRate() {
|
||||
const { threshold, lookbackDays } = this.getAutoDisableConfig()
|
||||
const now = new Date()
|
||||
const since = new Date(now.getTime() - lookbackDays * 24 * 60 * 60 * 1000)
|
||||
|
||||
const subscriptionCounts = await this.webhookNotificationModel.aggregate<{
|
||||
_id: mongoose.Types.ObjectId
|
||||
count: number
|
||||
}>([
|
||||
{
|
||||
$addFields: {
|
||||
_finalizedAt: {
|
||||
$ifNull: ['$lastDeliveryAttemptAt', '$createdAt'],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
$match: {
|
||||
deliveredAt: null,
|
||||
_finalizedAt: { $gte: since },
|
||||
$or: [
|
||||
{ deliveryAttemptAbortedAt: { $ne: null } },
|
||||
{ deliveryAttemptCount: { $gte: 10 } },
|
||||
],
|
||||
},
|
||||
},
|
||||
{ $group: { _id: '$webhookSubscription', count: { $sum: 1 } } },
|
||||
{ $match: { count: { $gte: threshold } } },
|
||||
])
|
||||
|
||||
if (subscriptionCounts.length === 0) {
|
||||
return
|
||||
}
|
||||
|
||||
const subscriptionIds = subscriptionCounts.map((s) => s._id)
|
||||
const countBySubscriptionId = new Map(
|
||||
subscriptionCounts.map((s) => [s._id.toString(), s.count]),
|
||||
)
|
||||
|
||||
const activeSubscriptions = await this.webhookSubscriptionModel.find({
|
||||
_id: { $in: subscriptionIds },
|
||||
isActive: true,
|
||||
})
|
||||
|
||||
const ctaUrlBase = process.env.FRONTEND_URL || 'https://app.textbee.dev'
|
||||
const disabledInThisRun: {
|
||||
subscriptionId: string
|
||||
deliveryUrl: string
|
||||
failureCount: number
|
||||
lookbackDays: number
|
||||
userName: string
|
||||
userEmail: string
|
||||
}[] = []
|
||||
|
||||
for (const subscription of activeSubscriptions) {
|
||||
const failureCount = countBySubscriptionId.get(
|
||||
subscription._id.toString(),
|
||||
) ?? threshold
|
||||
const noteText = `Auto-disabled: ${failureCount} failed deliveries in the last ${lookbackDays} days. Re-enable in dashboard when your endpoint is ready.`
|
||||
const noteEntry = { at: new Date(), text: noteText }
|
||||
|
||||
const result = await this.webhookSubscriptionModel.updateOne(
|
||||
{ _id: subscription._id, isActive: true },
|
||||
{
|
||||
$set: { isActive: false },
|
||||
$push: { notes: noteEntry },
|
||||
},
|
||||
)
|
||||
|
||||
if (result.modifiedCount === 0) {
|
||||
continue
|
||||
}
|
||||
|
||||
const user = await this.usersService.findOne({
|
||||
_id: subscription.user,
|
||||
})
|
||||
|
||||
disabledInThisRun.push({
|
||||
subscriptionId: subscription._id.toString(),
|
||||
deliveryUrl: subscription.deliveryUrl ?? '',
|
||||
failureCount,
|
||||
lookbackDays,
|
||||
userName: user?.name ?? '—',
|
||||
userEmail: user?.email ?? '—',
|
||||
})
|
||||
|
||||
if (!user?.email) {
|
||||
console.log(
|
||||
`Webhook subscription ${subscription._id} auto-disabled but no user/email to notify`,
|
||||
)
|
||||
continue
|
||||
}
|
||||
|
||||
try {
|
||||
await this.mailService.sendEmailFromTemplate({
|
||||
to: user.email,
|
||||
subject: 'Webhook subscription disabled – textbee',
|
||||
template: 'webhook-subscription-disabled',
|
||||
context: {
|
||||
name: user.name?.split(' ')?.[0] || 'there',
|
||||
title: 'Webhook subscription disabled',
|
||||
message: `Your webhook had ${failureCount} failed deliveries in the last ${lookbackDays} days and was automatically disabled. Re-enable it in the dashboard when your endpoint is ready.`,
|
||||
ctaUrl: `${ctaUrlBase}/dashboard/account`,
|
||||
ctaLabel: 'View webhooks',
|
||||
brandName: 'textbee.dev',
|
||||
},
|
||||
})
|
||||
} catch (e) {
|
||||
console.log(
|
||||
`Failed to send webhook-disabled email to ${user.email}:`,
|
||||
e,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
const adminEmail = process.env.ADMIN_EMAIL
|
||||
if (disabledInThisRun.length > 0 && adminEmail) {
|
||||
const runAt = now.toISOString()
|
||||
try {
|
||||
await this.mailService.sendEmailFromTemplate({
|
||||
to: adminEmail,
|
||||
subject: `Webhook auto-disable: ${disabledInThisRun.length} subscription(s) – ${runAt.slice(0, 10)}`,
|
||||
template: 'webhook-auto-disable-admin-summary',
|
||||
context: {
|
||||
title: 'Webhook auto-disable summary',
|
||||
runAt,
|
||||
count: disabledInThisRun.length,
|
||||
disabledList: disabledInThisRun,
|
||||
brandName: 'textbee.dev',
|
||||
},
|
||||
})
|
||||
} catch (e) {
|
||||
console.log(`Failed to send webhook auto-disable admin summary to ${adminEmail}:`, e)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user