From 7ff6ea23fce1c65f41a10d2b809e78191d7851aa Mon Sep 17 00:00:00 2001 From: isra el Date: Thu, 5 Mar 2026 10:33:20 +0300 Subject: [PATCH 1/4] chore(api): update webhook notification delivery timeout to 30ms --- api/src/webhook/webhook.service.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/api/src/webhook/webhook.service.ts b/api/src/webhook/webhook.service.ts index 43bf9be..060ae99 100644 --- a/api/src/webhook/webhook.service.ts +++ b/api/src/webhook/webhook.service.ts @@ -446,12 +446,17 @@ export class WebhookService { let responseBody: string | undefined let errorType: 'retryable' | 'non-retryable' | undefined + const deliveryTimeoutMs = Math.min( + 60000, + Math.max(10000, parseInt(process.env.WEBHOOK_DELIVERY_TIMEOUT_MS ?? '30000', 10) || 30000), + ) + try { const response = await axios.post(deliveryUrl, webhookNotification.payload, { headers: { 'X-Signature': signature, }, - timeout: 10000, + timeout: deliveryTimeoutMs, }) httpStatusCode = response.status From 0144cff16e7acb64093528f49629ddba192640a8 Mon Sep 17 00:00:00 2001 From: isra el Date: Thu, 5 Mar 2026 10:33:59 +0300 Subject: [PATCH 2/4] chore(api): update webhook notifications delivery check cron time and query --- api/src/webhook/webhook.service.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/api/src/webhook/webhook.service.ts b/api/src/webhook/webhook.service.ts index 060ae99..8697405 100644 --- a/api/src/webhook/webhook.service.ts +++ b/api/src/webhook/webhook.service.ts @@ -556,8 +556,8 @@ export class WebhookService { return new Date(Date.now() + delayInMinutes * 60 * 1000) } - // Check for notifications that need to be delivered every 3 minutes - @Cron('0 */3 * * * *', { + // Check for notifications that need to be delivered every 5 minutes + @Cron('0 */5 * * * *', { disabled: process.env.NODE_ENV !== 'production', }) async checkForNotificationsToDeliver() { @@ -571,7 +571,7 @@ export class WebhookService { deliveryAttemptAbortedAt: null, }) .sort({ nextDeliveryAttemptAt: 1 }) - .limit(30) + .limit(200) if (notifications.length === 0) { return From 8fd06f06d1d70c077ca563db3ea0c4645c12e3a4 Mon Sep 17 00:00:00 2001 From: isra el Date: Thu, 5 Mar 2026 11:01:51 +0300 Subject: [PATCH 3/4] feat(api): auto-disable webhook ubscriptions with high failure rate --- api/.env.example | 8 + .../webhook-auto-disable-admin-summary.hbs | 52 ++++++ .../webhook-subscription-disabled.hbs | 86 +++++++++ .../schemas/webhook-subscription.schema.ts | 6 + api/src/webhook/webhook.module.ts | 2 + api/src/webhook/webhook.service.ts | 165 +++++++++++++++++- 6 files changed, 315 insertions(+), 4 deletions(-) create mode 100644 api/src/mail/templates/webhook-auto-disable-admin-summary.hbs create mode 100644 api/src/mail/templates/webhook-subscription-disabled.hbs diff --git a/api/.env.example b/api/.env.example index 3835323..8fc0307 100644 --- a/api/.env.example +++ b/api/.env.example @@ -24,6 +24,14 @@ MAIL_USER= MAIL_PASS= MAIL_FROM= MAIL_REPLY_TO= +ADMIN_EMAIL= + +# Webhook delivery HTTP timeout in milliseconds (default 30000, min 10000, max 60000) +WEBHOOK_DELIVERY_TIMEOUT_MS=30000 + +# Auto-disable webhook subscriptions with high failure rate (cron runs daily) +WEBHOOK_AUTO_DISABLE_FAILURE_THRESHOLD=50 +WEBHOOK_AUTO_DISABLE_LOOKBACK_DAYS=30 # SMS Queue Configuration USE_SMS_QUEUE=false diff --git a/api/src/mail/templates/webhook-auto-disable-admin-summary.hbs b/api/src/mail/templates/webhook-auto-disable-admin-summary.hbs new file mode 100644 index 0000000..c0f33e6 --- /dev/null +++ b/api/src/mail/templates/webhook-auto-disable-admin-summary.hbs @@ -0,0 +1,52 @@ + + + + + {{title}} – {{runAt}} + + + +
+
+
{{title}}
+
Run at {{runAt}} · {{count}} subscription(s) auto-disabled
+
+ + + + + + + + + + + + + {{#each disabledList}} + + + + + + + + + {{/each}} + +
Subscription IDDelivery URLFailed countPeriod (days)User nameUser email
{{this.subscriptionId}}{{this.deliveryUrl}}{{this.failureCount}}{{this.lookbackDays}}{{this.userName}}{{this.userEmail}}
+
{{brandName}} – Webhook auto-disable cron
+
+ + diff --git a/api/src/mail/templates/webhook-subscription-disabled.hbs b/api/src/mail/templates/webhook-subscription-disabled.hbs new file mode 100644 index 0000000..0e4a374 --- /dev/null +++ b/api/src/mail/templates/webhook-subscription-disabled.hbs @@ -0,0 +1,86 @@ + + + + + + {{title}} • {{brandName}} + + + + +
Webhook subscription disabled – {{brandName}}
+ + + + + + + diff --git a/api/src/webhook/schemas/webhook-subscription.schema.ts b/api/src/webhook/schemas/webhook-subscription.schema.ts index 8083705..f4593e8 100644 --- a/api/src/webhook/schemas/webhook-subscription.schema.ts +++ b/api/src/webhook/schemas/webhook-subscription.schema.ts @@ -40,6 +40,12 @@ export class WebhookSubscription { @Prop({ type: Date }) lastDeliveryFailureAt: Date + + @Prop({ + type: [{ at: { type: Date }, text: { type: String } }], + default: [], + }) + notes: { at: Date; text: string }[] } export const WebhookSubscriptionSchema = diff --git a/api/src/webhook/webhook.module.ts b/api/src/webhook/webhook.module.ts index c04552f..2537e8f 100644 --- a/api/src/webhook/webhook.module.ts +++ b/api/src/webhook/webhook.module.ts @@ -13,6 +13,7 @@ import { } from './schemas/webhook-notification.schema' import { AuthModule } from 'src/auth/auth.module' import { UsersModule } from 'src/users/users.module' +import { MailModule } from 'src/mail/mail.module' import { WebhookQueueService } from './queue/webhook-queue.service' import { WebhookQueueProcessor } from './queue/webhook-queue.processor' @@ -38,6 +39,7 @@ import { WebhookQueueProcessor } from './queue/webhook-queue.processor' }), AuthModule, UsersModule, + MailModule, ], controllers: [WebhookController], providers: [WebhookService, WebhookQueueService, WebhookQueueProcessor], diff --git a/api/src/webhook/webhook.service.ts b/api/src/webhook/webhook.service.ts index 8697405..edfd2a7 100644 --- a/api/src/webhook/webhook.service.ts +++ b/api/src/webhook/webhook.service.ts @@ -13,11 +13,12 @@ import { import axios from 'axios' import { v4 as uuidv4 } from 'uuid' import { Cron } from '@nestjs/schedule' -import { CronExpression } from '@nestjs/schedule' import * as crypto from 'crypto' import mongoose from 'mongoose' import { SMS } from 'src/gateway/schemas/sms.schema' import { WebhookQueueService } from './queue/webhook-queue.service' +import { MailService } from 'src/mail/mail.service' +import { UsersService } from 'src/users/users.service' @Injectable() export class WebhookService { @@ -27,6 +28,8 @@ export class WebhookService { @InjectModel(WebhookNotification.name) private webhookNotificationModel: Model, private webhookQueueService: WebhookQueueService, + private mailService: MailService, + private usersService: UsersService, ) {} async findOne({ user, webhookId }) { @@ -557,9 +560,7 @@ export class WebhookService { } // Check for notifications that need to be delivered every 5 minutes - @Cron('0 */5 * * * *', { - disabled: process.env.NODE_ENV !== 'production', - }) + @Cron('0 */5 * * * *') async checkForNotificationsToDeliver() { const now = new Date() const fiveMinutesAgo = new Date(now.getTime() - 5 * 60 * 1000) @@ -585,4 +586,160 @@ export class WebhookService { ) } } + + private getAutoDisableConfig(): { + threshold: number + lookbackDays: number + } { + const threshold = Math.max( + 1, + parseInt(process.env.WEBHOOK_AUTO_DISABLE_FAILURE_THRESHOLD ?? '50', 10) || 50, + ) + const lookbackDays = Math.max( + 1, + Math.min( + 365, + parseInt(process.env.WEBHOOK_AUTO_DISABLE_LOOKBACK_DAYS ?? '30', 10) || 30, + ), + ) + return { threshold, lookbackDays } + } + + @Cron('0 6 * * *') + async autoDisableSubscriptionsWithHighFailureRate() { + const { threshold, lookbackDays } = this.getAutoDisableConfig() + const now = new Date() + const since = new Date(now.getTime() - lookbackDays * 24 * 60 * 60 * 1000) + + const subscriptionCounts = await this.webhookNotificationModel.aggregate<{ + _id: mongoose.Types.ObjectId + count: number + }>([ + { + $addFields: { + _finalizedAt: { + $ifNull: ['$lastDeliveryAttemptAt', '$createdAt'], + }, + }, + }, + { + $match: { + deliveredAt: null, + _finalizedAt: { $gte: since }, + $or: [ + { deliveryAttemptAbortedAt: { $ne: null } }, + { deliveryAttemptCount: { $gte: 10 } }, + ], + }, + }, + { $group: { _id: '$webhookSubscription', count: { $sum: 1 } } }, + { $match: { count: { $gte: threshold } } }, + ]) + + if (subscriptionCounts.length === 0) { + return + } + + const subscriptionIds = subscriptionCounts.map((s) => s._id) + const countBySubscriptionId = new Map( + subscriptionCounts.map((s) => [s._id.toString(), s.count]), + ) + + const activeSubscriptions = await this.webhookSubscriptionModel.find({ + _id: { $in: subscriptionIds }, + isActive: true, + }) + + const ctaUrlBase = process.env.FRONTEND_URL || 'https://app.textbee.dev' + const disabledInThisRun: { + subscriptionId: string + deliveryUrl: string + failureCount: number + lookbackDays: number + userName: string + userEmail: string + }[] = [] + + for (const subscription of activeSubscriptions) { + const failureCount = countBySubscriptionId.get( + subscription._id.toString(), + ) ?? threshold + const noteText = `Auto-disabled: ${failureCount} failed deliveries in the last ${lookbackDays} days. Re-enable in dashboard when your endpoint is ready.` + const noteEntry = { at: new Date(), text: noteText } + + const result = await this.webhookSubscriptionModel.updateOne( + { _id: subscription._id, isActive: true }, + { + $set: { isActive: false }, + $push: { notes: noteEntry }, + }, + ) + + if (result.modifiedCount === 0) { + continue + } + + const user = await this.usersService.findOne({ + _id: subscription.user, + }) + + disabledInThisRun.push({ + subscriptionId: subscription._id.toString(), + deliveryUrl: subscription.deliveryUrl ?? '', + failureCount, + lookbackDays, + userName: user?.name ?? '—', + userEmail: user?.email ?? '—', + }) + + if (!user?.email) { + console.log( + `Webhook subscription ${subscription._id} auto-disabled but no user/email to notify`, + ) + continue + } + + try { + await this.mailService.sendEmailFromTemplate({ + to: user.email, + subject: 'Webhook subscription disabled – textbee', + template: 'webhook-subscription-disabled', + context: { + name: user.name?.split(' ')?.[0] || 'there', + title: 'Webhook subscription disabled', + message: `Your webhook had ${failureCount} failed deliveries in the last ${lookbackDays} days and was automatically disabled. Re-enable it in the dashboard when your endpoint is ready.`, + ctaUrl: `${ctaUrlBase}/dashboard/account`, + ctaLabel: 'View webhooks', + brandName: 'textbee.dev', + }, + }) + } catch (e) { + console.log( + `Failed to send webhook-disabled email to ${user.email}:`, + e, + ) + } + } + + const adminEmail = process.env.ADMIN_EMAIL + if (disabledInThisRun.length > 0 && adminEmail) { + const runAt = now.toISOString() + try { + await this.mailService.sendEmailFromTemplate({ + to: adminEmail, + subject: `Webhook auto-disable: ${disabledInThisRun.length} subscription(s) – ${runAt.slice(0, 10)}`, + template: 'webhook-auto-disable-admin-summary', + context: { + title: 'Webhook auto-disable summary', + runAt, + count: disabledInThisRun.length, + disabledList: disabledInThisRun, + brandName: 'textbee.dev', + }, + }) + } catch (e) { + console.log(`Failed to send webhook auto-disable admin summary to ${adminEmail}:`, e) + } + } + } } From 6e7ed42fe829fca9666cc28c29f0a1ee599ff67c Mon Sep 17 00:00:00 2001 From: isra el Date: Thu, 5 Mar 2026 11:15:29 +0300 Subject: [PATCH 4/4] fix(api): fix failing test --- api/src/webhook/webhook.service.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/api/src/webhook/webhook.service.ts b/api/src/webhook/webhook.service.ts index edfd2a7..e869d8e 100644 --- a/api/src/webhook/webhook.service.ts +++ b/api/src/webhook/webhook.service.ts @@ -15,10 +15,10 @@ import { v4 as uuidv4 } from 'uuid' import { Cron } from '@nestjs/schedule' import * as crypto from 'crypto' import mongoose from 'mongoose' -import { SMS } from 'src/gateway/schemas/sms.schema' +import { SMS } from '../gateway/schemas/sms.schema' import { WebhookQueueService } from './queue/webhook-queue.service' -import { MailService } from 'src/mail/mail.service' -import { UsersService } from 'src/users/users.service' +import { MailService } from '../mail/mail.service' +import { UsersService } from '../users/users.service' @Injectable() export class WebhookService {