diff --git a/api/.env.example b/api/.env.example index 3835323..8fc0307 100644 --- a/api/.env.example +++ b/api/.env.example @@ -24,6 +24,14 @@ MAIL_USER= MAIL_PASS= MAIL_FROM= MAIL_REPLY_TO= +ADMIN_EMAIL= + +# Webhook delivery HTTP timeout in milliseconds (default 30000, min 10000, max 60000) +WEBHOOK_DELIVERY_TIMEOUT_MS=30000 + +# Auto-disable webhook subscriptions with high failure rate (cron runs daily) +WEBHOOK_AUTO_DISABLE_FAILURE_THRESHOLD=50 +WEBHOOK_AUTO_DISABLE_LOOKBACK_DAYS=30 # SMS Queue Configuration USE_SMS_QUEUE=false diff --git a/api/src/mail/templates/webhook-auto-disable-admin-summary.hbs b/api/src/mail/templates/webhook-auto-disable-admin-summary.hbs new file mode 100644 index 0000000..c0f33e6 --- /dev/null +++ b/api/src/mail/templates/webhook-auto-disable-admin-summary.hbs @@ -0,0 +1,52 @@ + + + + + {{title}} – {{runAt}} + + + +
+
+
{{title}}
+
Run at {{runAt}} · {{count}} subscription(s) auto-disabled
+
+ + + + + + + + + + + + + {{#each disabledList}} + + + + + + + + + {{/each}} + +
Subscription IDDelivery URLFailed countPeriod (days)User nameUser email
{{this.subscriptionId}}{{this.deliveryUrl}}{{this.failureCount}}{{this.lookbackDays}}{{this.userName}}{{this.userEmail}}
+
{{brandName}} – Webhook auto-disable cron
+
+ + diff --git a/api/src/mail/templates/webhook-subscription-disabled.hbs b/api/src/mail/templates/webhook-subscription-disabled.hbs new file mode 100644 index 0000000..0e4a374 --- /dev/null +++ b/api/src/mail/templates/webhook-subscription-disabled.hbs @@ -0,0 +1,86 @@ + + + + + + {{title}} • {{brandName}} + + + + +
Webhook subscription disabled – {{brandName}}
+ + + + + + + diff --git a/api/src/webhook/schemas/webhook-subscription.schema.ts b/api/src/webhook/schemas/webhook-subscription.schema.ts index 8083705..f4593e8 100644 --- a/api/src/webhook/schemas/webhook-subscription.schema.ts +++ b/api/src/webhook/schemas/webhook-subscription.schema.ts @@ -40,6 +40,12 @@ export class WebhookSubscription { @Prop({ type: Date }) lastDeliveryFailureAt: Date + + @Prop({ + type: [{ at: { type: Date }, text: { type: String } }], + default: [], + }) + notes: { at: Date; text: string }[] } export const WebhookSubscriptionSchema = diff --git a/api/src/webhook/webhook.module.ts b/api/src/webhook/webhook.module.ts index c04552f..2537e8f 100644 --- a/api/src/webhook/webhook.module.ts +++ b/api/src/webhook/webhook.module.ts @@ -13,6 +13,7 @@ import { } from './schemas/webhook-notification.schema' import { AuthModule } from 'src/auth/auth.module' import { UsersModule } from 'src/users/users.module' +import { MailModule } from 'src/mail/mail.module' import { WebhookQueueService } from './queue/webhook-queue.service' import { WebhookQueueProcessor } from './queue/webhook-queue.processor' @@ -38,6 +39,7 @@ import { WebhookQueueProcessor } from './queue/webhook-queue.processor' }), AuthModule, UsersModule, + MailModule, ], controllers: [WebhookController], providers: [WebhookService, WebhookQueueService, WebhookQueueProcessor], diff --git a/api/src/webhook/webhook.service.ts b/api/src/webhook/webhook.service.ts index 43bf9be..e869d8e 100644 --- a/api/src/webhook/webhook.service.ts +++ b/api/src/webhook/webhook.service.ts @@ -13,11 +13,12 @@ import { import axios from 'axios' import { v4 as uuidv4 } from 'uuid' import { Cron } from '@nestjs/schedule' -import { CronExpression } from '@nestjs/schedule' import * as crypto from 'crypto' import mongoose from 'mongoose' -import { SMS } from 'src/gateway/schemas/sms.schema' +import { SMS } from '../gateway/schemas/sms.schema' import { WebhookQueueService } from './queue/webhook-queue.service' +import { MailService } from '../mail/mail.service' +import { UsersService } from '../users/users.service' @Injectable() export class WebhookService { @@ -27,6 +28,8 @@ export class WebhookService { @InjectModel(WebhookNotification.name) private webhookNotificationModel: Model, private webhookQueueService: WebhookQueueService, + private mailService: MailService, + private usersService: UsersService, ) {} async findOne({ user, webhookId }) { @@ -446,12 +449,17 @@ export class WebhookService { let responseBody: string | undefined let errorType: 'retryable' | 'non-retryable' | undefined + const deliveryTimeoutMs = Math.min( + 60000, + Math.max(10000, parseInt(process.env.WEBHOOK_DELIVERY_TIMEOUT_MS ?? '30000', 10) || 30000), + ) + try { const response = await axios.post(deliveryUrl, webhookNotification.payload, { headers: { 'X-Signature': signature, }, - timeout: 10000, + timeout: deliveryTimeoutMs, }) httpStatusCode = response.status @@ -551,10 +559,8 @@ export class WebhookService { return new Date(Date.now() + delayInMinutes * 60 * 1000) } - // Check for notifications that need to be delivered every 3 minutes - @Cron('0 */3 * * * *', { - disabled: process.env.NODE_ENV !== 'production', - }) + // Check for notifications that need to be delivered every 5 minutes + @Cron('0 */5 * * * *') async checkForNotificationsToDeliver() { const now = new Date() const fiveMinutesAgo = new Date(now.getTime() - 5 * 60 * 1000) @@ -566,7 +572,7 @@ export class WebhookService { deliveryAttemptAbortedAt: null, }) .sort({ nextDeliveryAttemptAt: 1 }) - .limit(30) + .limit(200) if (notifications.length === 0) { return @@ -580,4 +586,160 @@ export class WebhookService { ) } } + + private getAutoDisableConfig(): { + threshold: number + lookbackDays: number + } { + const threshold = Math.max( + 1, + parseInt(process.env.WEBHOOK_AUTO_DISABLE_FAILURE_THRESHOLD ?? '50', 10) || 50, + ) + const lookbackDays = Math.max( + 1, + Math.min( + 365, + parseInt(process.env.WEBHOOK_AUTO_DISABLE_LOOKBACK_DAYS ?? '30', 10) || 30, + ), + ) + return { threshold, lookbackDays } + } + + @Cron('0 6 * * *') + async autoDisableSubscriptionsWithHighFailureRate() { + const { threshold, lookbackDays } = this.getAutoDisableConfig() + const now = new Date() + const since = new Date(now.getTime() - lookbackDays * 24 * 60 * 60 * 1000) + + const subscriptionCounts = await this.webhookNotificationModel.aggregate<{ + _id: mongoose.Types.ObjectId + count: number + }>([ + { + $addFields: { + _finalizedAt: { + $ifNull: ['$lastDeliveryAttemptAt', '$createdAt'], + }, + }, + }, + { + $match: { + deliveredAt: null, + _finalizedAt: { $gte: since }, + $or: [ + { deliveryAttemptAbortedAt: { $ne: null } }, + { deliveryAttemptCount: { $gte: 10 } }, + ], + }, + }, + { $group: { _id: '$webhookSubscription', count: { $sum: 1 } } }, + { $match: { count: { $gte: threshold } } }, + ]) + + if (subscriptionCounts.length === 0) { + return + } + + const subscriptionIds = subscriptionCounts.map((s) => s._id) + const countBySubscriptionId = new Map( + subscriptionCounts.map((s) => [s._id.toString(), s.count]), + ) + + const activeSubscriptions = await this.webhookSubscriptionModel.find({ + _id: { $in: subscriptionIds }, + isActive: true, + }) + + const ctaUrlBase = process.env.FRONTEND_URL || 'https://app.textbee.dev' + const disabledInThisRun: { + subscriptionId: string + deliveryUrl: string + failureCount: number + lookbackDays: number + userName: string + userEmail: string + }[] = [] + + for (const subscription of activeSubscriptions) { + const failureCount = countBySubscriptionId.get( + subscription._id.toString(), + ) ?? threshold + const noteText = `Auto-disabled: ${failureCount} failed deliveries in the last ${lookbackDays} days. Re-enable in dashboard when your endpoint is ready.` + const noteEntry = { at: new Date(), text: noteText } + + const result = await this.webhookSubscriptionModel.updateOne( + { _id: subscription._id, isActive: true }, + { + $set: { isActive: false }, + $push: { notes: noteEntry }, + }, + ) + + if (result.modifiedCount === 0) { + continue + } + + const user = await this.usersService.findOne({ + _id: subscription.user, + }) + + disabledInThisRun.push({ + subscriptionId: subscription._id.toString(), + deliveryUrl: subscription.deliveryUrl ?? '', + failureCount, + lookbackDays, + userName: user?.name ?? '—', + userEmail: user?.email ?? '—', + }) + + if (!user?.email) { + console.log( + `Webhook subscription ${subscription._id} auto-disabled but no user/email to notify`, + ) + continue + } + + try { + await this.mailService.sendEmailFromTemplate({ + to: user.email, + subject: 'Webhook subscription disabled – textbee', + template: 'webhook-subscription-disabled', + context: { + name: user.name?.split(' ')?.[0] || 'there', + title: 'Webhook subscription disabled', + message: `Your webhook had ${failureCount} failed deliveries in the last ${lookbackDays} days and was automatically disabled. Re-enable it in the dashboard when your endpoint is ready.`, + ctaUrl: `${ctaUrlBase}/dashboard/account`, + ctaLabel: 'View webhooks', + brandName: 'textbee.dev', + }, + }) + } catch (e) { + console.log( + `Failed to send webhook-disabled email to ${user.email}:`, + e, + ) + } + } + + const adminEmail = process.env.ADMIN_EMAIL + if (disabledInThisRun.length > 0 && adminEmail) { + const runAt = now.toISOString() + try { + await this.mailService.sendEmailFromTemplate({ + to: adminEmail, + subject: `Webhook auto-disable: ${disabledInThisRun.length} subscription(s) – ${runAt.slice(0, 10)}`, + template: 'webhook-auto-disable-admin-summary', + context: { + title: 'Webhook auto-disable summary', + runAt, + count: disabledInThisRun.length, + disabledList: disabledInThisRun, + brandName: 'textbee.dev', + }, + }) + } catch (e) { + console.log(`Failed to send webhook auto-disable admin summary to ${adminEmail}:`, e) + } + } + } }