From 42975a4168083d63fcd440ba01c0c2cbb162f3ed Mon Sep 17 00:00:00 2001 From: neo773 <62795688+neo773@users.noreply.github.com> Date: Thu, 14 May 2026 15:55:24 +0530 Subject: [PATCH] fix(server): decouple SDK client generation from workspace activation (#20514) `activateWorkspace` enqueues SDK gen job inside `WorkspaceManagerService.init()` introduced by https://github.com/twentyhq/twenty/pull/19271 But if enqueue call fails it crashes cuz it doesn't have try catch so created workspace is in corrupted state image FIx: Move SDK enqueue out of `init()` Call after `activateAndInitializeUpgradeState` succeeds, wrap in try catch. Mirror preInstalledAppsService.installOnWorkspace pattern. Assuming enqueue failure if Redis is unavailable we fallback to `SdkClientArchiveService.downloadArchiveBufferOrGenerate` which generates it on the fly Around 19 workspaces in prod affected with status `ONGOING_CREATION` --- .../sdk-client-generation.service.spec.ts | 125 ++++++++++++++++++ .../jobs/generate-sdk-client.job-constants.ts | 7 + .../jobs/generate-sdk-client.job.ts | 12 +- .../sdk-client-generation.service.ts | 39 ++++++ .../sdk-client/sdk-client.module.ts | 2 + .../__tests__/workspace.service.spec.ts | 2 + .../workspace/services/workspace.service.ts | 14 ++ .../workspace/workspace.module.ts | 2 + .../workspace-manager.service.ts | 32 +---- 9 files changed, 197 insertions(+), 38 deletions(-) create mode 100644 packages/twenty-server/src/engine/core-modules/sdk-client/__tests__/sdk-client-generation.service.spec.ts create mode 100644 packages/twenty-server/src/engine/core-modules/sdk-client/jobs/generate-sdk-client.job-constants.ts diff --git a/packages/twenty-server/src/engine/core-modules/sdk-client/__tests__/sdk-client-generation.service.spec.ts b/packages/twenty-server/src/engine/core-modules/sdk-client/__tests__/sdk-client-generation.service.spec.ts new file mode 100644 index 00000000000..7d4479594a9 --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/sdk-client/__tests__/sdk-client-generation.service.spec.ts @@ -0,0 +1,125 @@ +import { Test, type TestingModule } from '@nestjs/testing'; +import { getRepositoryToken } from '@nestjs/typeorm'; + +import { Repository } from 'typeorm'; + +import { WorkspaceSchemaFactory } from 'src/engine/api/graphql/workspace-schema.factory'; +import { ApplicationEntity } from 'src/engine/core-modules/application/application.entity'; +import { ApplicationService } from 'src/engine/core-modules/application/application.service'; +import { FileStorageService } from 'src/engine/core-modules/file-storage/file-storage.service'; +import { getQueueToken } from 'src/engine/core-modules/message-queue/utils/get-queue-token.util'; +import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants'; +import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service'; +import { GENERATE_SDK_CLIENT_JOB_NAME } from 'src/engine/core-modules/sdk-client/jobs/generate-sdk-client.job-constants'; +import { SdkClientGenerationService } from 'src/engine/core-modules/sdk-client/sdk-client-generation.service'; +import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity'; +import { WorkspaceCacheService } from 'src/engine/workspace-cache/services/workspace-cache.service'; + +describe('SdkClientGenerationService', () => { + let service: SdkClientGenerationService; + let applicationService: jest.Mocked< + Pick< + ApplicationService, + 'findWorkspaceTwentyStandardAndCustomApplicationOrThrow' + > + >; + let messageQueueService: jest.Mocked>; + + beforeEach(async () => { + applicationService = { + findWorkspaceTwentyStandardAndCustomApplicationOrThrow: jest.fn(), + }; + messageQueueService = { + add: jest.fn().mockResolvedValue(undefined), + }; + + const module: TestingModule = await Test.createTestingModule({ + providers: [ + SdkClientGenerationService, + { provide: FileStorageService, useValue: {} }, + { + provide: getRepositoryToken(ApplicationEntity), + useValue: {} as Repository, + }, + { + provide: getRepositoryToken(WorkspaceEntity), + useValue: {} as Repository, + }, + { provide: WorkspaceCacheService, useValue: {} }, + { provide: WorkspaceSchemaFactory, useValue: {} }, + { provide: ApplicationService, useValue: applicationService }, + { + provide: getQueueToken(MessageQueue.workspaceQueue), + useValue: messageQueueService, + }, + ], + }).compile(); + + service = module.get( + SdkClientGenerationService, + ); + }); + + describe('enqueueSdkClientGenerationForWorkspace', () => { + const workspaceId = 'workspace-1'; + const apps = { + twentyStandardFlatApplication: { + id: 'std-app-id', + universalIdentifier: 'twenty-standard', + }, + workspaceCustomFlatApplication: { + id: 'custom-app-id', + universalIdentifier: 'workspace-custom', + }, + }; + + it('enqueues one job per application with dedup id and retry limit', async () => { + applicationService.findWorkspaceTwentyStandardAndCustomApplicationOrThrow.mockResolvedValue( + apps as never, + ); + + await service.enqueueSdkClientGenerationForWorkspace(workspaceId); + + expect(messageQueueService.add).toHaveBeenCalledTimes(2); + expect(messageQueueService.add).toHaveBeenNthCalledWith( + 1, + GENERATE_SDK_CLIENT_JOB_NAME, + { + workspaceId, + applicationId: 'std-app-id', + applicationUniversalIdentifier: 'twenty-standard', + }, + { + id: `sdk-client:${workspaceId}:std-app-id`, + retryLimit: 3, + }, + ); + expect(messageQueueService.add).toHaveBeenNthCalledWith( + 2, + GENERATE_SDK_CLIENT_JOB_NAME, + { + workspaceId, + applicationId: 'custom-app-id', + applicationUniversalIdentifier: 'workspace-custom', + }, + { + id: `sdk-client:${workspaceId}:custom-app-id`, + retryLimit: 3, + }, + ); + }); + + it('propagates errors thrown by the message queue service', async () => { + applicationService.findWorkspaceTwentyStandardAndCustomApplicationOrThrow.mockResolvedValue( + apps as never, + ); + const failure = new Error('Redis unavailable'); + + messageQueueService.add.mockRejectedValueOnce(failure); + + await expect( + service.enqueueSdkClientGenerationForWorkspace(workspaceId), + ).rejects.toBe(failure); + }); + }); +}); diff --git a/packages/twenty-server/src/engine/core-modules/sdk-client/jobs/generate-sdk-client.job-constants.ts b/packages/twenty-server/src/engine/core-modules/sdk-client/jobs/generate-sdk-client.job-constants.ts new file mode 100644 index 00000000000..bdbf9edda0a --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/sdk-client/jobs/generate-sdk-client.job-constants.ts @@ -0,0 +1,7 @@ +export const GENERATE_SDK_CLIENT_JOB_NAME = 'GenerateSdkClientJob'; + +export type GenerateSdkClientJobData = { + workspaceId: string; + applicationId: string; + applicationUniversalIdentifier: string; +}; diff --git a/packages/twenty-server/src/engine/core-modules/sdk-client/jobs/generate-sdk-client.job.ts b/packages/twenty-server/src/engine/core-modules/sdk-client/jobs/generate-sdk-client.job.ts index 86fffe16952..cb20aaa03a2 100644 --- a/packages/twenty-server/src/engine/core-modules/sdk-client/jobs/generate-sdk-client.job.ts +++ b/packages/twenty-server/src/engine/core-modules/sdk-client/jobs/generate-sdk-client.job.ts @@ -1,21 +1,19 @@ import { Processor } from 'src/engine/core-modules/message-queue/decorators/processor.decorator'; import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants'; import { Process } from 'src/engine/core-modules/message-queue/decorators/process.decorator'; +import { + GENERATE_SDK_CLIENT_JOB_NAME, + type GenerateSdkClientJobData, +} from 'src/engine/core-modules/sdk-client/jobs/generate-sdk-client.job-constants'; import { SdkClientGenerationService } from 'src/engine/core-modules/sdk-client/sdk-client-generation.service'; -export type GenerateSdkClientJobData = { - workspaceId: string; - applicationId: string; - applicationUniversalIdentifier: string; -}; - @Processor(MessageQueue.workspaceQueue) export class GenerateSdkClientJob { constructor( private readonly sdkClientGenerationService: SdkClientGenerationService, ) {} - @Process(GenerateSdkClientJob.name) + @Process(GENERATE_SDK_CLIENT_JOB_NAME) async handle(data: GenerateSdkClientJobData): Promise { await this.sdkClientGenerationService.generateSdkClientForApplication({ workspaceId: data.workspaceId, diff --git a/packages/twenty-server/src/engine/core-modules/sdk-client/sdk-client-generation.service.ts b/packages/twenty-server/src/engine/core-modules/sdk-client/sdk-client-generation.service.ts index e8fe0a8b67c..b2e80595c25 100644 --- a/packages/twenty-server/src/engine/core-modules/sdk-client/sdk-client-generation.service.ts +++ b/packages/twenty-server/src/engine/core-modules/sdk-client/sdk-client-generation.service.ts @@ -11,19 +11,28 @@ import { Repository } from 'typeorm'; import { WorkspaceSchemaFactory } from 'src/engine/api/graphql/workspace-schema.factory'; import { ApplicationEntity } from 'src/engine/core-modules/application/application.entity'; +import { ApplicationService } from 'src/engine/core-modules/application/application.service'; import { FileStorageService } from 'src/engine/core-modules/file-storage/file-storage.service'; import { createZipFile } from 'src/engine/core-modules/logic-function/logic-function-drivers/utils/create-zip-file'; import { TemporaryDirManager } from 'src/engine/core-modules/logic-function/logic-function-drivers/utils/temporary-dir-manager'; +import { InjectMessageQueue } from 'src/engine/core-modules/message-queue/decorators/message-queue.decorator'; +import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants'; +import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service'; import { SDK_CLIENT_PACKAGE_DIRNAME } from 'src/engine/core-modules/sdk-client/constants/sdk-client-package-dirname'; import { SdkClientException, SdkClientExceptionCode, } from 'src/engine/core-modules/sdk-client/exceptions/sdk-client.exception'; +import { + GENERATE_SDK_CLIENT_JOB_NAME, + type GenerateSdkClientJobData, +} from 'src/engine/core-modules/sdk-client/jobs/generate-sdk-client.job-constants'; import { fromWorkspaceEntityToFlat } from 'src/engine/core-modules/workspace/utils/from-workspace-entity-to-flat.util'; import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity'; import { WorkspaceCacheService } from 'src/engine/workspace-cache/services/workspace-cache.service'; const SDK_CLIENT_ARCHIVE_NAME = 'twenty-client-sdk.zip'; +const SDK_CLIENT_GENERATION_RETRY_LIMIT = 3; @Injectable() export class SdkClientGenerationService { @@ -37,8 +46,38 @@ export class SdkClientGenerationService { private readonly workspaceRepository: Repository, private readonly workspaceCacheService: WorkspaceCacheService, private readonly workspaceSchemaFactory: WorkspaceSchemaFactory, + private readonly applicationService: ApplicationService, + @InjectMessageQueue(MessageQueue.workspaceQueue) + private readonly messageQueueService: MessageQueueService, ) {} + async enqueueSdkClientGenerationForWorkspace( + workspaceId: string, + ): Promise { + const { twentyStandardFlatApplication, workspaceCustomFlatApplication } = + await this.applicationService.findWorkspaceTwentyStandardAndCustomApplicationOrThrow( + { workspaceId }, + ); + + await Promise.all( + [twentyStandardFlatApplication, workspaceCustomFlatApplication].map( + (application) => + this.messageQueueService.add( + GENERATE_SDK_CLIENT_JOB_NAME, + { + workspaceId, + applicationId: application.id, + applicationUniversalIdentifier: application.universalIdentifier, + }, + { + id: `sdk-client:${workspaceId}:${application.id}`, + retryLimit: SDK_CLIENT_GENERATION_RETRY_LIMIT, + }, + ), + ), + ); + } + async generateSdkClientForApplication({ workspaceId, applicationId, diff --git a/packages/twenty-server/src/engine/core-modules/sdk-client/sdk-client.module.ts b/packages/twenty-server/src/engine/core-modules/sdk-client/sdk-client.module.ts index 0f7eff430ca..712090ea1d7 100644 --- a/packages/twenty-server/src/engine/core-modules/sdk-client/sdk-client.module.ts +++ b/packages/twenty-server/src/engine/core-modules/sdk-client/sdk-client.module.ts @@ -3,6 +3,7 @@ import { TypeOrmModule } from '@nestjs/typeorm'; import { CoreGraphQLApiModule } from 'src/engine/api/graphql/core-graphql-api.module'; import { ApplicationEntity } from 'src/engine/core-modules/application/application.entity'; +import { ApplicationModule } from 'src/engine/core-modules/application/application.module'; import { SdkClientController } from 'src/engine/core-modules/sdk-client/controllers/sdk-client.controller'; import { SdkClientArchiveService } from 'src/engine/core-modules/sdk-client/sdk-client-archive.service'; import { SdkClientGenerationService } from 'src/engine/core-modules/sdk-client/sdk-client-generation.service'; @@ -14,6 +15,7 @@ import { WorkspaceCacheModule } from 'src/engine/workspace-cache/workspace-cache TypeOrmModule.forFeature([ApplicationEntity, WorkspaceEntity]), WorkspaceCacheModule, CoreGraphQLApiModule, + ApplicationModule, ], controllers: [SdkClientController], providers: [SdkClientGenerationService, SdkClientArchiveService], diff --git a/packages/twenty-server/src/engine/core-modules/workspace/services/__tests__/workspace.service.spec.ts b/packages/twenty-server/src/engine/core-modules/workspace/services/__tests__/workspace.service.spec.ts index 1f4f422723f..82e9f4cf4c9 100644 --- a/packages/twenty-server/src/engine/core-modules/workspace/services/__tests__/workspace.service.spec.ts +++ b/packages/twenty-server/src/engine/core-modules/workspace/services/__tests__/workspace.service.spec.ts @@ -39,6 +39,7 @@ import { WorkspaceDataSourceService } from 'src/engine/workspace-datasource/work import { PrefillLogicFunctionService } from 'src/engine/workspace-manager/standard-objects-prefill-data/services/prefill-logic-function.service'; import { ApplicationService } from 'src/engine/core-modules/application/application.service'; import { PreInstalledAppsService } from 'src/engine/core-modules/application/pre-installed-apps/pre-installed-apps.service'; +import { SdkClientGenerationService } from 'src/engine/core-modules/sdk-client/sdk-client-generation.service'; import { WorkspaceMigrationValidateBuildAndRunService } from 'src/engine/workspace-manager/workspace-migration/services/workspace-migration-validate-build-and-run-service'; import { WorkspaceManagerService } from 'src/engine/workspace-manager/workspace-manager.service'; @@ -128,6 +129,7 @@ describe('WorkspaceService', () => { AiModelRegistryService, ApplicationService, PreInstalledAppsService, + SdkClientGenerationService, PrefillLogicFunctionService, WorkspaceMigrationValidateBuildAndRunService, UpgradeMigrationService, diff --git a/packages/twenty-server/src/engine/core-modules/workspace/services/workspace.service.ts b/packages/twenty-server/src/engine/core-modules/workspace/services/workspace.service.ts index 5dab5c1111a..a7f5c094d8a 100644 --- a/packages/twenty-server/src/engine/core-modules/workspace/services/workspace.service.ts +++ b/packages/twenty-server/src/engine/core-modules/workspace/services/workspace.service.ts @@ -58,6 +58,7 @@ import { PermissionsService } from 'src/engine/metadata-modules/permissions/perm import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service'; import { getWorkspaceSchemaName } from 'src/engine/workspace-datasource/utils/get-workspace-schema-name.util'; import { WorkspaceDataSourceService } from 'src/engine/workspace-datasource/workspace-datasource.service'; +import { SdkClientGenerationService } from 'src/engine/core-modules/sdk-client/sdk-client-generation.service'; import { PrefillLogicFunctionService } from 'src/engine/workspace-manager/standard-objects-prefill-data/services/prefill-logic-function.service'; import { prefillCompanies } from 'src/engine/workspace-manager/standard-objects-prefill-data/utils/prefill-companies.util'; import { prefillDashboards } from 'src/engine/workspace-manager/standard-objects-prefill-data/utils/prefill-dashboards.util'; @@ -136,6 +137,7 @@ export class WorkspaceService extends TypeOrmQueryService { private readonly coreEntityCacheService: CoreEntityCacheService, private readonly upgradeMigrationService: UpgradeMigrationService, private readonly upgradeSequenceReaderService: UpgradeSequenceReaderService, + private readonly sdkClientGenerationService: SdkClientGenerationService, ) { super(workspaceRepository); } @@ -369,6 +371,18 @@ export class WorkspaceService extends TypeOrmQueryService { displayName: data.displayName, }); + try { + await this.sdkClientGenerationService.enqueueSdkClientGenerationForWorkspace( + workspace.id, + ); + } catch (error) { + this.logger.error( + `failed to enqueue SDK client generation jobs for workspace ${workspace.id}`, + error, + ); + this.exceptionHandlerService.captureExceptions([error as Error]); + } + await this.coreEntityCacheService.invalidate( 'workspaceEntity', workspace.id, diff --git a/packages/twenty-server/src/engine/core-modules/workspace/workspace.module.ts b/packages/twenty-server/src/engine/core-modules/workspace/workspace.module.ts index 828a93a9fbf..9e58e342921 100644 --- a/packages/twenty-server/src/engine/core-modules/workspace/workspace.module.ts +++ b/packages/twenty-server/src/engine/core-modules/workspace/workspace.module.ts @@ -21,6 +21,7 @@ import { FileModule } from 'src/engine/core-modules/file/file.module'; import { MetricsModule } from 'src/engine/core-modules/metrics/metrics.module'; import { OnboardingModule } from 'src/engine/core-modules/onboarding/onboarding.module'; import { PublicDomainEntity } from 'src/engine/core-modules/public-domain/public-domain.entity'; +import { SdkClientModule } from 'src/engine/core-modules/sdk-client/sdk-client.module'; import { UserWorkspaceEntity } from 'src/engine/core-modules/user-workspace/user-workspace.entity'; import { UserWorkspaceModule } from 'src/engine/core-modules/user-workspace/user-workspace.module'; import { UserEntity } from 'src/engine/core-modules/user/user.entity'; @@ -89,6 +90,7 @@ import { StandardObjectsPrefillModule } from 'src/engine/workspace-manager/stand WorkspaceMigrationModule, CoreEntityCacheModule, UpgradeModule, + SdkClientModule, ], services: [WorkspaceService], resolvers: workspaceAutoResolverOpts, diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-manager.service.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-manager.service.ts index d9165596c27..8404f5e981d 100644 --- a/packages/twenty-server/src/engine/workspace-manager/workspace-manager.service.ts +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-manager.service.ts @@ -5,13 +5,6 @@ import { Repository } from 'typeorm'; import { ApplicationService } from 'src/engine/core-modules/application/application.service'; import { FlatApplication } from 'src/engine/core-modules/application/types/flat-application.type'; -import { InjectMessageQueue } from 'src/engine/core-modules/message-queue/decorators/message-queue.decorator'; -import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants'; -import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service'; -import { - GenerateSdkClientJob, - GenerateSdkClientJobData, -} from 'src/engine/core-modules/sdk-client/jobs/generate-sdk-client.job'; import { UserWorkspaceEntity } from 'src/engine/core-modules/user-workspace/user-workspace.entity'; import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity'; import { RoleEntity } from 'src/engine/metadata-modules/role/role.entity'; @@ -37,8 +30,6 @@ export class WorkspaceManagerService { @InjectRepository(RoleEntity) private readonly roleRepository: Repository, private readonly applicationService: ApplicationService, - @InjectMessageQueue(MessageQueue.workspaceQueue) - private readonly messageQueueService: MessageQueueService, ) {} public async init({ @@ -83,34 +74,13 @@ export class WorkspaceManagerService { `Metadata creation took ${dataSourceMetadataCreationEnd - dataSourceMetadataCreationStart}ms`, ); - const { workspaceCustomFlatApplication, twentyStandardFlatApplication } = + const { workspaceCustomFlatApplication } = await this.applicationService.findWorkspaceTwentyStandardAndCustomApplicationOrThrow( { workspaceId, }, ); - await Promise.all([ - this.messageQueueService.add( - GenerateSdkClientJob.name, - { - workspaceId, - applicationId: twentyStandardFlatApplication.id, - applicationUniversalIdentifier: - twentyStandardFlatApplication.universalIdentifier, - }, - ), - this.messageQueueService.add( - GenerateSdkClientJob.name, - { - workspaceId, - applicationId: workspaceCustomFlatApplication.id, - applicationUniversalIdentifier: - workspaceCustomFlatApplication.universalIdentifier, - }, - ), - ]); - await this.setupDefaultRoles({ workspaceId, userId,