From db9da3194fafc591fc53cd5dfb3b84865bfec91d Mon Sep 17 00:00:00 2001 From: Paul Rastoin <45004772+prastoin@users.noreply.github.com> Date: Fri, 27 Mar 2026 10:23:26 +0100 Subject: [PATCH] Refactor client auto heal (#19032) --- ...enerate-application-sdk-clients.command.ts | 67 --------------- .../1-20-upgrade-version-command.module.ts | 7 -- .../upgrade.command.ts | 3 - .../drivers/lambda.driver.ts | 1 + .../drivers/local.driver.ts | 1 + .../controllers/sdk-client.controller.ts | 1 + .../sdk-client/sdk-client-archive.service.ts | 86 ++++++++++--------- .../sdk-client-generation.service.ts | 14 ++- 8 files changed, 58 insertions(+), 122 deletions(-) delete mode 100644 packages/twenty-server/src/database/commands/upgrade-version-command/1-20/1-20-generate-application-sdk-clients.command.ts diff --git a/packages/twenty-server/src/database/commands/upgrade-version-command/1-20/1-20-generate-application-sdk-clients.command.ts b/packages/twenty-server/src/database/commands/upgrade-version-command/1-20/1-20-generate-application-sdk-clients.command.ts deleted file mode 100644 index e88572e5cc5..00000000000 --- a/packages/twenty-server/src/database/commands/upgrade-version-command/1-20/1-20-generate-application-sdk-clients.command.ts +++ /dev/null @@ -1,67 +0,0 @@ -import { InjectRepository } from '@nestjs/typeorm'; - -import { Command } from 'nest-commander'; -import { Repository } from 'typeorm'; - -import { ActiveOrSuspendedWorkspacesMigrationCommandRunner } from 'src/database/commands/command-runners/active-or-suspended-workspaces-migration.command-runner'; -import { RunOnWorkspaceArgs } from 'src/database/commands/command-runners/workspaces-migration.command-runner'; -import { ApplicationEntity } from 'src/engine/core-modules/application/application.entity'; -import { SdkClientGenerationService } from 'src/engine/core-modules/sdk-client/sdk-client-generation.service'; -import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity'; -import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service'; -import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager'; - -@Command({ - name: 'upgrade:1-20:generate-application-sdk-clients', - description: - 'Generate SDK client archives for all existing applications so drivers do not crash with ARCHIVE_NOT_FOUND', -}) -export class GenerateApplicationSdkClientsCommand extends ActiveOrSuspendedWorkspacesMigrationCommandRunner { - constructor( - @InjectRepository(WorkspaceEntity) - protected readonly workspaceRepository: Repository, - protected readonly twentyORMGlobalManager: GlobalWorkspaceOrmManager, - protected readonly dataSourceService: DataSourceService, - @InjectRepository(ApplicationEntity) - private readonly applicationRepository: Repository, - private readonly sdkClientGenerationService: SdkClientGenerationService, - ) { - super(workspaceRepository, twentyORMGlobalManager, dataSourceService); - } - - override async runOnWorkspace({ - workspaceId, - options, - }: RunOnWorkspaceArgs): Promise { - const dryRun = options.dryRun ?? false; - - const applications = await this.applicationRepository.find({ - where: { workspaceId }, - }); - - this.logger.log( - `Found ${applications.length} application(s) in workspace ${workspaceId}`, - ); - - for (const application of applications) { - if (dryRun) { - this.logger.log( - `[DRY RUN] Would generate SDK client for application ${application.universalIdentifier} (${application.id})`, - ); - continue; - } - - try { - await this.sdkClientGenerationService.generateSdkClientForApplication({ - workspaceId, - applicationId: application.id, - applicationUniversalIdentifier: application.universalIdentifier, - }); - } catch (error) { - this.logger.error( - `Failed to generate SDK client for application ${application.universalIdentifier} (${application.id}): ${error instanceof Error ? error.message : String(error)}`, - ); - } - } - } -} diff --git a/packages/twenty-server/src/database/commands/upgrade-version-command/1-20/1-20-upgrade-version-command.module.ts b/packages/twenty-server/src/database/commands/upgrade-version-command/1-20/1-20-upgrade-version-command.module.ts index 7e6da522501..c1d249e6a16 100644 --- a/packages/twenty-server/src/database/commands/upgrade-version-command/1-20/1-20-upgrade-version-command.module.ts +++ b/packages/twenty-server/src/database/commands/upgrade-version-command/1-20/1-20-upgrade-version-command.module.ts @@ -6,7 +6,6 @@ import { BackfillNavigationMenuItemTypeCommand } from 'src/database/commands/upg import { BackfillPageLayoutsAndFieldsWidgetViewFieldsCommand } from 'src/database/commands/upgrade-version-command/1-20/1-20-backfill-page-layouts-and-fields-widget-view-fields.command'; import { BackfillSelectFieldOptionIdsCommand } from 'src/database/commands/upgrade-version-command/1-20/1-20-backfill-select-field-option-ids.command'; import { DeleteOrphanNavigationMenuItemsCommand } from 'src/database/commands/upgrade-version-command/1-20/1-20-delete-orphan-navigation-menu-items.command'; -import { GenerateApplicationSdkClientsCommand } from 'src/database/commands/upgrade-version-command/1-20/1-20-generate-application-sdk-clients.command'; import { IdentifyObjectPermissionMetadataCommand } from 'src/database/commands/upgrade-version-command/1-20/1-20-identify-object-permission-metadata.command'; import { IdentifyPermissionFlagMetadataCommand } from 'src/database/commands/upgrade-version-command/1-20/1-20-identify-permission-flag-metadata.command'; import { MakeObjectPermissionUniversalIdentifierAndApplicationIdNotNullableMigrationCommand } from 'src/database/commands/upgrade-version-command/1-20/1-20-make-object-permission-universal-identifier-and-application-id-not-nullable-migration.command'; @@ -17,10 +16,8 @@ import { MigrateRichTextToTextCommand } from 'src/database/commands/upgrade-vers import { SeedCliApplicationRegistrationCommand } from 'src/database/commands/upgrade-version-command/1-20/1-20-seed-cli-application-registration.command'; import { UpdateStandardIndexViewNamesCommand } from 'src/database/commands/upgrade-version-command/1-20/1-20-update-standard-index-view-names.command'; import { ApplicationRegistrationModule } from 'src/engine/core-modules/application/application-registration/application-registration.module'; -import { ApplicationEntity } from 'src/engine/core-modules/application/application.entity'; import { ApplicationModule } from 'src/engine/core-modules/application/application.module'; import { FeatureFlagModule } from 'src/engine/core-modules/feature-flag/feature-flag.module'; -import { SdkClientModule } from 'src/engine/core-modules/sdk-client/sdk-client.module'; import { UserWorkspaceEntity } from 'src/engine/core-modules/user-workspace/user-workspace.entity'; import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity'; import { CalendarChannelEntity } from 'src/engine/metadata-modules/calendar-channel/entities/calendar-channel.entity'; @@ -40,7 +37,6 @@ import { WorkflowCommonModule } from 'src/modules/workflow/common/workflow-commo imports: [ TypeOrmModule.forFeature([ WorkspaceEntity, - ApplicationEntity, ConnectedAccountEntity, MessageChannelEntity, CalendarChannelEntity, @@ -57,7 +53,6 @@ import { WorkflowCommonModule } from 'src/modules/workflow/common/workflow-commo ApplicationRegistrationModule, WorkspaceMigrationModule, FeatureFlagModule, - SdkClientModule, WorkflowCommonModule, ], providers: [ @@ -70,7 +65,6 @@ import { WorkflowCommonModule } from 'src/modules/workflow/common/workflow-commo BackfillPageLayoutsAndFieldsWidgetViewFieldsCommand, BackfillSelectFieldOptionIdsCommand, DeleteOrphanNavigationMenuItemsCommand, - GenerateApplicationSdkClientsCommand, SeedCliApplicationRegistrationCommand, MigrateRichTextToTextCommand, MigrateMessagingInfrastructureToMetadataCommand, @@ -87,7 +81,6 @@ import { WorkflowCommonModule } from 'src/modules/workflow/common/workflow-commo BackfillPageLayoutsAndFieldsWidgetViewFieldsCommand, BackfillSelectFieldOptionIdsCommand, DeleteOrphanNavigationMenuItemsCommand, - GenerateApplicationSdkClientsCommand, SeedCliApplicationRegistrationCommand, MigrateRichTextToTextCommand, MigrateMessagingInfrastructureToMetadataCommand, diff --git a/packages/twenty-server/src/database/commands/upgrade-version-command/upgrade.command.ts b/packages/twenty-server/src/database/commands/upgrade-version-command/upgrade.command.ts index 017724acac6..e63e62031bc 100644 --- a/packages/twenty-server/src/database/commands/upgrade-version-command/upgrade.command.ts +++ b/packages/twenty-server/src/database/commands/upgrade-version-command/upgrade.command.ts @@ -38,7 +38,6 @@ import { BackfillNavigationMenuItemTypeCommand } from 'src/database/commands/upg import { BackfillPageLayoutsAndFieldsWidgetViewFieldsCommand } from 'src/database/commands/upgrade-version-command/1-20/1-20-backfill-page-layouts-and-fields-widget-view-fields.command'; import { BackfillSelectFieldOptionIdsCommand } from 'src/database/commands/upgrade-version-command/1-20/1-20-backfill-select-field-option-ids.command'; import { DeleteOrphanNavigationMenuItemsCommand } from 'src/database/commands/upgrade-version-command/1-20/1-20-delete-orphan-navigation-menu-items.command'; -import { GenerateApplicationSdkClientsCommand } from 'src/database/commands/upgrade-version-command/1-20/1-20-generate-application-sdk-clients.command'; import { IdentifyObjectPermissionMetadataCommand } from 'src/database/commands/upgrade-version-command/1-20/1-20-identify-object-permission-metadata.command'; import { IdentifyPermissionFlagMetadataCommand } from 'src/database/commands/upgrade-version-command/1-20/1-20-identify-permission-flag-metadata.command'; import { MakeObjectPermissionUniversalIdentifierAndApplicationIdNotNullableMigrationCommand } from 'src/database/commands/upgrade-version-command/1-20/1-20-make-object-permission-universal-identifier-and-application-id-not-nullable-migration.command'; @@ -106,7 +105,6 @@ export class UpgradeCommand extends UpgradeCommandRunner { protected readonly backfillCommandMenuItemsCommand: BackfillCommandMenuItemsCommand, protected readonly deleteOrphanNavigationMenuItemsCommand: DeleteOrphanNavigationMenuItemsCommand, protected readonly backfillPageLayoutsAndFieldsWidgetViewFieldsCommand: BackfillPageLayoutsAndFieldsWidgetViewFieldsCommand, - protected readonly generateApplicationSdkClientsCommand: GenerateApplicationSdkClientsCommand, protected readonly seedCliApplicationRegistrationCommand: SeedCliApplicationRegistrationCommand, protected readonly migrateRichTextToTextCommand: MigrateRichTextToTextCommand, protected readonly migrateMessagingInfrastructureToMetadataCommand: MigrateMessagingInfrastructureToMetadataCommand, @@ -176,7 +174,6 @@ export class UpgradeCommand extends UpgradeCommandRunner { this.backfillSelectFieldOptionIdsCommand, this.updateStandardIndexViewNamesCommand, this.makeWorkflowSearchableCommand, - this.generateApplicationSdkClientsCommand, ]; this.allCommands = { diff --git a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda.driver.ts b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda.driver.ts index 5b75757c8ae..126c1d042b9 100644 --- a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda.driver.ts +++ b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda.driver.ts @@ -721,6 +721,7 @@ export class LambdaDriver implements LogicFunctionDriver { const sdkArchiveBuffer = await this.sdkClientArchiveService.downloadArchiveBuffer({ workspaceId: flatApplication.workspaceId, + applicationId: flatApplication.id, applicationUniversalIdentifier, }); diff --git a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/local.driver.ts b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/local.driver.ts index 72aacd97cd9..9216f296433 100644 --- a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/local.driver.ts +++ b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/local.driver.ts @@ -113,6 +113,7 @@ export class LocalDriver implements LogicFunctionDriver { await this.sdkClientArchiveService.downloadAndExtractToPackage({ workspaceId: flatApplication.workspaceId, + applicationId: flatApplication.id, applicationUniversalIdentifier, targetPackagePath: sdkPackagePath, }); diff --git a/packages/twenty-server/src/engine/core-modules/sdk-client/controllers/sdk-client.controller.ts b/packages/twenty-server/src/engine/core-modules/sdk-client/controllers/sdk-client.controller.ts index 026735eb733..b8752fcab3b 100644 --- a/packages/twenty-server/src/engine/core-modules/sdk-client/controllers/sdk-client.controller.ts +++ b/packages/twenty-server/src/engine/core-modules/sdk-client/controllers/sdk-client.controller.ts @@ -59,6 +59,7 @@ export class SdkClientController { const fileBuffer = await this.sdkClientArchiveService.getClientModuleFromArchive({ workspaceId: workspace.id, + applicationId, applicationUniversalIdentifier: application.universalIdentifier, moduleName, }); diff --git a/packages/twenty-server/src/engine/core-modules/sdk-client/sdk-client-archive.service.ts b/packages/twenty-server/src/engine/core-modules/sdk-client/sdk-client-archive.service.ts index 657c255732d..27402eb90db 100644 --- a/packages/twenty-server/src/engine/core-modules/sdk-client/sdk-client-archive.service.ts +++ b/packages/twenty-server/src/engine/core-modules/sdk-client/sdk-client-archive.service.ts @@ -1,11 +1,7 @@ -import { Injectable } from '@nestjs/common'; +import { Injectable, Logger } from '@nestjs/common'; import { InjectRepository } from '@nestjs/typeorm'; -import { createWriteStream } from 'fs'; import * as fs from 'fs/promises'; -import { join } from 'path'; -import { type Readable } from 'stream'; -import { pipeline } from 'stream/promises'; import { FileFolder } from 'twenty-shared/types'; import { Repository } from 'typeorm'; @@ -16,12 +12,12 @@ import { FileStorageException, FileStorageExceptionCode, } from 'src/engine/core-modules/file-storage/interfaces/file-storage-exception'; -import { TemporaryDirManager } from 'src/engine/core-modules/logic-function/logic-function-drivers/utils/temporary-dir-manager'; import { type SdkModuleName } from 'src/engine/core-modules/sdk-client/constants/allowed-sdk-modules'; import { SdkClientException, SdkClientExceptionCode, } from 'src/engine/core-modules/sdk-client/exceptions/sdk-client.exception'; +import { SdkClientGenerationService } from 'src/engine/core-modules/sdk-client/sdk-client-generation.service'; import { WorkspaceCacheService } from 'src/engine/workspace-cache/services/workspace-cache.service'; import { streamToBuffer } from 'src/utils/stream-to-buffer'; @@ -29,75 +25,74 @@ const SDK_CLIENT_ARCHIVE_NAME = 'twenty-client-sdk.zip'; @Injectable() export class SdkClientArchiveService { + private readonly logger = new Logger(SdkClientArchiveService.name); + constructor( private readonly fileStorageService: FileStorageService, @InjectRepository(ApplicationEntity) private readonly applicationRepository: Repository, private readonly workspaceCacheService: WorkspaceCacheService, + private readonly sdkClientGenerationService: SdkClientGenerationService, ) {} async downloadAndExtractToPackage({ workspaceId, + applicationId, applicationUniversalIdentifier, targetPackagePath, }: { workspaceId: string; + applicationId: string; applicationUniversalIdentifier: string; targetPackagePath: string; }): Promise { - const temporaryDirManager = new TemporaryDirManager(); + const archiveBuffer = await this.downloadArchiveBufferOrGenerate({ + workspaceId, + applicationId, + applicationUniversalIdentifier, + }); - try { - const { sourceTemporaryDir } = await temporaryDirManager.init(); - const archivePath = join(sourceTemporaryDir, SDK_CLIENT_ARCHIVE_NAME); + await fs.rm(targetPackagePath, { recursive: true, force: true }); + await fs.mkdir(targetPackagePath, { recursive: true }); - const archiveStream = await this.readArchiveStream({ - workspaceId, - applicationUniversalIdentifier, - }); + const { default: unzipper } = await import('unzipper'); + const directory = await unzipper.Open.buffer(archiveBuffer); - await pipeline(archiveStream, createWriteStream(archivePath)); - - await fs.rm(targetPackagePath, { recursive: true, force: true }); - await fs.mkdir(targetPackagePath, { recursive: true }); - - const { default: unzipper } = await import('unzipper'); - const directory = await unzipper.Open.file(archivePath); - - await directory.extract({ path: targetPackagePath }); - } finally { - await temporaryDirManager.clean(); - } + await directory.extract({ path: targetPackagePath }); } async downloadArchiveBuffer({ workspaceId, + applicationId, applicationUniversalIdentifier, }: { workspaceId: string; + applicationId: string; applicationUniversalIdentifier: string; }): Promise { - const archiveStream = await this.readArchiveStream({ + return this.downloadArchiveBufferOrGenerate({ workspaceId, + applicationId, applicationUniversalIdentifier, }); - - return streamToBuffer(archiveStream); } async getClientModuleFromArchive({ workspaceId, + applicationId, applicationUniversalIdentifier, moduleName, }: { workspaceId: string; + applicationId: string; applicationUniversalIdentifier: string; moduleName: SdkModuleName; }): Promise { const filePath = `dist/${moduleName}.mjs`; - const archiveBuffer = await this.downloadArchiveBuffer({ + const archiveBuffer = await this.downloadArchiveBufferOrGenerate({ workspaceId, + applicationId, applicationUniversalIdentifier, }); @@ -135,32 +130,41 @@ export class SdkClientArchiveService { ]); } - private async readArchiveStream({ + private async downloadArchiveBufferOrGenerate({ workspaceId, + applicationId, applicationUniversalIdentifier, }: { workspaceId: string; + applicationId: string; applicationUniversalIdentifier: string; - }): Promise { + }): Promise { try { - return await this.fileStorageService.readFile({ + const stream = await this.fileStorageService.readFile({ workspaceId, applicationUniversalIdentifier, fileFolder: FileFolder.GeneratedSdkClient, resourcePath: SDK_CLIENT_ARCHIVE_NAME, }); + + return await streamToBuffer(stream); } catch (error) { if ( - error instanceof FileStorageException && - error.code === FileStorageExceptionCode.FILE_NOT_FOUND + !(error instanceof FileStorageException) || + error.code !== FileStorageExceptionCode.FILE_NOT_FOUND ) { - throw new SdkClientException( - `SDK client archive "${SDK_CLIENT_ARCHIVE_NAME}" not found for application "${applicationUniversalIdentifier}" in workspace "${workspaceId}".`, - SdkClientExceptionCode.ARCHIVE_NOT_FOUND, - ); + throw error; } - - throw error; } + + this.logger.warn( + `SDK client archive missing for application "${applicationUniversalIdentifier}" in workspace "${workspaceId}", generating on-the-fly`, + ); + + return this.sdkClientGenerationService.generateSdkClientForApplication({ + workspaceId, + applicationId, + applicationUniversalIdentifier, + }); } } diff --git a/packages/twenty-server/src/engine/core-modules/sdk-client/sdk-client-generation.service.ts b/packages/twenty-server/src/engine/core-modules/sdk-client/sdk-client-generation.service.ts index 0392df0e774..0e074835a26 100644 --- a/packages/twenty-server/src/engine/core-modules/sdk-client/sdk-client-generation.service.ts +++ b/packages/twenty-server/src/engine/core-modules/sdk-client/sdk-client-generation.service.ts @@ -44,13 +44,13 @@ export class SdkClientGenerationService { workspaceId: string; applicationId: string; applicationUniversalIdentifier: string; - }): Promise { + }): Promise { const graphqlSchema = await this.workspaceSchemaFactory.createGraphQLSchema( { id: workspaceId } as WorkspaceEntity, applicationId, ); - await this.generateAndStore({ + const archiveBuffer = await this.generateAndStore({ workspaceId, applicationId, applicationUniversalIdentifier, @@ -60,6 +60,8 @@ export class SdkClientGenerationService { this.logger.log( `Generated SDK client for application ${applicationUniversalIdentifier}`, ); + + return archiveBuffer; } private async generateAndStore({ @@ -72,7 +74,7 @@ export class SdkClientGenerationService { applicationId: string; applicationUniversalIdentifier: string; schema: string; - }): Promise { + }): Promise { const temporaryDirManager = new TemporaryDirManager(); try { @@ -101,12 +103,14 @@ export class SdkClientGenerationService { await createZipFile(tempPackageRoot, archivePath); + const archiveBuffer = await fs.readFile(archivePath); + await this.fileStorageService.writeFile({ workspaceId, applicationUniversalIdentifier, fileFolder: FileFolder.GeneratedSdkClient, resourcePath: SDK_CLIENT_ARCHIVE_NAME, - sourceFile: await fs.readFile(archivePath), + sourceFile: archiveBuffer, mimeType: 'application/zip', settings: { isTemporaryFile: false, toDelete: false }, }); @@ -119,6 +123,8 @@ export class SdkClientGenerationService { await this.workspaceCacheService.invalidateAndRecompute(workspaceId, [ 'flatApplicationMaps', ]); + + return archiveBuffer; } catch (error) { throw new SdkClientException( `Failed to generate SDK client for application "${applicationUniversalIdentifier}" in workspace "${workspaceId}": ${error instanceof Error ? error.message : String(error)}`,