Refactor client auto heal (#19032)

packages/twenty-server/src/database/commands/upgrade-version-command/1-20/1-20-generate-application-sdk-clients.command.ts

@@ -1,67 +0,0 @@
-import { InjectRepository } from '@nestjs/typeorm';
-
-import { Command } from 'nest-commander';
-import { Repository } from 'typeorm';
-
-import { ActiveOrSuspendedWorkspacesMigrationCommandRunner } from 'src/database/commands/command-runners/active-or-suspended-workspaces-migration.command-runner';
-import { RunOnWorkspaceArgs } from 'src/database/commands/command-runners/workspaces-migration.command-runner';
-import { ApplicationEntity } from 'src/engine/core-modules/application/application.entity';
-import { SdkClientGenerationService } from 'src/engine/core-modules/sdk-client/sdk-client-generation.service';
-import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
-import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service';
-import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
-
-@Command({
-  name: 'upgrade:1-20:generate-application-sdk-clients',
-  description:
-    'Generate SDK client archives for all existing applications so drivers do not crash with ARCHIVE_NOT_FOUND',
-})
-export class GenerateApplicationSdkClientsCommand extends ActiveOrSuspendedWorkspacesMigrationCommandRunner {
-  constructor(
-    @InjectRepository(WorkspaceEntity)
-    protected readonly workspaceRepository: Repository<WorkspaceEntity>,
-    protected readonly twentyORMGlobalManager: GlobalWorkspaceOrmManager,
-    protected readonly dataSourceService: DataSourceService,
-    @InjectRepository(ApplicationEntity)
-    private readonly applicationRepository: Repository<ApplicationEntity>,
-    private readonly sdkClientGenerationService: SdkClientGenerationService,
-  ) {
-    super(workspaceRepository, twentyORMGlobalManager, dataSourceService);
-  }
-
-  override async runOnWorkspace({
-    workspaceId,
-    options,
-  }: RunOnWorkspaceArgs): Promise<void> {
-    const dryRun = options.dryRun ?? false;
-
-    const applications = await this.applicationRepository.find({
-      where: { workspaceId },
-    });
-
-    this.logger.log(
-      `Found ${applications.length} application(s) in workspace ${workspaceId}`,
-    );
-
-    for (const application of applications) {
-      if (dryRun) {
-        this.logger.log(
-          `[DRY RUN] Would generate SDK client for application ${application.universalIdentifier} (${application.id})`,
-        );
-        continue;
-      }
-
-      try {
-        await this.sdkClientGenerationService.generateSdkClientForApplication({
-          workspaceId,
-          applicationId: application.id,
-          applicationUniversalIdentifier: application.universalIdentifier,
-        });
-      } catch (error) {
-        this.logger.error(
-          `Failed to generate SDK client for application ${application.universalIdentifier} (${application.id}): ${error instanceof Error ? error.message : String(error)}`,
-        );
-      }
-    }
-  }
-}

packages/twenty-server/src/database/commands/upgrade-version-command/1-20/1-20-upgrade-version-command.module.ts

@@ -6,7 +6,6 @@ import { BackfillNavigationMenuItemTypeCommand } from 'src/database/commands/upg
 import { BackfillPageLayoutsAndFieldsWidgetViewFieldsCommand } from 'src/database/commands/upgrade-version-command/1-20/1-20-backfill-page-layouts-and-fields-widget-view-fields.command';
 import { BackfillSelectFieldOptionIdsCommand } from 'src/database/commands/upgrade-version-command/1-20/1-20-backfill-select-field-option-ids.command';
 import { DeleteOrphanNavigationMenuItemsCommand } from 'src/database/commands/upgrade-version-command/1-20/1-20-delete-orphan-navigation-menu-items.command';
-import { GenerateApplicationSdkClientsCommand } from 'src/database/commands/upgrade-version-command/1-20/1-20-generate-application-sdk-clients.command';
 import { IdentifyObjectPermissionMetadataCommand } from 'src/database/commands/upgrade-version-command/1-20/1-20-identify-object-permission-metadata.command';
 import { IdentifyPermissionFlagMetadataCommand } from 'src/database/commands/upgrade-version-command/1-20/1-20-identify-permission-flag-metadata.command';
 import { MakeObjectPermissionUniversalIdentifierAndApplicationIdNotNullableMigrationCommand } from 'src/database/commands/upgrade-version-command/1-20/1-20-make-object-permission-universal-identifier-and-application-id-not-nullable-migration.command';
@@ -17,10 +16,8 @@ import { MigrateRichTextToTextCommand } from 'src/database/commands/upgrade-vers
 import { SeedCliApplicationRegistrationCommand } from 'src/database/commands/upgrade-version-command/1-20/1-20-seed-cli-application-registration.command';
 import { UpdateStandardIndexViewNamesCommand } from 'src/database/commands/upgrade-version-command/1-20/1-20-update-standard-index-view-names.command';
 import { ApplicationRegistrationModule } from 'src/engine/core-modules/application/application-registration/application-registration.module';
-import { ApplicationEntity } from 'src/engine/core-modules/application/application.entity';
 import { ApplicationModule } from 'src/engine/core-modules/application/application.module';
 import { FeatureFlagModule } from 'src/engine/core-modules/feature-flag/feature-flag.module';
-import { SdkClientModule } from 'src/engine/core-modules/sdk-client/sdk-client.module';
 import { UserWorkspaceEntity } from 'src/engine/core-modules/user-workspace/user-workspace.entity';
 import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
 import { CalendarChannelEntity } from 'src/engine/metadata-modules/calendar-channel/entities/calendar-channel.entity';
@@ -40,7 +37,6 @@ import { WorkflowCommonModule } from 'src/modules/workflow/common/workflow-commo
   imports: [
     TypeOrmModule.forFeature([
       WorkspaceEntity,
-      ApplicationEntity,
       ConnectedAccountEntity,
       MessageChannelEntity,
       CalendarChannelEntity,
@@ -57,7 +53,6 @@ import { WorkflowCommonModule } from 'src/modules/workflow/common/workflow-commo
     ApplicationRegistrationModule,
     WorkspaceMigrationModule,
     FeatureFlagModule,
-    SdkClientModule,
     WorkflowCommonModule,
   ],
   providers: [
@@ -70,7 +65,6 @@ import { WorkflowCommonModule } from 'src/modules/workflow/common/workflow-commo
     BackfillPageLayoutsAndFieldsWidgetViewFieldsCommand,
     BackfillSelectFieldOptionIdsCommand,
     DeleteOrphanNavigationMenuItemsCommand,
-    GenerateApplicationSdkClientsCommand,
     SeedCliApplicationRegistrationCommand,
     MigrateRichTextToTextCommand,
     MigrateMessagingInfrastructureToMetadataCommand,
@@ -87,7 +81,6 @@ import { WorkflowCommonModule } from 'src/modules/workflow/common/workflow-commo
     BackfillPageLayoutsAndFieldsWidgetViewFieldsCommand,
     BackfillSelectFieldOptionIdsCommand,
     DeleteOrphanNavigationMenuItemsCommand,
-    GenerateApplicationSdkClientsCommand,
     SeedCliApplicationRegistrationCommand,
     MigrateRichTextToTextCommand,
     MigrateMessagingInfrastructureToMetadataCommand,

packages/twenty-server/src/database/commands/upgrade-version-command/upgrade.command.ts

@@ -38,7 +38,6 @@ import { BackfillNavigationMenuItemTypeCommand } from 'src/database/commands/upg
 import { BackfillPageLayoutsAndFieldsWidgetViewFieldsCommand } from 'src/database/commands/upgrade-version-command/1-20/1-20-backfill-page-layouts-and-fields-widget-view-fields.command';
 import { BackfillSelectFieldOptionIdsCommand } from 'src/database/commands/upgrade-version-command/1-20/1-20-backfill-select-field-option-ids.command';
 import { DeleteOrphanNavigationMenuItemsCommand } from 'src/database/commands/upgrade-version-command/1-20/1-20-delete-orphan-navigation-menu-items.command';
-import { GenerateApplicationSdkClientsCommand } from 'src/database/commands/upgrade-version-command/1-20/1-20-generate-application-sdk-clients.command';
 import { IdentifyObjectPermissionMetadataCommand } from 'src/database/commands/upgrade-version-command/1-20/1-20-identify-object-permission-metadata.command';
 import { IdentifyPermissionFlagMetadataCommand } from 'src/database/commands/upgrade-version-command/1-20/1-20-identify-permission-flag-metadata.command';
 import { MakeObjectPermissionUniversalIdentifierAndApplicationIdNotNullableMigrationCommand } from 'src/database/commands/upgrade-version-command/1-20/1-20-make-object-permission-universal-identifier-and-application-id-not-nullable-migration.command';
@@ -106,7 +105,6 @@ export class UpgradeCommand extends UpgradeCommandRunner {
     protected readonly backfillCommandMenuItemsCommand: BackfillCommandMenuItemsCommand,
     protected readonly deleteOrphanNavigationMenuItemsCommand: DeleteOrphanNavigationMenuItemsCommand,
     protected readonly backfillPageLayoutsAndFieldsWidgetViewFieldsCommand: BackfillPageLayoutsAndFieldsWidgetViewFieldsCommand,
-    protected readonly generateApplicationSdkClientsCommand: GenerateApplicationSdkClientsCommand,
     protected readonly seedCliApplicationRegistrationCommand: SeedCliApplicationRegistrationCommand,
     protected readonly migrateRichTextToTextCommand: MigrateRichTextToTextCommand,
     protected readonly migrateMessagingInfrastructureToMetadataCommand: MigrateMessagingInfrastructureToMetadataCommand,
@@ -176,7 +174,6 @@ export class UpgradeCommand extends UpgradeCommandRunner {
       this.backfillSelectFieldOptionIdsCommand,
       this.updateStandardIndexViewNamesCommand,
       this.makeWorkflowSearchableCommand,
-      this.generateApplicationSdkClientsCommand,
     ];
 
     this.allCommands = {

packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda.driver.ts

@@ -721,6 +721,7 @@ export class LambdaDriver implements LogicFunctionDriver {
     const sdkArchiveBuffer =
       await this.sdkClientArchiveService.downloadArchiveBuffer({
         workspaceId: flatApplication.workspaceId,
+        applicationId: flatApplication.id,
         applicationUniversalIdentifier,
       });
 

packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/local.driver.ts

@@ -113,6 +113,7 @@ export class LocalDriver implements LogicFunctionDriver {
 
     await this.sdkClientArchiveService.downloadAndExtractToPackage({
       workspaceId: flatApplication.workspaceId,
+      applicationId: flatApplication.id,
       applicationUniversalIdentifier,
       targetPackagePath: sdkPackagePath,
     });

packages/twenty-server/src/engine/core-modules/sdk-client/controllers/sdk-client.controller.ts

@@ -59,6 +59,7 @@ export class SdkClientController {
     const fileBuffer =
       await this.sdkClientArchiveService.getClientModuleFromArchive({
         workspaceId: workspace.id,
+        applicationId,
         applicationUniversalIdentifier: application.universalIdentifier,
         moduleName,
       });

packages/twenty-server/src/engine/core-modules/sdk-client/sdk-client-archive.service.ts

@@ -1,11 +1,7 @@
-import { Injectable } from '@nestjs/common';
+import { Injectable, Logger } from '@nestjs/common';
 import { InjectRepository } from '@nestjs/typeorm';
 
-import { createWriteStream } from 'fs';
 import * as fs from 'fs/promises';
-import { join } from 'path';
-import { type Readable } from 'stream';
-import { pipeline } from 'stream/promises';
 
 import { FileFolder } from 'twenty-shared/types';
 import { Repository } from 'typeorm';
@@ -16,12 +12,12 @@ import {
   FileStorageException,
   FileStorageExceptionCode,
 } from 'src/engine/core-modules/file-storage/interfaces/file-storage-exception';
-import { TemporaryDirManager } from 'src/engine/core-modules/logic-function/logic-function-drivers/utils/temporary-dir-manager';
 import { type SdkModuleName } from 'src/engine/core-modules/sdk-client/constants/allowed-sdk-modules';
 import {
   SdkClientException,
   SdkClientExceptionCode,
 } from 'src/engine/core-modules/sdk-client/exceptions/sdk-client.exception';
+import { SdkClientGenerationService } from 'src/engine/core-modules/sdk-client/sdk-client-generation.service';
 import { WorkspaceCacheService } from 'src/engine/workspace-cache/services/workspace-cache.service';
 import { streamToBuffer } from 'src/utils/stream-to-buffer';
 
@@ -29,75 +25,74 @@ const SDK_CLIENT_ARCHIVE_NAME = 'twenty-client-sdk.zip';
 
 @Injectable()
 export class SdkClientArchiveService {
+  private readonly logger = new Logger(SdkClientArchiveService.name);
+
   constructor(
     private readonly fileStorageService: FileStorageService,
     @InjectRepository(ApplicationEntity)
     private readonly applicationRepository: Repository<ApplicationEntity>,
     private readonly workspaceCacheService: WorkspaceCacheService,
+    private readonly sdkClientGenerationService: SdkClientGenerationService,
   ) {}
 
   async downloadAndExtractToPackage({
     workspaceId,
+    applicationId,
     applicationUniversalIdentifier,
     targetPackagePath,
   }: {
     workspaceId: string;
+    applicationId: string;
     applicationUniversalIdentifier: string;
     targetPackagePath: string;
   }): Promise<void> {
-    const temporaryDirManager = new TemporaryDirManager();
+    const archiveBuffer = await this.downloadArchiveBufferOrGenerate({
+      workspaceId,
+      applicationId,
+      applicationUniversalIdentifier,
+    });
 
-    try {
-      const { sourceTemporaryDir } = await temporaryDirManager.init();
-      const archivePath = join(sourceTemporaryDir, SDK_CLIENT_ARCHIVE_NAME);
+    await fs.rm(targetPackagePath, { recursive: true, force: true });
+    await fs.mkdir(targetPackagePath, { recursive: true });
 
-      const archiveStream = await this.readArchiveStream({
-        workspaceId,
-        applicationUniversalIdentifier,
-      });
+    const { default: unzipper } = await import('unzipper');
+    const directory = await unzipper.Open.buffer(archiveBuffer);
 
-      await pipeline(archiveStream, createWriteStream(archivePath));
-
-      await fs.rm(targetPackagePath, { recursive: true, force: true });
-      await fs.mkdir(targetPackagePath, { recursive: true });
-
-      const { default: unzipper } = await import('unzipper');
-      const directory = await unzipper.Open.file(archivePath);
-
-      await directory.extract({ path: targetPackagePath });
-    } finally {
-      await temporaryDirManager.clean();
-    }
+    await directory.extract({ path: targetPackagePath });
   }
 
   async downloadArchiveBuffer({
     workspaceId,
+    applicationId,
     applicationUniversalIdentifier,
   }: {
     workspaceId: string;
+    applicationId: string;
     applicationUniversalIdentifier: string;
   }): Promise<Buffer> {
-    const archiveStream = await this.readArchiveStream({
+    return this.downloadArchiveBufferOrGenerate({
       workspaceId,
+      applicationId,
       applicationUniversalIdentifier,
     });
-
-    return streamToBuffer(archiveStream);
   }
 
   async getClientModuleFromArchive({
     workspaceId,
+    applicationId,
     applicationUniversalIdentifier,
     moduleName,
   }: {
     workspaceId: string;
+    applicationId: string;
     applicationUniversalIdentifier: string;
     moduleName: SdkModuleName;
   }): Promise<Buffer> {
     const filePath = `dist/${moduleName}.mjs`;
 
-    const archiveBuffer = await this.downloadArchiveBuffer({
+    const archiveBuffer = await this.downloadArchiveBufferOrGenerate({
       workspaceId,
+      applicationId,
       applicationUniversalIdentifier,
     });
 
@@ -135,32 +130,41 @@ export class SdkClientArchiveService {
     ]);
   }
 
-  private async readArchiveStream({
+  private async downloadArchiveBufferOrGenerate({
     workspaceId,
+    applicationId,
     applicationUniversalIdentifier,
   }: {
     workspaceId: string;
+    applicationId: string;
     applicationUniversalIdentifier: string;
-  }): Promise<Readable> {
+  }): Promise<Buffer> {
     try {
-      return await this.fileStorageService.readFile({
+      const stream = await this.fileStorageService.readFile({
         workspaceId,
         applicationUniversalIdentifier,
         fileFolder: FileFolder.GeneratedSdkClient,
         resourcePath: SDK_CLIENT_ARCHIVE_NAME,
       });
+
+      return await streamToBuffer(stream);
     } catch (error) {
       if (
-        error instanceof FileStorageException &&
-        error.code === FileStorageExceptionCode.FILE_NOT_FOUND
+        !(error instanceof FileStorageException) ||
+        error.code !== FileStorageExceptionCode.FILE_NOT_FOUND
       ) {
-        throw new SdkClientException(
-          `SDK client archive "${SDK_CLIENT_ARCHIVE_NAME}" not found for application "${applicationUniversalIdentifier}" in workspace "${workspaceId}".`,
-          SdkClientExceptionCode.ARCHIVE_NOT_FOUND,
-        );
+        throw error;
       }
-
-      throw error;
     }
+
+    this.logger.warn(
+      `SDK client archive missing for application "${applicationUniversalIdentifier}" in workspace "${workspaceId}", generating on-the-fly`,
+    );
+
+    return this.sdkClientGenerationService.generateSdkClientForApplication({
+      workspaceId,
+      applicationId,
+      applicationUniversalIdentifier,
+    });
   }
 }

packages/twenty-server/src/engine/core-modules/sdk-client/sdk-client-generation.service.ts

@@ -44,13 +44,13 @@ export class SdkClientGenerationService {
     workspaceId: string;
     applicationId: string;
     applicationUniversalIdentifier: string;
-  }): Promise<void> {
+  }): Promise<Buffer> {
     const graphqlSchema = await this.workspaceSchemaFactory.createGraphQLSchema(
       { id: workspaceId } as WorkspaceEntity,
       applicationId,
     );
 
-    await this.generateAndStore({
+    const archiveBuffer = await this.generateAndStore({
       workspaceId,
       applicationId,
       applicationUniversalIdentifier,
@@ -60,6 +60,8 @@ export class SdkClientGenerationService {
     this.logger.log(
       `Generated SDK client for application ${applicationUniversalIdentifier}`,
     );
+
+    return archiveBuffer;
   }
 
   private async generateAndStore({
@@ -72,7 +74,7 @@ export class SdkClientGenerationService {
     applicationId: string;
     applicationUniversalIdentifier: string;
     schema: string;
-  }): Promise<void> {
+  }): Promise<Buffer> {
     const temporaryDirManager = new TemporaryDirManager();
 
     try {
@@ -101,12 +103,14 @@ export class SdkClientGenerationService {
 
       await createZipFile(tempPackageRoot, archivePath);
 
+      const archiveBuffer = await fs.readFile(archivePath);
+
       await this.fileStorageService.writeFile({
         workspaceId,
         applicationUniversalIdentifier,
         fileFolder: FileFolder.GeneratedSdkClient,
         resourcePath: SDK_CLIENT_ARCHIVE_NAME,
-        sourceFile: await fs.readFile(archivePath),
+        sourceFile: archiveBuffer,
         mimeType: 'application/zip',
         settings: { isTemporaryFile: false, toDelete: false },
       });
@@ -119,6 +123,8 @@ export class SdkClientGenerationService {
       await this.workspaceCacheService.invalidateAndRecompute(workspaceId, [
         'flatApplicationMaps',
       ]);
+
+      return archiveBuffer;
     } catch (error) {
       throw new SdkClientException(
         `Failed to generate SDK client for application "${applicationUniversalIdentifier}" in workspace "${workspaceId}": ${error instanceof Error ? error.message : String(error)}`,