feat(downloads): add resume-safe pause action (#11222)

Give gallery operations distinct pause and cancel paths. Pause preserves partial download data so reinstalling the same model or backend resumes through HTTP Range, while cancel keeps its destructive semantics. Surface the action in the Activity UI and document the API behavior.

Assisted-by: Codex:gpt-5

Co-authored-by: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com>

core/http/react-ui/e2e/activity-page.spec.js

@@ -43,6 +43,40 @@ test('lists live operations and cancels one from a labelled button', async ({ pa
   expect(cancelledPath).toBe('/api/operations/job-gemma/cancel')
 })
 
+test('pauses a model download without invoking destructive cancel', async ({ page }) => {
+  await stub(page, {
+    operations: [{
+      id: 'gemma-3-27b-it',
+      name: 'gemma-3-27b-it',
+      jobID: 'job-gemma',
+      progress: 22,
+      taskType: 'installation',
+      isBackend: false,
+      isQueued: false,
+      isDeletion: false,
+      cancellable: true,
+      phase: 'downloading',
+    }],
+  })
+
+  const requests = []
+  await page.route('**/api/operations/job-gemma/pause', (route) => {
+    requests.push(new URL(route.request().url()).pathname)
+    return route.fulfill({ contentType: 'application/json', body: '{}' })
+  })
+  await page.route('**/api/operations/job-gemma/cancel', (route) => {
+    requests.push(new URL(route.request().url()).pathname)
+    return route.fulfill({ contentType: 'application/json', body: '{}' })
+  })
+
+  await page.goto('/app/activity')
+
+  const card = page.locator('.operation-card').filter({ hasText: 'gemma-3-27b-it' })
+  await card.locator('.operation-card__pause').click()
+
+  await expect.poll(() => requests).toEqual(['/api/operations/job-gemma/pause'])
+})
+
 test('separates an unacknowledged failure from the record', async ({ page }) => {
   await stub(page, {
     operations: [{

core/http/react-ui/public/locales/de/admin.json

@@ -12,6 +12,8 @@
     "timeLeft": "{{value}} left",
     "cancel": "Cancel",
     "cancelLabel": "Cancel {{name}}",
+    "pause": "Pause",
+    "pauseLabel": "Pause {{name}} and keep downloaded data",
     "retry": "Retry",
     "retryLabel": "Retry {{name}}",
     "nodeCount": "{{count}} nodes",

core/http/react-ui/public/locales/en/admin.json

@@ -12,6 +12,8 @@
     "timeLeft": "{{value}} left",
     "cancel": "Cancel",
     "cancelLabel": "Cancel {{name}}",
+    "pause": "Pause",
+    "pauseLabel": "Pause {{name}} and keep downloaded data",
     "retry": "Retry",
     "retryLabel": "Retry {{name}}",
     "nodeCount": "{{count}} nodes",

core/http/react-ui/public/locales/es/admin.json

@@ -12,6 +12,8 @@
     "timeLeft": "{{value}} left",
     "cancel": "Cancel",
     "cancelLabel": "Cancel {{name}}",
+    "pause": "Pause",
+    "pauseLabel": "Pause {{name}} and keep downloaded data",
     "retry": "Retry",
     "retryLabel": "Retry {{name}}",
     "nodeCount": "{{count}} nodes",

core/http/react-ui/public/locales/id/admin.json

@@ -12,6 +12,8 @@
     "timeLeft": "{{value}} left",
     "cancel": "Cancel",
     "cancelLabel": "Cancel {{name}}",
+    "pause": "Pause",
+    "pauseLabel": "Pause {{name}} and keep downloaded data",
     "retry": "Retry",
     "retryLabel": "Retry {{name}}",
     "nodeCount": "{{count}} nodes",

core/http/react-ui/public/locales/it/admin.json

@@ -12,6 +12,8 @@
     "timeLeft": "{{value}} left",
     "cancel": "Cancel",
     "cancelLabel": "Cancel {{name}}",
+    "pause": "Pause",
+    "pauseLabel": "Pause {{name}} and keep downloaded data",
     "retry": "Retry",
     "retryLabel": "Retry {{name}}",
     "nodeCount": "{{count}} nodes",

core/http/react-ui/public/locales/ko/admin.json

@@ -12,6 +12,8 @@
     "timeLeft": "{{value}} left",
     "cancel": "Cancel",
     "cancelLabel": "Cancel {{name}}",
+    "pause": "Pause",
+    "pauseLabel": "Pause {{name}} and keep downloaded data",
     "retry": "Retry",
     "retryLabel": "Retry {{name}}",
     "nodeCount": "{{count}} nodes",

core/http/react-ui/public/locales/zh-CN/admin.json

@@ -12,6 +12,8 @@
     "timeLeft": "{{value}} left",
     "cancel": "Cancel",
     "cancelLabel": "Cancel {{name}}",
+    "pause": "Pause",
+    "pauseLabel": "Pause {{name}} and keep downloaded data",
     "retry": "Retry",
     "retryLabel": "Retry {{name}}",
     "nodeCount": "{{count}} nodes",

core/http/react-ui/src/components/OperationCard.jsx

@@ -29,7 +29,7 @@ function formatEta(seconds) {
   return `${Math.floor(minutes / 60)}h ${minutes % 60}m`
 }
 
-export default function OperationCard({ operation, onCancel, onDismiss, onRetry }) {
+export default function OperationCard({ operation, onCancel, onPause, onDismiss, onRetry }) {
   const { t } = useTranslation('admin')
   const nodes = Array.isArray(operation.nodes) ? operation.nodes : []
   // Holds only what the user chose. The default has to stay a live
@@ -144,6 +144,16 @@ export default function OperationCard({ operation, onCancel, onDismiss, onRetry
 
         <div className="operation-card__actions">
           {showProgress && <span className="operation-card__pct" aria-hidden="true">{Math.round(operation.progress)}%</span>}
+          {canCancel && (
+            <button
+              type="button"
+              className="btn btn-sm btn-secondary operation-card__pause"
+              onClick={() => onPause?.(operation.jobID)}
+              aria-label={t('activity.pauseLabel', { name })}
+            >
+              {t('activity.pause')}
+            </button>
+          )}
           {canCancel && (
             // A page of cards would otherwise hand a screen reader a list of
             // identical "Cancel" buttons with nothing to tell them apart.

core/http/react-ui/src/contexts/OperationsContext.jsx

@@ -137,6 +137,16 @@ export function OperationsProvider({ children, pollInterval = 1000 }) {
     }
   }, [fetchOperations])
 
+  const pauseOperation = useCallback(async (jobID) => {
+    try {
+      await operationsApi.pause(jobID)
+      cancelledRef.current.set(jobID, Date.now())
+      await fetchOperations()
+    } catch (err) {
+      setError(err.message)
+    }
+  }, [fetchOperations])
+
   // Whether this tab cancelled the job. Read by the strip to tell "the last
   // operation finished" from "the user called it off": both look identical in
   // /api/operations, which lists neither.
@@ -226,6 +236,7 @@ export function OperationsProvider({ children, pollInterval = 1000 }) {
     fetchHistory,
     clearHistory,
     cancelOperation,
+    pauseOperation,
     wasCancelled,
     dismissFailedOp,
     refetch: fetchOperations,

core/http/react-ui/src/pages/Activity.jsx

@@ -83,7 +83,7 @@ export default function Activity() {
   const { t } = useTranslation('admin')
   const outlet = useOutletContext()
   const addToast = outlet?.addToast
-  const { operations, history, fetchHistory, clearHistory, cancelOperation, dismissFailedOp } = useOperations()
+  const { operations, history, fetchHistory, clearHistory, cancelOperation, pauseOperation, dismissFailedOp } = useOperations()
   const [filter, setFilter] = useState('all')
 
   useEffect(() => { fetchHistory() }, [fetchHistory])
@@ -195,7 +195,7 @@ export default function Activity() {
             {t('activity.inProgress')} <span className="activity-section__count">{live.length}</span>
           </h2>
           {live.map((op) => (
-            <OperationCard key={op.jobID || op.id} operation={op} onCancel={cancelOperation} />
+            <OperationCard key={op.jobID || op.id} operation={op} onCancel={cancelOperation} onPause={pauseOperation} />
           ))}
         </section>
       )}

core/http/react-ui/src/utils/api.js

@@ -173,6 +173,7 @@ export const resourcesApi = {
 export const operationsApi = {
   list: () => fetchJSON(API_CONFIG.endpoints.operations),
   cancel: (jobID) => postJSON(API_CONFIG.endpoints.cancelOperation(jobID), {}),
+  pause: (jobID) => postJSON(API_CONFIG.endpoints.pauseOperation(jobID), {}),
   dismiss: (jobID) => postJSON(API_CONFIG.endpoints.dismissOperation(jobID), {}),
   history: () => fetchJSON(API_CONFIG.endpoints.operationsHistory),
   clearHistory: () => fetchJSON(API_CONFIG.endpoints.operationsHistory, { method: 'DELETE' }),

core/http/react-ui/src/utils/config.js

@@ -4,6 +4,7 @@ export const API_CONFIG = {
     operations: '/api/operations',
     operationsHistory: '/api/operations/history',
     cancelOperation: (jobID) => `/api/operations/${jobID}/cancel`,
+    pauseOperation: (jobID) => `/api/operations/${jobID}/pause`,
     dismissOperation: (jobID) => `/api/operations/${jobID}/dismiss`,
 
     // Models gallery

core/http/routes/ui_api.go

@@ -356,6 +356,24 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model
 		})
 	}, adminMiddleware)
 
+	// Pause operation endpoint (admin only). Unlike cancel, pause preserves a
+	// partial download so submitting the same install later resumes it.
+	app.POST("/api/operations/:jobID/pause", func(c echo.Context) error {
+		jobID := c.Param("jobID")
+		xlog.Debug("API request to pause operation", "jobID", jobID)
+
+		if err := galleryService.PauseOperation(jobID); err != nil {
+			xlog.Error("Failed to pause operation", "error", err, "jobID", jobID)
+			return c.JSON(http.StatusBadRequest, map[string]any{"error": err.Error()})
+		}
+
+		opcache.DeleteUUID(jobID)
+		return c.JSON(200, map[string]any{
+			"success": true,
+			"message": "Operation paused",
+		})
+	}, adminMiddleware)
+
 	// Dismiss a failed operation (acknowledge the error and remove it from the list)
 	app.POST("/api/operations/:jobID/dismiss", func(c echo.Context) error {
 		jobID := c.Param("jobID")
@@ -970,7 +988,7 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model
 		uid := id.String()
 		opcache.Set(galleryID, uid)
 
-		ctx, cancelFunc := context.WithCancel(context.Background())
+		ctx, cancelFunc, pauseFunc := galleryop.NewUserCancellableContext(context.Background())
 		op := galleryop.ManagementOp[gallery.GalleryModel, gallery.ModelConfig]{
 			ID:                 uid,
 			GalleryElementName: galleryID,
@@ -979,9 +997,10 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model
 			BackendGalleries:   appConfig.BackendGalleries,
 			Context:            ctx,
 			CancelFunc:         cancelFunc,
+			PauseFunc:          pauseFunc,
 		}
 		// Store cancellation function immediately so queued operations can be cancelled
-		galleryService.StoreCancellation(uid, cancelFunc)
+		galleryService.StoreCancellationActions(uid, cancelFunc, pauseFunc)
 		galleryService.EnqueueModelOp(op)
 
 		return c.JSON(200, map[string]any{
@@ -1017,7 +1036,7 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model
 
 		opcache.Set(galleryID, uid)
 
-		ctx, cancelFunc := context.WithCancel(context.Background())
+		ctx, cancelFunc, pauseFunc := galleryop.NewUserCancellableContext(context.Background())
 		op := galleryop.ManagementOp[gallery.GalleryModel, gallery.ModelConfig]{
 			ID:                 uid,
 			Delete:             true,
@@ -1026,9 +1045,10 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model
 			BackendGalleries:   appConfig.BackendGalleries,
 			Context:            ctx,
 			CancelFunc:         cancelFunc,
+			PauseFunc:          pauseFunc,
 		}
 		// Store cancellation function immediately so queued operations can be cancelled
-		galleryService.StoreCancellation(uid, cancelFunc)
+		galleryService.StoreCancellationActions(uid, cancelFunc, pauseFunc)
 		galleryService.EnqueueModelOp(op)
 		cl.RemoveModelConfig(galleryName)
 
@@ -1423,19 +1443,20 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model
 		uid := id.String()
 		opcache.SetBackend(backendID, uid)
 
-		ctx, cancelFunc := context.WithCancel(context.Background())
+		ctx, cancelFunc, pauseFunc := galleryop.NewUserCancellableContext(context.Background())
 		op := galleryop.ManagementOp[gallery.GalleryBackend, any]{
 			ID:                 uid,
 			GalleryElementName: backendID,
 			Galleries:          appConfig.BackendGalleries,
 			Context:            ctx,
 			CancelFunc:         cancelFunc,
+			PauseFunc:          pauseFunc,
 			// The React UI's "Reinstall backend" action reuses this route, so
 			// the op must force even when the backend is already installed.
 			Force: true,
 		}
 		// Store cancellation function immediately so queued operations can be cancelled
-		galleryService.StoreCancellation(uid, cancelFunc)
+		galleryService.StoreCancellationActions(uid, cancelFunc, pauseFunc)
 		galleryService.EnqueueBackendOp(op)
 
 		return c.JSON(200, map[string]any{
@@ -1485,19 +1506,20 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model
 		}
 		opcache.SetBackend(cacheKey, uid)
 
-		ctx, cancelFunc := context.WithCancel(context.Background())
+		ctx, cancelFunc, pauseFunc := galleryop.NewUserCancellableContext(context.Background())
 		op := galleryop.ManagementOp[gallery.GalleryBackend, any]{
 			ID:                 uid,
 			GalleryElementName: req.Name, // May be empty, will be derived during installation
 			Galleries:          appConfig.BackendGalleries,
 			Context:            ctx,
 			CancelFunc:         cancelFunc,
+			PauseFunc:          pauseFunc,
 			ExternalURI:        req.URI,
 			ExternalName:       req.Name,
 			ExternalAlias:      req.Alias,
 		}
 		// Store cancellation function immediately so queued operations can be cancelled
-		galleryService.StoreCancellation(uid, cancelFunc)
+		galleryService.StoreCancellationActions(uid, cancelFunc, pauseFunc)
 		galleryService.EnqueueBackendOp(op)
 
 		return c.JSON(200, map[string]any{
@@ -1533,7 +1555,7 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model
 
 		opcache.SetBackend(backendID, uid)
 
-		ctx, cancelFunc := context.WithCancel(context.Background())
+		ctx, cancelFunc, pauseFunc := galleryop.NewUserCancellableContext(context.Background())
 		op := galleryop.ManagementOp[gallery.GalleryBackend, any]{
 			ID:                 uid,
 			Delete:             true,
@@ -1541,9 +1563,10 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model
 			Galleries:          appConfig.BackendGalleries,
 			Context:            ctx,
 			CancelFunc:         cancelFunc,
+			PauseFunc:          pauseFunc,
 		}
 		// Store cancellation function immediately so queued operations can be cancelled
-		galleryService.StoreCancellation(uid, cancelFunc)
+		galleryService.StoreCancellationActions(uid, cancelFunc, pauseFunc)
 		galleryService.EnqueueBackendOp(op)
 
 		return c.JSON(200, map[string]any{
@@ -1652,7 +1675,7 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model
 		// and the Backends UI can reflect progress on the affected row.
 		opcache.SetBackend(backendName, uid)
 
-		ctx, cancelFunc := context.WithCancel(context.Background())
+		ctx, cancelFunc, pauseFunc := galleryop.NewUserCancellableContext(context.Background())
 		op := galleryop.ManagementOp[gallery.GalleryBackend, any]{
 			ID:                 uid,
 			GalleryElementName: backendName,
@@ -1660,9 +1683,10 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model
 			Upgrade:            true,
 			Context:            ctx,
 			CancelFunc:         cancelFunc,
+			PauseFunc:          pauseFunc,
 		}
 		// Store cancellation function immediately so queued operations can be cancelled
-		galleryService.StoreCancellation(uid, cancelFunc)
+		galleryService.StoreCancellationActions(uid, cancelFunc, pauseFunc)
 		galleryService.EnqueueBackendOp(op)
 
 		return c.JSON(200, map[string]any{

core/http/routes/ui_api_operations_test.go

@@ -339,6 +339,33 @@ var _ = Describe("/api/operations with node-scoped backend ops", func() {
 		Expect(envelope.Operations[0]).ToNot(HaveKey("isCancelled"))
 	})
 
+	It("pauses through the resume-safe operation callback", func() {
+		state, err := system.GetSystemState(system.WithModelPath(GinkgoT().TempDir()))
+		Expect(err).NotTo(HaveOccurred())
+		appCfg := &config.ApplicationConfig{SystemState: state}
+		galleryService := galleryop.NewGalleryService(appCfg, nil)
+		opcache := galleryop.NewOpCache(galleryService)
+		opcache.Set("localai@gemma", "job-pause")
+
+		var cancelled, paused bool
+		galleryService.StoreCancellationActions(
+			"job-pause",
+			func() { cancelled = true },
+			func() { paused = true },
+		)
+
+		e := echo.New()
+		routes.RegisterUIAPIRoutes(e, nil, nil, appCfg, galleryService, opcache, &application.Application{}, noopMw)
+		req := httptest.NewRequest(http.MethodPost, "/api/operations/job-pause/pause", nil)
+		rec := httptest.NewRecorder()
+		e.ServeHTTP(rec, req)
+
+		Expect(rec.Code).To(Equal(http.StatusOK))
+		Expect(paused).To(BeTrue())
+		Expect(cancelled).To(BeFalse())
+		Expect(opcache.Get("localai@gemma")).To(BeEmpty())
+	})
+
 	It("reports a running removal as a deletion", func() {
 		state, err := system.GetSystemState(system.WithModelPath(GinkgoT().TempDir()))
 		Expect(err).NotTo(HaveOccurred())