feat(Git Sync): Downgrade -> Upgrade path (#9882)

* feat: add mechanism to flush newer DB workspaces to disk during downgrade

* feat: implement effective Git repository ID handling for project connections

* feat: enhance Git repository ID handling for improved project queries and updates
This commit is contained in:
James Gatz
2026-04-30 20:24:00 +02:00
committed by GitHub
parent c8dff92dcc
commit bdf74ddea0
13 changed files with 197 additions and 56 deletions

View File

@@ -225,7 +225,8 @@ async function _removeAllCredentials() {
*
*/
async function _removeGitRepository(repo: GitRepository) {
const projects = await database.find<Project>(models.project.type, { gitRepositoryId: repo._id });
const queryIds = models.project.getQueryableGitRepositoryIds(repo._id);
const projects = await database.find<Project>(models.project.type, { gitRepositoryId: { $in: queryIds } });
for (const p of projects) {
await services.project.update(p, { gitRepositoryId: models.project.EMPTY_GIT_PROJECT_ID });
}

View File

@@ -16,8 +16,9 @@ export function getByRemoteId(remoteId: string) {
}
export function getAllByGitRepositoryIds(gitRepositoryIds: string[]) {
const queryIds = gitRepositoryIds.flatMap(id => models.project.getQueryableGitRepositoryIds(id));
return db.find<Project>(type, {
gitRepositoryId: { $in: gitRepositoryIds },
gitRepositoryId: { $in: queryIds },
});
}

View File

@@ -15,10 +15,69 @@ export const SCRATCHPAD_PROJECT_ID = `${prefix}_scratchpad`;
// This is used to identify Git Projects that are not connected to a remote yet
export const EMPTY_GIT_PROJECT_ID = 'empty';
// Prefix used when encoding a GitRepository._id into gitRepositoryId for downgrade protection.
// The real GitRepository doc has _id = 'git_xxx'. We store 'gr_xxx' on the project so that the
// old app's getById('gr_xxx') returns null — preventing it from touching the git folder.
// The new app swaps the prefix back to recover the real ID.
export const PROTECTED_GIT_REPO_PREFIX = 'gr_';
const REAL_GIT_REPO_PREFIX = 'git_';
/**
* Decode a raw gitRepositoryId string to the real GitRepository._id.
* Handles both the protected ('gr_xxx') and legacy ('git_xxx') formats.
*/
export function decodeRepoId(id: string): string {
if (id.startsWith(PROTECTED_GIT_REPO_PREFIX)) {
return REAL_GIT_REPO_PREFIX + id.slice(PROTECTED_GIT_REPO_PREFIX.length);
}
return id;
}
/**
* Given a connected GitProject, return the real GitRepository._id.
* Returns null when the project is not connected (gitRepositoryId is 'empty').
*
* Handles two formats:
* - 'gr_xxx' → protected (new format) → returns 'git_xxx'
* - 'git_xxx' → legacy (pre-migration) → returns 'git_xxx' as-is
*/
export function getEffectiveRepoId(project: GitProject): string | null {
const id = project.gitRepositoryId;
if (id === EMPTY_GIT_PROJECT_ID) return null;
return decodeRepoId(id);
}
/**
* Encode a real GitRepository._id ('git_xxx') into the protected format ('gr_xxx')
* that is stored on the project's gitRepositoryId field.
*/
export function toProtectedRepoId(gitRepositoryId: string): string {
if (gitRepositoryId.startsWith(REAL_GIT_REPO_PREFIX)) {
return PROTECTED_GIT_REPO_PREFIX + gitRepositoryId.slice(REAL_GIT_REPO_PREFIX.length);
}
return gitRepositoryId; // already protected or unexpected format — pass through
}
/**
* Return all values that may be stored in Project.gitRepositoryId for a given real
* GitRepository._id, covering both legacy ('git_xxx') and protected ('gr_xxx') forms.
* Use this when building DB queries that must match projects regardless of which
* storage format they were written with.
*/
export function getQueryableGitRepositoryIds(gitRepositoryId: string): string[] {
const realId = decodeRepoId(gitRepositoryId);
const protectedId = toProtectedRepoId(realId);
return Array.from(new Set([realId, protectedId]));
}
export function isEmptyGitProject(project: Project) {
return project.gitRepositoryId === EMPTY_GIT_PROJECT_ID;
}
export function isConnectedGitProject(project: Project): project is GitProject {
return isGitProject(project) && getEffectiveRepoId(project) !== null;
}
export const isScratchpadProject = (project: Pick<Project, '_id'>) => project._id === SCRATCHPAD_PROJECT_ID;
export const isLocalProject = (project: Pick<Project, 'remoteId'>): project is LocalProject =>
project.remoteId === null;

View File

@@ -218,12 +218,10 @@ async function getGitRepository({ projectId, workspaceId }: { projectId: string;
invariant(projectId, 'Project ID is required');
const project = await services.project.getById(projectId);
invariant(project, 'Project not found');
invariant(project.gitRepositoryId, 'Project is not linked to a git repository');
invariant(
project.gitRepositoryId && !models.project.isEmptyGitProject(project),
'Project is not linked to a git repository',
);
const gitRepository = await services.gitRepository.getById(project.gitRepositoryId);
invariant(models.project.isConnectedGitProject(project), 'Project is not linked to a git repository');
const repoId = models.project.getEffectiveRepoId(project);
invariant(repoId, 'Project is not linked to a git repository');
const gitRepository = await services.gitRepository.getById(repoId);
invariant(gitRepository, 'Git Repository not found');
return gitRepository;
}
@@ -293,22 +291,18 @@ export async function getProjectGitFileIssues({
gitRepositoryId,
}: GetProjectGitFileIssuesOptions): Promise<WorkspaceFileIssue[]> {
const project = await services.project.getById(projectId);
if (
!project ||
!models.project.isGitProject(project) ||
!project.gitRepositoryId ||
models.project.isEmptyGitProject(project)
) {
if (!project || !models.project.isConnectedGitProject(project)) {
return [];
}
if (gitRepositoryId && gitRepositoryId !== project.gitRepositoryId) {
const effectiveRepoId = models.project.getEffectiveRepoId(project);
if (gitRepositoryId && gitRepositoryId !== effectiveRepoId) {
return [];
}
return mapWorkspaceFileIssues({
issues: repoFileWatcherRegistry.getProblems(project.gitRepositoryId),
repoId: project.gitRepositoryId,
issues: repoFileWatcherRegistry.getProblems(effectiveRepoId!),
repoId: effectiveRepoId!,
metas: await getProjectWorkspacesWithMeta(projectId),
workspaceId,
});
@@ -1168,7 +1162,7 @@ export const cloneGitRepoAction = async ({
await services.project.update(project, {
remoteId: null,
gitRepositoryId: gitRepository._id,
gitRepositoryId: models.project.toProtectedRepoId(gitRepository._id),
});
return project;
@@ -1177,7 +1171,7 @@ export const cloneGitRepoAction = async ({
const project = await services.project.create({
name: name || gitRepository.uri.split('/').pop() || 'New Git Project',
parentId: organizationId,
gitRepositoryId: gitRepository._id,
gitRepositoryId: models.project.toProtectedRepoId(gitRepository._id),
});
return project;
@@ -1483,7 +1477,8 @@ export const updateGitRepoAction = async ({
let gitRepository: GitRepository | undefined;
if (gitRepositoryId && gitRepositoryId !== models.project.EMPTY_GIT_PROJECT_ID) {
gitRepository = await services.gitRepository.getById(gitRepositoryId);
const effectiveId = models.project.decodeRepoId(gitRepositoryId);
gitRepository = await services.gitRepository.getById(effectiveId);
invariant(gitRepository, 'GitRepository not found');
} else {
const newRepo: Partial<GitRepository> = {
@@ -1505,7 +1500,7 @@ export const updateGitRepoAction = async ({
const project = await services.project.getById(projectId);
invariant(project, 'Project not found');
await services.project.update(project, {
gitRepositoryId: gitRepository._id,
gitRepositoryId: models.project.toProtectedRepoId(gitRepository._id),
});
}
@@ -2898,14 +2893,12 @@ export async function runAllGitRepoMigrations(): Promise<MigrationSummary> {
const failedProjects: { id: string; name: string }[] = [];
const allProjects = await services.project.all();
const gitProjects = allProjects.filter(
(p): p is GitProject => models.project.isGitProject(p) && !models.project.isEmptyGitProject(p),
);
const gitProjects = allProjects.filter((p): p is GitProject => models.project.isConnectedGitProject(p));
if (gitProjects.length === 0) return { logs, failedProjects };
// Batch-fetch all git repositories in one query instead of N individual lookups.
const repoIds = gitProjects.map(p => p.gitRepositoryId);
const repoIds = gitProjects.map(p => models.project.getEffectiveRepoId(p)).filter(Boolean) as string[];
const gitRepositories = await database.find<GitRepository>(models.gitRepository.type, {
_id: { $in: repoIds },
});
@@ -2922,7 +2915,7 @@ export async function runAllGitRepoMigrations(): Promise<MigrationSummary> {
await Promise.all(
gitProjects.map(async project => {
const gitRepository = repoById.get(project.gitRepositoryId);
const gitRepository = repoById.get(models.project.getEffectiveRepoId(project)!);
if (!gitRepository) return;
const repoId = gitRepository._id;
@@ -2950,15 +2943,16 @@ export async function runAllGitRepoMigrations(): Promise<MigrationSummary> {
logs.push(`${ts()} [INFO] ["${name}"] Converting to local project`);
try {
const project = await services.project.getById(id);
if (!project || !project.gitRepositoryId) {
if (!project || !models.project.isConnectedGitProject(project)) {
logs.push(`${ts()} [WARN] ["${name}"] Project not found or already local — skipping`);
return;
}
const gitRepository = await services.gitRepository.getById(project.gitRepositoryId);
const effectiveRepoId = models.project.getEffectiveRepoId(project as GitProject);
const gitRepository = effectiveRepoId ? await services.gitRepository.getById(effectiveRepoId) : null;
if (gitRepository) {
await services.gitRepository.remove(gitRepository);
logs.push(`${ts()} [INFO] ["${name}"] Removed git repository ${project.gitRepositoryId}`);
logs.push(`${ts()} [INFO] ["${name}"] Removed git repository ${effectiveRepoId}`);
}
await services.project.update(project, { name, gitRepositoryId: null });

View File

@@ -22,18 +22,17 @@ export async function clientLoader() {
const organizationMap = Object.fromEntries(organizations.map(o => [o.id, o]));
const allConnectedGitProjects = allProjects.filter(
project => models.project.isGitProject(project) && !models.project.isEmptyGitProject(project),
);
const allConnectedGitProjects = allProjects.filter(project => models.project.isConnectedGitProject(project));
const gitRepoURIInfoMap: Record<string, { organizationName: string; projectName: string }> = {};
await Promise.all(
allConnectedGitProjects.map(async ({ gitRepositoryId, name, parentId }) => {
allConnectedGitProjects.map(async project => {
const gitRepositoryId = models.project.isGitProject(project) ? models.project.getEffectiveRepoId(project) : null;
if (gitRepositoryId) {
const gitRepository = await services.gitRepository.getById(gitRepositoryId);
if (gitRepository) {
gitRepoURIInfoMap[gitRepository.uri] = {
organizationName: organizationMap[parentId]?.name || '',
projectName: name,
organizationName: organizationMap[project.parentId]?.name || '',
projectName: project.name,
};
}
}

View File

@@ -136,7 +136,9 @@ export async function getProjectsWithGitRepositories({
parentId: organizationId,
});
const gitRepositoryIds = projects.map(p => p.gitRepositoryId).filter(isNotNullOrUndefined);
const gitRepositoryIds = projects
.map(p => (models.project.isConnectedGitProject(p) ? models.project.getEffectiveRepoId(p) : null))
.filter(isNotNullOrUndefined);
const gitRepositories = await database.find<GitRepository>('GitRepository', {
_id: {
@@ -145,7 +147,10 @@ export async function getProjectsWithGitRepositories({
});
return projects.map(project => {
const gitRepository = gitRepositories.find(gr => gr._id === project.gitRepositoryId);
const effectiveId = models.project.isConnectedGitProject(project)
? models.project.getEffectiveRepoId(project)
: null;
const gitRepository = gitRepositories.find(gr => gr._id === effectiveId);
return {
...project,
gitRepository,
@@ -403,8 +408,8 @@ export async function clientLoader({ params }: LoaderFunctionArgs) {
const projectsSyncStatusPromise = CheckAllProjectSyncStatus(projects);
const activeProjectGitRepository =
project && models.project.isGitProject(project)
? await services.gitRepository.getById(project.gitRepositoryId || '')
project && models.project.isConnectedGitProject(project)
? await services.gitRepository.getById(models.project.getEffectiveRepoId(project) || '')
: null;
return {

View File

@@ -4,6 +4,7 @@ import { href, redirect } from 'react-router';
import { database } from '~/common/database';
import { projectLock } from '~/common/project';
import { services } from '~/insomnia-data';
import * as models from '~/models';
import { reportGitProjectCount } from '~/routes/organization.$organizationId.project.new';
import { invariant } from '~/utils/invariant';
import { createFetcherSubmitHook, getInitialRouteForOrganization } from '~/utils/router';
@@ -32,8 +33,9 @@ export async function clientAction({ params }: Route.ClientActionArgs) {
});
}
if (project.gitRepositoryId) {
const gitRepository = await services.gitRepository.getById(project.gitRepositoryId);
if (models.project.isConnectedGitProject(project)) {
const effectiveRepoId = models.project.isGitProject(project) ? models.project.getEffectiveRepoId(project) : null;
const gitRepository = effectiveRepoId ? await services.gitRepository.getById(effectiveRepoId) : null;
gitRepository && (await services.gitRepository.remove(gitRepository));
}

View File

@@ -29,8 +29,8 @@ export function useProjectLoaderData() {
const Component = () => {
const data = useProjectLoaderData();
const gitRepositoryId =
data && models.project.isGitProject(data.activeProject) && !models.project.isEmptyGitProject(data.activeProject)
? data.activeProject.gitRepositoryId
data && models.project.isConnectedGitProject(data.activeProject)
? models.project.getEffectiveRepoId(data.activeProject)
: null;
const gitFileIssues = useProjectGitFileIssues({
projectId: data?.activeProject._id,

View File

@@ -40,7 +40,8 @@ export async function clientAction({ request, params }: Route.ClientActionArgs)
const project = await services.project.getById(projectId);
invariant(project, 'Project not found');
const gitRepository = project.gitRepositoryId ? await services.gitRepository.getById(project.gitRepositoryId) : null;
const effectiveRepoId = models.project.isGitProject(project) ? models.project.getEffectiveRepoId(project) : null;
const gitRepository = effectiveRepoId ? await services.gitRepository.getById(effectiveRepoId) : null;
const user = await services.userSession.getOrCreate();
const sessionId = user.id;
@@ -165,8 +166,8 @@ export async function clientAction({ request, params }: Route.ClientActionArgs)
},
});
if (project.gitRepositoryId) {
const gitRepository = await services.gitRepository.getById(project.gitRepositoryId);
if (models.project.isConnectedGitProject(project)) {
const gitRepository = await services.gitRepository.getById(models.project.getEffectiveRepoId(project) || '');
gitRepository && (await services.gitRepository.remove(gitRepository));
}
@@ -337,7 +338,8 @@ export async function clientAction({ request, params }: Route.ClientActionArgs)
// convert from git to local
if (storageType === 'local' && project.gitRepositoryId) {
const gitRepository = await services.gitRepository.getById(project.gitRepositoryId);
const effectiveId = models.project.isGitProject(project) ? models.project.getEffectiveRepoId(project) : null;
const gitRepository = effectiveId ? await services.gitRepository.getById(effectiveId) : null;
gitRepository && (await services.gitRepository.remove(gitRepository));
await services.project.update(project, { name, gitRepositoryId: null });

View File

@@ -27,8 +27,8 @@ export async function clientAction({ params }: Route.ClientActionArgs) {
const isLintError = (result: IRuleResult) => result.severity === 0;
const gitRepositoryId = models.project.isGitProject(project)
? project.gitRepositoryId
const gitRepositoryId = models.project.isConnectedGitProject(project)
? models.project.getEffectiveRepoId(project)
: workspaceMeta?.gitRepositoryId;
const rulesetPath = gitRepositoryId

View File

@@ -84,8 +84,8 @@ export async function clientLoader({ params }: Route.ClientLoaderArgs) {
const workspaceMeta = await services.workspaceMeta.getByParentId(workspaceId);
const gitRepositoryId = models.project.isGitProject(project)
? project.gitRepositoryId
const gitRepositoryId = models.project.isConnectedGitProject(project)
? models.project.getEffectiveRepoId(project)
: workspaceMeta?.gitRepositoryId;
// we don't run the lint here because it is expensive and slows first render too much
// TODO: add this in once we run this loader outside the renderer

View File

@@ -103,8 +103,8 @@ export async function clientLoader({ params, request }: Route.ClientLoaderArgs)
const activeWorkspaceMeta = await services.workspaceMeta.getOrCreateByParentId(workspaceId);
const gitRepositoryId = models.project.isGitProject(activeProject)
? activeProject.gitRepositoryId
const gitRepositoryId = models.project.isConnectedGitProject(activeProject)
? models.project.getEffectiveRepoId(activeProject)
: activeWorkspaceMeta.gitRepositoryId;
const gitRepository = await services.gitRepository.getById(gitRepositoryId || '');

View File

@@ -147,6 +147,11 @@ class RepoFileWatcher {
// 1. Load workspace-to-file mappings from the DB for rename detection.
await watcher.loadKnownGitFilePaths();
// 1b. If the DB has newer data than whats on disk (e.g. the user edited
// requests on the old app during a downgrade), write fresh YAML to
// disk BEFORE importing so those edits are not silently overwritten.
await watcher.flushNewerDbWorkspacesToDisk();
// 2. Import all YAML files into the DB so it reflects disk state.
// This populates lastSyncMtime + lastWrittenHash as a side-effect,
// which prevents step 3's watchers from re-importing the same files.
@@ -222,10 +227,83 @@ class RepoFileWatcher {
}
/**
* Import all YAML files in the repo directory into the DB.
* For each workspace linked to this project, if the DB was modified more
* recently than the on-disk YAML, write fresh YAML to disk before the
* initial `importAllFiles` scan.
*
* Called during watcher creation and after bulk git operations (clone, pull,
* merge, checkout) so the DB reflects the current disk state.
* This prevents the stale-YAML-wins problem that occurs when:
* 1. User downgrades (old app has no RepoFileWatcher — DB changes aren\u2019t flushed to disk).
* 2. User edits requests via the old app (DB updated, no YAML written).
* 3. User re-upgrades; without this guard those edits would be silently lost.
*
* Written files are recorded in `lastWrittenHash` / `lastSyncMtime` so that
* `importAllFiles` skips them (they are already up-to-date).
*/
private async flushNewerDbWorkspacesToDisk(): Promise<void> {
const workspaces = await services.workspace.findByParentId(this.projectId);
await Promise.all(
workspaces.map(async workspace => {
try {
const meta = await services.workspaceMeta.getByParentId(workspace._id);
const gitFilePath = meta?.gitFilePath ?? `insomnia.${workspace._id}.yaml`;
const absPath = path.resolve(this.repoDir, gitFilePath);
// Path-traversal guard
const rel = path.relative(this.repoDir, absPath);
if (rel.startsWith('..') || path.isAbsolute(rel)) return;
// Get the most recently modified DB document in this workspace\u2019s tree
const allDocs = await db.getWithDescendants(workspace);
let maxDbModified: number = workspace.modified ?? 0;
for (const doc of allDocs) {
const m = (doc as { modified?: number }).modified ?? 0;
if (m > maxDbModified) maxDbModified = m;
}
// Compare against the on-disk mtime
let fileMtime = 0;
try {
const stat = await fs.promises.stat(absPath);
fileMtime = stat.mtimeMs;
} catch {
// File doesn\u2019t exist yet \u2014 nothing to do; importAllFiles will handle creation.
return;
}
if (maxDbModified <= fileMtime) return; // disk is up-to-date
// DB is newer \u2014 write fresh YAML so importAllFiles doesn\u2019t overwrite it
const yamlContent = await getInsomniaV5DataExport({
workspaceId: workspace._id,
includePrivateEnvironments: false,
});
if (!yamlContent?.trim()) return;
await fs.promises.mkdir(path.dirname(absPath), { recursive: true });
await fs.promises.writeFile(absPath, yamlContent, 'utf8');
const hash = contentHash(yamlContent);
const normalised = path.normalize(absPath);
this.lastWrittenHash.set(normalised, hash);
const newStat = await fs.promises.stat(absPath);
this.lastSyncMtime.set(normalised, newStat.mtimeMs);
console.log(
'[repo-file-watcher] DB newer than disk for workspace',
workspace._id,
'— flushed to',
gitFilePath,
);
} catch (err) {
console.warn('[repo-file-watcher] flushNewerDbWorkspacesToDisk error for workspace', workspace._id, err);
}
}),
);
}
/**
* Import all YAML files in the repo directory into the DB.
*
* Always bypasses the mtime fast-path (`forceRead`) so every file is read
* and compared by content-hash. This makes the method safe to call at any