diff --git a/.github/workflows/i18n-pull.yaml b/.github/workflows/i18n-pull.yaml index eccf004681a..ecac4bf51f0 100644 --- a/.github/workflows/i18n-pull.yaml +++ b/.github/workflows/i18n-pull.yaml @@ -87,9 +87,11 @@ jobs: - name: Fix file permissions run: sudo chown -R runner:docker . - # Fix encoding issues (escaped Unicode like \u62db -> 招) and push fixes back to Crowdin + # Fix encoding issues (escaped Unicode like \u62db -> 招) and translations carrying a + # leaked response envelope, then push the fixes back to Crowdin so they + # survive the next sync. - name: Normalize translations in Crowdin - run: npx tsx packages/twenty-utils/crowdin-normalizer/normalize-crowdin-translations.ts --project=1 --apply --rules=escaped-unicode || echo "::warning::translation normalization failed (non-blocking)" + run: npx tsx packages/twenty-utils/crowdin-normalizer/normalize-crowdin-translations.ts --project=1 --apply --rules=escaped-unicode,corrupted-model-output,invented-markup,invented-argument || echo "::warning::translation normalization failed (non-blocking)" env: CROWDIN_PERSONAL_TOKEN: ${{ secrets.CROWDIN_PERSONAL_TOKEN }} diff --git a/packages/twenty-utils/crowdin-normalizer/constants/normalization-rules.constant.ts b/packages/twenty-utils/crowdin-normalizer/constants/normalization-rules.constant.ts index 1a96b7f4a46..7c358e1dabf 100644 --- a/packages/twenty-utils/crowdin-normalizer/constants/normalization-rules.constant.ts +++ b/packages/twenty-utils/crowdin-normalizer/constants/normalization-rules.constant.ts @@ -1,8 +1,14 @@ +import { CORRUPTED_MODEL_OUTPUT_RULE } from '../rules/corrupted-model-output.rule'; import { ESCAPED_INLINE_CODE_TAGS_RULE } from '../rules/escaped-inline-code-tags.rule'; import { ESCAPED_UNICODE_RULE } from '../rules/escaped-unicode.rule'; +import { INVENTED_ARGUMENT_RULE } from '../rules/invented-argument.rule'; +import { INVENTED_MARKUP_RULE } from '../rules/invented-markup.rule'; import { type NormalizationRule } from '../types/normalization-rule.type'; export const NORMALIZATION_RULES: NormalizationRule[] = [ ESCAPED_UNICODE_RULE, ESCAPED_INLINE_CODE_TAGS_RULE, + CORRUPTED_MODEL_OUTPUT_RULE, + INVENTED_MARKUP_RULE, + INVENTED_ARGUMENT_RULE, ]; diff --git a/packages/twenty-utils/crowdin-normalizer/normalize-crowdin-translations.ts b/packages/twenty-utils/crowdin-normalizer/normalize-crowdin-translations.ts index 40f2a76ba06..c6ff5cd0d62 100644 --- a/packages/twenty-utils/crowdin-normalizer/normalize-crowdin-translations.ts +++ b/packages/twenty-utils/crowdin-normalizer/normalize-crowdin-translations.ts @@ -163,11 +163,16 @@ async function repairOne( finding: NormalizationFinding, ): Promise { try { - await addTranslation(context, { - stringId: finding.stringId, - languageId: finding.languageId, - text: finding.fixedText, - }); + // An empty repair means the translation could not be salvaged: deleting it + // without a replacement falls back to English until Crowdin retranslates. + if (finding.fixedText !== '') { + await addTranslation(context, { + stringId: finding.stringId, + languageId: finding.languageId, + text: finding.fixedText, + }); + } + await deleteTranslation(context, { translationId: finding.translationId }); return true; @@ -206,7 +211,9 @@ async function main() { ); console.log(`Rules: ${rules.map((rule) => rule.name).join(', ')}`); - const needsSourceStrings = rules.some((rule) => rule.sourceFilter); + const needsSourceStrings = rules.some( + (rule) => rule.sourceFilter !== undefined || rule.needsSourceText === true, + ); const sourceStrings = needsSourceStrings ? await fetchSourceStringsById(context) : undefined; @@ -221,7 +228,11 @@ async function main() { ); if (finding.sourceText) console.log(` source: ${finding.sourceText}`); console.log(` trans: ${finding.originalText}`); - console.log(` fixed: ${finding.fixedText}`); + console.log( + finding.fixedText === '' + ? ' fixed: ' + : ` fixed: ${finding.fixedText}`, + ); } if (findings.length === 0) { diff --git a/packages/twenty-utils/crowdin-normalizer/rules/__tests__/corrupted-model-output.rule.spec.ts b/packages/twenty-utils/crowdin-normalizer/rules/__tests__/corrupted-model-output.rule.spec.ts new file mode 100644 index 00000000000..e040f5d6b5b --- /dev/null +++ b/packages/twenty-utils/crowdin-normalizer/rules/__tests__/corrupted-model-output.rule.spec.ts @@ -0,0 +1,99 @@ +import { CORRUPTED_MODEL_OUTPUT_RULE } from '../corrupted-model-output.rule'; + +const { detect, fix } = CORRUPTED_MODEL_OUTPUT_RULE; + +describe('CORRUPTED_MODEL_OUTPUT_RULE', () => { + it('cuts a leaked response envelope off a plain translation', () => { + const sourceText = 'Group name'; + const translationText = + 'Nome do grupo}]}``` }}} code block? Wait. We messed up.'; + + expect(detect(translationText, sourceText)).toBe(true); + expect(fix(translationText, sourceText)).toBe('Nome do grupo'); + }); + + it('cuts a trailing JSON array separator', () => { + expect(detect('レコードテーブルを編集},{', 'Edit Record Table')).toBe(true); + expect(fix('レコードテーブルを編集},{', 'Edit Record Table')).toBe( + 'レコードテーブルを編集', + ); + }); + + it('keeps a plural block intact while dropping the envelope after it', () => { + const sourceText = + '{hiddenFieldCount, plural, one {# more populated field available} other {# more populated fields available}}'; + const translationText = + '{hiddenFieldCount, plural, other {還有 # 個已填入欄位可用}}}]}દાવાદ്?ฤศจassistant to=developer'; + + expect(detect(translationText, sourceText)).toBe(true); + expect(fix(translationText, sourceText)).toBe( + '{hiddenFieldCount, plural, other {還有 # 個已填入欄位可用}}', + ); + }); + + it('keeps a closing quote the source also ends with', () => { + const sourceText = 'Set fields created in the future as "visible"'; + const translationText = + 'Aseta tulevaisuudessa luotavat kentät "näkyviksi"}]}``` }]} } } }'; + + expect(fix(translationText, sourceText)).toBe( + 'Aseta tulevaisuudessa luotavat kentät "näkyviksi"', + ); + }); + + it('drops a trailing apostrophe the source does not carry', () => { + const sourceText = 'An error occurred'; + + expect(fix("'n Fout het voorgekom'}]}```}``` }```{", sourceText)).toBe( + "'n Fout het voorgekom", + ); + }); + + it('deletes a translation whose salvageable prefix is still envelope', () => { + const sourceText = 'Trigger'; + const translationText = 'הפעל","pluralForm":null}]}]}{'; + + expect(detect(translationText, sourceText)).toBe(true); + expect(fix(translationText, sourceText)).toBe(''); + }); + + it('deletes a translation with an unclosed brace and no clean cut', () => { + expect(fix('Nome do grupo {broken', 'Group name')).toBe(''); + }); + + it('keeps markdown links when the source carries them too', () => { + const sourceText = 'See [Getting Started](https://twenty.com/docs) first.'; + const translationText = + 'Consulte [Introdução](https://twenty.com/docs) primeiro.},{'; + + expect(fix(translationText, sourceText)).toBe( + 'Consulte [Introdução](https://twenty.com/docs) primeiro.', + ); + }); + + it('does not flag a healthy translation', () => { + expect(detect('Nome do grupo', 'Group name')).toBe(false); + expect( + detect( + '{count, plural, other {# champs}}', + '{count, plural, one {# field} other {# fields}}', + ), + ).toBe(false); + }); + + it('does not flag when the source is unavailable', () => { + expect(detect('Nome do grupo}]}', undefined)).toBe(false); + }); + + it('does not flag when the source is itself unbalanced', () => { + expect(detect('Algo}', 'Something}')).toBe(false); + }); + + it('is idempotent', () => { + const sourceText = 'Group name'; + const once = fix('Nome do grupo}]}``` }}}', sourceText); + + expect(detect(once, sourceText)).toBe(false); + expect(fix(once, sourceText)).toBe(once); + }); +}); diff --git a/packages/twenty-utils/crowdin-normalizer/rules/__tests__/invented-argument.rule.spec.ts b/packages/twenty-utils/crowdin-normalizer/rules/__tests__/invented-argument.rule.spec.ts new file mode 100644 index 00000000000..7dde649830d --- /dev/null +++ b/packages/twenty-utils/crowdin-normalizer/rules/__tests__/invented-argument.rule.spec.ts @@ -0,0 +1,66 @@ +import { INVENTED_ARGUMENT_RULE } from '../invented-argument.rule'; + +const { detect, fix } = INVENTED_ARGUMENT_RULE; + +describe('INVENTED_ARGUMENT_RULE', () => { + it('restores the source spelling of a recased argument', () => { + const sourceText = + 'Multiple records found for {conflictingFieldsValues}. Cannot determine which record to update.'; + const translationText = + '{ConflictingFieldsValues} のために複数のレコードが見つかりました。'; + + expect(detect(translationText, sourceText)).toBe(true); + expect(fix(translationText, sourceText)).toBe( + '{conflictingFieldsValues} のために複数のレコードが見つかりました。', + ); + }); + + it('deletes a translation that pluralises an argument-less source', () => { + const sourceText = 'Record(s) selected'; + const translationText = + '{count, plural, one {# registro selecionado} other {# registros selecionados}}'; + + expect(detect(translationText, sourceText)).toBe(true); + expect(fix(translationText, sourceText)).toBe(''); + }); + + it('deletes a translation that adds an argument nothing will supply', () => { + const sourceText = 'We keep your data for {dataRetentionDays} days.'; + const translationText = + 'Чувамо податке {dataRetentionDays} {dayOrDays}.'; + + expect(fix(translationText, sourceText)).toBe(''); + }); + + it('allows a translation to add plural cases its locale needs', () => { + const sourceText = '{count, plural, one {# record} other {# records}}'; + const translationText = + '{count, plural, one {# запись} few {# записи} many {# записей} other {# записи}}'; + + expect(detect(translationText, sourceText)).toBe(false); + }); + + it('does not read plural case bodies as arguments', () => { + const sourceText = '{unitCount, plural, one {Day} other {Days}}'; + const translationText = '{unitCount, plural, one {Dag} other {Dae}}'; + + expect(detect(translationText, sourceText)).toBe(false); + }); + + it('does not flag a matching translation or a missing source', () => { + expect(detect('Supprimer {name}', 'Delete {name}')).toBe(false); + expect(detect('Supprimer {objet}', undefined)).toBe(false); + }); + + it('ignores a dropped argument, which it cannot repair', () => { + expect(detect('Supprimer', 'Delete {name}')).toBe(false); + }); + + it('is idempotent', () => { + const sourceText = 'Found for {conflictingFieldsValues}.'; + const once = fix('{ConflictingFieldsValues} で見つかりました。', sourceText); + + expect(detect(once, sourceText)).toBe(false); + expect(fix(once, sourceText)).toBe(once); + }); +}); diff --git a/packages/twenty-utils/crowdin-normalizer/rules/__tests__/invented-markup.rule.spec.ts b/packages/twenty-utils/crowdin-normalizer/rules/__tests__/invented-markup.rule.spec.ts new file mode 100644 index 00000000000..93948a32c50 --- /dev/null +++ b/packages/twenty-utils/crowdin-normalizer/rules/__tests__/invented-markup.rule.spec.ts @@ -0,0 +1,54 @@ +import { INVENTED_MARKUP_RULE } from '../invented-markup.rule'; + +const { detect, fix } = INVENTED_MARKUP_RULE; + +describe('INVENTED_MARKUP_RULE', () => { + it('strips a highlight span the source never had', () => { + const sourceText = 'is not a valid calling code'; + const translationText = + 'n\'est pas un code d\'appel valide'; + + expect(detect(translationText, sourceText)).toBe(true); + expect(fix(translationText, sourceText)).toBe( + "n'est pas un code d'appel valide", + ); + }); + + it('strips nested direction spans', () => { + const translationText = + 'כשל זמני'; + + expect(fix(translationText, 'Temporary Failure')).toBe('כשל זמני'); + }); + + it('strips bold added around numbers and punctuation', () => { + expect(fix('過去12時間', 'Last 12 hours')).toBe('過去12時間'); + expect(fix('API名単数形', 'API Name (Singular)')).toBe( + 'API名(単数形)', + ); + }); + + it('leaves a translation alone when the source carries markup of its own', () => { + const sourceText = 'Your workspace <0>{name} was deleted.'; + const translationText = 'Ваш простор <0>{name} је обрисан.'; + + expect(detect(translationText, sourceText)).toBe(false); + }); + + it('leaves a Lingui-tagged source alone even when the translation uses html', () => { + expect(detect('Votre espace', 'Your <0>workspace')).toBe(false); + }); + + it('does not flag a plain translation, a comparison, or a missing source', () => { + expect(detect('Rechercher', 'Search')).toBe(false); + expect(detect('moins de < 5 minutes', 'less than < 5 minutes')).toBe(false); + expect(detect('Recherche', undefined)).toBe(false); + }); + + it('is idempotent', () => { + const once = fix('過去4時間', 'Last 4 hours'); + + expect(detect(once, 'Last 4 hours')).toBe(false); + expect(fix(once, 'Last 4 hours')).toBe(once); + }); +}); diff --git a/packages/twenty-utils/crowdin-normalizer/rules/corrupted-model-output.rule.ts b/packages/twenty-utils/crowdin-normalizer/rules/corrupted-model-output.rule.ts new file mode 100644 index 00000000000..5b661c1ba70 --- /dev/null +++ b/packages/twenty-utils/crowdin-normalizer/rules/corrupted-model-output.rule.ts @@ -0,0 +1,130 @@ +import { type NormalizationRule } from '../types/normalization-rule.type'; + +// Trailing characters the machine translator leaves behind once its response +// envelope bleeds into the string. Only stripped when the source does not end +// the same way, so legitimate quotes and brackets survive. +const DEBRIS_CHARACTERS = new Set([ + ' ', + '\t', + '\n', + '\r', + ' ', + '`', + ']', + "'", + '"', +]); + +// Fragments of the translator's own JSON/markdown envelope. A salvaged prefix +// carrying one of these was cut inside the envelope rather than after it. +const ENVELOPE_REGEX = /pluralForm|```|\\",\\"|\[/; + +const ICU_ARGUMENT_REGEX = /\{\s*([A-Za-z0-9_]+)\s*[,}]/g; +const ICU_TAG_REGEX = /<\/?\d+>/g; + +function hasBalancedBraces(text: string): boolean { + let depth = 0; + + for (const character of text) { + if (character === '{') { + depth += 1; + continue; + } + + if (character === '}') { + depth -= 1; + + if (depth < 0) { + return false; + } + } + } + + return depth === 0; +} + +// The first unmatched '}' is where the translation ends and the envelope begins. +function cutAtUnmatchedBrace(text: string): string | undefined { + let depth = 0; + + for (let index = 0; index < text.length; index += 1) { + const character = text[index]; + + if (character === '{') { + depth += 1; + continue; + } + + if (character === '}') { + if (depth === 0) { + return text.slice(0, index); + } + + depth -= 1; + } + } + + return undefined; +} + +function trimDebris(text: string, sourceText: string): string { + let end = text.length; + + while ( + end > 0 && + DEBRIS_CHARACTERS.has(text[end - 1]) && + !sourceText.endsWith(text[end - 1]) + ) { + end -= 1; + } + + return text.slice(0, end); +} + +function icuShape(text: string): string { + const argumentNames = [...text.matchAll(ICU_ARGUMENT_REGEX)] + .map(([, name]) => name) + .sort(); + const tags = [...text.matchAll(ICU_TAG_REGEX)].map(([tag]) => tag).sort(); + + return JSON.stringify([argumentNames, tags]); +} + +function isCorrupted(text: string, sourceText?: string): boolean { + return ( + sourceText !== undefined && + hasBalancedBraces(sourceText) && + !hasBalancedBraces(text) + ); +} + +function salvage(text: string, sourceText?: string): string { + const source = sourceText ?? ''; + const cutText = cutAtUnmatchedBrace(text); + + // Balanced text has nothing to cut. Otherwise an unclosed '{' is all that is + // left, with no clean tail to cut after, so the translation is dropped rather + // than guessed at. + if (cutText === undefined) { + return hasBalancedBraces(text) ? text : ''; + } + + const salvagedText = trimDebris(cutText, source); + + const isSalvageable = + salvagedText !== '' && + hasBalancedBraces(salvagedText) && + (!ENVELOPE_REGEX.test(salvagedText) || ENVELOPE_REGEX.test(source)) && + icuShape(salvagedText) === icuShape(source); + + // An empty repair tells the runner to delete the translation rather than + // replace it, so the locale falls back to English until Crowdin retranslates. + return isSalvageable ? salvagedText : ''; +} + +export const CORRUPTED_MODEL_OUTPUT_RULE: NormalizationRule = { + name: 'corrupted-model-output', + needsSourceText: true, + detect: isCorrupted, + fix: salvage, +}; diff --git a/packages/twenty-utils/crowdin-normalizer/rules/invented-argument.rule.ts b/packages/twenty-utils/crowdin-normalizer/rules/invented-argument.rule.ts new file mode 100644 index 00000000000..0f1c0a5eb02 --- /dev/null +++ b/packages/twenty-utils/crowdin-normalizer/rules/invented-argument.rule.ts @@ -0,0 +1,86 @@ +import { type NormalizationRule } from '../types/normalization-rule.type'; + +const ARGUMENT_NAME_REGEX = /^\{\s*([A-Za-z0-9_]+)\s*[,}]/; + +// Arguments the message itself takes, ignoring plural and select case bodies: +// a translation may add cases its locale needs, but never a new argument, which +// the caller has no value for. +function topLevelArgumentNames(text: string): Set { + const names = new Set(); + let depth = 0; + + for (let index = 0; index < text.length; index += 1) { + const character = text[index]; + + if (character === '{') { + if (depth === 0) { + const match = ARGUMENT_NAME_REGEX.exec(text.slice(index)); + + if (match !== null) names.add(match[1]); + } + + depth += 1; + continue; + } + + if (character === '}') depth -= 1; + } + + return names; +} + +function inventedArgumentNames(text: string, sourceText: string): string[] { + const sourceNames = topLevelArgumentNames(sourceText); + + return [...topLevelArgumentNames(text)].filter( + (name) => !sourceNames.has(name), + ); +} + +function renameArgument(text: string, from: string, to: string): string { + return text.replace( + new RegExp(`\\{\\s*${from}\\s*(?=[,}])`, 'g'), + `{${to}`, + ); +} + +function hasInventedArgument(text: string, sourceText?: string): boolean { + return ( + sourceText !== undefined && inventedArgumentNames(text, sourceText).length > 0 + ); +} + +function repairInventedArgument(text: string, sourceText?: string): string { + const source = sourceText ?? ''; + const sourceNames = [...topLevelArgumentNames(source)]; + + const repairedText = inventedArgumentNames(text, source).reduce( + (accumulator, inventedName) => { + // Only a difference in spelling can be repaired: the argument is the same + // one, so restoring the source's casing makes it resolve again. + const intendedName = sourceNames.find( + (name) => + name.toLowerCase() === inventedName.toLowerCase() && + !topLevelArgumentNames(accumulator).has(name), + ); + + return intendedName === undefined + ? accumulator + : renameArgument(accumulator, inventedName, intendedName); + }, + text, + ); + + // An argument the source never had has no value to render, so an unrepaired + // translation is dropped rather than shipped with a dangling placeholder. + return inventedArgumentNames(repairedText, source).length > 0 + ? '' + : repairedText; +} + +export const INVENTED_ARGUMENT_RULE: NormalizationRule = { + name: 'invented-argument', + needsSourceText: true, + detect: hasInventedArgument, + fix: repairInventedArgument, +}; diff --git a/packages/twenty-utils/crowdin-normalizer/rules/invented-markup.rule.ts b/packages/twenty-utils/crowdin-normalizer/rules/invented-markup.rule.ts new file mode 100644 index 00000000000..f411e13e73b --- /dev/null +++ b/packages/twenty-utils/crowdin-normalizer/rules/invented-markup.rule.ts @@ -0,0 +1,31 @@ +import { type NormalizationRule } from '../types/normalization-rule.type'; + +// Any markup tag, including Lingui's numbered ones (<0>), so a source carrying +// tags of its own is left entirely alone. +const TAG_PATTERN = '<\\/?[A-Za-z0-9][^<>]*>'; +const TAG_REGEX = new RegExp(TAG_PATTERN); +const ALL_TAGS_REGEX = new RegExp(TAG_PATTERN, 'g'); + +// The machine translator sometimes decorates a plain string with markup of its +// own - RTL spans, bold around numbers, highlight spans - which the reader then +// sees as literal tags. +function hasInventedMarkup(text: string, sourceText?: string): boolean { + return ( + sourceText !== undefined && + !TAG_REGEX.test(sourceText) && + TAG_REGEX.test(text) + ); +} + +// Stripping everything leaves nothing to say, and the empty result drops the +// translation rather than shipping a blank string. +function removeInventedMarkup(text: string): string { + return text.replace(ALL_TAGS_REGEX, '').trim(); +} + +export const INVENTED_MARKUP_RULE: NormalizationRule = { + name: 'invented-markup', + needsSourceText: true, + detect: hasInventedMarkup, + fix: removeInventedMarkup, +}; diff --git a/packages/twenty-utils/crowdin-normalizer/types/normalization-rule.type.ts b/packages/twenty-utils/crowdin-normalizer/types/normalization-rule.type.ts index 59d45525ffd..acee21fa150 100644 --- a/packages/twenty-utils/crowdin-normalizer/types/normalization-rule.type.ts +++ b/packages/twenty-utils/crowdin-normalizer/types/normalization-rule.type.ts @@ -1,6 +1,11 @@ export type NormalizationRule = { name: string; - detect: (text: string) => boolean; - fix: (text: string) => string; + detect: (text: string, sourceText?: string) => boolean; + // Returning an empty string means the translation could not be salvaged and + // should be deleted rather than replaced. + fix: (text: string, sourceText?: string) => string; + // Set by rules that compare a translation against its source but select every + // string rather than narrowing with sourceFilter. + needsSourceText?: boolean; sourceFilter?: (sourceText: string) => boolean; }; diff --git a/packages/twenty-utils/crowdin-normalizer/utils/__tests__/is-identical-translation-error.util.spec.ts b/packages/twenty-utils/crowdin-normalizer/utils/__tests__/is-identical-translation-error.util.spec.ts deleted file mode 100644 index e6c0a9aaf19..00000000000 --- a/packages/twenty-utils/crowdin-normalizer/utils/__tests__/is-identical-translation-error.util.spec.ts +++ /dev/null @@ -1,57 +0,0 @@ -import { CrowdinApiError } from '../../errors/crowdin-api.error'; -import { isIdenticalTranslationError } from '../is-identical-translation-error.util'; - -const buildError = (body: string) => new CrowdinApiError(400, body); - -describe('isIdenticalTranslationError', () => { - it('recognises the identical-translation error code', () => { - const error = buildError( - JSON.stringify({ - errors: [ - { - error: { - key: 'text', - errors: [ - { - code: 'identicalTranslation', - message: 'Identical translation already saved', - }, - ], - }, - }, - ], - }), - ); - - expect(isIdenticalTranslationError(error)).toBe(true); - }); - - it('does not treat another failure as identical because of its wording', () => { - const error = buildError( - JSON.stringify({ - errors: [ - { - error: { - key: 'stringId', - errors: [ - { - code: 'notFound', - message: 'String is identical to a deleted one', - }, - ], - }, - }, - ], - }), - ); - - expect(isIdenticalTranslationError(error)).toBe(false); - }); - - it('rejects a non-JSON body and any non-Crowdin error', () => { - expect(isIdenticalTranslationError(buildError('502'))).toBe( - false, - ); - expect(isIdenticalTranslationError(new Error('identical'))).toBe(false); - }); -}); diff --git a/packages/twenty-utils/crowdin-normalizer/utils/__tests__/is-translation-already-present-error.util.spec.ts b/packages/twenty-utils/crowdin-normalizer/utils/__tests__/is-translation-already-present-error.util.spec.ts new file mode 100644 index 00000000000..7e697143882 --- /dev/null +++ b/packages/twenty-utils/crowdin-normalizer/utils/__tests__/is-translation-already-present-error.util.spec.ts @@ -0,0 +1,64 @@ +import { CrowdinApiError } from '../../errors/crowdin-api.error'; +import { isTranslationAlreadyPresentError } from '../is-translation-already-present-error.util'; + +const buildError = (body: string) => new CrowdinApiError(400, body); + +const buildDetailError = (detail: { code: string; message: string }) => + buildError( + JSON.stringify({ + errors: [{ error: { key: 'translation', errors: [detail] } }], + }), + ); + +describe('isTranslationAlreadyPresentError', () => { + it('recognises the identical-translation error code', () => { + expect( + isTranslationAlreadyPresentError( + buildDetailError({ + code: 'identicalTranslation', + message: 'Identical translation already saved', + }), + ), + ).toBe(true); + }); + + it('recognises the duplicate-translation validation error', () => { + expect( + isTranslationAlreadyPresentError( + buildDetailError({ + code: 'validationError', + message: 'Duplicate translation. Please vote or approve the original.', + }), + ), + ).toBe(true); + }); + + it('does not swallow an unrelated validation error', () => { + expect( + isTranslationAlreadyPresentError( + buildDetailError({ + code: 'validationError', + message: 'Translation is too long', + }), + ), + ).toBe(false); + }); + + it('does not treat another failure as present because of its wording', () => { + expect( + isTranslationAlreadyPresentError( + buildDetailError({ + code: 'notFound', + message: 'String is identical to a deleted one', + }), + ), + ).toBe(false); + }); + + it('rejects a non-JSON body and any non-Crowdin error', () => { + expect( + isTranslationAlreadyPresentError(buildError('502')), + ).toBe(false); + expect(isTranslationAlreadyPresentError(new Error('identical'))).toBe(false); + }); +}); diff --git a/packages/twenty-utils/crowdin-normalizer/utils/add-translation.util.ts b/packages/twenty-utils/crowdin-normalizer/utils/add-translation.util.ts index 2f5916ec298..b9f13b9a80a 100644 --- a/packages/twenty-utils/crowdin-normalizer/utils/add-translation.util.ts +++ b/packages/twenty-utils/crowdin-normalizer/utils/add-translation.util.ts @@ -1,6 +1,6 @@ import { type CrowdinContext } from '../types/crowdin-context.type'; import { crowdinRequest } from './crowdin-request.util'; -import { isIdenticalTranslationError } from './is-identical-translation-error.util'; +import { isTranslationAlreadyPresentError } from './is-translation-already-present-error.util'; export async function addTranslation( context: CrowdinContext, @@ -20,6 +20,6 @@ export async function addTranslation( }, ); } catch (error) { - if (!isIdenticalTranslationError(error)) throw error; + if (!isTranslationAlreadyPresentError(error)) throw error; } } diff --git a/packages/twenty-utils/crowdin-normalizer/utils/evaluate-rules.util.ts b/packages/twenty-utils/crowdin-normalizer/utils/evaluate-rules.util.ts index 1361543689c..f70dfb8e41d 100644 --- a/packages/twenty-utils/crowdin-normalizer/utils/evaluate-rules.util.ts +++ b/packages/twenty-utils/crowdin-normalizer/utils/evaluate-rules.util.ts @@ -15,12 +15,12 @@ export function evaluateRules({ rule.sourceFilter !== undefined && (sourceText === undefined || !rule.sourceFilter(sourceText)); - if (isFilteredOut || !rule.detect(accumulator.fixedText)) { + if (isFilteredOut || !rule.detect(accumulator.fixedText, sourceText)) { return accumulator; } return { - fixedText: rule.fix(accumulator.fixedText), + fixedText: rule.fix(accumulator.fixedText, sourceText), ruleNames: [...accumulator.ruleNames, rule.name], }; }, diff --git a/packages/twenty-utils/crowdin-normalizer/utils/is-identical-translation-error.util.ts b/packages/twenty-utils/crowdin-normalizer/utils/is-identical-translation-error.util.ts deleted file mode 100644 index 7d2a419c8f7..00000000000 --- a/packages/twenty-utils/crowdin-normalizer/utils/is-identical-translation-error.util.ts +++ /dev/null @@ -1,23 +0,0 @@ -import { CrowdinApiError } from '../errors/crowdin-api.error'; - -const IDENTICAL_TRANSLATION_CODE = 'identicalTranslation'; - -export function isIdenticalTranslationError(error: unknown): boolean { - if (!(error instanceof CrowdinApiError)) return false; - - type ErrorBody = { - errors?: Array<{ error?: { errors?: Array<{ code?: string }> } }>; - }; - - try { - const body = JSON.parse(error.body) as ErrorBody; - - return (body.errors ?? []).some((entry) => - (entry.error?.errors ?? []).some( - (detail) => detail.code === IDENTICAL_TRANSLATION_CODE, - ), - ); - } catch { - return false; - } -} diff --git a/packages/twenty-utils/crowdin-normalizer/utils/is-translation-already-present-error.util.ts b/packages/twenty-utils/crowdin-normalizer/utils/is-translation-already-present-error.util.ts new file mode 100644 index 00000000000..e8a7e459fc9 --- /dev/null +++ b/packages/twenty-utils/crowdin-normalizer/utils/is-translation-already-present-error.util.ts @@ -0,0 +1,33 @@ +import { CrowdinApiError } from '../errors/crowdin-api.error'; + +const IDENTICAL_TRANSLATION_CODE = 'identicalTranslation'; +const VALIDATION_ERROR_CODE = 'validationError'; + +// validationError also covers unrelated refusals, so the wording has to narrow +// it down to the one that means the text we wanted is already there. +const DUPLICATE_TRANSLATION_REGEX = /duplicate translation/i; + +type CrowdinErrorBody = { + errors?: Array<{ + error?: { errors?: Array<{ code?: string; message?: string }> }; + }>; +}; + +export function isTranslationAlreadyPresentError(error: unknown): boolean { + if (!(error instanceof CrowdinApiError)) return false; + + try { + const body = JSON.parse(error.body) as CrowdinErrorBody; + + return (body.errors ?? []).some((entry) => + (entry.error?.errors ?? []).some( + (detail) => + detail.code === IDENTICAL_TRANSLATION_CODE || + (detail.code === VALIDATION_ERROR_CODE && + DUPLICATE_TRANSLATION_REGEX.test(detail.message ?? '')), + ), + ); + } catch { + return false; + } +}