Sync with Crowdin (#2195)

Github Actions and scripts to

* push strings.ftl to Crowdin when it changes
* pull translations from Crowdin every day and a PR should get made if there are changes
* FTL validation and normalization happens in the action and will prevent a PR from being created

Closes #2103
Closes #2104

.github/workflows/crowdin-pull.yml

@@ -0,0 +1,90 @@
+name: Crowdin Pull
+
+on:
+  schedule:
+    - cron: '0 6 * * *'
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  pull-from-crowdin:
+    runs-on: ubuntu-latest
+
+    # Apparently needed to rename files, though maybe there's a better way
+    container:
+      image: node:18
+      options: --user root
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Install Node.js, NPM and Yarn
+        uses: actions/setup-node@v4
+        with:
+          node-version: 18
+
+      - name: Cache node modules
+        uses: actions/cache@v4
+        id: cache
+        with:
+          path: node_modules
+          key: node-modules-${{ hashFiles('**/package-lock.json') }}
+
+      - name: Install node dependencies
+        if: steps.cache.outputs.cache-hit != 'true'
+        run: npm install
+
+      - name: Sync with Crowdin
+        uses: crowdin/github-action@v2
+        with:
+          # Note: a lot of the way this behaves is controlled in crowdin.yml,
+          # e.g. the path to the source files and paths to translations
+          # Upload options
+          upload_sources: false
+          upload_translations: false
+          # Download options
+          download_translations: true
+          skip_untranslated_strings: true
+          export_only_approved: true
+          # Pull request options
+          create_pull_request: false
+        env:
+          GITHUB_TOKEN: ${{ secrets.CROWDIN_GITHUB_TOKEN }}
+          CROWDIN_PROJECT_ID: ${{ secrets.CROWDIN_PROJECT_ID }}
+          CROWDIN_PERSONAL_TOKEN: ${{ secrets.CROWDIN_PERSONAL_TOKEN }}
+
+      # Move files named with crowdin locales to files named with iNat
+      # locales. This will also fail if there's invalid FTL. We might want to
+      # separate those things in the future if it's easier to notice
+      # validation failures in the PR
+      - name: Normalize and validate
+        run: npm run translate
+
+      - name: Make pull request
+        # The original doesn't support making a pull request without
+        # downloading, which we don't want to do b/c that would recreate the
+        # Crowdin-named files that we just removed in the previous step, so
+        # we're using a fork.
+        uses: inaturalist/crowdin-github-action@pr-without-download
+        with:
+          # Note: a lot of the way this behaves is controlled in crowdin.yml,
+          # e.g. the path to the source files and paths to translations
+          # Upload options
+          upload_sources: false
+          upload_translations: false
+          # Download options
+          download_translations: false
+          # Pull request options
+          create_pull_request: true
+          localization_branch_name: l10n_main
+          pull_request_title: 'New Crowdin Translations'
+          pull_request_body: 'New Crowdin translations by [Crowdin GH Action](https://github.com/crowdin/github-action)'
+          pull_request_base_branch_name: 'main'
+        env:
+          GITHUB_TOKEN: ${{ secrets.CROWDIN_GITHUB_TOKEN }}
+          CROWDIN_PROJECT_ID: ${{ secrets.CROWDIN_PROJECT_ID }}
+          CROWDIN_PERSONAL_TOKEN: ${{ secrets.CROWDIN_PERSONAL_TOKEN }}

.github/workflows/crowdin-push.yml

@@ -0,0 +1,36 @@
+name: Crowdin Push
+
+on:
+  push:
+    branches: [ main ]
+    paths: ["src/i18n/strings.ftl"]
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  push-to-crowdin:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Sync with Crowdin
+        uses: crowdin/github-action@v2
+        with:
+          # Note: a lot of the way this behaves is controlled in crowdin.yml,
+          # e.g. the path to the source files and paths to translations
+          # Upload options
+          upload_sources: true
+          upload_translations: false
+          # Download options
+          download_translations: false
+          # Pull request options
+          create_pull_request: false
+        env:
+          GITHUB_TOKEN: ${{ secrets.CROWDIN_GITHUB_TOKEN }}
+          CROWDIN_PROJECT_ID: ${{ secrets.CROWDIN_PROJECT_ID }}
+          CROWDIN_PERSONAL_TOKEN: ${{ secrets.CROWDIN_PERSONAL_TOKEN }}

crowdin.yml

@@ -1,13 +1,11 @@
-base_path" : "."
-base_url : "https://api.crowdin.com"
+project_id_env: CROWDIN_PROJECT_ID
+api_token_env: CROWDIN_PERSONAL_TOKEN
 preserve_hierarchy: true
 
 files:
-  - source: "/src/i18n/strings.ftl"
-    dest: "/iNaturalistReactNative/%original_file_name%"
-    translation: "/src/i18n/l10n/%locale%.ftl"
-    export_only_approved: true
-    languages_mapping:
-      locale:
-        es-ES: es
-        ru: ru
+  - source: /src/i18n/strings.ftl
+    dest: /ReactNative/strings.ftl
+    # Note: this uses the Crowdin locale which always has a region. We need to
+    # mess with this later in i18ncli
+    translation: /src/i18n/l10n/%locale%.ftl
+    type: ftl

src/i18n/i18ncli.js

@@ -8,8 +8,7 @@ const {
   serialize: serializeFtl,
   Resource
 } = require( "@fluent/syntax" );
-
-const { readFile, writeFile } = fs.promises;
+const fsp = require( "fs/promises" );
 const path = require( "path" );
 const util = require( "util" );
 const { glob } = require( "glob" );
@@ -20,6 +19,23 @@ const {
   uniq
 } = require( "lodash" );
 
+// Exceptions to the rule that all locales should be specified as two-letter
+// language code only
+const SUPPORTED_REGIONAL_LOCALES = [
+  "en-GB",
+  "en-NZ",
+  "es-AR",
+  "es-CO",
+  "es-CR",
+  "es-MX",
+  "fr-CA",
+  "pt-BR",
+  "zh-CN",
+  "zh-HK",
+  "zh-TW"
+];
+
+// Prepends an FTL translation with a checkmark for testing
 function checkifyText( ftlTxt ) {
   if ( ftlTxt.indexOf( "<0>" ) >= 0 ) {
     return ftlTxt.replace( "<0>", "<0>✅" );
@@ -52,11 +68,16 @@ function checkifyLocalizations( localizations ) {
   }, {} );
 }
 
+// Paths to all existing localizations
+async function l10nFtlPaths() {
+  return glob( path.join( __dirname, "l10n", "*.ftl" ) );
+}
+
 // Convert a single FTL file to JSON
 const jsonifyPath = async ( inPath, outPath, options = { } ) => {
   let ftlTxt;
   try {
-    ftlTxt = await readFile( inPath );
+    ftlTxt = await fsp.readFile( inPath );
   } catch ( readFileErr ) {
     console.error( `Could not read ${inPath}, skipping...` );
     if ( options.debug ) {
@@ -77,7 +98,7 @@ const jsonifyPath = async ( inPath, outPath, options = { } ) => {
     ? checkifyLocalizations( localizations )
     : localizations;
   try {
-    await writeFile( outPath, `${JSON.stringify( massagedLocalizations, null, 2 )}\n` );
+    await fsp.writeFile( outPath, `${JSON.stringify( massagedLocalizations, null, 2 )}\n` );
   } catch ( writeFileErr ) {
     console.error( `Failed to write ${outPath} with error:` );
     console.error( writeFileErr );
@@ -89,7 +110,7 @@ const jsonifyPath = async ( inPath, outPath, options = { } ) => {
 
 // Assume all existing localized locales are supported
 const supportedLocales = async ( ) => {
-  const paths = await glob( path.join( __dirname, "l10n", "*.ftl" ) );
+  const paths = await l10nFtlPaths( );
   return paths.map( f => path.basename( f, ".ftl" ) );
 };
 
@@ -139,12 +160,8 @@ const writeLoadTranslations = async ( ) => {
   out.write( "};\n" );
 };
 
-async function l10nFtlPaths() {
-  return glob( path.join( __dirname, "l10n", "*.ftl" ) );
-}
-
 async function validateFtlFile( ftlPath, options = {} ) {
-  const ftlTxt = await readFile( ftlPath );
+  const ftlTxt = await fsp.readFile( ftlPath );
   const ftl = parseFtl( ftlTxt.toString( ) );
   const errors = [];
   // Chalk does not expose a CommonJS module, so we have to do this
@@ -203,7 +220,7 @@ async function validate() {
 }
 
 async function normalizeFtlFile( ftlPath, options = {} ) {
-  const ftlTxt = await readFile( ftlPath );
+  const ftlTxt = await fsp.readFile( ftlPath );
   const ftl = parseFtl( ftlTxt.toString( ) );
   const resourceComments = [];
   const messages = [];
@@ -225,7 +242,7 @@ async function normalizeFtlFile( ftlPath, options = {} ) {
     ...sortedMessages
   ] );
   const newFtlTxt = serializeFtl( newResource );
-  await writeFile( ftlPath, newFtlTxt );
+  await fsp.writeFile( ftlPath, newFtlTxt );
   if ( !options.quiet ) {
     console.log( `✅ ${ftlPath} normalized` );
   }
@@ -240,7 +257,7 @@ async function normalize( ) {
 
 async function getKeys( ) {
   const stringsPath = path.join( __dirname, "strings.ftl" );
-  const ftlTxt = await readFile( stringsPath );
+  const ftlTxt = await fsp.readFile( stringsPath );
   const ftl = parseFtl( ftlTxt.toString( ) );
   return ftl.body.filter( item => item.type === "Message" ).map( msg => msg.id.name );
 }
@@ -248,7 +265,7 @@ async function getKeys( ) {
 async function getKeysInUse( ) {
   const paths = await glob( path.join( __dirname, "..", "**", "*.{js,ts,jsx,tsx}" ) );
   const allKeys = await Promise.all( paths.map( async srcPath => {
-    const src = await readFile( srcPath );
+    const src = await fsp.readFile( srcPath );
     const tMatches = [...src.toString( ).matchAll( /[^a-z]t\(\s*["']([\w-_\s]+?)["']/g )];
     const i18nkeyMatches = [...src.toString( ).matchAll( /i18nKey=["']([\w-_\s]+?)["']/g )];
     return [...tMatches, ...i18nkeyMatches].map( match => match[1] );
@@ -268,6 +285,7 @@ async function unused( ) {
   }
 }
 
+// Look for keys in the code that aren't in the source strings
 async function untranslatable( ) {
   const keys = await getKeys( );
   const keysInUse = await getKeysInUse( );
@@ -282,6 +300,25 @@ async function untranslatable( ) {
   }
 }
 
+// Ensure localization file names match iNat convention and not Crowdin, e.g.
+// we use "fr" instead of "fr-FR"
+async function normalizeFileNames( ) {
+  const paths = await l10nFtlPaths();
+  return Promise.all( paths.map( async l10nPath => {
+    const locale = path.basename( l10nPath, ".ftl" );
+    const [lng, region] = locale.split( "-" );
+    // No need to move anything if there's no region
+    if ( !region ) return;
+    // We need to keep some regions
+    if ( SUPPORTED_REGIONAL_LOCALES.indexOf( locale ) >= 0 ) {
+      return;
+    }
+    // Everything else needs to be regionless
+    const newPath = path.join( path.dirname( l10nPath ), `${lng}.ftl` );
+    await fsp.rename( l10nPath, newPath );
+  } ) );
+}
+
 // eslint-disable-next-line no-unused-expressions
 yargs
   .usage( "Usage: $0 <cmd> [args]" )
@@ -309,6 +346,9 @@ yargs
     // eslint-disable-next-line @typescript-eslint/no-empty-function
     ( ) => {},
     async argv => {
+      // Make sure all files are iNat locales before validating and
+      // normalizing FT
+      await normalizeFileNames( );
       await validate( );
       await normalize( );
       await untranslatable( );

src/i18n/initI18next.js

@@ -48,6 +48,19 @@ export const I18NEXT_CONFIG = {
         )
       }
     }
+  },
+  // All languages should fallback to English, some regional variants should
+  // fall back to another region
+  fallbackLng: code => {
+    const fallbacks = [];
+    if ( code.match( /^es-/ ) ) {
+      fallbacks.push( "es" );
+    } else if ( code.match( /^fr-/ ) ) {
+      fallbacks.push( "fr" );
+    } else if ( code.match( /^pt-/ ) ) {
+      fallbacks.push( "pt" );
+    }
+    return [...fallbacks, "en"];
   }
 };
 

src/i18n/l10n/af.ftl

@@ -0,0 +1,22 @@
+### Source strings for iNaturalistReactNative
+###
+### Notes
+### * GroupComments (comments beginning w/ ##) are not allowed because all
+###   strings in this file will be alphabetized and it's impossible to
+###   determine where group comments should fit in.
+### * Keys should match their content closesly but not exceed 100 chars
+### * Try to annotate all strings with comments to provide context for
+###   translators, especially for fragments and any situation where the
+###   meaning is open to interpretation without context
+### * Use different strings for synonyms, e.g. stop-noun and stop-verb, as
+###   these might have different translations in different languages
+### * Accessibility hints are used by screen readers to describe what happens
+###   when the user interacts with an element
+###   (https://developer.apple.com/documentation/uikit/uiaccessibilityelement/1619585-accessibilityhint).
+###   The iOS Guidelines defines it as "A string that briefly describes the
+###   result of performing an action on the accessibility element." We write
+###   them in third person singular ending with a period.
+
+Welcome-to-iNaturalist = Welkom by iNaturalist!
+# Welcome user back to app
+Welcome-user = <0>Welkom terug,</0><1>{ $userHandle }</1>

src/i18n/l10n/af.ftl.json

@@ -0,0 +1,7 @@
+{
+  "Welcome-to-iNaturalist": "Welkom by iNaturalist!",
+  "Welcome-user": {
+    "comment": "Welcome user back to app",
+    "val": "<0>Welkom terug,</0><1>{ $userHandle }</1>"
+  }
+}

src/i18n/l10n/ar.ftl

@@ -0,0 +1,22 @@
+### Source strings for iNaturalistReactNative
+###
+### Notes
+### * GroupComments (comments beginning w/ ##) are not allowed because all
+###   strings in this file will be alphabetized and it's impossible to
+###   determine where group comments should fit in.
+### * Keys should match their content closesly but not exceed 100 chars
+### * Try to annotate all strings with comments to provide context for
+###   translators, especially for fragments and any situation where the
+###   meaning is open to interpretation without context
+### * Use different strings for synonyms, e.g. stop-noun and stop-verb, as
+###   these might have different translations in different languages
+### * Accessibility hints are used by screen readers to describe what happens
+###   when the user interacts with an element
+###   (https://developer.apple.com/documentation/uikit/uiaccessibilityelement/1619585-accessibilityhint).
+###   The iOS Guidelines defines it as "A string that briefly describes the
+###   result of performing an action on the accessibility element." We write
+###   them in third person singular ending with a period.
+
+Welcome-to-iNaturalist = مرحبا بكم في iNaturalist!
+# Welcome user back to app
+Welcome-user = <0>مرحبا بعودتك،</0><1>{ $userHandle }</1>

src/i18n/l10n/ar.ftl.json

@@ -0,0 +1,7 @@
+{
+  "Welcome-to-iNaturalist": "مرحبا بكم في iNaturalist!",
+  "Welcome-user": {
+    "comment": "Welcome user back to app",
+    "val": "<0>مرحبا بعودتك،</0><1>{ $userHandle }</1>"
+  }
+}

src/i18n/loadTranslations.js

@@ -5,5 +5,7 @@ export default locale => {
   if ( locale === "es" ) { return require( "./l10n/es.ftl.json" ); }
   if ( locale === "es-MX" ) { return require( "./l10n/es-MX.ftl.json" ); }
   if ( locale === "en" ) { return require( "./l10n/en.ftl.json" ); }
+  if ( locale === "ar" ) { return require( "./l10n/ar.ftl.json" ); }
+  if ( locale === "af" ) { return require( "./l10n/af.ftl.json" ); }
   return require( "./l10n/en.ftl.json" );
 };