Protect non-embedded CID text layers from PDF/A corruption (closes #1561)

Ghostscript's PDF/A conversion re-embeds non-embedded CID (CJK) fonts by
substituting a system font, which corrupts the character-to-Unicode
mapping and silently destroys an existing text layer -- commonly the OCR
layer Adobe Acrobat adds to scanned CJK documents.

Detect non-embedded CID/Type0 fonts before conversion: with
--output-type auto (the default) downgrade to a regular PDF and preserve
the text layer; with an explicit --output-type pdfa* stop with an error
rather than emit corrupted output. Simple non-embedded fonts (e.g. Latin)
are left alone -- Ghostscript substitutes them without corrupting the
text, and they are far too common to treat as conversion blockers.

Use --output-type pdf to keep the existing text layer, or --force-ocr to
rebuild it with embedded fonts.

docs/advanced.md

@@ -523,6 +523,36 @@ OCRmyPDF can produce PDF/A compliant output for long-term archival. The
 | `pdf` | Standard PDF, no PDF/A conversion |
 | `none` | No output file (useful with `--sidecar`) |
 
+### Non-embedded fonts and PDF/A
+
+:::{versionadded} 17.8.0
+OCRmyPDF now refuses to corrupt non-embedded CID text layers during PDF/A
+conversion.
+:::
+
+PDF/A requires every font to be embedded. If your input already has a text
+layer that uses *non-embedded* CID fonts — most commonly a CJK
+(Chinese-Japanese-Korean) OCR layer
+produced by Adobe Acrobat, which relies on the reader's system fonts —
+Ghostscript would have to substitute and re-embed a replacement font to make
+the file PDF/A. For CID-keyed (CJK) fonts this routinely corrupts the
+character-to-Unicode mapping, so the text silently becomes garbage or stops
+being searchable even though the page still *looks* correct.
+
+Rather than emit corrupted output, OCRmyPDF detects this situation and:
+
+- with `--output-type auto` (the default), produces a regular PDF instead of
+  PDF/A, preserving the existing text layer exactly;
+- with an explicit `--output-type pdfa` (or `pdfa-1`/`pdfa-2`/`pdfa-3`), stops
+  with an error.
+
+This is a Ghostscript limitation that OCRmyPDF cannot repair, because a
+non-embedded font cannot be made PDF/A-compliant without re-embedding it. To
+keep the existing text layer, use `--output-type pdf`. To produce PDF/A anyway,
+re-run OCR with `--force-ocr`, which discards the original text layer and
+rebuilds it with embedded fonts. Text layers whose fonts are *already embedded*
+are converted to PDF/A normally.
+
 ### Speculative PDF/A conversion
 
 :::{versionadded} 17.0.0

docs/releasenotes/version17.md

@@ -5,6 +5,16 @@
 
 ## v17.8.0
 
+- OCRmyPDF no longer silently corrupts a non-embedded CID (CJK) text layer when
+  producing PDF/A ({issue}`1561`). PDF/A requires all fonts to be embedded, so
+  Ghostscript substitutes and re-embeds non-embedded CID fonts — such as the OCR
+  text layer Adobe Acrobat adds to scanned CJK documents — which mangles the
+  text and destroys searchability. OCRmyPDF now detects non-embedded CID fonts
+  before conversion: with `--output-type auto` (the default) it produces a
+  regular PDF and preserves the existing text layer, and with an explicit
+  `--output-type pdfa*` it stops with an error rather than emit corrupted
+  output. Use `--output-type pdf` to keep the text layer, or `--force-ocr` to
+  rebuild it with embedded fonts.
 - Writing the output PDF to standard output (`ocrmypdf input.pdf -`) is now
   protected against corruption at the operating system level. Previously
   OCRmyPDF relied on no in-process code — third-party libraries, plugins, or

src/ocrmypdf/_pipeline.py

@@ -36,6 +36,7 @@ from ocrmypdf.exceptions import (
     DpiError,
     EncryptedPdfError,
     InputFileError,
+    NonEmbeddedFontsError,
     PriorOcrFoundError,
     TaggedPDFError,
     UnsupportedImageFormatError,
@@ -43,6 +44,7 @@ from ocrmypdf.exceptions import (
 from ocrmypdf.helpers import IMG2PDF_KWARGS, Resolution, safe_symlink
 from ocrmypdf.pdfa import (
     file_claims_pdfa,
+    find_nonembedded_cid_fonts,
     generate_pdfa_ps,
     speculative_pdfa_conversion,
 )
@@ -977,6 +979,12 @@ def convert_to_pdfa(input_pdf: Path, input_ps_stub: Path, context: PdfContext) -
     # pikepdf can deal with this, but we make the world a better place by
     # stamping them out as soon as possible.
     with pikepdf.open(input_pdf) as pdf_file:
+        # Ghostscript would substitute and re-embed any non-embedded CID font to
+        # satisfy PDF/A, corrupting CJK text (e.g. an Acrobat OCR layer) in the
+        # process. Refuse rather than silently damage the user's text layer.
+        nonembedded = find_nonembedded_cid_fonts(pdf_file)
+        if nonembedded:
+            raise NonEmbeddedFontsError(nonembedded)
         if repair_docinfo_nuls(pdf_file):
             pdf_file.save(fix_docinfo_file)
         else:
@@ -1089,6 +1097,21 @@ def try_auto_pdfa(input_pdf: Path, context: PdfContext) -> tuple[Path, str]:
     """
     from ocrmypdf._exec import verapdf
 
+    # Non-embedded CID fonts cannot be made PDF/A without Ghostscript font
+    # substitution that corrupts CID/CJK text. Rather than risk an existing
+    # text layer, downgrade to a regular PDF (the same outcome as any other
+    # case where best-effort PDF/A is not achievable).
+    with pikepdf.open(input_pdf) as pdf_file:
+        nonembedded = find_nonembedded_cid_fonts(pdf_file)
+    if nonembedded:
+        log.info(
+            "Auto mode: input has non-embedded CID fonts (%s) that cannot be "
+            "converted to PDF/A without corrupting the text; outputting a "
+            "regular PDF. Use --output-type pdf to select this explicitly.",
+            ', '.join(sorted(nonembedded)),
+        )
+        return (input_pdf, 'pdf')
+
     # If verapdf available, try speculative conversion with validation
     if verapdf.available():
         result = try_speculative_pdfa(input_pdf, context)

src/ocrmypdf/exceptions.py

@@ -139,6 +139,37 @@ class TaggedPDFError(InputFileError):
     )
 
 
+class NonEmbeddedFontsError(InputFileError):
+    """Input has non-embedded CID fonts that PDF/A conversion would corrupt.
+
+    PDF/A requires all fonts to be embedded. Ghostscript substitutes and embeds
+    a replacement for non-embedded CID (CJK) fonts, which corrupts the
+    character-to-Unicode mapping and silently destroys an existing text layer
+    (commonly an Adobe Acrobat CJK OCR layer). OCRmyPDF refuses to produce such
+    output rather than damage the user's data
+    (see https://github.com/ocrmypdf/OCRmyPDF/issues/1561).
+    """
+
+    def __init__(self, fonts: set[str]):
+        """Build guidance naming the offending fonts."""
+        super().__init__()
+        font_list = ', '.join(sorted(fonts))
+        self.message = dedent(
+            f"""\
+            The input PDF contains non-embedded CID (character ID) fonts: {font_list}.
+
+            PDF/A requires all fonts to be embedded. Converting to PDF/A would
+            make Ghostscript substitute and embed replacement fonts, which
+            corrupts CID (e.g. CJK/Chinese-Japanese-Korean) text and silently
+            destroys an existing text layer such as one produced by Adobe Acrobat.
+
+            Use --output-type pdf to keep the existing text layer intact without
+            PDF/A conversion, or --force-ocr to discard the existing layer and
+            rebuild it with embedded fonts.
+            """
+        )
+
+
 class ColorConversionNeededError(BadArgsError):
     """PDF needs color conversion to a standard color space.
 

src/ocrmypdf/pdfa.py

@@ -137,6 +137,65 @@ def file_claims_pdfa(filename: Path):
     return pdfa_dict
 
 
+def _cid_font_is_embedded(type0_font: Dictionary) -> bool:
+    """Return True if a Type0 font's CID descendant carries embedded glyphs."""
+    for descendant in type0_font.get(Name.DescendantFonts, []):
+        descriptor = descendant.get(Name.FontDescriptor, None)
+        if descriptor is not None and any(
+            key in descriptor for key in (Name.FontFile, Name.FontFile2, Name.FontFile3)
+        ):
+            return True
+    return False
+
+
+def find_nonembedded_cid_fonts(pdf: Pdf) -> set[str]:
+    """Find CID-keyed (Type0) fonts that lack embedded glyph data.
+
+    PDF/A requires every font to be embedded. When Ghostscript converts a PDF
+    to PDF/A it must substitute and embed a replacement for any non-embedded
+    font. For CID-keyed fonts -- which is how CJK text is encoded, including the
+    OCR text layers produced by Adobe Acrobat -- this substitution routinely
+    corrupts the character-to-Unicode mapping, silently destroying the
+    searchable text. Detecting these fonts lets the caller refuse PDF/A
+    conversion rather than emit corrupted output.
+
+    Simple (non-CID) non-embedded fonts are not reported: Ghostscript
+    substitutes standard encodings for them without corrupting the text, and
+    they are far too common to treat as conversion blockers.
+
+    Args:
+        pdf: An open ``pikepdf.Pdf`` to scan.
+
+    Returns:
+        The set of ``BaseFont`` names of non-embedded CID fonts found.
+    """
+    found: set[str] = set()
+
+    def scan_resources(resources, depth: int = 0) -> None:
+        if resources is None or depth > 10:
+            return
+        fonts = resources.get(Name.Font, None)
+        if fonts is not None:
+            for font in fonts.values():
+                try:
+                    if font.get(Name.Subtype) != Name.Type0:
+                        continue
+                    if not _cid_font_is_embedded(font):
+                        basefont = str(font.get(Name.BaseFont, '/(unnamed)'))
+                        found.add(basefont.lstrip('/'))
+                except (AttributeError, TypeError, KeyError):
+                    continue
+        xobjects = resources.get(Name.XObject, None)
+        if xobjects is not None:
+            for xobj in xobjects.values():
+                if xobj.get(Name.Subtype) == Name.Form and Name.Resources in xobj:
+                    scan_resources(xobj[Name.Resources], depth + 1)
+
+    for page in pdf.pages:
+        scan_resources(page.get(Name.Resources, None))
+    return found
+
+
 def _load_srgb_icc_profile() -> bytes:
     """Load the sRGB ICC profile from package data."""
     return (package_files('ocrmypdf.data') / SRGB_ICC_PROFILE_NAME).read_bytes()
@@ -191,12 +250,14 @@ def add_srgb_output_intent(pdf: Pdf) -> None:
     icc_stream[Name.N] = 3  # RGB has 3 components
 
     # Create OutputIntent dictionary
-    output_intent = Dictionary({
-        '/Type': Name.OutputIntent,
-        '/S': Name('/GTS_PDFA1'),
-        '/OutputConditionIdentifier': 'sRGB',
-        '/DestOutputProfile': icc_stream,
-    })
+    output_intent = Dictionary(
+        {
+            '/Type': Name.OutputIntent,
+            '/S': Name('/GTS_PDFA1'),
+            '/OutputConditionIdentifier': 'sRGB',
+            '/DestOutputProfile': icc_stream,
+        }
+    )
 
     # Add to catalog's OutputIntents array
     if Name.OutputIntents not in pdf.Root:

tests/test_pdfa.py

@@ -7,10 +7,147 @@ import os
 
 import pikepdf
 import pytest
+from pikepdf import Name
 
-from ocrmypdf.exceptions import MissingDependencyError
+from ocrmypdf.exceptions import ExitCode, MissingDependencyError
+from ocrmypdf.pdfa import file_claims_pdfa, find_nonembedded_cid_fonts
 
-from .conftest import check_ocrmypdf
+from .conftest import check_ocrmypdf, run_ocrmypdf_api
+
+
+def _make_cid_font(
+    pdf: pikepdf.Pdf, *, embedded: bool, basefont: str
+) -> pikepdf.Object:
+    """Build a Type0/CID font object, optionally embedding glyph data."""
+    descriptor = pikepdf.Dictionary(
+        Type=Name.FontDescriptor, FontName=Name(basefont), Flags=4
+    )
+    if embedded:
+        # The actual bytes do not matter; only the presence of FontFile2 marks
+        # the CID font as embedded.
+        descriptor.FontFile2 = pdf.make_stream(b'\x00\x01\x00\x00 fake font program')
+    cidfont = pdf.make_indirect(
+        pikepdf.Dictionary(
+            Type=Name.Font,
+            Subtype=Name.CIDFontType2,
+            BaseFont=Name(basefont),
+            FontDescriptor=descriptor,
+            CIDSystemInfo=pikepdf.Dictionary(
+                Registry='Adobe', Ordering='Identity', Supplement=0
+            ),
+        )
+    )
+    return pdf.make_indirect(
+        pikepdf.Dictionary(
+            Type=Name.Font,
+            Subtype=Name.Type0,
+            BaseFont=Name(basefont),
+            Encoding=Name.Identity_H,
+            DescendantFonts=pikepdf.Array([cidfont]),
+        )
+    )
+
+
+def _write_cid_font_pdf(path, *, embedded: bool, basefont='/ABCDEF+TestCID'):
+    with pikepdf.new() as pdf:
+        page = pdf.add_blank_page()
+        font = _make_cid_font(pdf, embedded=embedded, basefont=basefont)
+        page.Resources = pikepdf.Dictionary(Font=pikepdf.Dictionary(F0=font))
+        pdf.save(path)
+
+
+class TestFindNonembeddedCidFonts:
+    def test_blank_page_reports_nothing(self, tmp_path):
+        path = tmp_path / 'blank.pdf'
+        with pikepdf.new() as pdf:
+            pdf.add_blank_page()
+            pdf.save(path)
+        with pikepdf.open(path) as pdf:
+            assert find_nonembedded_cid_fonts(pdf) == set()
+
+    def test_detects_nonembedded_cid_font(self, tmp_path):
+        path = tmp_path / 'nonembedded.pdf'
+        _write_cid_font_pdf(path, embedded=False)
+        with pikepdf.open(path) as pdf:
+            assert find_nonembedded_cid_fonts(pdf) == {'ABCDEF+TestCID'}
+
+    def test_ignores_embedded_cid_font(self, tmp_path):
+        path = tmp_path / 'embedded.pdf'
+        _write_cid_font_pdf(path, embedded=True)
+        with pikepdf.open(path) as pdf:
+            assert find_nonembedded_cid_fonts(pdf) == set()
+
+    def test_detects_nonembedded_cid_font_in_form_xobject(self, tmp_path):
+        path = tmp_path / 'xobject.pdf'
+        with pikepdf.new() as pdf:
+            page = pdf.add_blank_page()
+            font = _make_cid_font(pdf, embedded=False, basefont='/ZZZ+Hidden')
+            form = pdf.make_stream(
+                b'',
+                Type=Name.XObject,
+                Subtype=Name.Form,
+                BBox=pikepdf.Array([0, 0, 1, 1]),
+                Resources=pikepdf.Dictionary(Font=pikepdf.Dictionary(F0=font)),
+            )
+            page.Resources = pikepdf.Dictionary(XObject=pikepdf.Dictionary(Fm0=form))
+            pdf.save(path)
+        with pikepdf.open(path) as pdf:
+            assert find_nonembedded_cid_fonts(pdf) == {'ZZZ+Hidden'}
+
+
+@pytest.fixture
+def nonembedded_cid_pdf(tmp_path):
+    """A PDF with a real, non-embedded CID (CJK) text layer, as Acrobat produces."""
+    reportlab = pytest.importorskip('reportlab')
+    del reportlab
+    from reportlab.lib.pagesizes import letter
+    from reportlab.pdfbase import pdfmetrics
+    from reportlab.pdfbase.cidfonts import UnicodeCIDFont
+    from reportlab.pdfgen import canvas
+
+    path = tmp_path / 'cjk_nonembedded.pdf'
+    pdfmetrics.registerFont(UnicodeCIDFont('STSong-Light'))  # Adobe-GB1, not embedded
+    c = canvas.Canvas(str(path), pagesize=letter)
+    c.setFont('STSong-Light', 24)
+    c.drawString(60, 650, '你好世界')
+    c.showPage()
+    c.save()
+    # Sanity check that we built the structure under test.
+    with pikepdf.open(path) as pdf:
+        assert find_nonembedded_cid_fonts(pdf)
+    return path
+
+
+def test_pdfa_rejects_nonembedded_cid_font(nonembedded_cid_pdf, outpdf):
+    """Explicit PDF/A on a non-embedded CID layer must error, not corrupt it."""
+    exitcode = run_ocrmypdf_api(
+        nonembedded_cid_pdf,
+        outpdf,
+        '--plugin',
+        'tests/plugins/tesseract_noop.py',
+        '--skip-text',
+        '--output-type',
+        'pdfa',
+    )
+    assert exitcode == ExitCode.input_file
+    assert not outpdf.exists() or outpdf.stat().st_size == 0
+
+
+def test_auto_downgrades_nonembedded_cid_font_to_pdf(nonembedded_cid_pdf, outpdf):
+    """Auto mode preserves the text layer by outputting a regular PDF."""
+    check_ocrmypdf(
+        nonembedded_cid_pdf,
+        outpdf,
+        '--plugin',
+        'tests/plugins/tesseract_noop.py',
+        '--skip-text',
+        '--output-type',
+        'auto',
+    )
+    # Not PDF/A, and the original non-embedded layer survived untouched.
+    assert not file_claims_pdfa(outpdf)['pass']
+    with pikepdf.open(outpdf) as pdf:
+        assert find_nonembedded_cid_fonts(pdf)
 
 
 @pytest.mark.parametrize('optimize', (0, 3))