From efe83e8c54c7740e1c63e231a67652f4d5fb2dc7 Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Tue, 30 Jun 2026 00:01:42 -0700 Subject: [PATCH] Protect non-embedded CID text layers from PDF/A corruption (closes #1561) Ghostscript's PDF/A conversion re-embeds non-embedded CID (CJK) fonts by substituting a system font, which corrupts the character-to-Unicode mapping and silently destroys an existing text layer -- commonly the OCR layer Adobe Acrobat adds to scanned CJK documents. Detect non-embedded CID/Type0 fonts before conversion: with --output-type auto (the default) downgrade to a regular PDF and preserve the text layer; with an explicit --output-type pdfa* stop with an error rather than emit corrupted output. Simple non-embedded fonts (e.g. Latin) are left alone -- Ghostscript substitutes them without corrupting the text, and they are far too common to treat as conversion blockers. Use --output-type pdf to keep the existing text layer, or --force-ocr to rebuild it with embedded fonts. --- docs/advanced.md | 30 +++++++ docs/releasenotes/version17.md | 10 +++ src/ocrmypdf/_pipeline.py | 23 ++++++ src/ocrmypdf/exceptions.py | 31 ++++++++ src/ocrmypdf/pdfa.py | 73 +++++++++++++++-- tests/test_pdfa.py | 141 ++++++++++++++++++++++++++++++++- 6 files changed, 300 insertions(+), 8 deletions(-) diff --git a/docs/advanced.md b/docs/advanced.md index efabea1b..17312ac6 100644 --- a/docs/advanced.md +++ b/docs/advanced.md @@ -523,6 +523,36 @@ OCRmyPDF can produce PDF/A compliant output for long-term archival. The | `pdf` | Standard PDF, no PDF/A conversion | | `none` | No output file (useful with `--sidecar`) | +### Non-embedded fonts and PDF/A + +:::{versionadded} 17.8.0 +OCRmyPDF now refuses to corrupt non-embedded CID text layers during PDF/A +conversion. +::: + +PDF/A requires every font to be embedded. If your input already has a text +layer that uses *non-embedded* CID fonts — most commonly a CJK +(Chinese-Japanese-Korean) OCR layer +produced by Adobe Acrobat, which relies on the reader's system fonts — +Ghostscript would have to substitute and re-embed a replacement font to make +the file PDF/A. For CID-keyed (CJK) fonts this routinely corrupts the +character-to-Unicode mapping, so the text silently becomes garbage or stops +being searchable even though the page still *looks* correct. + +Rather than emit corrupted output, OCRmyPDF detects this situation and: + +- with `--output-type auto` (the default), produces a regular PDF instead of + PDF/A, preserving the existing text layer exactly; +- with an explicit `--output-type pdfa` (or `pdfa-1`/`pdfa-2`/`pdfa-3`), stops + with an error. + +This is a Ghostscript limitation that OCRmyPDF cannot repair, because a +non-embedded font cannot be made PDF/A-compliant without re-embedding it. To +keep the existing text layer, use `--output-type pdf`. To produce PDF/A anyway, +re-run OCR with `--force-ocr`, which discards the original text layer and +rebuilds it with embedded fonts. Text layers whose fonts are *already embedded* +are converted to PDF/A normally. + ### Speculative PDF/A conversion :::{versionadded} 17.0.0 diff --git a/docs/releasenotes/version17.md b/docs/releasenotes/version17.md index 9b96cae2..9809d751 100644 --- a/docs/releasenotes/version17.md +++ b/docs/releasenotes/version17.md @@ -5,6 +5,16 @@ ## v17.8.0 +- OCRmyPDF no longer silently corrupts a non-embedded CID (CJK) text layer when + producing PDF/A ({issue}`1561`). PDF/A requires all fonts to be embedded, so + Ghostscript substitutes and re-embeds non-embedded CID fonts — such as the OCR + text layer Adobe Acrobat adds to scanned CJK documents — which mangles the + text and destroys searchability. OCRmyPDF now detects non-embedded CID fonts + before conversion: with `--output-type auto` (the default) it produces a + regular PDF and preserves the existing text layer, and with an explicit + `--output-type pdfa*` it stops with an error rather than emit corrupted + output. Use `--output-type pdf` to keep the text layer, or `--force-ocr` to + rebuild it with embedded fonts. - Writing the output PDF to standard output (`ocrmypdf input.pdf -`) is now protected against corruption at the operating system level. Previously OCRmyPDF relied on no in-process code — third-party libraries, plugins, or diff --git a/src/ocrmypdf/_pipeline.py b/src/ocrmypdf/_pipeline.py index a01a7e61..08ee2220 100644 --- a/src/ocrmypdf/_pipeline.py +++ b/src/ocrmypdf/_pipeline.py @@ -36,6 +36,7 @@ from ocrmypdf.exceptions import ( DpiError, EncryptedPdfError, InputFileError, + NonEmbeddedFontsError, PriorOcrFoundError, TaggedPDFError, UnsupportedImageFormatError, @@ -43,6 +44,7 @@ from ocrmypdf.exceptions import ( from ocrmypdf.helpers import IMG2PDF_KWARGS, Resolution, safe_symlink from ocrmypdf.pdfa import ( file_claims_pdfa, + find_nonembedded_cid_fonts, generate_pdfa_ps, speculative_pdfa_conversion, ) @@ -977,6 +979,12 @@ def convert_to_pdfa(input_pdf: Path, input_ps_stub: Path, context: PdfContext) - # pikepdf can deal with this, but we make the world a better place by # stamping them out as soon as possible. with pikepdf.open(input_pdf) as pdf_file: + # Ghostscript would substitute and re-embed any non-embedded CID font to + # satisfy PDF/A, corrupting CJK text (e.g. an Acrobat OCR layer) in the + # process. Refuse rather than silently damage the user's text layer. + nonembedded = find_nonembedded_cid_fonts(pdf_file) + if nonembedded: + raise NonEmbeddedFontsError(nonembedded) if repair_docinfo_nuls(pdf_file): pdf_file.save(fix_docinfo_file) else: @@ -1089,6 +1097,21 @@ def try_auto_pdfa(input_pdf: Path, context: PdfContext) -> tuple[Path, str]: """ from ocrmypdf._exec import verapdf + # Non-embedded CID fonts cannot be made PDF/A without Ghostscript font + # substitution that corrupts CID/CJK text. Rather than risk an existing + # text layer, downgrade to a regular PDF (the same outcome as any other + # case where best-effort PDF/A is not achievable). + with pikepdf.open(input_pdf) as pdf_file: + nonembedded = find_nonembedded_cid_fonts(pdf_file) + if nonembedded: + log.info( + "Auto mode: input has non-embedded CID fonts (%s) that cannot be " + "converted to PDF/A without corrupting the text; outputting a " + "regular PDF. Use --output-type pdf to select this explicitly.", + ', '.join(sorted(nonembedded)), + ) + return (input_pdf, 'pdf') + # If verapdf available, try speculative conversion with validation if verapdf.available(): result = try_speculative_pdfa(input_pdf, context) diff --git a/src/ocrmypdf/exceptions.py b/src/ocrmypdf/exceptions.py index 6f3b3af4..402b2b24 100644 --- a/src/ocrmypdf/exceptions.py +++ b/src/ocrmypdf/exceptions.py @@ -139,6 +139,37 @@ class TaggedPDFError(InputFileError): ) +class NonEmbeddedFontsError(InputFileError): + """Input has non-embedded CID fonts that PDF/A conversion would corrupt. + + PDF/A requires all fonts to be embedded. Ghostscript substitutes and embeds + a replacement for non-embedded CID (CJK) fonts, which corrupts the + character-to-Unicode mapping and silently destroys an existing text layer + (commonly an Adobe Acrobat CJK OCR layer). OCRmyPDF refuses to produce such + output rather than damage the user's data + (see https://github.com/ocrmypdf/OCRmyPDF/issues/1561). + """ + + def __init__(self, fonts: set[str]): + """Build guidance naming the offending fonts.""" + super().__init__() + font_list = ', '.join(sorted(fonts)) + self.message = dedent( + f"""\ + The input PDF contains non-embedded CID (character ID) fonts: {font_list}. + + PDF/A requires all fonts to be embedded. Converting to PDF/A would + make Ghostscript substitute and embed replacement fonts, which + corrupts CID (e.g. CJK/Chinese-Japanese-Korean) text and silently + destroys an existing text layer such as one produced by Adobe Acrobat. + + Use --output-type pdf to keep the existing text layer intact without + PDF/A conversion, or --force-ocr to discard the existing layer and + rebuild it with embedded fonts. + """ + ) + + class ColorConversionNeededError(BadArgsError): """PDF needs color conversion to a standard color space. diff --git a/src/ocrmypdf/pdfa.py b/src/ocrmypdf/pdfa.py index 3e73c394..6d79722a 100644 --- a/src/ocrmypdf/pdfa.py +++ b/src/ocrmypdf/pdfa.py @@ -137,6 +137,65 @@ def file_claims_pdfa(filename: Path): return pdfa_dict +def _cid_font_is_embedded(type0_font: Dictionary) -> bool: + """Return True if a Type0 font's CID descendant carries embedded glyphs.""" + for descendant in type0_font.get(Name.DescendantFonts, []): + descriptor = descendant.get(Name.FontDescriptor, None) + if descriptor is not None and any( + key in descriptor for key in (Name.FontFile, Name.FontFile2, Name.FontFile3) + ): + return True + return False + + +def find_nonembedded_cid_fonts(pdf: Pdf) -> set[str]: + """Find CID-keyed (Type0) fonts that lack embedded glyph data. + + PDF/A requires every font to be embedded. When Ghostscript converts a PDF + to PDF/A it must substitute and embed a replacement for any non-embedded + font. For CID-keyed fonts -- which is how CJK text is encoded, including the + OCR text layers produced by Adobe Acrobat -- this substitution routinely + corrupts the character-to-Unicode mapping, silently destroying the + searchable text. Detecting these fonts lets the caller refuse PDF/A + conversion rather than emit corrupted output. + + Simple (non-CID) non-embedded fonts are not reported: Ghostscript + substitutes standard encodings for them without corrupting the text, and + they are far too common to treat as conversion blockers. + + Args: + pdf: An open ``pikepdf.Pdf`` to scan. + + Returns: + The set of ``BaseFont`` names of non-embedded CID fonts found. + """ + found: set[str] = set() + + def scan_resources(resources, depth: int = 0) -> None: + if resources is None or depth > 10: + return + fonts = resources.get(Name.Font, None) + if fonts is not None: + for font in fonts.values(): + try: + if font.get(Name.Subtype) != Name.Type0: + continue + if not _cid_font_is_embedded(font): + basefont = str(font.get(Name.BaseFont, '/(unnamed)')) + found.add(basefont.lstrip('/')) + except (AttributeError, TypeError, KeyError): + continue + xobjects = resources.get(Name.XObject, None) + if xobjects is not None: + for xobj in xobjects.values(): + if xobj.get(Name.Subtype) == Name.Form and Name.Resources in xobj: + scan_resources(xobj[Name.Resources], depth + 1) + + for page in pdf.pages: + scan_resources(page.get(Name.Resources, None)) + return found + + def _load_srgb_icc_profile() -> bytes: """Load the sRGB ICC profile from package data.""" return (package_files('ocrmypdf.data') / SRGB_ICC_PROFILE_NAME).read_bytes() @@ -191,12 +250,14 @@ def add_srgb_output_intent(pdf: Pdf) -> None: icc_stream[Name.N] = 3 # RGB has 3 components # Create OutputIntent dictionary - output_intent = Dictionary({ - '/Type': Name.OutputIntent, - '/S': Name('/GTS_PDFA1'), - '/OutputConditionIdentifier': 'sRGB', - '/DestOutputProfile': icc_stream, - }) + output_intent = Dictionary( + { + '/Type': Name.OutputIntent, + '/S': Name('/GTS_PDFA1'), + '/OutputConditionIdentifier': 'sRGB', + '/DestOutputProfile': icc_stream, + } + ) # Add to catalog's OutputIntents array if Name.OutputIntents not in pdf.Root: diff --git a/tests/test_pdfa.py b/tests/test_pdfa.py index 9edd0654..a93cd361 100644 --- a/tests/test_pdfa.py +++ b/tests/test_pdfa.py @@ -7,10 +7,147 @@ import os import pikepdf import pytest +from pikepdf import Name -from ocrmypdf.exceptions import MissingDependencyError +from ocrmypdf.exceptions import ExitCode, MissingDependencyError +from ocrmypdf.pdfa import file_claims_pdfa, find_nonembedded_cid_fonts -from .conftest import check_ocrmypdf +from .conftest import check_ocrmypdf, run_ocrmypdf_api + + +def _make_cid_font( + pdf: pikepdf.Pdf, *, embedded: bool, basefont: str +) -> pikepdf.Object: + """Build a Type0/CID font object, optionally embedding glyph data.""" + descriptor = pikepdf.Dictionary( + Type=Name.FontDescriptor, FontName=Name(basefont), Flags=4 + ) + if embedded: + # The actual bytes do not matter; only the presence of FontFile2 marks + # the CID font as embedded. + descriptor.FontFile2 = pdf.make_stream(b'\x00\x01\x00\x00 fake font program') + cidfont = pdf.make_indirect( + pikepdf.Dictionary( + Type=Name.Font, + Subtype=Name.CIDFontType2, + BaseFont=Name(basefont), + FontDescriptor=descriptor, + CIDSystemInfo=pikepdf.Dictionary( + Registry='Adobe', Ordering='Identity', Supplement=0 + ), + ) + ) + return pdf.make_indirect( + pikepdf.Dictionary( + Type=Name.Font, + Subtype=Name.Type0, + BaseFont=Name(basefont), + Encoding=Name.Identity_H, + DescendantFonts=pikepdf.Array([cidfont]), + ) + ) + + +def _write_cid_font_pdf(path, *, embedded: bool, basefont='/ABCDEF+TestCID'): + with pikepdf.new() as pdf: + page = pdf.add_blank_page() + font = _make_cid_font(pdf, embedded=embedded, basefont=basefont) + page.Resources = pikepdf.Dictionary(Font=pikepdf.Dictionary(F0=font)) + pdf.save(path) + + +class TestFindNonembeddedCidFonts: + def test_blank_page_reports_nothing(self, tmp_path): + path = tmp_path / 'blank.pdf' + with pikepdf.new() as pdf: + pdf.add_blank_page() + pdf.save(path) + with pikepdf.open(path) as pdf: + assert find_nonembedded_cid_fonts(pdf) == set() + + def test_detects_nonembedded_cid_font(self, tmp_path): + path = tmp_path / 'nonembedded.pdf' + _write_cid_font_pdf(path, embedded=False) + with pikepdf.open(path) as pdf: + assert find_nonembedded_cid_fonts(pdf) == {'ABCDEF+TestCID'} + + def test_ignores_embedded_cid_font(self, tmp_path): + path = tmp_path / 'embedded.pdf' + _write_cid_font_pdf(path, embedded=True) + with pikepdf.open(path) as pdf: + assert find_nonembedded_cid_fonts(pdf) == set() + + def test_detects_nonembedded_cid_font_in_form_xobject(self, tmp_path): + path = tmp_path / 'xobject.pdf' + with pikepdf.new() as pdf: + page = pdf.add_blank_page() + font = _make_cid_font(pdf, embedded=False, basefont='/ZZZ+Hidden') + form = pdf.make_stream( + b'', + Type=Name.XObject, + Subtype=Name.Form, + BBox=pikepdf.Array([0, 0, 1, 1]), + Resources=pikepdf.Dictionary(Font=pikepdf.Dictionary(F0=font)), + ) + page.Resources = pikepdf.Dictionary(XObject=pikepdf.Dictionary(Fm0=form)) + pdf.save(path) + with pikepdf.open(path) as pdf: + assert find_nonembedded_cid_fonts(pdf) == {'ZZZ+Hidden'} + + +@pytest.fixture +def nonembedded_cid_pdf(tmp_path): + """A PDF with a real, non-embedded CID (CJK) text layer, as Acrobat produces.""" + reportlab = pytest.importorskip('reportlab') + del reportlab + from reportlab.lib.pagesizes import letter + from reportlab.pdfbase import pdfmetrics + from reportlab.pdfbase.cidfonts import UnicodeCIDFont + from reportlab.pdfgen import canvas + + path = tmp_path / 'cjk_nonembedded.pdf' + pdfmetrics.registerFont(UnicodeCIDFont('STSong-Light')) # Adobe-GB1, not embedded + c = canvas.Canvas(str(path), pagesize=letter) + c.setFont('STSong-Light', 24) + c.drawString(60, 650, '你好世界') + c.showPage() + c.save() + # Sanity check that we built the structure under test. + with pikepdf.open(path) as pdf: + assert find_nonembedded_cid_fonts(pdf) + return path + + +def test_pdfa_rejects_nonembedded_cid_font(nonembedded_cid_pdf, outpdf): + """Explicit PDF/A on a non-embedded CID layer must error, not corrupt it.""" + exitcode = run_ocrmypdf_api( + nonembedded_cid_pdf, + outpdf, + '--plugin', + 'tests/plugins/tesseract_noop.py', + '--skip-text', + '--output-type', + 'pdfa', + ) + assert exitcode == ExitCode.input_file + assert not outpdf.exists() or outpdf.stat().st_size == 0 + + +def test_auto_downgrades_nonembedded_cid_font_to_pdf(nonembedded_cid_pdf, outpdf): + """Auto mode preserves the text layer by outputting a regular PDF.""" + check_ocrmypdf( + nonembedded_cid_pdf, + outpdf, + '--plugin', + 'tests/plugins/tesseract_noop.py', + '--skip-text', + '--output-type', + 'auto', + ) + # Not PDF/A, and the original non-embedded layer survived untouched. + assert not file_claims_pdfa(outpdf)['pass'] + with pikepdf.open(outpdf) as pdf: + assert find_nonembedded_cid_fonts(pdf) @pytest.mark.parametrize('optimize', (0, 3))