tounicode.rs

//! ToUnicode CMap parsing for PDF text extraction
//!
//! This module parses ToUnicode CMaps to convert CID-encoded text to Unicode.

use log::{debug, warn};
use lopdf::{Document, Object, ObjectId};
use std::borrow::Cow;
use std::collections::{HashMap, HashSet};
#[cfg(not(target_arch = "wasm32"))]
use std::path::{Path, PathBuf};

use crate::glyph_names::glyph_to_char;

#[cfg(target_arch = "wasm32")]
static BUILTIN_CMAPS: include_dir::Dir<'_> =
    include_dir::include_dir!("$CARGO_MANIFEST_DIR/external/bcmaps");

/// A parsed ToUnicode CMap mapping CIDs to Unicode strings
#[derive(Debug, Default, Clone)]
pub struct ToUnicodeCMap {
    /// Direct character mappings (CID -> Unicode codepoint(s))
    pub char_map: HashMap<u16, String>,
    /// Range mappings (start_cid, end_cid) -> base_unicode
    pub ranges: Vec<(u16, u16, u32)>,
    /// Byte width of source codes (1 or 2), determined from codespace and CMap entries
    pub code_byte_length: u8,
    /// When true, unmapped CIDs are interpreted as Unicode codepoints directly.
    /// Used as a last resort for Identity-H fonts without ToUnicode/cmap/glyph names.
    pub cid_passthrough: bool,
}

/// Detect whether a parsed ToUnicode CMap is a degenerate full-range identity
/// bfrange (`<0000> <FFFF> <0000>`) with no other mappings.
///
/// Some PDF producers (notably dompdf) emit this single range as the entire
/// ToUnicode, asserting that every CID equals its Unicode codepoint.  For
/// subset CIDFontType2 fonts where CIDs are renumbered GIDs, that identity is
/// false, and the range silently corrupts roughly a third of CJK characters
/// into Latin-1 mojibake.  The correct mapping must come from the embedded
/// TrueType `cmap` table instead.
///
/// Returns `true` when the CMap consists of exactly one range covering the
/// full 16-bit CID space with a base offset of 0, and has no `char_map`
/// entries.  The caller should treat such a CMap as absent.
fn is_degenerate_identity_bfrange(cmap: &ToUnicodeCMap) -> bool {
    cmap.char_map.is_empty()
        && cmap.ranges.len() == 1
        && cmap.ranges[0].0 == 0
        && cmap.ranges[0].1 == 0xFFFF
        && cmap.ranges[0].2 == 0
}

pub(crate) fn build_cmap_entry_from_stream(
    data: &[u8],
    font_dict: &lopdf::Dictionary,
    doc: &Document,
    obj_num: u32,
) -> Option<CMapEntry> {
    if let Some(cmap) = ToUnicodeCMap::parse(data) {
        if is_degenerate_identity_bfrange(&cmap) {
            debug!(
                "ToUnicode obj={}: full-range identity bfrange detected; treating as absent",
                obj_num
            );
        } else {
            let (mut primary, mut remapped) = try_remap_subset_cmap(cmap, font_dict, doc, obj_num);
            let mut fallback = build_fallback_tounicode_from_encoding(font_dict, doc)
                .or_else(|| build_fallback_cmap_for_type0(font_dict, doc))
                .or_else(|| build_fallback_cmap_for_simple(font_dict, doc));

            let primary_entries = primary.char_map.len() + primary.ranges.len();
            if primary_entries < 10 {
                if let Some(fb) = fallback.take() {
                    debug!(
                        "ToUnicode CMap obj={} too sparse ({} entries); using fallback",
                        obj_num, primary_entries
                    );
                    remapped = Some(primary);
                    primary = fb;
                }
            }

            // When a sequential remap was applied and a TrueType fallback has more
            // entries than the primary ToUnicode CMap, prefer the TrueType cmap.
            // Subset fonts number GIDs by document encounter order, so the sorted
            // sequential remap scrambles characters.  The TrueType cmap table maps
            // the real GID→Unicode and is authoritative.
            if remapped.is_some() {
                if let Some(ref fb) = fallback {
                    let fb_entries = fb.char_map.len() + fb.ranges.len();
                    if fb_entries > primary_entries {
                        debug!(
                            "ToUnicode CMap obj={}: TrueType fallback ({} entries) > primary ({}); promoting over sequential remap",
                            obj_num, fb_entries, primary_entries
                        );
                        let old_remap = remapped.take().unwrap();
                        remapped = fallback.take();
                        fallback = Some(old_remap);
                    }
                }
            }

            return Some(CMapEntry {
                primary,
                remapped,
                fallback,
            });
        } // else: degenerate identity bfrange - fall through to fallback
    }

    let fallback = build_fallback_cmap_for_type0(font_dict, doc)
        .or_else(|| build_fallback_cmap_for_simple(font_dict, doc))?;
    debug!(
        "ToUnicode CMap obj={} parse failed; using fallback (entries={})",
        obj_num,
        fallback.char_map.len()
    );
    Some(CMapEntry {
        primary: fallback,
        remapped: None,
        fallback: None,
    })
}

impl ToUnicodeCMap {
    /// Create a new empty CMap
    pub fn new() -> Self {
        Self::default()
    }

    /// Parse a ToUnicode CMap from its decompressed content
    pub fn parse(content: &[u8]) -> Option<Self> {
        let text = String::from_utf8_lossy(content);
        let mut cmap = ToUnicodeCMap::new();
        let mut src_hex_lengths: Vec<usize> = Vec::new();
        let mut use_cmap_name: Option<String> = None;

        // Parse begincodespacerange ... endcodespacerange to determine byte width
        let mut codespace_byte_len: Option<u8> = None;
        if let Some(cs_start) = text.find("begincodespacerange") {
            let section_start = cs_start + "begincodespacerange".len();
            if let Some(cs_end) = text[section_start..].find("endcodespacerange") {
                let section = &text[section_start..section_start + cs_end];
                // Parse hex values to determine byte length
                let mut in_hex = false;
                let mut hex_len = 0;
                for c in section.chars() {
                    if c == '<' {
                        in_hex = true;
                        hex_len = 0;
                    } else if c == '>' {
                        if in_hex && hex_len > 0 {
                            let byte_len = (hex_len + 1) / 2; // 2 hex digits = 1 byte
                            codespace_byte_len = Some(byte_len as u8);
                        }
                        in_hex = false;
                    } else if in_hex && c.is_ascii_hexdigit() {
                        hex_len += 1;
                    }
                }
            }
        }

        // Parse "usecmap" if present
        if let Some(name) = find_usecmap_name(&text) {
            use_cmap_name = Some(name);
        }

        // Parse beginbfchar ... endbfchar sections
        let mut pos = 0;
        while let Some(start) = text[pos..].find("beginbfchar") {
            let section_start = pos + start + "beginbfchar".len();
            if let Some(end) = text[section_start..].find("endbfchar") {
                let section = &text[section_start..section_start + end];
                cmap.parse_bfchar_section(section, &mut src_hex_lengths);
                pos = section_start + end;
            } else {
                break;
            }
        }

        // Parse beginbfrange ... endbfrange sections
        pos = 0;
        while let Some(start) = text[pos..].find("beginbfrange") {
            let section_start = pos + start + "beginbfrange".len();
            if let Some(end) = text[section_start..].find("endbfrange") {
                let section = &text[section_start..section_start + end];
                cmap.parse_bfrange_section(section, &mut src_hex_lengths);
                pos = section_start + end;
            } else {
                break;
            }
        }

        if cmap.char_map.is_empty() && cmap.ranges.is_empty() {
            return None;
        }

        // Determine byte width: use codespace if available, otherwise infer from entries
        cmap.code_byte_length = if let Some(cs_len) = codespace_byte_len {
            // If codespace says 2-byte but ALL entries use 1-byte source codes
            // (hex length <= 2), treat as 1-byte. This handles the common case where
            // codespace is <0000><FFFF> but entries are <20>, <41>, etc.
            if cs_len == 2 && !src_hex_lengths.is_empty() && src_hex_lengths.iter().all(|&l| l <= 2)
            {
                1
            } else {
                cs_len
            }
        } else if !src_hex_lengths.is_empty() {
            // No codespace declaration: infer from entry hex lengths
            let max_hex_len = src_hex_lengths.iter().max().copied().unwrap_or(4);
            if max_hex_len <= 2 {
                1
            } else {
                2
            }
        } else {
            2 // Default to 2-byte
        };

        // Sort ranges by start CID for binary search in lookup()
        cmap.ranges.sort_unstable_by_key(|&(start, _, _)| start);

        if let Some(name) = use_cmap_name {
            if let Some(base) = load_builtin_cmap_by_name(&name) {
                cmap = merge_cmaps(base, cmap);
            } else {
                warn!("usecmap={} could not be loaded", name);
            }
        }

        Some(cmap)
    }

    /// Parse a bfchar section: <src> <dst> pairs
    fn parse_bfchar_section(&mut self, section: &str, src_hex_lengths: &mut Vec<usize>) {
        // Match pairs of hex values: <XXXX> <YYYY>
        let mut chars = section.chars().peekable();

        loop {
            // Skip whitespace
            while chars.peek().is_some_and(|c| c.is_whitespace()) {
                chars.next();
            }

            // Look for opening <
            if chars.peek() != Some(&'<') {
                break;
            }
            chars.next(); // consume <

            // Read source hex
            let mut src_hex = String::new();
            while chars.peek().is_some_and(|&c| c != '>') {
                if let Some(c) = chars.next() {
                    src_hex.push(c);
                }
            }
            chars.next(); // consume >

            // Track source hex length for byte width detection
            let trimmed_src = src_hex.trim();
            if !trimmed_src.is_empty() {
                src_hex_lengths.push(trimmed_src.len());
            }

            // Skip whitespace
            while chars.peek().is_some_and(|c| c.is_whitespace()) {
                chars.next();
            }

            // Look for opening <
            if chars.peek() != Some(&'<') {
                continue;
            }
            chars.next(); // consume <

            // Read destination hex
            let mut dst_hex = String::new();
            while chars.peek().is_some_and(|&c| c != '>') {
                if let Some(c) = chars.next() {
                    dst_hex.push(c);
                }
            }
            chars.next(); // consume >

            // Parse and store mapping
            if let (Some(src), Some(dst)) =
                (parse_hex_u16(&src_hex), hex_to_unicode_string(&dst_hex))
            {
                self.char_map.insert(src, dst);
            }
        }
    }

    /// Parse a bfrange section: <start> <end> <base> or <start> <end> [<u1> <u2> ...] triplets
    fn parse_bfrange_section(&mut self, section: &str, src_hex_lengths: &mut Vec<usize>) {
        let mut chars = section.chars().peekable();

        loop {
            // Skip whitespace
            while chars.peek().is_some_and(|c| c.is_whitespace()) {
                chars.next();
            }

            // Look for opening <
            if chars.peek() != Some(&'<') {
                break;
            }
            chars.next(); // consume <

            // Read start hex
            let mut start_hex = String::new();
            while chars.peek().is_some_and(|&c| c != '>') {
                if let Some(c) = chars.next() {
                    start_hex.push(c);
                }
            }
            chars.next(); // consume >

            // Track source hex length
            let trimmed_start = start_hex.trim();
            if !trimmed_start.is_empty() {
                src_hex_lengths.push(trimmed_start.len());
            }

            // Skip whitespace
            while chars.peek().is_some_and(|c| c.is_whitespace()) {
                chars.next();
            }

            // Read end hex
            if chars.peek() != Some(&'<') {
                continue;
            }
            chars.next();
            let mut end_hex = String::new();
            while chars.peek().is_some_and(|&c| c != '>') {
                if let Some(c) = chars.next() {
                    end_hex.push(c);
                }
            }
            chars.next();

            // Skip whitespace
            while chars.peek().is_some_and(|c| c.is_whitespace()) {
                chars.next();
            }

            // Read base - could be <hex> or [array]
            if chars.peek() == Some(&'<') {
                chars.next();
                let mut base_hex = String::new();
                while chars.peek().is_some_and(|&c| c != '>') {
                    if let Some(c) = chars.next() {
                        base_hex.push(c);
                    }
                }
                chars.next();

                // Store range mapping
                if let (Some(start), Some(end), Some(base)) = (
                    parse_hex_u16(&start_hex),
                    parse_hex_u16(&end_hex),
                    hex_to_unicode_scalar(&base_hex),
                ) {
                    self.ranges.push((start, end, base));
                }
            } else if chars.peek() == Some(&'[') {
                // Array format: [<unicode1> <unicode2> ...]
                // Each entry maps to start_cid + index
                chars.next(); // consume [
                if let (Some(start), Some(end)) =
                    (parse_hex_u16(&start_hex), parse_hex_u16(&end_hex))
                {
                    let mut cid = start;
                    loop {
                        // Skip whitespace
                        while chars.peek().is_some_and(|c| c.is_whitespace()) {
                            chars.next();
                        }
                        if chars.peek() == Some(&']') {
                            chars.next();
                            break;
                        }
                        if chars.peek() != Some(&'<') {
                            break;
                        }
                        chars.next(); // consume <
                        let mut hex = String::new();
                        while chars.peek().is_some_and(|&c| c != '>') {
                            if let Some(c) = chars.next() {
                                hex.push(c);
                            }
                        }
                        chars.next(); // consume >
                        if let Some(unicode_str) = hex_to_unicode_string(&hex) {
                            self.char_map.insert(cid, unicode_str);
                        }
                        if cid >= end {
                            // Skip remaining entries and closing bracket
                            while chars.peek().is_some_and(|&c| c != ']') {
                                chars.next();
                            }
                            if chars.peek() == Some(&']') {
                                chars.next();
                            }
                            break;
                        }
                        cid = cid.saturating_add(1);
                    }
                } else {
                    // Couldn't parse start/end, skip the array
                    while chars.peek().is_some_and(|&c| c != ']') {
                        chars.next();
                    }
                    if chars.peek() == Some(&']') {
                        chars.next();
                    }
                }
            }
        }
    }

    /// Look up a CID and return the Unicode string
    pub fn lookup(&self, cid: u16) -> Option<String> {
        // First check direct mappings
        if let Some(s) = self.char_map.get(&cid) {
            return Some(s.clone());
        }

        // Binary search through sorted ranges
        let idx = self
            .ranges
            .binary_search_by(|&(start, _, _)| start.cmp(&cid))
            .unwrap_or_else(|i| i);

        // Check the range at idx (where start == cid)
        if idx < self.ranges.len() {
            let (start, end, base) = self.ranges[idx];
            if cid >= start && cid <= end {
                let unicode = base + (cid - start) as u32;
                if let Some(c) = char::from_u32(unicode) {
                    return Some(c.to_string());
                }
            }
        }

        // Check the range before idx (cid may fall within a range that starts before it)
        if idx > 0 {
            let (start, end, base) = self.ranges[idx - 1];
            if cid >= start && cid <= end {
                let unicode = base + (cid - start) as u32;
                if let Some(c) = char::from_u32(unicode) {
                    return Some(c.to_string());
                }
            }
        }

        None
    }

    /// Per-byte CMap lookup without Latin-1 fallback.
    /// Returns `(raw_byte, Option<cmap_result>)` for each byte.
    /// Only meaningful for single-byte (code_byte_length==1) CMaps.
    pub fn lookup_bytes(&self, bytes: &[u8]) -> Vec<(u8, Option<String>)> {
        bytes
            .iter()
            .map(|&b| {
                let code = b as u16;
                let result = self.lookup(code).filter(|s| !s.contains('\u{FFFD}'));
                (b, result)
            })
            .collect()
    }

    /// Decode a byte slice to a Unicode string, respecting the CMap's code byte width
    pub fn decode_cids(&self, bytes: &[u8]) -> String {
        let mut result = String::new();
        let mut unmapped_count = 0usize;

        if self.code_byte_length == 1 {
            // Single-byte codes: each byte is a code
            for &b in bytes {
                let code = b as u16;
                match self.lookup(code) {
                    Some(s) if !s.contains('\u{FFFD}') => result.push_str(&s),
                    _ => {
                        // For single-byte unmapped codes, try as Latin-1
                        // (the byte IS the character code in most legacy encodings)
                        if b >= 0x20 {
                            result.push(b as char);
                        }
                        unmapped_count += 1;
                    }
                }
            }
        } else {
            // Two-byte codes: CIDs are 2 bytes each (big-endian)
            for chunk in bytes.chunks(2) {
                if chunk.len() == 2 {
                    let cid = u16::from_be_bytes([chunk[0], chunk[1]]);
                    match self.lookup(cid) {
                        Some(s) if !s.contains('\u{FFFD}') => result.push_str(&s),
                        _ => {
                            if self.cid_passthrough {
                                // Last-resort: treat CID as Unicode codepoint.
                                // Valid for Identity-H fonts where the PDF generator
                                // used Unicode values as CIDs but stripped the cmap.
                                if let Some(ch) = char::from_u32(cid as u32) {
                                    if !ch.is_control() || ch == '\t' || ch == '\n' {
                                        result.push(ch);
                                    } else {
                                        unmapped_count += 1;
                                    }
                                } else {
                                    unmapped_count += 1;
                                }
                            } else {
                                // CIDs are font-internal indices, not Unicode values.
                                // Unmapped 2-byte CIDs are skipped to avoid CJK garbage.
                                unmapped_count += 1;
                            }
                        }
                    }
                }
            }
        }

        // If too many codes were unmapped, signal failure by returning empty
        // so the caller can fall through to other decoding methods
        let total = if self.code_byte_length == 1 {
            bytes.len()
        } else {
            bytes.len() / 2
        };
        if total > 0 && unmapped_count > total / 2 {
            return String::new();
        }

        result
    }

    /// Get the minimum source CID across all mappings (char_map + ranges).
    fn min_source_cid(&self) -> Option<u16> {
        let char_min = self.char_map.keys().copied().min();
        let range_min = self.ranges.iter().map(|&(start, _, _)| start).min();
        match (char_min, range_min) {
            (Some(a), Some(b)) => Some(a.min(b)),
            (a @ Some(_), None) => a,
            (None, b @ Some(_)) => b,
            (None, None) => None,
        }
    }

    /// Get the maximum source CID across all mappings (char_map + ranges).
    fn max_source_cid(&self) -> Option<u16> {
        let char_max = self.char_map.keys().copied().max();
        let range_max = self.ranges.iter().map(|&(_, end, _)| end).max();
        match (char_max, range_max) {
            (Some(a), Some(b)) => Some(a.max(b)),
            (a @ Some(_), None) => a,
            (None, b @ Some(_)) => b,
            (None, None) => None,
        }
    }

    /// Remap a CMap that references pre-subsetting GIDs to sequential post-subsetting GIDs.
    /// Collects all source CIDs, sorts them, and reassigns to 1, 2, 3, ...
    ///
    /// Range expansion stops after `MAX_CID_W_EXPANSION` CID visits, counting
    /// overwrites, so repeated full-width `bfrange`s cannot re-expand the
    /// 16-bit domain. Later overlapping ranges that would have introduced new
    /// CIDs after that many visits are truncated.
    pub fn remap_to_sequential(&self) -> ToUnicodeCMap {
        let mut cid_to_unicode: HashMap<u16, String> = HashMap::new();
        expand_bfranges_for_remap(&self.ranges, &mut cid_to_unicode, MAX_CID_W_EXPANSION);

        // char_map entries override range entries
        for (&cid, unicode) in &self.char_map {
            cid_to_unicode.insert(cid, unicode.clone());
        }

        // Sort old CIDs ascending
        let mut old_cids: Vec<u16> = cid_to_unicode.keys().copied().collect();
        old_cids.sort_unstable();

        // Build new CMap with sequential CIDs starting at 1
        let mut new_cmap = ToUnicodeCMap::new();
        for (i, &old_cid) in old_cids.iter().enumerate() {
            let new_cid = (i + 1) as u16; // GID 0 is .notdef, content CIDs start at 1
            if let Some(unicode) = cid_to_unicode.get(&old_cid) {
                new_cmap.char_map.insert(new_cid, unicode.clone());
            }
        }
        new_cmap.code_byte_length = self.code_byte_length;

        new_cmap
    }
}

/// Expand `bfrange` entries into individual CID→Unicode inserts.
/// Returns how many CIDs were visited. Counts overwrites so a repeated
/// full-width range cannot keep working after `max_assignments`.
fn expand_bfranges_for_remap(
    ranges: &[(u16, u16, u32)],
    cid_to_unicode: &mut HashMap<u16, String>,
    max_assignments: usize,
) -> usize {
    let mut assigned = 0usize;
    'ranges: for &(start, end, base) in ranges {
        if start > end {
            continue;
        }
        for cid in start..=end {
            if assigned >= max_assignments {
                break 'ranges;
            }
            assigned += 1;
            let unicode_cp = base + (cid - start) as u32;
            if let Some(ch) = char::from_u32(unicode_cp) {
                cid_to_unicode.insert(cid, ch.to_string());
            }
        }
    }
    assigned
}

/// Parse a hex string to u16
fn parse_hex_u16(hex: &str) -> Option<u16> {
    u16::from_str_radix(hex.trim(), 16).ok()
}

/// Convert a ToUnicode destination hex string to Unicode.
///
/// PDF ToUnicode destinations are UTF-16BE strings. Supplementary-plane
/// characters are encoded as surrogate pairs, so treating each 4-hex chunk as
/// a scalar drops emoji like D83CDF1F.
fn hex_to_unicode_string(hex: &str) -> Option<String> {
    let hex: String = hex.chars().filter(|ch| !ch.is_ascii_whitespace()).collect();
    if hex.is_empty() || !hex.len().is_multiple_of(2) {
        return None;
    }

    let bytes: Option<Vec<u8>> = (0..hex.len())
        .step_by(2)
        .map(|i| u8::from_str_radix(hex.get(i..i + 2)?, 16).ok())
        .collect();
    let bytes = bytes?;

    if bytes.len().is_multiple_of(2) {
        let units: Vec<u16> = bytes
            .chunks_exact(2)
            .map(|chunk| u16::from_be_bytes([chunk[0], chunk[1]]))
            .collect();
        if let Ok(result) = String::from_utf16(&units) {
            if !result.is_empty() {
                return Some(normalize_tounicode_destination(result));
            }
        }
    }

    // Be permissive for non-standard one-byte destinations.
    if bytes.len() == 1 {
        let ch = bytes[0] as char;
        if !ch.is_control() || ch == '\t' || ch == '\n' {
            return Some(ch.to_string());
        }
    }

    None
}

fn normalize_tounicode_destination(text: String) -> String {
    let is_multi_char = text.chars().nth(1).is_some();

    // Some malformed producer CMaps put a list of alternative whitespace or
    // hyphen codepoints into one destination. Keep ordinary multi-character
    // mappings intact unless that malformed signature is present.
    if is_multi_char
        && text.chars().all(char::is_whitespace)
        && text.chars().any(|ch| matches!(ch, '\t' | '\n' | '\r'))
    {
        return if text.contains('\t') {
            "\t".to_string()
        } else {
            " ".to_string()
        };
    }

    if is_multi_char
        && text.contains('\u{00ad}')
        && text.chars().all(|ch| {
            matches!(
                ch,
                '-' | '\u{00ad}' | '\u{2010}' | '\u{2011}' | '\u{2012}' | '\u{2013}' | '\u{2212}'
            )
        })
    {
        return "-".to_string();
    }

    text
}

fn hex_to_unicode_scalar(hex: &str) -> Option<u32> {
    let text = hex_to_unicode_string(hex)?;
    let mut chars = text.chars();
    let ch = chars.next()?;
    if chars.next().is_none() {
        Some(ch as u32)
    } else {
        None
    }
}

fn find_usecmap_name(text: &str) -> Option<String> {
    for line in text.lines() {
        if line.contains("usecmap") {
            let parts: Vec<&str> = line.split_whitespace().collect();
            for i in 0..parts.len() {
                if parts[i] == "usecmap" && i > 0 {
                    let name = parts[i - 1].trim();
                    if let Some(stripped) = name.strip_prefix('/') {
                        return Some(stripped.to_string());
                    }
                }
            }
        }
    }
    None
}

/// Navigate to the first DescendantFont dictionary of a Type0 font.
fn get_descendant_cid_font<'a>(
    font_dict: &'a lopdf::Dictionary,
    doc: &'a Document,
) -> Option<&'a lopdf::Dictionary> {
    let desc_fonts_obj = font_dict.get(b"DescendantFonts").ok()?;
    let arr = match desc_fonts_obj {
        Object::Array(arr) => arr,
        Object::Reference(r) => match doc.get_object(*r) {
            Ok(Object::Array(arr)) => arr,
            _ => return None,
        },
        _ => return None,
    };
    if arr.is_empty() {
        return None;
    }
    match &arr[0] {
        Object::Reference(r) => doc.get_dictionary(*r).ok(),
        Object::Dictionary(d) => Some(d),
        _ => None,
    }
}

/// Get the starting CID from a CIDFont's W (widths) array.
fn get_w_array_start_cid(cid_font_dict: &lopdf::Dictionary, doc: &Document) -> Option<u16> {
    let w_obj = cid_font_dict.get(b"W").ok()?;
    let arr = match w_obj {
        Object::Array(arr) => arr,
        Object::Reference(r) => match doc.get_object(*r) {
            Ok(Object::Array(arr)) => arr,
            _ => return None,
        },
        _ => return None,
    };
    if arr.is_empty() {
        return None;
    }
    match &arr[0] {
        Object::Integer(n) => Some(*n as u16),
        Object::Reference(r) => match doc.get_object(*r) {
            Ok(Object::Integer(n)) => Some(*n as u16),
            _ => None,
        },
        _ => None,
    }
}

/// Return true if the CIDFont's W (widths) array explicitly covers the given CID.
///
/// The W array uses two formats (PDF 32000-1:2008, §9.7.4.3):
///   1. `c [w1 w2 ... wn]` — widths for CIDs c, c+1, ..., c+n-1
///   2. `c_first c_last w` — CIDs c_first..c_last all have width w
fn w_array_covers_cid(cid_font_dict: &lopdf::Dictionary, doc: &Document, target: u16) -> bool {
    let Ok(w_obj) = cid_font_dict.get(b"W") else {
        return false;
    };
    let arr = match w_obj {
        Object::Array(arr) => arr,
        Object::Reference(r) => match doc.get_object(*r) {
            Ok(Object::Array(arr)) => arr,
            _ => return false,
        },
        _ => return false,
    };

    let resolve_int = |o: &Object| -> Option<i64> {
        match o {
            Object::Integer(n) => Some(*n),
            Object::Reference(r) => match doc.get_object(*r) {
                Ok(Object::Integer(n)) => Some(*n),
                _ => None,
            },
            _ => None,
        }
    };

    let resolve_arr = |o: &Object| -> Option<Vec<Object>> {
        match o {
            Object::Array(a) => Some(a.clone()),
            Object::Reference(r) => match doc.get_object(*r) {
                Ok(Object::Array(a)) => Some(a.clone()),
                _ => None,
            },
            _ => None,
        }
    };

    let target = target as i64;
    let mut i = 0usize;
    while i < arr.len() {
        let Some(first) = resolve_int(&arr[i]) else {
            break;
        };
        i += 1;
        if i >= arr.len() {
            break;
        }
        // Peek at arr[i] to decide format.
        if let Some(widths) = resolve_arr(&arr[i]) {
            // Format 1: c [w1 ... wn]
            let last = first + widths.len() as i64 - 1;
            if target >= first && target <= last {
                return true;
            }
            i += 1;
        } else if let Some(last) = resolve_int(&arr[i]) {
            // Format 2: c_first c_last w
            i += 1;
            if i < arr.len() {
                i += 1; // skip the width value
            }
            if target >= first && target <= last {
                return true;
            }
        } else {
            // Unknown token — abort parsing safely
            break;
        }
    }
    false
}

/// Extract CIDToGIDMap as a vector of GIDs (u16) indexed by CID.
fn get_cid_to_gid_map(cid_font_dict: &lopdf::Dictionary, doc: &Document) -> Option<Vec<u16>> {
    let obj = cid_font_dict.get(b"CIDToGIDMap").ok()?;
    match obj {
        Object::Name(n) if n.as_slice() == b"Identity" => None,
        Object::Reference(r) => match doc.get_object(*r) {
            Ok(Object::Stream(s)) => parse_cid_to_gid_stream(&s.decompressed_content().ok()?),
            _ => None,
        },
        Object::Stream(s) => parse_cid_to_gid_stream(&s.decompressed_content().ok()?),
        _ => None,
    }
}

fn parse_cid_to_gid_stream(data: &[u8]) -> Option<Vec<u16>> {
    if data.len() < 2 {
        return None;
    }
    let mut map = Vec::with_capacity(data.len() / 2);
    for chunk in data.chunks_exact(2) {
        map.push(u16::from_be_bytes([chunk[0], chunk[1]]));
    }
    Some(map)
}

/// Build a CID→Unicode CMap by applying a CIDToGIDMap to an existing CMap that maps GID→Unicode.
fn build_cmap_with_cid_to_gid_map(
    cmap: &ToUnicodeCMap,
    cid_to_gid: &[u16],
) -> Option<ToUnicodeCMap> {
    let mut new_cmap = ToUnicodeCMap::new();
    for (cid, &gid) in cid_to_gid.iter().enumerate() {
        if let Some(s) = cmap.lookup(gid) {
            new_cmap.char_map.insert(cid as u16, s);
        }
    }
    if new_cmap.char_map.is_empty() {
        None
    } else {
        new_cmap.code_byte_length = 2;
        Some(new_cmap)
    }
}

/// Detect and fix broken ToUnicode CMaps from subset fonts with GID mismatch.
///
/// Some PDF generators subset-embed fonts by renumbering GIDs sequentially (1, 2, 3...)
/// but fail to update the ToUnicode CMap, which still references original GID values.
/// This detects the mismatch and remaps the CMap to sequential positions.
fn try_remap_subset_cmap(
    cmap: ToUnicodeCMap,
    font_dict: &lopdf::Dictionary,
    doc: &Document,
    obj_num: u32,
) -> (ToUnicodeCMap, Option<ToUnicodeCMap>) {
    // Only applies to Identity-H/V CID fonts
    let encoding = font_dict
        .get(b"Encoding")
        .ok()
        .and_then(|o| o.as_name().ok());
    if encoding != Some(b"Identity-H") && encoding != Some(b"Identity-V") {
        return (cmap, None);
    }

    // CMap's minimum source CID must be > 2 (indicating old, non-sequential GIDs)
    let min_cid = match cmap.min_source_cid() {
        Some(c) if c > 2 => c,
        _ => return (cmap, None),
    };

    // Navigate to DescendantFonts[0]
    let cid_font_dict = match get_descendant_cid_font(font_dict, doc) {
        Some(d) => d,
        None => return (cmap, None),
    };

    // Both repair paths below assume CIDs are glyph indices that a subsetter can
    // renumber, which is only true for CIDFontType2 (TrueType). For CIDFontType0
    // (CFF), CIDs are resolved through the CFF charset, so a valid CMap stays valid
    // after subsetting and renumbering it corrupts otherwise-correct text.
    // CIDToGIDMap is likewise CIDFontType2-only (PDF 32000-1:2008, 9.7.4.2), so this
    // also ignores a CIDToGIDMap that a malformed producer attached to a CFF font.
    // /Subtype may be an indirect reference, so resolve it through the document.
    // Only bail out when the descendant is *explicitly* something other than
    // CIDFontType2: a missing or unresolvable /Subtype keeps the previous
    // behaviour rather than silently disabling the repair.
    let subtype = cid_font_dict.get(b"Subtype").ok().and_then(|o| match o {
        Object::Reference(r) => doc.get_object(*r).ok().and_then(|o| o.as_name().ok()),
        other => other.as_name().ok(),
    });
    if subtype.is_some_and(|name| name != b"CIDFontType2") {
        debug!("Subset remap skipped for obj={obj_num}: descendant is not CIDFontType2");
        return (cmap, None);
    }

    // If there's an explicit CIDToGIDMap, build a repaired CMap using it.
    if let Some(cid_to_gid) = get_cid_to_gid_map(cid_font_dict, doc) {
        if let Some(repaired) = build_cmap_with_cid_to_gid_map(&cmap, &cid_to_gid) {
            debug!(
                "CIDToGIDMap repair applied for obj={}: {} entries",
                obj_num,
                repaired.char_map.len()
            );
            return (cmap, Some(repaired));
        }
        // Fall through to sequential remap if repair failed.
    }

    // W array must start at a low CID (≤ 2), indicating sequential post-subset GIDs
    let w_start = match get_w_array_start_cid(cid_font_dict, doc) {
        Some(c) if c <= 2 => c,
        _ => return (cmap, None),
    };

    // If the W array actually covers the CMap's max source CID, the CMap is
    // aligned with the font — no sequential renumbering happened. A sparse W
    // array starting at CID 0 (for .notdef) with additional high-CID entries
    // matching the CMap is the normal subset layout, not a mismatch.
    if let Some(max_cid) = cmap.max_source_cid() {
        if w_array_covers_cid(cid_font_dict, doc, max_cid) {
            debug!(
                "Subset remap skipped for obj={}: W array covers CMap max CID {}",
                obj_num, max_cid
            );
            return (cmap, None);
        }
    }

    debug!(
        "Subset GID mismatch detected for obj={}: W starts at CID {}, CMap min CID {}. Remapping to sequential.",
        obj_num, w_start, min_cid
    );

    let remapped = cmap.remap_to_sequential();
    (cmap, Some(remapped))
}

/// Build a ToUnicodeCMap from an embedded TrueType font's cmap table.
///
/// For Identity-H CID fonts, CID == GID. The TrueType cmap maps Unicode→GID,
/// so we reverse it to get GID→Unicode (i.e. CID→Unicode).
pub fn build_cmap_from_truetype(font_data: &[u8]) -> Option<ToUnicodeCMap> {
    let face = ttf_parser::Face::parse(font_data, 0).ok()?;
    let gid_to_unicode = build_gid_to_unicode(&face)?;

    debug!(
        "TrueType cmap: {} GID→Unicode entries",
        gid_to_unicode.len()
    );

    let mut cmap = ToUnicodeCMap::new();
    for (gid, ch) in &gid_to_unicode {
        cmap.char_map.insert(*gid, ch.to_string());
    }
    cmap.code_byte_length = 2; // Identity-H uses 2-byte CIDs

    Some(cmap)
}

/// Build a single-byte CMap for simple fonts by treating the character code
/// as a glyph id (best-effort fallback when no usable ToUnicode exists).
fn build_simple_cmap_from_truetype(font_data: &[u8]) -> Option<ToUnicodeCMap> {
    let face = ttf_parser::Face::parse(font_data, 0).ok()?;
    let gid_to_unicode = build_gid_to_unicode(&face)?;

    let mut cmap = ToUnicodeCMap::new();

    // Use the font's encoding cmap subtable for proper code→GID→Unicode mapping.
    // In subsetted TrueType fonts, GID ≠ character code, so we need the cmap table
    // to translate byte codes (as used in the PDF content stream) to GIDs.
    let mut used_encoding_cmap = false;
    if let Some(cmap_table) = face.tables().cmap {
        // Prefer Mac Roman (1,0): maps byte codes 0–255 directly to GIDs.
        for subtable in cmap_table.subtables {
            if subtable.platform_id == ttf_parser::PlatformId::Macintosh
                && subtable.encoding_id == 0
            {
                for code in 0x20..=0xFF_u32 {
                    if let Some(gid) = subtable.glyph_index(code) {
                        if let Some(&ch) = gid_to_unicode.get(&gid.0) {
                            let ch = strip_pua_char(ch);
                            cmap.char_map.entry(code as u16).or_insert(ch.to_string());
                        }
                    }
                }
                used_encoding_cmap = true;
                break;
            }
        }
        // Fallback: Windows Symbol (3,0) — maps F000+byte to GIDs.
        if !used_encoding_cmap {
            for subtable in cmap_table.subtables {
                if subtable.platform_id == ttf_parser::PlatformId::Windows
                    && subtable.encoding_id == 0
                {
                    for code in 0x20..=0xFF_u32 {
                        if let Some(gid) = subtable.glyph_index(code + 0xF000) {
                            if let Some(&ch) = gid_to_unicode.get(&gid.0) {
                                let ch = strip_pua_char(ch);
                                cmap.char_map.entry(code as u16).or_insert(ch.to_string());
                            }
                        }
                    }
                    used_encoding_cmap = true;
                    break;
                }
            }
        }
        // Fallback: Windows Unicode BMP (3,1) — maps Unicode codepoints to GIDs.
        // For single-byte fonts, try each byte value as a Unicode codepoint.
        // Common in OCR-generated PDFs where byte values correspond to Unicode
        // codepoints but the declared encoding (WinAnsiEncoding) is wrong.
        if !used_encoding_cmap {
            for subtable in cmap_table.subtables {
                if subtable.platform_id == ttf_parser::PlatformId::Windows
                    && subtable.encoding_id == 1
                {
                    for code in 0x20..=0xFF_u32 {
                        if let Some(gid) = subtable.glyph_index(code) {
                            if let Some(&ch) = gid_to_unicode.get(&gid.0) {
                                let ch = strip_pua_char(ch);
                                cmap.char_map.entry(code as u16).or_insert(ch.to_string());
                            }
                        }
                    }
                    used_encoding_cmap = true;
                    break;
                }
            }
        }
    }

    if !used_encoding_cmap {
        // No encoding cmap found — fall back to treating GID as code.
        for (&gid, &ch) in &gid_to_unicode {
            if gid <= 0xFF {
                cmap.char_map.insert(gid, ch.to_string());
            }
        }
        // Fill missing single-byte codes from glyph names (helps with ligatures like "t_i").
        for gid_idx in 0..face.number_of_glyphs() {
            let gid = ttf_parser::GlyphId(gid_idx);
            let gid_val = gid.0;
            if gid_val > 0xFF || cmap.char_map.contains_key(&gid_val) {
                continue;
            }
            if let Some(name) = face.glyph_name(gid) {
                if let Some(s) = glyph_name_to_string(name) {
                    cmap.char_map.insert(gid_val, s);
                }
            }
        }
    }

    if cmap.char_map.is_empty() {
        return None;
    }
    debug!(
        "TrueType simple cmap: {} code→Unicode entries",
        cmap.char_map.len()
    );
    cmap.code_byte_length = 1;
    Some(cmap)
}

/// Strip Private Use Area F000 offset (Windows Symbol encoding convention).
fn strip_pua_char(ch: char) -> char {
    let cp = ch as u32;
    if (0xF000..=0xF0FF).contains(&cp) {
        char::from_u32(cp - 0xF000).unwrap_or(ch)
    } else {
        ch
    }
}

fn glyph_name_to_string(name: &str) -> Option<String> {
    let base = name.split('.').next().unwrap_or(name);
    if let Some(ch) = glyph_to_char(base) {
        return Some(ch.to_string());
    }
    if base.contains('_') {
        let mut out = String::new();
        for part in base.split('_') {
            if part.is_empty() {
                return None;
            }
            if let Some(ch) = glyph_to_char(part) {
                out.push(ch);
            } else if part.len() == 1 {
                out.push(part.chars().next().unwrap());
            } else {
                return None;
            }
        }
        if !out.is_empty() {
            return Some(out);
        }
    }
    if matches!(base, "ti" | "tt" | "tz") {
        return Some(base.to_string());
    }
    None
}

/// Build a ToUnicodeCMap from a font's glyph names (post table).
/// Uses Adobe Glyph List to map glyph names to Unicode.
fn build_cmap_from_glyph_names(face: &ttf_parser::Face<'_>) -> Option<ToUnicodeCMap> {
    let mut cmap = ToUnicodeCMap::new();

    for gid in 0..face.number_of_glyphs() {
        let gid = ttf_parser::GlyphId(gid);
        if let Some(name) = face.glyph_name(gid) {
            if let Some(ch) = glyph_to_char(name) {
                cmap.char_map.insert(gid.0, ch.to_string());
            }
        }
    }

    if cmap.char_map.is_empty() {
        return None;
    }

    debug!(
        "TrueType post glyph names: {} GID→Unicode entries",
        cmap.char_map.len()
    );
    cmap.code_byte_length = 2;
    Some(cmap)
}

fn build_gid_to_unicode(face: &ttf_parser::Face<'_>) -> Option<HashMap<u16, char>> {
    let mut gid_to_unicode: HashMap<u16, char> = HashMap::new();

    // Iterate all Unicode codepoints that have a glyph mapping.
    // For each codepoint, the face gives us a GlyphId; reverse that to GID→Unicode.
    // We prefer the first (lowest) codepoint for each GID to handle duplicates.
    for subtable in face.tables().cmap.iter().flat_map(|cmap| cmap.subtables) {
        let is_symbol =
            subtable.platform_id == ttf_parser::PlatformId::Windows && subtable.encoding_id == 0;
        if !subtable.is_unicode() && !is_symbol {
            continue;
        }
        subtable.codepoints(|cp| {
            if let Some(ch) = char::from_u32(cp) {
                if let Some(gid) = subtable.glyph_index(cp) {
                    let gid_val = gid.0;
                    gid_to_unicode.entry(gid_val).or_insert(ch);
                }
            }
        });
    }

    if gid_to_unicode.is_empty() {
        return build_cmap_from_glyph_names(face).map(|cmap| {
            let mut map = HashMap::new();
            for (gid, s) in cmap.char_map {
                if let Some(ch) = s.chars().next() {
                    map.insert(gid, ch);
                }
            }
            map
        });
    }

    Some(gid_to_unicode)
}

/// Build a ToUnicodeCMap from pdf.js built-in binary CMaps (bcmaps).
fn build_cmap_from_builtin_cmap(ordering: &str) -> Option<ToUnicodeCMap> {
    let name = format!("Adobe-{}-UCS2.bcmap", ordering);
    let data = read_builtin_cmap_file(&name)?;
    let mut cmap = parse_binary_cmap(&data).ok()?;
    if cmap.char_map.is_empty() && cmap.ranges.is_empty() {
        return None;
    }
    cmap.code_byte_length = 2;
    debug!(
        "Built-in CMap {}: char_map={} ranges={}",
        name,
        cmap.char_map.len(),
        cmap.ranges.len()
    );
    Some(cmap)
}

#[cfg(not(target_arch = "wasm32"))]
fn find_bcmaps_dir() -> Option<PathBuf> {
    if let Ok(dir) = std::env::var("PDF_INSPECTOR_BCMAPS_DIR") {
        let p = PathBuf::from(dir);
        if p.is_dir() {
            return Some(p);
        }
    }
    let default = Path::new(env!("CARGO_MANIFEST_DIR"))
        .join("external")
        .join("bcmaps");
    if default.is_dir() {
        return Some(default);
    }
    None
}

#[cfg(not(target_arch = "wasm32"))]
fn read_builtin_cmap_file(name: &str) -> Option<Cow<'static, [u8]>> {
    let path = find_bcmaps_dir()?.join(name);
    std::fs::read(path).ok().map(Cow::Owned)
}

#[cfg(target_arch = "wasm32")]
fn read_builtin_cmap_file(name: &str) -> Option<Cow<'static, [u8]>> {
    let file = BUILTIN_CMAPS.get_file(name)?;
    Some(Cow::Borrowed(file.contents()))
}

fn parse_binary_cmap(data: &[u8]) -> Result<ToUnicodeCMap, String> {
    let mut stream = BinaryCMapStream::new(data);
    let _header = stream.read_byte().ok_or("unexpected EOF in bcmap header")?;

    let mut cmap = ToUnicodeCMap::new();
    let mut use_cmap: Option<String> = None;

    while let Some(b) = stream.read_byte() {
        let typ = b >> 5;
        if typ == 7 {
            match b & 0x1f {
                0 => {
                    stream.read_string()?;
                }
                1 => {
                    let name = stream.read_string()?;
                    use_cmap = Some(name);
                }
                _ => {}
            }
            continue;
        }
        let sequence = (b & 0x10) != 0;
        let data_size = (b & 0x0f) as usize;
        if data_size + 1 > 16 {
            return Err("invalid dataSize in bcmap".to_string());
        }
        let subitems = stream.read_number()? as usize;
        match typ {
            4 => {
                // bfchar
                for i in 0..subitems {
                    let src = stream.read_hex_number(1)?;
                    let dst = stream.read_hex_bytes(data_size + 1)?;
                    let src_code = hex_to_u32(&src) as u16;
                    if let Some(s) = bytes_to_unicode_string(&dst) {
                        cmap.char_map.insert(src_code, s);
                    }
                    if i + 1 < subitems && sequence {
                        // sequence handled by encoded data, nothing to do
                    }
                }
            }
            5 => {
                // bfrange
                for _ in 0..subitems {
                    let start = stream.read_hex_number(1)?;
                    let end_delta = stream.read_hex_number(1)?;
                    let mut end = start.clone();
                    add_hex(&mut end, &end_delta);
                    let dst = stream.read_hex_bytes(data_size + 1)?;
                    let start_code = hex_to_u32(&start) as u16;
                    let end_code = hex_to_u32(&end) as u16;
                    if let Some(s) = bytes_to_unicode_string(&dst) {
                        if s.chars().count() == 1 {
                            let base = s.chars().next().unwrap() as u32;
                            cmap.ranges.push((start_code, end_code, base));
                        } else {
                            // Expand multi-char sequences
                            let mut cid = start_code;
                            for ch in s.chars() {
                                cmap.char_map.insert(cid, ch.to_string());
                                if cid == end_code {
                                    break;
                                }
                                cid = cid.saturating_add(1);
                            }
                        }
                    }
                }
            }
            _ => {
                // Skip unsupported types by consuming their payload.
                // We only implement bfchar/bfrange for UCS2 maps.
                for _ in 0..subitems {
                    // Best-effort skip: read a few fields based on type.
                    if typ <= 3 {
                        let _ = stream.read_hex_number(data_size)?;
                        let _ = stream.read_hex_number(data_size)?;
                        if typ >= 1 {
                            let _ = stream.read_number()?;
                        }
                    }
                }
            }
        }
    }

    cmap.ranges.sort_unstable_by_key(|&(start, _, _)| start);
    if let Some(name) = use_cmap {
        if let Some(base) = load_builtin_cmap_by_name(&name) {
            cmap = merge_cmaps(base, cmap);
        } else {
            warn!("bcmap usecmap={} could not be loaded", name);
        }
    }
    Ok(cmap)
}

struct BinaryCMapStream<'a> {
    data: &'a [u8],
    pos: usize,
}

impl<'a> BinaryCMapStream<'a> {
    fn new(data: &'a [u8]) -> Self {
        Self { data, pos: 0 }
    }

    fn read_byte(&mut self) -> Option<u8> {
        if self.pos >= self.data.len() {
            None
        } else {
            let b = self.data[self.pos];
            self.pos += 1;
            Some(b)
        }
    }

    fn read_number(&mut self) -> Result<u32, String> {
        let mut n = 0u32;
        loop {
            let b = self.read_byte().ok_or("unexpected EOF in bcmap")?;
            let last = (b & 0x80) == 0;
            n = (n << 7) | (b & 0x7f) as u32;
            if last {
                break;
            }
        }
        Ok(n)
    }

    fn read_hex_number(&mut self, size: usize) -> Result<Vec<u8>, String> {
        // encoded 7-bit number into size+1 bytes
        let mut stack = Vec::new();
        loop {
            let b = self.read_byte().ok_or("unexpected EOF in bcmap")?;
            let last = (b & 0x80) == 0;
            stack.push(b & 0x7f);
            if last {
                break;
            }
        }
        let mut out = vec![0u8; size + 1];
        let mut buffer = 0u32;
        let mut buffer_size = 0u32;
        let mut i: i32 = size as i32;
        while i >= 0 {
            while buffer_size < 8 && !stack.is_empty() {
                buffer |= (stack.pop().unwrap() as u32) << buffer_size;
                buffer_size += 7;
            }
            out[i as usize] = (buffer & 0xff) as u8;
            buffer >>= 8;
            buffer_size = buffer_size.saturating_sub(8);
            i -= 1;
        }
        Ok(out)
    }

    fn read_hex_bytes(&mut self, len: usize) -> Result<Vec<u8>, String> {
        if self.pos + len > self.data.len() {
            return Err("unexpected EOF in bcmap".to_string());
        }
        let out = self.data[self.pos..self.pos + len].to_vec();
        self.pos += len;
        Ok(out)
    }

    fn read_string(&mut self) -> Result<String, String> {
        let len = self.read_number()? as usize;
        let mut buf = Vec::with_capacity(len);
        for _ in 0..len {
            let v = self.read_number()? as u8;
            buf.push(v);
        }
        String::from_utf8(buf).map_err(|e| e.to_string())
    }
}

fn hex_to_u32(bytes: &[u8]) -> u32 {
    let mut n = 0u32;
    for &b in bytes {
        n = (n << 8) | b as u32;
    }
    n
}

fn add_hex(a: &mut [u8], b: &[u8]) {
    let mut c = 0u16;
    for i in (0..a.len()).rev() {
        c += a[i] as u16 + b[i] as u16;
        a[i] = (c & 0xff) as u8;
        c >>= 8;
    }
}

fn bytes_to_unicode_string(bytes: &[u8]) -> Option<String> {
    if bytes.is_empty() {
        return None;
    }
    if !bytes.len().is_multiple_of(2) {
        // Treat as latin-1 bytes
        return Some(bytes.iter().map(|&b| b as char).collect());
    }
    let mut out = String::new();
    for chunk in bytes.chunks_exact(2) {
        let cp = u16::from_be_bytes([chunk[0], chunk[1]]) as u32;
        if let Some(ch) = char::from_u32(cp) {
            out.push(ch);
        }
    }
    if out.is_empty() {
        None
    } else {
        Some(out)
    }
}

#[derive(Debug, Clone)]
struct EncodingCMap {
    map: HashMap<u16, u16>,
    code_byte_length: u8,
    is_identity: bool,
}

fn build_fallback_tounicode_from_encoding(
    font_dict: &lopdf::Dictionary,
    doc: &Document,
) -> Option<ToUnicodeCMap> {
    let encoding = build_encoding_cmap_from_font(font_dict, doc)?;
    let ordering = get_cid_system_info_ordering(font_dict, doc)?;
    let ucs2 = build_cmap_from_builtin_cmap(&ordering)?;

    if encoding.is_identity {
        // Identity mapping: charcode == CID
        return Some(ucs2);
    }

    let mut cmap = ToUnicodeCMap::new();
    for (charcode, cid) in encoding.map {
        if let Some(s) = ucs2.lookup(cid) {
            cmap.char_map.insert(charcode, s);
        }
    }
    if cmap.char_map.is_empty() {
        return None;
    }
    cmap.code_byte_length = encoding.code_byte_length;
    Some(cmap)
}

fn get_cid_system_info_ordering(font_dict: &lopdf::Dictionary, doc: &Document) -> Option<String> {
    let cid_font_dict = get_descendant_cid_font(font_dict, doc)?;
    let csi_obj = cid_font_dict.get(b"CIDSystemInfo").ok()?;
    let csi_dict = match csi_obj {
        Object::Reference(r) => doc.get_dictionary(*r).ok()?,
        Object::Dictionary(d) => d,
        _ => return None,
    };
    let ordering = csi_dict.get(b"Ordering").ok().and_then(|o| {
        if let Object::String(bytes, _) = o {
            Some(String::from_utf8_lossy(bytes).to_string())
        } else {
            None
        }
    })?;
    Some(ordering)
}

fn build_encoding_cmap_from_font(
    font_dict: &lopdf::Dictionary,
    doc: &Document,
) -> Option<EncodingCMap> {
    let encoding_obj = font_dict.get(b"Encoding").ok()?;
    match encoding_obj {
        Object::Name(name) => {
            let enc = name.as_slice();
            if enc == b"Identity-H" || enc == b"Identity-V" {
                return Some(EncodingCMap {
                    map: HashMap::new(),
                    code_byte_length: 2,
                    is_identity: true,
                });
            }
            let enc_name = String::from_utf8_lossy(enc).to_string();
            load_builtin_encoding_cmap(&enc_name)
        }
        Object::Reference(r) => {
            let obj = doc.get_object(*r).ok()?;
            parse_encoding_cmap_object(obj, doc)
        }
        Object::Stream(s) => parse_encoding_cmap_stream(&s.decompressed_content().ok()?),
        Object::Dictionary(_) => None,
        _ => None,
    }
}

fn parse_encoding_cmap_object(obj: &Object, doc: &Document) -> Option<EncodingCMap> {
    match obj {
        Object::Stream(s) => parse_encoding_cmap_stream(&s.decompressed_content().ok()?),
        Object::Reference(r) => {
            let obj = doc.get_object(*r).ok()?;
            parse_encoding_cmap_object(obj, doc)
        }
        _ => None,
    }
}

fn load_builtin_encoding_cmap(name: &str) -> Option<EncodingCMap> {
    let data = read_builtin_cmap_file(&format!("{}.bcmap", name))?;
    parse_binary_cmap_encoding(&data).ok()
}

fn parse_encoding_cmap_stream(data: &[u8]) -> Option<EncodingCMap> {
    let text = String::from_utf8_lossy(data);
    let mut src_hex_lengths: Vec<usize> = Vec::new();
    let mut codespace_byte_len: Option<u8> = None;

    if let Some(cs_start) = text.find("begincodespacerange") {
        let section_start = cs_start + "begincodespacerange".len();
        if let Some(cs_end) = text[section_start..].find("endcodespacerange") {
            let section = &text[section_start..section_start + cs_end];
            let mut in_hex = false;
            let mut hex_len = 0;
            for c in section.chars() {
                if c == '<' {
                    in_hex = true;
                    hex_len = 0;
                } else if c == '>' {
                    if in_hex && hex_len > 0 {
                        let byte_len = (hex_len + 1) / 2;
                        codespace_byte_len = Some(byte_len as u8);
                    }
                    in_hex = false;
                } else if in_hex && c.is_ascii_hexdigit() {
                    hex_len += 1;
                }
            }
        }
    }

    let mut map = HashMap::new();
    let mut assigned = 0usize;
    let mut pos = 0;
    while let Some(start) = text[pos..].find("begincidchar") {
        let section_start = pos + start + "begincidchar".len();
        if let Some(end) = text[section_start..].find("endcidchar") {
            let section = &text[section_start..section_start + end];
            if !parse_cidchar_section(section, &mut map, &mut src_hex_lengths, &mut assigned) {
                break;
            }
            pos = section_start + end;
        } else {
            break;
        }
    }
    pos = 0;
    while assigned < MAX_CID_W_EXPANSION {
        let Some(start) = text[pos..].find("begincidrange") else {
            break;
        };
        let section_start = pos + start + "begincidrange".len();
        if let Some(end) = text[section_start..].find("endcidrange") {
            let section = &text[section_start..section_start + end];
            if !parse_cidrange_section(section, &mut map, &mut src_hex_lengths, &mut assigned) {
                break;
            }
            pos = section_start + end;
        } else {
            break;
        }
    }

    if map.is_empty() {
        return None;
    }

    let code_byte_length = if let Some(cs_len) = codespace_byte_len {
        cs_len
    } else if !src_hex_lengths.is_empty() {
        let max_hex_len = src_hex_lengths.iter().max().copied().unwrap_or(4);
        if max_hex_len <= 2 {
            1
        } else {
            2
        }
    } else {
        2
    };

    Some(EncodingCMap {
        map,
        code_byte_length,
        is_identity: false,
    })
}

fn parse_cidchar_section(
    section: &str,
    map: &mut HashMap<u16, u16>,
    src_hex_lengths: &mut Vec<usize>,
    assigned: &mut usize,
) -> bool {
    let mut chars = section.chars().peekable();
    loop {
        while chars.peek().is_some_and(|c| c.is_whitespace()) {
            chars.next();
        }
        if chars.peek() != Some(&'<') {
            break;
        }
        chars.next();
        let mut src_hex = String::new();
        while chars.peek().is_some_and(|&c| c != '>') {
            if let Some(c) = chars.next() {
                src_hex.push(c);
            }
        }
        chars.next();
        let trimmed_src = src_hex.trim();
        if !trimmed_src.is_empty() {
            src_hex_lengths.push(trimmed_src.len());
        }
        while chars.peek().is_some_and(|c| c.is_whitespace()) {
            chars.next();
        }
        let mut cid_str = String::new();
        while chars.peek().is_some_and(|c| !c.is_whitespace()) {
            if let Some(c) = chars.next() {
                cid_str.push(c);
            }
        }
        if let (Some(code), Ok(cid)) = (parse_hex_u16(&src_hex), cid_str.parse::<u16>()) {
            if !assign_encoding_cid(map, code, cid, assigned) {
                return false;
            }
        }
    }
    true
}

fn parse_cidrange_section(
    section: &str,
    map: &mut HashMap<u16, u16>,
    src_hex_lengths: &mut Vec<usize>,
    assigned: &mut usize,
) -> bool {
    let mut chars = section.chars().peekable();
    loop {
        while chars.peek().is_some_and(|c| c.is_whitespace()) {
            chars.next();
        }
        if chars.peek() != Some(&'<') {
            break;
        }
        chars.next();
        let mut start_hex = String::new();
        while chars.peek().is_some_and(|&c| c != '>') {
            if let Some(c) = chars.next() {
                start_hex.push(c);
            }
        }
        chars.next();
        let trimmed_start = start_hex.trim();
        if !trimmed_start.is_empty() {
            src_hex_lengths.push(trimmed_start.len());
        }
        while chars.peek().is_some_and(|c| c.is_whitespace()) {
            chars.next();
        }
        if chars.peek() != Some(&'<') {
            continue;
        }
        chars.next();
        let mut end_hex = String::new();
        while chars.peek().is_some_and(|&c| c != '>') {
            if let Some(c) = chars.next() {
                end_hex.push(c);
            }
        }
        chars.next();
        while chars.peek().is_some_and(|c| c.is_whitespace()) {
            chars.next();
        }
        let mut cid_str = String::new();
        while chars.peek().is_some_and(|c| !c.is_whitespace()) {
            if let Some(c) = chars.next() {
                cid_str.push(c);
            }
        }
        let (Some(start), Some(end), Ok(start_cid)) = (
            parse_hex_u16(&start_hex),
            parse_hex_u16(&end_hex),
            cid_str.parse::<u16>(),
        ) else {
            continue;
        };
        if start > end {
            continue;
        }
        let mut cid = start_cid;
        for code in start..=end {
            if !assign_encoding_cid(map, code, cid, assigned) {
                return false;
            }
            cid = cid.saturating_add(1);
        }
    }
    true
}

fn assign_encoding_cid(
    map: &mut HashMap<u16, u16>,
    code: u16,
    cid: u16,
    assigned: &mut usize,
) -> bool {
    // Count overwrites: unique-key coverage alone would not stop a repeated
    // full-width range from re-inserting all 65,536 codes.
    if *assigned >= MAX_CID_W_EXPANSION {
        return false;
    }
    map.insert(code, cid);
    *assigned += 1;
    true
}

fn parse_binary_cmap_encoding(data: &[u8]) -> Result<EncodingCMap, String> {
    let mut stream = BinaryCMapStream::new(data);
    let _header = stream.read_byte().ok_or("unexpected EOF in bcmap header")?;
    let mut map: HashMap<u16, u16> = HashMap::new();
    let mut max_code_size: u8 = 1;
    let mut use_cmap: Option<String> = None;

    while let Some(b) = stream.read_byte() {
        let typ = b >> 5;
        if typ == 7 {
            match b & 0x1f {
                0 => {
                    stream.read_string()?;
                }
                1 => {
                    let name = stream.read_string()?;
                    use_cmap = Some(name);
                }
                _ => {}
            }
            continue;
        }
        let _sequence = (b & 0x10) != 0;
        let data_size = (b & 0x0f) as usize;
        if data_size + 1 > 16 {
            return Err("invalid dataSize in bcmap".to_string());
        }
        max_code_size = max_code_size.max((data_size + 1) as u8);
        let subitems = stream.read_number()? as usize;
        match typ {
            2 => {
                // cidchar
                let mut prev_code: u32 = 0;
                for i in 0..subitems {
                    let code_bytes = stream.read_hex_number(data_size)?;
                    let code = hex_to_u32(&code_bytes);
                    let cid = stream.read_number()? as u16;
                    if i == 0 {
                        prev_code = code;
                        map.insert(code as u16, cid);
                        continue;
                    }
                    if _sequence {
                        prev_code = prev_code.saturating_add(1);
                        map.insert(prev_code as u16, cid);
                    } else {
                        map.insert(code as u16, cid);
                        prev_code = code;
                    }
                }
            }
            3 => {
                // cidrange
                for _ in 0..subitems {
                    let start = stream.read_hex_number(data_size)?;
                    let end_delta = stream.read_hex_number(data_size)?;
                    let mut end = start.clone();
                    add_hex(&mut end, &end_delta);
                    let cid_start = stream.read_number()? as u16;
                    let start_code = hex_to_u32(&start) as u16;
                    let end_code = hex_to_u32(&end) as u16;
                    let mut cid = cid_start;
                    for code in start_code..=end_code {
                        map.insert(code, cid);
                        cid = cid.saturating_add(1);
                    }
                }
            }
            _ => {
                // Skip other types
                for _ in 0..subitems {
                    let _ = stream.read_hex_number(data_size)?;
                    let _ = stream.read_hex_number(data_size)?;
                    let _ = stream.read_number()?;
                }
            }
        }
    }

    if let Some(name) = use_cmap {
        if let Some(base) = load_builtin_encoding_cmap(&name) {
            let mut merged = base.map;
            merged.extend(map);
            return Ok(EncodingCMap {
                map: merged,
                code_byte_length: base.code_byte_length.max(max_code_size),
                is_identity: false,
            });
        }
    }

    Ok(EncodingCMap {
        map,
        code_byte_length: max_code_size,
        is_identity: false,
    })
}

fn load_builtin_cmap_by_name(name: &str) -> Option<ToUnicodeCMap> {
    if !name.ends_with("UCS2") {
        return None;
    }
    let data = read_builtin_cmap_file(&format!("{}.bcmap", name))?;
    let mut cmap = parse_binary_cmap(&data).ok()?;
    if cmap.char_map.is_empty() && cmap.ranges.is_empty() {
        return None;
    }
    cmap.code_byte_length = 2;
    Some(cmap)
}

fn merge_cmaps(mut base: ToUnicodeCMap, overlay: ToUnicodeCMap) -> ToUnicodeCMap {
    for (cid, s) in overlay.char_map {
        base.char_map.insert(cid, s);
    }
    base.ranges.extend(overlay.ranges);
    base.ranges.sort_unstable_by_key(|&(start, _, _)| start);
    base.code_byte_length = base.code_byte_length.max(overlay.code_byte_length);
    base
}

/// Shared 16-bit CID expansion cap (65,536).
/// Encoding `begincidrange`, `/W` width assignment, and ToUnicode sequential
/// remap count every insert, including overwrites, so a repeated full-width
/// range cannot keep working after the domain is filled. The `/W` unicode
/// heuristic caps unique CIDs with the same number.
pub(crate) const MAX_CID_W_EXPANSION: usize = 65_536;

/// Check if a CIDFont's /W (widths) array contains CID values that look like
/// Unicode codepoints rather than low-value GIDs.
///
/// Returns true if the median CID is >= 0x41 (letter 'A'), indicating
/// the PDF generator likely used Unicode codepoints as CIDs.
pub(crate) fn cid_values_look_like_unicode(cid_font_dict: &lopdf::Dictionary) -> bool {
    let w_arr = match cid_font_dict.get(b"W").ok() {
        Some(Object::Array(arr)) => arr,
        _ => return false,
    };

    // The /W array format: [cid [w1 w2 ...]] or [cid_start cid_end w].
    // Collect unique CIDs only: repeating a full-width range must not grow a
    // temporary vector (or the sort) with the range length on every copy.
    let mut seen = HashSet::new();
    let mut i = 0;
    while i < w_arr.len() && seen.len() < MAX_CID_W_EXPANSION {
        if let Ok(cid) = w_arr[i].as_i64() {
            let start = cid as u16;
            if i + 1 < w_arr.len() {
                match &w_arr[i + 1] {
                    Object::Array(widths) => {
                        // [cid [w1 w2 ...]] — CIDs are cid, cid+1, ..., cid+len-1
                        for j in 0..widths.len() {
                            if seen.len() >= MAX_CID_W_EXPANSION {
                                break;
                            }
                            seen.insert(start.wrapping_add(j as u16));
                        }
                        i += 2;
                    }
                    _ => {
                        // [cid_start cid_end w] — range of CIDs
                        if i + 2 < w_arr.len() {
                            if let Ok(cid_end) = w_arr[i + 1].as_i64() {
                                record_unique_cid_range(start, cid_end as u16, &mut seen);
                            }
                            i += 3;
                        } else {
                            i += 1;
                        }
                    }
                }
            } else {
                seen.insert(start);
                i += 1;
            }
        } else {
            i += 1;
        }
    }

    if seen.is_empty() {
        return false;
    }

    let mut cids: Vec<u16> = seen.into_iter().collect();
    cids.sort_unstable();
    let median = cids[cids.len() / 2];
    // Unicode text CIDs are typically >= 0x20 (space) with letters at 0x41+.
    // GID-based subsets typically start at low values (0-based).
    // Use median >= 0x41 as a heuristic for Unicode CIDs.
    median >= 0x41
}

fn record_unique_cid_range(start: u16, end: u16, seen: &mut HashSet<u16>) {
    if start > end {
        return;
    }
    for cid in start..=end {
        if seen.len() >= MAX_CID_W_EXPANSION {
            return;
        }
        seen.insert(cid);
    }
}

/// Build a ToUnicodeCMap from predefined CID→Unicode mapping based on CIDSystemInfo.
///
/// Supports Adobe-Korea1 (Korean) character collection. Can be extended for
/// Adobe-Japan1, Adobe-GB1, Adobe-CNS1 in the future.
fn build_cmap_from_cid_system_info(
    cid_font_dict: &lopdf::Dictionary,
    doc: &Document,
) -> Option<ToUnicodeCMap> {
    let csi_obj = cid_font_dict.get(b"CIDSystemInfo").ok()?;
    let csi_dict = match csi_obj {
        Object::Reference(r) => doc.get_dictionary(*r).ok()?,
        Object::Dictionary(d) => d,
        _ => return None,
    };
    let ordering = csi_dict.get(b"Ordering").ok().and_then(|o| {
        if let Object::String(bytes, _) = o {
            Some(String::from_utf8_lossy(bytes).to_string())
        } else {
            None
        }
    })?;

    match ordering.as_str() {
        "Korea1" => {
            use crate::adobe_korea1::ADOBE_KOREA1_CID_TO_UNICODE;
            let mut cmap = ToUnicodeCMap::new();
            for &(cid, unicode) in ADOBE_KOREA1_CID_TO_UNICODE.iter() {
                if let Some(ch) = char::from_u32(unicode as u32) {
                    cmap.char_map.insert(cid, ch.to_string());
                }
            }
            cmap.code_byte_length = 2;
            debug!(
                "Adobe-Korea1 predefined CMap: {} entries",
                cmap.char_map.len()
            );
            Some(cmap)
        }
        "Japan1" | "GB1" | "CNS1" => build_cmap_from_builtin_cmap(&ordering),
        _ => None,
    }
}

/// Collection of ToUnicode CMaps indexed by ToUnicode stream object number
#[derive(Debug, Default, Clone)]
pub struct FontCMaps {
    /// Map of ToUnicode object number to CMap
    by_obj_num: HashMap<u32, CMapEntry>,
}

/// Primary CMap plus optional alternative variants.
#[derive(Debug, Clone)]
pub struct CMapEntry {
    pub primary: ToUnicodeCMap,
    pub remapped: Option<ToUnicodeCMap>,
    pub fallback: Option<ToUnicodeCMap>,
}

impl FontCMaps {
    /// Build FontCMaps from a lopdf Document model.
    ///
    /// Iterates every page, collects fonts (including Form XObject fonts),
    /// and parses any `/ToUnicode` streams via lopdf's decompression.
    pub fn from_doc(doc: &Document) -> Self {
        Self::from_doc_pages(doc, None)
    }

    /// Build FontCMaps for specific pages only. Pass `None` for all pages.
    pub fn from_doc_pages(doc: &Document, page_filter: Option<&HashSet<u32>>) -> Self {
        Self::from_doc_pages_inner(doc, page_filter, false)
    }

    /// Build FontCMaps in fast mode: skip expensive TrueType font fallback
    /// parsing. Fonts that can't be decoded from their ToUnicode CMap alone
    /// will be missing, causing text extraction to produce empty/garbage text
    /// which triggers `needs_ocr` fallback. This is ideal for hybrid OCR
    /// pipelines where GPU OCR is always available as a fallback.
    pub fn from_doc_pages_fast(doc: &Document, page_filter: Option<&HashSet<u32>>) -> Self {
        Self::from_doc_pages_inner(doc, page_filter, true)
    }

    fn from_doc_pages_inner(
        doc: &Document,
        page_filter: Option<&HashSet<u32>>,
        skip_truetype_fallback: bool,
    ) -> Self {
        let mut by_obj_num: HashMap<u32, CMapEntry> = HashMap::new();

        for (page_num, &page_id) in doc.get_pages().iter() {
            if let Some(filter) = page_filter {
                if !filter.contains(page_num) {
                    continue;
                }
            }
            // Page-level fonts (includes inherited parent resources)
            let fonts = doc.get_page_fonts(page_id).unwrap_or_default();
            Self::collect_cmaps_from_fonts_inner(
                &fonts,
                doc,
                &mut by_obj_num,
                skip_truetype_fallback,
            );

            if !skip_truetype_fallback {
                // Fonts inside Form XObjects referenced by this page
                Self::collect_cmaps_from_xobjects(doc, page_id, &mut by_obj_num);
            }
        }

        FontCMaps { by_obj_num }
    }

    /// Parse ToUnicode CMaps from a set of font dictionaries.
    /// Also handles Identity-H/V CID fonts without ToUnicode by parsing
    /// the embedded TrueType cmap from FontFile2.
    fn collect_cmaps_from_fonts(
        fonts: &std::collections::BTreeMap<Vec<u8>, &lopdf::Dictionary>,
        doc: &Document,
        by_obj_num: &mut HashMap<u32, CMapEntry>,
    ) {
        Self::collect_cmaps_from_fonts_inner(fonts, doc, by_obj_num, false);
    }

    fn collect_cmaps_from_fonts_inner(
        fonts: &std::collections::BTreeMap<Vec<u8>, &lopdf::Dictionary>,
        doc: &Document,
        by_obj_num: &mut HashMap<u32, CMapEntry>,
        skip_truetype_fallback: bool,
    ) {
        // First pass: collect ToUnicode CMaps
        for font_dict in fonts.values() {
            let obj_ref = match font_dict
                .get(b"ToUnicode")
                .ok()
                .and_then(|o| o.as_reference().ok())
            {
                Some(r) => r,
                None => continue,
            };
            let obj_num = obj_ref.0;
            if by_obj_num.contains_key(&obj_num) {
                continue;
            }
            let stream = match doc.get_object(obj_ref).and_then(Object::as_stream) {
                Ok(s) => s,
                Err(_) => continue,
            };
            let data = match stream.decompressed_content() {
                Ok(d) => d,
                Err(_) => stream.content.clone(),
            };
            if let Some(cmap) = ToUnicodeCMap::parse(&data) {
                if is_degenerate_identity_bfrange(&cmap) {
                    let fallback = if skip_truetype_fallback {
                        build_fallback_cmap_for_simple(font_dict, doc)
                    } else {
                        build_fallback_cmap_for_type0(font_dict, doc)
                            .or_else(|| build_fallback_cmap_for_simple(font_dict, doc))
                    };
                    if let Some(fb) = fallback {
                        debug!(
                            "ToUnicode CMap obj={} is a full-range identity bfrange; using fallback (entries={})",
                            obj_num,
                            fb.char_map.len()
                        );
                        by_obj_num.insert(
                            obj_num,
                            CMapEntry {
                                primary: fb,
                                remapped: None,
                                fallback: None,
                            },
                        );
                    }
                    continue;
                }
                debug!(
                    "CMap obj={:<6} code_byte_length={} char_map={} ranges={}",
                    obj_num,
                    cmap.code_byte_length,
                    cmap.char_map.len(),
                    cmap.ranges.len()
                );
                let (mut primary, mut remapped) =
                    try_remap_subset_cmap(cmap, font_dict, doc, obj_num);

                // Only build expensive fallbacks when the primary CMap is sparse.
                // build_fallback_cmap_for_type0 can take seconds on large embedded
                // TrueType fonts (decompressing + parsing 100K+ byte font files).
                // Skip entirely when the primary CMap is sufficient.
                let primary_entries = primary.char_map.len() + primary.ranges.len();
                let mut fallback = if primary_entries < 10 && !skip_truetype_fallback {
                    // Try cheap fallback first; only attempt expensive TrueType
                    // parsing if cheap fallbacks don't yield results.
                    let cheap = build_fallback_tounicode_from_encoding(font_dict, doc)
                        .or_else(|| build_fallback_cmap_for_simple(font_dict, doc));
                    if cheap.is_some() {
                        cheap
                    } else {
                        build_fallback_cmap_for_type0(font_dict, doc)
                    }
                } else if primary_entries < 10 {
                    // Fast mode: only try cheap fallbacks, skip TrueType parsing.
                    // Regions using this font will get needs_ocr=true.
                    build_fallback_tounicode_from_encoding(font_dict, doc)
                        .or_else(|| build_fallback_cmap_for_simple(font_dict, doc))
                } else {
                    // Primary is rich enough; only try the cheap encoding fallback
                    build_fallback_tounicode_from_encoding(font_dict, doc)
                };

                if primary_entries < 10 {
                    if let Some(fb) = fallback.take() {
                        debug!(
                            "ToUnicode CMap obj={} too sparse ({} entries); using fallback",
                            obj_num, primary_entries
                        );
                        remapped = Some(primary);
                        primary = fb;
                    }
                }
                by_obj_num.insert(
                    obj_num,
                    CMapEntry {
                        primary,
                        remapped,
                        fallback,
                    },
                );
            } else {
                // ToUnicode present but parse failed; try fallbacks to avoid empty decoding.
                let fallback = if skip_truetype_fallback {
                    build_fallback_cmap_for_simple(font_dict, doc)
                } else {
                    build_fallback_cmap_for_type0(font_dict, doc)
                        .or_else(|| build_fallback_cmap_for_simple(font_dict, doc))
                };
                if let Some(fb) = fallback {
                    debug!(
                        "ToUnicode CMap obj={} parse failed; using fallback (entries={})",
                        obj_num,
                        fb.char_map.len()
                    );
                    by_obj_num.insert(
                        obj_num,
                        CMapEntry {
                            primary: fb,
                            remapped: None,
                            fallback: None,
                        },
                    );
                }
            }
        }

        // Second pass: Identity-H/V fonts without ToUnicode
        // Try: (1) embedded TrueType/OpenType cmap, (2) predefined CID→Unicode mapping
        // Skip entirely in fast mode — these fonts require expensive TrueType parsing.
        if skip_truetype_fallback {
            return;
        }
        for font_dict in fonts.values() {
            if font_dict.get(b"ToUnicode").is_ok() {
                continue;
            }
            let encoding = match font_dict
                .get(b"Encoding")
                .ok()
                .and_then(|o| o.as_name().ok())
            {
                Some(name) => name,
                None => continue,
            };
            if encoding != b"Identity-H" && encoding != b"Identity-V" {
                continue;
            }
            // Navigate: DescendantFonts[0]
            let desc_fonts_obj = match font_dict.get(b"DescendantFonts").ok() {
                Some(obj) => obj,
                None => continue,
            };
            let desc_fonts = match desc_fonts_obj {
                Object::Array(arr) => arr.clone(),
                Object::Reference(r) => match doc.get_object(*r) {
                    Ok(Object::Array(arr)) => arr.clone(),
                    _ => continue,
                },
                _ => continue,
            };
            if desc_fonts.is_empty() {
                continue;
            }
            let cid_font_dict = match &desc_fonts[0] {
                Object::Reference(r) => match doc.get_dictionary(*r) {
                    Ok(d) => d,
                    _ => continue,
                },
                Object::Dictionary(d) => d,
                _ => continue,
            };

            // Try to build CMap from embedded font (FontFile2 or FontFile3)
            let font_descriptor = cid_font_dict
                .get(b"FontDescriptor")
                .ok()
                .and_then(|o| match o {
                    Object::Reference(r) => doc.get_dictionary(*r).ok(),
                    Object::Dictionary(d) => Some(d),
                    _ => None,
                });

            let mut resolved = false;

            // Determine the font file reference (FontFile2 or FontFile3)
            let font_file_ref = font_descriptor.and_then(|fd| {
                fd.get(b"FontFile2")
                    .ok()
                    .and_then(|o| o.as_reference().ok())
                    .or_else(|| {
                        fd.get(b"FontFile3")
                            .ok()
                            .and_then(|o| o.as_reference().ok())
                    })
            });

            // The lookup key must match what get_font_file2_obj_num() returns:
            // font file obj_num if present, else CIDFont dict obj_num
            let lookup_key = font_file_ref
                .map(|r| r.0)
                .unwrap_or_else(|| match &desc_fonts[0] {
                    Object::Reference(r) => r.0,
                    _ => 0,
                });
            if lookup_key == 0 || by_obj_num.contains_key(&lookup_key) {
                continue;
            }

            // Try parsing embedded TrueType/OpenType cmap
            if let Some(ff_ref) = font_file_ref {
                if let Ok(stream) = doc.get_object(ff_ref).and_then(Object::as_stream) {
                    let data = match stream.decompressed_content() {
                        Ok(d) => d,
                        Err(_) => stream.content.clone(),
                    };
                    if let Some(cmap) = build_cmap_from_truetype(&data) {
                        debug!(
                            "TrueType CMap obj={:<6} (embedded font) char_map={}",
                            lookup_key,
                            cmap.char_map.len()
                        );
                        by_obj_num.insert(
                            lookup_key,
                            CMapEntry {
                                primary: cmap,
                                remapped: None,
                                fallback: None,
                            },
                        );
                        resolved = true;
                    }
                }
            }

            // Fallback: predefined CID→Unicode mapping from CIDSystemInfo
            if !resolved {
                if let Some(cmap) = build_cmap_from_cid_system_info(cid_font_dict, doc) {
                    debug!(
                        "Predefined CMap obj={:<6} (CIDSystemInfo) char_map={}",
                        lookup_key,
                        cmap.char_map.len()
                    );
                    by_obj_num.insert(
                        lookup_key,
                        CMapEntry {
                            primary: cmap,
                            remapped: None,
                            fallback: None,
                        },
                    );
                    resolved = true;
                }
            }

            // Last resort: CID-as-Unicode passthrough.
            // Many PDF generators (Chromium, wkhtmltopdf) use Identity-H encoding where
            // CID values ARE Unicode codepoints, but strip the cmap table and omit
            // ToUnicode. We detect this by checking the /W (widths) array: if CID values
            // fall in typical Unicode letter/digit ranges (0x41+), CIDs are likely Unicode.
            // If CIDs are low values (< 0x41), they're GIDs in a subset font.
            if !resolved {
                if cid_values_look_like_unicode(cid_font_dict) {
                    debug!(
                        "Identity-H font obj={}: W array CIDs look like Unicode — using passthrough",
                        lookup_key
                    );
                    let mut cmap = ToUnicodeCMap::new();
                    cmap.code_byte_length = 2;
                    cmap.cid_passthrough = true;
                    by_obj_num.insert(
                        lookup_key,
                        CMapEntry {
                            primary: cmap,
                            remapped: None,
                            fallback: None,
                        },
                    );
                } else {
                    debug!(
                        "Identity-H font obj={}: no decoding possible (stripped cmap, GID-based CIDs)",
                        lookup_key
                    );
                }
            }
        }

        // Third pass: simple fonts without ToUnicode (use embedded font cmap as fallback)
        for font_dict in fonts.values() {
            if font_dict.get(b"ToUnicode").is_ok() {
                continue;
            }
            // Skip fonts with explicit encoding — they can be decoded by the
            // standard encoding path (lopdf) and don't need a fallback CMap.
            if let Ok(enc) = font_dict.get(b"Encoding") {
                if enc.as_name().is_ok() || enc.as_dict().is_ok() || enc.as_reference().is_ok() {
                    continue;
                }
            }
            let subtype = match font_dict
                .get(b"Subtype")
                .ok()
                .and_then(|o| o.as_name().ok())
            {
                Some(name) => name,
                None => continue,
            };
            if subtype == b"Type0" {
                continue;
            }

            let font_descriptor = font_dict.get(b"FontDescriptor").ok().and_then(|o| match o {
                Object::Reference(r) => doc.get_dictionary(*r).ok(),
                Object::Dictionary(d) => Some(d),
                _ => None,
            });
            let font_file_ref = font_descriptor.and_then(|fd| {
                fd.get(b"FontFile2")
                    .ok()
                    .and_then(|o| o.as_reference().ok())
                    .or_else(|| {
                        fd.get(b"FontFile3")
                            .ok()
                            .and_then(|o| o.as_reference().ok())
                    })
            });
            let ff_ref = match font_file_ref {
                Some(r) => r,
                None => continue,
            };
            let lookup_key = ff_ref.0;
            if by_obj_num.contains_key(&lookup_key) {
                continue;
            }
            if let Ok(stream) = doc.get_object(ff_ref).and_then(Object::as_stream) {
                if let Ok(data) = stream.decompressed_content() {
                    if let Some(cmap) = build_simple_cmap_from_truetype(&data) {
                        debug!(
                            "Simple font cmap obj={:<6} (embedded font) char_map={}",
                            lookup_key,
                            cmap.char_map.len()
                        );
                        by_obj_num.insert(
                            lookup_key,
                            CMapEntry {
                                primary: cmap,
                                remapped: None,
                                fallback: None,
                            },
                        );
                    }
                }
            }
        }
    }

    /// Walk Form XObjects in a page's resources and collect their font CMaps.
    fn collect_cmaps_from_xobjects(
        doc: &Document,
        page_id: ObjectId,
        by_obj_num: &mut HashMap<u32, CMapEntry>,
    ) {
        let (resource_dict, resource_ids) = match doc.get_page_resources(page_id) {
            Ok(r) => r,
            Err(_) => return,
        };

        let mut visited = HashSet::new();

        if let Some(resources) = resource_dict {
            Self::walk_xobject_fonts(resources, doc, by_obj_num, &mut visited);
        }
        for resource_id in resource_ids {
            if let Ok(resources) = doc.get_dictionary(resource_id) {
                Self::walk_xobject_fonts(resources, doc, by_obj_num, &mut visited);
            }
        }
    }

    /// Recursively collect font CMaps from XObjects in a resource dictionary.
    fn walk_xobject_fonts(
        resources: &lopdf::Dictionary,
        doc: &Document,
        by_obj_num: &mut HashMap<u32, CMapEntry>,
        visited: &mut HashSet<ObjectId>,
    ) {
        let xobject_dict = match resources.get(b"XObject") {
            Ok(Object::Reference(id)) => doc.get_object(*id).and_then(Object::as_dict).ok(),
            Ok(Object::Dictionary(dict)) => Some(dict),
            _ => None,
        };
        let xobject_dict = match xobject_dict {
            Some(d) => d,
            None => return,
        };

        for (_name, value) in xobject_dict.iter() {
            let id = match value {
                Object::Reference(id) => *id,
                _ => continue,
            };
            if !visited.insert(id) {
                continue;
            }
            let stream = match doc.get_object(id).and_then(Object::as_stream) {
                Ok(s) => s,
                Err(_) => continue,
            };
            let is_form = stream
                .dict
                .get(b"Subtype")
                .and_then(|o| o.as_name())
                .is_ok_and(|n| n == b"Form");
            if !is_form {
                continue;
            }
            // Collect fonts from this Form XObject's Resources
            if let Ok(form_resources) = stream.dict.get(b"Resources").and_then(Object::as_dict) {
                // Extract font dict from the Form's resources
                let font_dict_obj = match form_resources.get(b"Font") {
                    Ok(Object::Reference(id)) => doc.get_object(*id).and_then(Object::as_dict).ok(),
                    Ok(Object::Dictionary(dict)) => Some(dict),
                    _ => None,
                };
                if let Some(font_dict) = font_dict_obj {
                    let mut fonts = std::collections::BTreeMap::new();
                    for (name, value) in font_dict.iter() {
                        let font = match value {
                            Object::Reference(id) => doc.get_dictionary(*id).ok(),
                            Object::Dictionary(dict) => Some(dict),
                            _ => None,
                        };
                        if let Some(font) = font {
                            fonts.insert(name.clone(), font);
                        }
                    }
                    Self::collect_cmaps_from_fonts(&fonts, doc, by_obj_num);
                }
                // Recurse into nested XObjects
                Self::walk_xobject_fonts(form_resources, doc, by_obj_num, visited);
            }
        }
    }

    /// Get a CMap by ToUnicode object number
    pub fn get_by_obj(&self, obj_num: u32) -> Option<&CMapEntry> {
        self.by_obj_num.get(&obj_num)
    }
}

/// For Type0 CID fonts, try to build a fallback CMap from embedded font data
/// or CIDSystemInfo when a ToUnicode CMap is present but incomplete.
fn build_fallback_cmap_for_type0(
    font_dict: &lopdf::Dictionary,
    doc: &Document,
) -> Option<ToUnicodeCMap> {
    let subtype = font_dict.get(b"Subtype").ok()?.as_name().ok()?;
    if subtype != b"Type0" {
        return None;
    }
    let encoding = font_dict
        .get(b"Encoding")
        .ok()
        .and_then(|o| o.as_name().ok())?;
    if encoding != b"Identity-H" && encoding != b"Identity-V" {
        return None;
    }

    let desc_fonts_obj = font_dict.get(b"DescendantFonts").ok()?;
    let desc_fonts = match desc_fonts_obj {
        Object::Array(arr) => arr,
        Object::Reference(r) => match doc.get_object(*r) {
            Ok(Object::Array(arr)) => arr,
            _ => return None,
        },
        _ => return None,
    };
    if desc_fonts.is_empty() {
        return None;
    }
    let cid_font_dict = match &desc_fonts[0] {
        Object::Reference(r) => doc.get_dictionary(*r).ok()?,
        Object::Dictionary(d) => d,
        _ => return None,
    };

    let font_descriptor = cid_font_dict
        .get(b"FontDescriptor")
        .ok()
        .and_then(|o| match o {
            Object::Reference(r) => doc.get_dictionary(*r).ok(),
            Object::Dictionary(d) => Some(d),
            _ => None,
        });

    let font_file_ref = font_descriptor.and_then(|fd| {
        fd.get(b"FontFile2")
            .ok()
            .and_then(|o| o.as_reference().ok())
            .or_else(|| {
                fd.get(b"FontFile3")
                    .ok()
                    .and_then(|o| o.as_reference().ok())
            })
    });

    if let Some(ff_ref) = font_file_ref {
        if let Ok(stream) = doc.get_object(ff_ref).and_then(Object::as_stream) {
            if let Ok(data) = stream.decompressed_content() {
                if let Some(cmap) = build_cmap_from_truetype(&data) {
                    if let Some(cid_to_gid) = get_cid_to_gid_map(cid_font_dict, doc) {
                        if let Some(repaired) = build_cmap_with_cid_to_gid_map(&cmap, &cid_to_gid) {
                            debug!(
                                "Fallback TrueType CMap repaired with CIDToGIDMap: {} entries",
                                repaired.char_map.len()
                            );
                            return Some(repaired);
                        }
                    }
                    debug!(
                        "Fallback TrueType CMap (Type0+ToUnicode) char_map={}",
                        cmap.char_map.len()
                    );
                    return Some(cmap);
                }
            }
        }
    }

    if let Some(cmap) = build_cmap_from_cid_system_info(cid_font_dict, doc) {
        debug!(
            "Fallback CIDSystemInfo CMap (Type0+ToUnicode) char_map={}",
            cmap.char_map.len()
        );
        return Some(cmap);
    }

    None
}

fn build_fallback_cmap_for_simple(
    font_dict: &lopdf::Dictionary,
    doc: &Document,
) -> Option<ToUnicodeCMap> {
    let subtype = font_dict.get(b"Subtype").ok()?.as_name().ok()?;
    if subtype == b"Type0" {
        return None;
    }
    let font_descriptor = font_dict
        .get(b"FontDescriptor")
        .ok()
        .and_then(|o| match o {
            Object::Reference(r) => doc.get_dictionary(*r).ok(),
            Object::Dictionary(d) => Some(d),
            _ => None,
        })?;
    let font_file_ref = font_descriptor
        .get(b"FontFile2")
        .ok()
        .and_then(|o| o.as_reference().ok())
        .or_else(|| {
            font_descriptor
                .get(b"FontFile3")
                .ok()
                .and_then(|o| o.as_reference().ok())
        })?;
    if let Ok(stream) = doc.get_object(font_file_ref).and_then(Object::as_stream) {
        if let Ok(data) = stream.decompressed_content() {
            if let Some(cmap) = build_simple_cmap_from_truetype(&data) {
                debug!(
                    "Fallback simple font cmap (ToUnicode present) char_map={}",
                    cmap.char_map.len()
                );
                return Some(cmap);
            }
        }
    }
    None
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_parse_bfchar_2byte() {
        let cmap_content = r#"
/CIDInit /ProcSet findresource begin
12 dict begin
begincmap
1 begincodespacerange
<0000><FFFF>
endcodespacerange
3 beginbfchar
<0003> <0020>
<0024> <0041>
<0025> <0042>
endbfchar
endcmap
"#;
        let cmap = ToUnicodeCMap::parse(cmap_content.as_bytes()).unwrap();

        assert_eq!(cmap.code_byte_length, 2);
        assert_eq!(cmap.lookup(0x0003), Some(" ".to_string()));
        assert_eq!(cmap.lookup(0x0024), Some("A".to_string()));
        assert_eq!(cmap.lookup(0x0025), Some("B".to_string()));
    }

    #[test]
    fn test_hex_to_unicode_non_ascii_no_panic() {
        // A destination containing a multi-byte char makes the byte length even
        // while a byte offset can land inside a char. Slicing must not panic;
        // it should be rejected gracefully.
        assert_eq!(hex_to_unicode_string("XéY"), None);
        assert_eq!(hex_to_unicode_string("\u{fffd}0"), None);
    }

    #[test]
    fn test_parse_bfchar_non_ascii_destination_no_panic() {
        // Crafted /ToUnicode CMap: a non-hex, non-ASCII destination previously
        // triggered a char-boundary panic in hex_to_unicode_string.
        let cmap_content = "beginbfchar <0041> <XéY> endbfchar";
        // Must not panic; the malformed entry is simply skipped.
        let _ = ToUnicodeCMap::parse(cmap_content.as_bytes());
    }

    #[test]
    fn test_parse_bfchar_1byte() {
        // This is the pattern that caused the CJK bug: codespace is <0000><FFFF>
        // but all source codes are 1-byte hex (e.g., <20>, <41>)
        let cmap_content = r#"
1 begincodespacerange
<0000> <FFFF>
endcodespacerange
3 beginbfchar
<20> <0020>
<41> <0041>
<42> <0042>
endbfchar
"#;
        let cmap = ToUnicodeCMap::parse(cmap_content.as_bytes()).unwrap();

        // Should detect as 1-byte because all source codes are 1-byte hex
        assert_eq!(cmap.code_byte_length, 1);
        assert_eq!(cmap.lookup(0x0020), Some(" ".to_string()));
        assert_eq!(cmap.lookup(0x0041), Some("A".to_string()));
    }

    #[test]
    fn test_decode_cids_2byte() {
        let cmap_content = r#"
1 begincodespacerange
<0000><FFFF>
endcodespacerange
3 beginbfchar
<0003> <0020>
<0024> <0041>
<0025> <0042>
endbfchar
"#;
        let cmap = ToUnicodeCMap::parse(cmap_content.as_bytes()).unwrap();

        // "AB " in 2-byte CID encoding
        let cids = [0x00, 0x24, 0x00, 0x25, 0x00, 0x03];
        assert_eq!(cmap.decode_cids(&cids), "AB ");
    }

    #[test]
    fn test_decode_cids_1byte_no_cjk_garbage() {
        // Simulates the bug: CMap with 1-byte source codes
        let cmap_content = r#"
1 begincodespacerange
<0000> <FFFF>
endcodespacerange
5 beginbfchar
<20> <0020>
<42> <0042>
<79> <0079>
<50> <0050>
<52> <0052>
endbfchar
"#;
        let cmap = ToUnicodeCMap::parse(cmap_content.as_bytes()).unwrap();
        assert_eq!(cmap.code_byte_length, 1);

        // "By" should decode to "By", NOT to CJK character 䉹
        let bytes = [0x42, 0x79];
        let result = cmap.decode_cids(&bytes);
        assert_eq!(result, "By");
        assert!(!result.contains('䉹'), "Should not produce CJK garbage");

        // "PR" should decode to "PR"
        let bytes2 = [0x50, 0x52];
        assert_eq!(cmap.decode_cids(&bytes2), "PR");
    }

    #[test]
    fn test_bfrange_array_format() {
        let cmap_content = r#"
1 begincodespacerange
<0000> <FFFF>
endcodespacerange
1 beginbfrange
<0003> <0005> [<0041> <0042> <0043>]
endbfrange
"#;
        let cmap = ToUnicodeCMap::parse(cmap_content.as_bytes()).unwrap();

        assert_eq!(cmap.lookup(0x0003), Some("A".to_string()));
        assert_eq!(cmap.lookup(0x0004), Some("B".to_string()));
        assert_eq!(cmap.lookup(0x0005), Some("C".to_string()));
    }

    #[test]
    fn test_parse_bfchar_surrogate_pair_emoji() {
        let cmap_content = r#"
1 begincodespacerange
<00> <FF>
endcodespacerange
2 beginbfchar
<16> <D83CDF1F>
<9D> <D83CDFAD>
endbfchar
"#;
        let cmap = ToUnicodeCMap::parse(cmap_content.as_bytes()).unwrap();

        assert_eq!(cmap.code_byte_length, 1);
        assert_eq!(cmap.lookup(0x16), Some("🌟".to_string()));
        assert_eq!(cmap.lookup(0x9D), Some("🎭".to_string()));
    }

    #[test]
    fn test_parse_bfrange_surrogate_pair_base() {
        let cmap_content = r#"
1 begincodespacerange
<00> <FF>
endcodespacerange
1 beginbfrange
<C8> <C9> <D83CDFD8>
endbfrange
"#;
        let cmap = ToUnicodeCMap::parse(cmap_content.as_bytes()).unwrap();

        assert_eq!(cmap.code_byte_length, 1);
        assert_eq!(cmap.lookup(0xC8), Some("🏘".to_string()));
        assert_eq!(cmap.lookup(0xC9), Some("🏙".to_string()));
    }

    #[test]
    fn test_parse_bfrange_preserves_single_hyphen_like_base() {
        let cmap_content = r#"
1 begincodespacerange
<00> <FF>
endcodespacerange
1 beginbfrange
<21> <22> <2013>
endbfrange
"#;
        let cmap = ToUnicodeCMap::parse(cmap_content.as_bytes()).unwrap();

        assert_eq!(cmap.lookup(0x21), Some("–".to_string()));
        assert_eq!(cmap.lookup(0x22), Some("—".to_string()));
    }

    #[test]
    fn test_parse_spaced_destination_hex_without_control_noise() {
        let cmap_content = r#"
1 begincodespacerange
<00> <FF>
endcodespacerange
3 beginbfchar
<21> < 0009 000d 0020 00a0 >
<22> < 002d 00ad 2010 >
<23> <00a0>
endbfchar
"#;
        let cmap = ToUnicodeCMap::parse(cmap_content.as_bytes()).unwrap();

        assert_eq!(cmap.lookup(0x21), Some("\t".to_string()));
        assert_eq!(cmap.lookup(0x22), Some("-".to_string()));
        assert_eq!(cmap.lookup(0x23), Some("\u{00a0}".to_string()));
    }

    #[test]
    fn test_parse_preserves_valid_multi_character_destinations() {
        let cmap_content = r#"
1 begincodespacerange
<00> <FF>
endcodespacerange
4 beginbfchar
<21> <002d002d>
<22> <20132013>
<23> <002000a0>
<24> <00660069>
endbfchar
"#;
        let cmap = ToUnicodeCMap::parse(cmap_content.as_bytes()).unwrap();

        assert_eq!(cmap.lookup(0x21), Some("--".to_string()));
        assert_eq!(cmap.lookup(0x22), Some("––".to_string()));
        assert_eq!(cmap.lookup(0x23), Some(" \u{00a0}".to_string()));
        assert_eq!(cmap.lookup(0x24), Some("fi".to_string()));
    }

    #[test]
    fn test_remap_to_sequential() {
        // Simulate a broken CMap where GIDs are from pre-subsetting:
        // Old GID 3 → space, old GID 36 → 'A', old GID 37 → 'B'
        // The subset font has sequential GIDs: 1=space, 2='A', 3='B'
        let cmap_content = r#"
1 begincodespacerange
<0000><FFFF>
endcodespacerange
3 beginbfchar
<0003> <0020>
<0024> <0041>
<0025> <0042>
endbfchar
"#;
        let cmap = ToUnicodeCMap::parse(cmap_content.as_bytes()).unwrap();

        // Original CMap: CID 3 → space, CID 36 → 'A', CID 37 → 'B'
        assert_eq!(cmap.lookup(0x0003), Some(" ".to_string()));
        assert_eq!(cmap.lookup(0x0024), Some("A".to_string()));
        assert_eq!(cmap.lookup(0x0025), Some("B".to_string()));
        assert_eq!(cmap.lookup(0x0001), None);
        assert_eq!(cmap.lookup(0x0002), None);

        // After remapping: CID 1 → space, CID 2 → 'A', CID 3 → 'B'
        let remapped = cmap.remap_to_sequential();
        assert_eq!(remapped.lookup(0x0001), Some(" ".to_string()));
        assert_eq!(remapped.lookup(0x0002), Some("A".to_string()));
        assert_eq!(remapped.lookup(0x0003), Some("B".to_string()));
        assert_eq!(remapped.lookup(0x0024), None);
        assert_eq!(remapped.lookup(0x0025), None);
    }

    #[test]
    fn test_remap_to_sequential_with_ranges() {
        // CMap with a bfrange: old GIDs 100-102 → 'X', 'Y', 'Z'
        // Plus a bfchar: old GID 50 → space
        let cmap_content = r#"
1 begincodespacerange
<0000><FFFF>
endcodespacerange
1 beginbfchar
<0032> <0020>
endbfchar
1 beginbfrange
<0064> <0066> <0058>
endbfrange
"#;
        let cmap = ToUnicodeCMap::parse(cmap_content.as_bytes()).unwrap();

        assert_eq!(cmap.lookup(0x0032), Some(" ".to_string())); // CID 50
        assert_eq!(cmap.lookup(0x0064), Some("X".to_string())); // CID 100
        assert_eq!(cmap.lookup(0x0065), Some("Y".to_string())); // CID 101
        assert_eq!(cmap.lookup(0x0066), Some("Z".to_string())); // CID 102

        let remapped = cmap.remap_to_sequential();
        // Sorted old CIDs: 50, 100, 101, 102 → new CIDs: 1, 2, 3, 4
        assert_eq!(remapped.lookup(0x0001), Some(" ".to_string()));
        assert_eq!(remapped.lookup(0x0002), Some("X".to_string()));
        assert_eq!(remapped.lookup(0x0003), Some("Y".to_string()));
        assert_eq!(remapped.lookup(0x0004), Some("Z".to_string()));
        // Ranges should be cleared (all in char_map now)
        assert!(remapped.ranges.is_empty());
    }

    #[test]
    fn remap_to_sequential_repeated_full_bfranges_stay_bounded() {
        // 5,000 copies of `<0003> <ffff>` must stop after 65,536 CID visits,
        // not 5,000 × ~65,533 expansions.
        let ranges = vec![(3u16, 65535u16, 0x41u32); 5_000];
        let mut map = std::collections::HashMap::new();
        let assigned = expand_bfranges_for_remap(&ranges, &mut map, MAX_CID_W_EXPANSION);
        assert_eq!(assigned, MAX_CID_W_EXPANSION);
        assert!(map.len() <= MAX_CID_W_EXPANSION);

        let mut body = String::new();
        let mut remaining = 5_000usize;
        while remaining > 0 {
            let n = remaining.min(100);
            body.push_str(&format!("{n} beginbfrange\n"));
            for _ in 0..n {
                body.push_str("<0003> <ffff> <0041>\n");
            }
            body.push_str("endbfrange\n");
            remaining -= n;
        }
        let data = format!("1 begincodespacerange\n<0000> <ffff>\nendcodespacerange\n{body}");
        let cmap = ToUnicodeCMap::parse(data.as_bytes()).unwrap();
        let remapped = cmap.remap_to_sequential();
        assert_eq!(remapped.lookup(1), Some("A".to_string()));
    }

    #[test]
    fn test_min_source_cid() {
        let cmap_content = r#"
1 begincodespacerange
<0000><FFFF>
endcodespacerange
2 beginbfchar
<0003> <0020>
<0024> <0041>
endbfchar
1 beginbfrange
<0030> <0032> <0058>
endbfrange
"#;
        let cmap = ToUnicodeCMap::parse(cmap_content.as_bytes()).unwrap();
        assert_eq!(cmap.min_source_cid(), Some(3));
    }

    #[test]
    fn test_unmapped_2byte_cids_skipped() {
        let cmap_content = r#"
1 begincodespacerange
<0000><FFFF>
endcodespacerange
1 beginbfchar
<0041> <0041>
endbfchar
"#;
        let cmap = ToUnicodeCMap::parse(cmap_content.as_bytes()).unwrap();
        assert_eq!(cmap.code_byte_length, 2);

        // CID 0x4279 is unmapped - should NOT produce CJK character
        let bytes = [0x42, 0x79];
        let result = cmap.decode_cids(&bytes);
        assert!(
            !result.contains('䉹'),
            "Unmapped 2-byte CIDs should not produce CJK"
        );
    }

    #[test]
    fn fallback_promotion_when_larger_than_primary() {
        // Simulate: primary has 5 char_map entries, remapped exists (sequential),
        // fallback has 20 entries (TrueType cmap).  The fallback should be
        // promoted to `remapped` and the old remap demoted to `fallback`.
        let mut primary = ToUnicodeCMap::new();
        for i in 0..5u16 {
            primary
                .char_map
                .insert(100 + i, char::from(b'A' + i as u8).to_string());
        }
        primary.code_byte_length = 2;

        let mut sequential_remap = ToUnicodeCMap::new();
        for i in 0..5u16 {
            sequential_remap
                .char_map
                .insert(i, char::from(b'A' + i as u8).to_string());
        }
        sequential_remap.code_byte_length = 2;

        let mut truetype_fb = ToUnicodeCMap::new();
        for i in 0..20u16 {
            truetype_fb
                .char_map
                .insert(i, format!("U+{:04X}", 0x4E00 + i));
        }
        truetype_fb.code_byte_length = 2;

        let primary_entries = primary.char_map.len() + primary.ranges.len();
        let mut remapped: Option<ToUnicodeCMap> = Some(sequential_remap);
        let mut fallback: Option<ToUnicodeCMap> = Some(truetype_fb);

        // Apply the same promotion logic as build_cmap_entry_from_stream
        if remapped.is_some() {
            if let Some(ref fb) = fallback {
                let fb_entries = fb.char_map.len() + fb.ranges.len();
                if fb_entries > primary_entries {
                    let old_remap = remapped.take().unwrap();
                    remapped = fallback.take();
                    fallback = Some(old_remap);
                }
            }
        }

        // The TrueType fallback (20 entries) should now be in `remapped`
        let r = remapped.unwrap();
        assert_eq!(
            r.char_map.len(),
            20,
            "TrueType cmap should be promoted to remapped"
        );

        // The old sequential remap (5 entries) should now be in `fallback`
        let f = fallback.unwrap();
        assert_eq!(
            f.char_map.len(),
            5,
            "Sequential remap should be demoted to fallback"
        );
    }

    #[test]
    fn no_fallback_promotion_when_smaller() {
        // When fallback has fewer entries than primary, no swap should occur.
        let mut primary = ToUnicodeCMap::new();
        for i in 0..50u16 {
            primary
                .char_map
                .insert(100 + i, format!("U+{:04X}", 0x0041 + i));
        }
        primary.code_byte_length = 2;

        let mut sequential_remap = ToUnicodeCMap::new();
        for i in 0..50u16 {
            sequential_remap
                .char_map
                .insert(i, format!("U+{:04X}", 0x0041 + i));
        }
        sequential_remap.code_byte_length = 2;

        let mut small_fb = ToUnicodeCMap::new();
        for i in 0..10u16 {
            small_fb.char_map.insert(i, format!("U+{:04X}", 0x4E00 + i));
        }
        small_fb.code_byte_length = 2;

        let primary_entries = primary.char_map.len() + primary.ranges.len();
        let mut remapped: Option<ToUnicodeCMap> = Some(sequential_remap);
        let mut fallback: Option<ToUnicodeCMap> = Some(small_fb);

        if remapped.is_some() {
            if let Some(ref fb) = fallback {
                let fb_entries = fb.char_map.len() + fb.ranges.len();
                if fb_entries > primary_entries {
                    let old_remap = remapped.take().unwrap();
                    remapped = fallback.take();
                    fallback = Some(old_remap);
                }
            }
        }

        // No swap: remapped should still have 50 entries
        assert_eq!(remapped.unwrap().char_map.len(), 50);
        assert_eq!(fallback.unwrap().char_map.len(), 10);
    }

    #[test]
    fn test_max_source_cid() {
        let cmap_content = r#"
1 begincodespacerange
<0000><FFFF>
endcodespacerange
2 beginbfchar
<0003> <0020>
<0031> <004E>
endbfchar
1 beginbfrange
<0208> <0227> <0430>
endbfrange
"#;
        let cmap = ToUnicodeCMap::parse(cmap_content.as_bytes()).unwrap();
        assert_eq!(cmap.min_source_cid(), Some(0x0003));
        assert_eq!(cmap.max_source_cid(), Some(0x0227));
    }

    /// Helper: build a minimal CIDFont dict with a W array and check coverage.
    fn cid_font_dict_with_w(w_items: Vec<lopdf::Object>) -> lopdf::Dictionary {
        let mut d = lopdf::Dictionary::new();
        d.set("W", lopdf::Object::Array(w_items));
        d
    }

    #[test]
    fn test_w_array_covers_cid_format1() {
        // Format 1: `c [w1 w2 ... wn]` — widths for CIDs c..c+n-1.
        // Mimics the 16.pdf Tahoma W array: 0[1000] 3[313] 5[401] 11[383 383] 16[363 303 382]
        let doc = Document::new();
        let d = cid_font_dict_with_w(vec![
            lopdf::Object::Integer(0),
            lopdf::Object::Array(vec![lopdf::Object::Integer(1000)]),
            lopdf::Object::Integer(3),
            lopdf::Object::Array(vec![lopdf::Object::Integer(313)]),
            lopdf::Object::Integer(5),
            lopdf::Object::Array(vec![lopdf::Object::Integer(401)]),
            lopdf::Object::Integer(11),
            lopdf::Object::Array(vec![
                lopdf::Object::Integer(383),
                lopdf::Object::Integer(383),
            ]),
            lopdf::Object::Integer(16),
            lopdf::Object::Array(vec![
                lopdf::Object::Integer(363),
                lopdf::Object::Integer(303),
                lopdf::Object::Integer(382),
            ]),
            lopdf::Object::Integer(570),
            lopdf::Object::Array(vec![lopdf::Object::Integer(667); 26]),
        ]);

        assert!(w_array_covers_cid(&d, &doc, 0));
        assert!(w_array_covers_cid(&d, &doc, 3));
        assert!(w_array_covers_cid(&d, &doc, 5));
        assert!(w_array_covers_cid(&d, &doc, 11));
        assert!(w_array_covers_cid(&d, &doc, 12));
        assert!(w_array_covers_cid(&d, &doc, 16));
        assert!(w_array_covers_cid(&d, &doc, 18));
        assert!(w_array_covers_cid(&d, &doc, 570));
        assert!(w_array_covers_cid(&d, &doc, 595));
        // Gaps are NOT covered
        assert!(!w_array_covers_cid(&d, &doc, 1));
        assert!(!w_array_covers_cid(&d, &doc, 4));
        assert!(!w_array_covers_cid(&d, &doc, 19));
        assert!(!w_array_covers_cid(&d, &doc, 596));
    }

    #[test]
    fn test_w_array_covers_cid_format2() {
        // Format 2: `c_first c_last w` — CIDs c_first..c_last all have width w.
        let doc = Document::new();
        let d = cid_font_dict_with_w(vec![
            lopdf::Object::Integer(100),
            lopdf::Object::Integer(120),
            lopdf::Object::Integer(500),
        ]);

        assert!(w_array_covers_cid(&d, &doc, 100));
        assert!(w_array_covers_cid(&d, &doc, 110));
        assert!(w_array_covers_cid(&d, &doc, 120));
        assert!(!w_array_covers_cid(&d, &doc, 99));
        assert!(!w_array_covers_cid(&d, &doc, 121));
    }

    #[test]
    fn test_w_array_covers_cid_missing_w() {
        let doc = Document::new();
        let d = lopdf::Dictionary::new();
        assert!(!w_array_covers_cid(&d, &doc, 3));
    }

    #[test]
    fn test_try_remap_skipped_when_w_covers_cmap() {
        // Simulates 16.pdf: CMap's max source CID (0x0279 = 633) is explicitly
        // in the W array, so no subset-renumbering happened — remap must NOT fire.
        let cmap_content = r#"
1 begincodespacerange
<0000><FFFF>
endcodespacerange
2 beginbfchar
<0003> <0020>
<0031> <004E>
endbfchar
2 beginbfrange
<023A> <0253> <0410>
<0255> <0279> <042B>
endbfrange
"#;
        let cmap = ToUnicodeCMap::parse(cmap_content.as_bytes()).unwrap();

        let mut doc = Document::new();
        // Build a CIDFont dict with Identity CIDToGIDMap and a W array that
        // covers CID 633 via `597 [widths...]`.
        let mut cid_font = lopdf::Dictionary::new();
        cid_font.set("CIDToGIDMap", lopdf::Object::Name(b"Identity".to_vec()));
        cid_font.set(
            "W",
            lopdf::Object::Array(vec![
                lopdf::Object::Integer(0),
                lopdf::Object::Array(vec![lopdf::Object::Integer(750)]),
                lopdf::Object::Integer(597),
                lopdf::Object::Array(vec![lopdf::Object::Integer(500); 37]), // 597..633
            ]),
        );
        let cid_font_id = doc.add_object(cid_font);

        // Build the Type0 font dict with Identity-H + DescendantFonts ref.
        let mut font_dict = lopdf::Dictionary::new();
        font_dict.set("Encoding", lopdf::Object::Name(b"Identity-H".to_vec()));
        font_dict.set(
            "DescendantFonts",
            lopdf::Object::Array(vec![lopdf::Object::Reference(cid_font_id)]),
        );

        let (primary, remapped) = try_remap_subset_cmap(cmap, &font_dict, &doc, 123);
        assert!(
            remapped.is_none(),
            "Remap must be skipped when W covers CMap max CID (this is 16.pdf)"
        );
        assert_eq!(primary.lookup(0x0003), Some(" ".to_string()));
    }

    #[test]
    fn test_try_remap_fires_for_true_subset_mismatch() {
        // True mismatch: CMap has high CIDs (512-544) but W only lists low sequential CIDs.
        let cmap_content = r#"
1 begincodespacerange
<0000><FFFF>
endcodespacerange
1 beginbfrange
<0200> <0220> <0410>
endbfrange
"#;
        let cmap = ToUnicodeCMap::parse(cmap_content.as_bytes()).unwrap();

        let mut doc = Document::new();
        let mut cid_font = lopdf::Dictionary::new();
        cid_font.set("CIDToGIDMap", lopdf::Object::Name(b"Identity".to_vec()));
        cid_font.set(
            "W",
            lopdf::Object::Array(vec![
                lopdf::Object::Integer(0),
                lopdf::Object::Array(vec![lopdf::Object::Integer(500); 34]), // 0..33
            ]),
        );
        let cid_font_id = doc.add_object(cid_font);

        let mut font_dict = lopdf::Dictionary::new();
        font_dict.set("Encoding", lopdf::Object::Name(b"Identity-H".to_vec()));
        font_dict.set(
            "DescendantFonts",
            lopdf::Object::Array(vec![lopdf::Object::Reference(cid_font_id)]),
        );

        let (_primary, remapped) = try_remap_subset_cmap(cmap, &font_dict, &doc, 456);
        assert!(
            remapped.is_some(),
            "Remap must fire when CMap's CIDs are outside W array coverage"
        );
    }

    #[test]
    fn test_try_remap_skipped_for_cid_font_type0() {
        // Same W/CMap mismatch as the CIDFontType2 case above, but the descendant is
        // CIDFontType0 (CFF). There CIDs are resolved through the CFF charset, so the
        // ToUnicode CIDs stay valid after subsetting and must not be renumbered.
        // Real-world case: Japanese Adobe-Japan1 PDFs (e.g. National Diet Library
        // minutes) where remapping turned correct text into unrelated glyphs.
        let cmap_content = r#"
1 begincodespacerange
<0000><FFFF>
endcodespacerange
1 beginbfrange
<0200> <0220> <0410>
endbfrange
"#;
        let cmap = ToUnicodeCMap::parse(cmap_content.as_bytes()).unwrap();

        let mut doc = Document::new();

        // CIDToGIDMap is CIDFontType2-only, but a malformed producer can still emit
        // one on a CFF font. Use a real stream (not /Identity, which is treated as
        // "no map") so this also fails if the guard is moved back below the
        // CIDToGIDMap branch: cid 1 -> gid 0x0200, which the CMap resolves.
        let mut cid_to_gid = vec![0u8; 68];
        cid_to_gid[2] = 0x02;
        cid_to_gid[3] = 0x00;
        let cid_to_gid_id =
            doc.add_object(lopdf::Stream::new(lopdf::Dictionary::new(), cid_to_gid));

        let mut cid_font = lopdf::Dictionary::new();
        cid_font.set("Subtype", lopdf::Object::Name(b"CIDFontType0".to_vec()));
        cid_font.set("CIDToGIDMap", lopdf::Object::Reference(cid_to_gid_id));
        cid_font.set(
            "W",
            lopdf::Object::Array(vec![
                lopdf::Object::Integer(1),
                lopdf::Object::Array(vec![lopdf::Object::Integer(500); 34]),
            ]),
        );
        let cid_font_id = doc.add_object(cid_font);

        let mut font_dict = lopdf::Dictionary::new();
        font_dict.set("Encoding", lopdf::Object::Name(b"Identity-H".to_vec()));
        font_dict.set(
            "DescendantFonts",
            lopdf::Object::Array(vec![lopdf::Object::Reference(cid_font_id)]),
        );

        let (primary, remapped) = try_remap_subset_cmap(cmap, &font_dict, &doc, 789);
        assert!(
            remapped.is_none(),
            "Remap must be skipped for CIDFontType0 (CFF) descendants, including a \
             CIDToGIDMap a malformed producer attached to one"
        );
        // The original CMap must still resolve its own CIDs.
        assert_eq!(primary.lookup(0x0200), Some("\u{0410}".to_string()));
    }

    #[test]
    fn test_try_remap_resolves_indirect_subtype() {
        // /Subtype may be stored as an indirect reference. A genuine CIDFontType2
        // font must still get the repair, so the guard has to dereference it rather
        // than treat the unresolved value as "not CIDFontType2".
        let cmap_content = r#"
1 begincodespacerange
<0000><FFFF>
endcodespacerange
1 beginbfrange
<0200> <0220> <0410>
endbfrange
"#;
        let cmap = ToUnicodeCMap::parse(cmap_content.as_bytes()).unwrap();

        let mut doc = Document::new();
        let subtype_id = doc.add_object(lopdf::Object::Name(b"CIDFontType2".to_vec()));

        let mut cid_font = lopdf::Dictionary::new();
        cid_font.set("Subtype", lopdf::Object::Reference(subtype_id));
        cid_font.set("CIDToGIDMap", lopdf::Object::Name(b"Identity".to_vec()));
        cid_font.set(
            "W",
            lopdf::Object::Array(vec![
                lopdf::Object::Integer(0),
                lopdf::Object::Array(vec![lopdf::Object::Integer(500); 34]),
            ]),
        );
        let cid_font_id = doc.add_object(cid_font);

        let mut font_dict = lopdf::Dictionary::new();
        font_dict.set("Encoding", lopdf::Object::Name(b"Identity-H".to_vec()));
        font_dict.set(
            "DescendantFonts",
            lopdf::Object::Array(vec![lopdf::Object::Reference(cid_font_id)]),
        );

        let (_primary, remapped) = try_remap_subset_cmap(cmap, &font_dict, &doc, 790);
        assert!(
            remapped.is_some(),
            "An indirect /Subtype naming CIDFontType2 must still reach the remap"
        );
    }

    #[test]
    fn cid_values_look_like_unicode_letter_range() {
        let mut dict = lopdf::Dictionary::new();
        dict.set(
            "W",
            Object::Array(vec![
                Object::Integer(0x41),
                Object::Integer(0x5A),
                Object::Integer(500),
            ]),
        );
        assert!(cid_values_look_like_unicode(&dict));
    }

    #[test]
    fn cid_values_look_like_unicode_low_gids() {
        let mut dict = lopdf::Dictionary::new();
        dict.set(
            "W",
            Object::Array(vec![
                Object::Integer(0),
                Object::Array(vec![Object::Integer(500); 10]),
            ]),
        );
        assert!(!cid_values_look_like_unicode(&dict));
    }

    #[test]
    fn cid_values_look_like_unicode_repeated_full_ranges_stay_bounded() {
        // Repeating `[0 65535 w]` must not materialize 65,536 CIDs per copy.
        let mut w = Vec::new();
        for _ in 0..5_000 {
            w.push(Object::Integer(0));
            w.push(Object::Integer(65535));
            w.push(Object::Integer(500));
        }
        let mut dict = lopdf::Dictionary::new();
        dict.set("W", Object::Array(w));
        assert!(cid_values_look_like_unicode(&dict));
    }

    #[test]
    fn encoding_cidrange_maps_a_normal_range() {
        let data = b"1 begincodespacerange\n<0000> <FFFF>\nendcodespacerange\n\
1 begincidrange\n<0041> <0043> 65\nendcidrange\n";
        let enc = parse_encoding_cmap_stream(data).unwrap();
        assert_eq!(enc.map.get(&0x41), Some(&65));
        assert_eq!(enc.map.get(&0x42), Some(&66));
        assert_eq!(enc.map.get(&0x43), Some(&67));
        assert_eq!(enc.map.len(), 3);
        assert_eq!(enc.code_byte_length, 2);
    }

    #[test]
    fn encoding_cidrange_repeated_full_ranges_stay_bounded() {
        // 5,000 copies of `<0000> <ffff> 0` must not re-expand the 16-bit
        // domain on every declaration.
        let mut body = String::new();
        let mut remaining = 5_000usize;
        while remaining > 0 {
            let n = remaining.min(100);
            body.push_str(&format!("{n} begincidrange\n"));
            for _ in 0..n {
                body.push_str("<0000> <ffff> 0\n");
            }
            body.push_str("endcidrange\n");
            remaining -= n;
        }
        let data = format!("1 begincodespacerange\n<0000> <FFFF>\nendcodespacerange\n{body}");
        let enc = parse_encoding_cmap_stream(data.as_bytes()).unwrap();
        assert!(enc.map.len() <= MAX_CID_W_EXPANSION);
        assert_eq!(enc.map.get(&0), Some(&0));
        assert_eq!(enc.map.get(&65535), Some(&65535));
    }

    #[test]
    fn is_degenerate_identity_bfrange_detects_full_range_identity() {
        let cmap_content = r#"
1 begincodespacerange
<0000><FFFF>
endcodespacerange
1 beginbfrange
<0000> <FFFF> <0000>
endbfrange
"#;
        let cmap = ToUnicodeCMap::parse(cmap_content.as_bytes()).unwrap();
        assert!(is_degenerate_identity_bfrange(&cmap));
    }

    #[test]
    fn degenerate_identity_bfrange_is_treated_as_absent() {
        let cmap_content = r#"
1 begincodespacerange
<0000><FFFF>
endcodespacerange
1 beginbfrange
<0000> <FFFF> <0000>
endbfrange
"#;
        let font_dict = lopdf::Dictionary::new();
        let doc = lopdf::Document::new();

        assert!(
            build_cmap_entry_from_stream(cmap_content.as_bytes(), &font_dict, &doc, 1).is_none()
        );
    }

    #[test]
    fn is_degenerate_identity_bfrange_rejects_nonzero_base() {
        // Same range but with a non-zero base offset: not identity
        let cmap_content = r#"
1 begincodespacerange
<0000><FFFF>
endcodespacerange
1 beginbfrange
<0000> <FFFF> <0020>
endbfrange
"#;
        let cmap = ToUnicodeCMap::parse(cmap_content.as_bytes()).unwrap();
        assert!(!is_degenerate_identity_bfrange(&cmap));
    }

    #[test]
    fn is_degenerate_identity_bfrange_rejects_partial_range() {
        // Partial range: not the degenerate full-range identity
        let cmap_content = r#"
1 begincodespacerange
<0000><FFFF>
endcodespacerange
1 beginbfrange
<0020> <007E> <0020>
endbfrange
"#;
        let cmap = ToUnicodeCMap::parse(cmap_content.as_bytes()).unwrap();
        assert!(!is_degenerate_identity_bfrange(&cmap));
    }

    #[test]
    fn is_degenerate_identity_bfrange_rejects_bfchar_entries() {
        // Full-range identity bfrange PLUS bfchar entries: not degenerate
        let cmap_content = r#"
1 begincodespacerange
<0000><FFFF>
endcodespacerange
1 beginbfchar
<0041> <0041>
endbfchar
1 beginbfrange
<0000> <FFFF> <0000>
endbfrange
"#;
        let cmap = ToUnicodeCMap::parse(cmap_content.as_bytes()).unwrap();
        assert!(!is_degenerate_identity_bfrange(&cmap));
    }

    #[test]
    fn is_degenerate_identity_bfrange_rejects_normal_tounicode() {
        // Normal CJK ToUnicode with real mappings
        let cmap_content = r#"
1 begincodespacerange
<0000><FFFF>
endcodespacerange
3 beginbfchar
<0001> <4E00>
<0002> <4E01>
<0003> <4E02>
endbfchar
"#;
        let cmap = ToUnicodeCMap::parse(cmap_content.as_bytes()).unwrap();
        assert!(!is_degenerate_identity_bfrange(&cmap));
    }
}