rust_pdf_box/cos/
lexer.rs

1use crate::cos::object::StringFormat;
2use crate::error::{PdfError, PdfResult};
3
4#[derive(Debug, Clone, PartialEq)]
5pub enum Token {
6    Keyword(String),
7    Integer(i64),
8    Real(f64),
9    String(Vec<u8>, StringFormat),
10    Name(String),
11    DictStart,  // <<
12    DictEnd,    // >>
13    ArrayStart, // [
14    ArrayEnd,   // ]
15    EOF,
16}
17
18pub struct Lexer<'a> {
19    data: &'a [u8],
20    pub pos: usize,
21}
22
23impl<'a> Lexer<'a> {
24    pub fn new(data: &'a [u8]) -> Self {
25        Self { data, pos: 0 }
26    }
27
28    pub fn with_pos(data: &'a [u8], pos: usize) -> Self {
29        Self { data, pos }
30    }
31
32    pub fn pos(&self) -> usize {
33        self.pos
34    }
35
36    pub fn set_pos(&mut self, pos: usize) {
37        self.pos = pos.min(self.data.len());
38    }
39
40    pub fn peek_byte(&self) -> Option<u8> {
41        self.data.get(self.pos).copied()
42    }
43
44    pub fn next_byte(&mut self) -> Option<u8> {
45        if self.pos < self.data.len() {
46            let b = self.data[self.pos];
47            self.pos += 1;
48            Some(b)
49        } else {
50            None
51        }
52    }
53
54    pub fn skip_whitespace_and_comments(&mut self) {
55        while self.pos < self.data.len() {
56            let b = self.data[self.pos];
57            if matches!(b, 0 | 9 | 10 | 12 | 13 | 32) {
58                self.pos += 1;
59            } else if b == b'%' {
60                // Comment until \r or \n
61                self.pos += 1;
62                while self.pos < self.data.len() {
63                    let c = self.data[self.pos];
64                    self.pos += 1;
65                    if c == b'\r' || c == b'\n' {
66                        break;
67                    }
68                }
69            } else {
70                break;
71            }
72        }
73    }
74
75    pub fn next_token(&mut self) -> PdfResult<Token> {
76        self.skip_whitespace_and_comments();
77        if self.pos >= self.data.len() {
78            return Ok(Token::EOF);
79        }
80
81        let start_pos = self.pos;
82        let b = self.data[self.pos];
83
84        match b {
85            b'<' => {
86                self.pos += 1;
87                if self.pos < self.data.len() && self.data[self.pos] == b'<' {
88                    self.pos += 1;
89                    Ok(Token::DictStart)
90                } else {
91                    self.read_hex_string(start_pos)
92                }
93            }
94            b'>' => {
95                self.pos += 1;
96                if self.pos < self.data.len() && self.data[self.pos] == b'>' {
97                    self.pos += 1;
98                    Ok(Token::DictEnd)
99                } else {
100                    Err(PdfError::Parse {
101                        offset: start_pos,
102                        message: "Unexpected single '>' character".to_string(),
103                    })
104                }
105            }
106            b'[' => {
107                self.pos += 1;
108                Ok(Token::ArrayStart)
109            }
110            b']' => {
111                self.pos += 1;
112                Ok(Token::ArrayEnd)
113            }
114            b'(' => self.read_literal_string(start_pos),
115            b'/' => self.read_name(),
116            _ => self.read_number_or_keyword(start_pos),
117        }
118    }
119
120    fn read_name(&mut self) -> PdfResult<Token> {
121        self.pos += 1; // skip leading '/'
122        let mut name_bytes = Vec::new();
123
124        while self.pos < self.data.len() {
125            let b = self.data[self.pos];
126            if matches!(
127                b,
128                0 | 9
129                    | 10
130                    | 12
131                    | 13
132                    | 32
133                    | b'('
134                    | b')'
135                    | b'<'
136                    | b'>'
137                    | b'['
138                    | b']'
139                    | b'{'
140                    | b'}'
141                    | b'/'
142                    | b'%'
143            ) {
144                break;
145            }
146            if b == b'#' {
147                self.pos += 1;
148                if self.pos + 1 < self.data.len() {
149                    let hex_chars = std::str::from_utf8(&self.data[self.pos..self.pos + 2])
150                        .map_err(|_| PdfError::Parse {
151                            offset: self.pos,
152                            message: "Invalid hex sequence in PDF Name".to_string(),
153                        })?;
154                    let val = u8::from_str_radix(hex_chars, 16).map_err(|_| PdfError::Parse {
155                        offset: self.pos,
156                        message: format!("Invalid hex digits in PDF Name: #{hex_chars}"),
157                    })?;
158                    name_bytes.push(val);
159                    self.pos += 2;
160                } else {
161                    name_bytes.push(b'#');
162                }
163            } else {
164                name_bytes.push(b);
165                self.pos += 1;
166            }
167        }
168
169        let name_str = String::from_utf8_lossy(&name_bytes).into_owned();
170        Ok(Token::Name(name_str))
171    }
172
173    fn read_literal_string(&mut self, start_pos: usize) -> PdfResult<Token> {
174        self.pos += 1; // skip '('
175        let mut depth = 1;
176        let mut result = Vec::new();
177
178        while self.pos < self.data.len() {
179            let b = self.data[self.pos];
180            self.pos += 1;
181
182            if b == b'\\' {
183                if self.pos >= self.data.len() {
184                    break;
185                }
186                let esc = self.data[self.pos];
187                self.pos += 1;
188                match esc {
189                    b'n' => result.push(b'\n'),
190                    b'r' => result.push(b'\r'),
191                    b't' => result.push(b'\t'),
192                    b'b' => result.push(0x08),
193                    b'f' => result.push(0x0C),
194                    b'(' => result.push(b'('),
195                    b')' => result.push(b')'),
196                    b'\\' => result.push(b'\\'),
197                    b'\r' => {
198                        // line continuation, skip following \n if present
199                        if self.pos < self.data.len() && self.data[self.pos] == b'\n' {
200                            self.pos += 1;
201                        }
202                    }
203                    b'\n' => {} // line continuation
204                    b'0'..=b'7' => {
205                        // Octal sequence up to 3 digits
206                        let mut oct_val = (esc - b'0') as u32;
207                        for _ in 0..2 {
208                            if self.pos < self.data.len()
209                                && (b'0'..=b'7').contains(&self.data[self.pos])
210                            {
211                                oct_val = oct_val * 8 + (self.data[self.pos] - b'0') as u32;
212                                self.pos += 1;
213                            } else {
214                                break;
215                            }
216                        }
217                        result.push((oct_val & 0xFF) as u8);
218                    }
219                    other => result.push(other),
220                }
221            } else if b == b'(' {
222                depth += 1;
223                result.push(b);
224            } else if b == b')' {
225                depth -= 1;
226                if depth == 0 {
227                    return Ok(Token::String(result, StringFormat::Literal));
228                } else {
229                    result.push(b);
230                }
231            } else {
232                result.push(b);
233            }
234        }
235
236        Err(PdfError::Parse {
237            offset: start_pos,
238            message: "Unterminated literal string".to_string(),
239        })
240    }
241
242    fn read_hex_string(&mut self, start_pos: usize) -> PdfResult<Token> {
243        let mut hex_digits = Vec::new();
244
245        while self.pos < self.data.len() {
246            let b = self.data[self.pos];
247            self.pos += 1;
248
249            if b == b'>' {
250                break;
251            } else if matches!(b, 0 | 9 | 10 | 12 | 13 | 32) {
252                continue;
253            } else if (b'0'..=b'9').contains(&b)
254                || (b'a'..=b'f').contains(&b)
255                || (b'A'..=b'F').contains(&b)
256            {
257                hex_digits.push(b);
258            } else {
259                return Err(PdfError::Parse {
260                    offset: self.pos - 1,
261                    message: format!("Invalid character in hex string: {:02X}", b),
262                });
263            }
264        }
265
266        if hex_digits.len() % 2 != 0 {
267            hex_digits.push(b'0'); // odd length padded with 0
268        }
269
270        let mut bytes = Vec::with_capacity(hex_digits.len() / 2);
271        for chunk in hex_digits.chunks_exact(2) {
272            let s = std::str::from_utf8(chunk).unwrap();
273            let byte = u8::from_str_radix(s, 16).map_err(|_| PdfError::Parse {
274                offset: start_pos,
275                message: "Failed parsing hex byte".to_string(),
276            })?;
277            bytes.push(byte);
278        }
279
280        Ok(Token::String(bytes, StringFormat::Hex))
281    }
282
283    fn read_number_or_keyword(&mut self, _start_pos: usize) -> PdfResult<Token> {
284        let mut token_bytes = Vec::new();
285
286        while self.pos < self.data.len() {
287            let b = self.data[self.pos];
288            if matches!(
289                b,
290                0 | 9
291                    | 10
292                    | 12
293                    | 13
294                    | 32
295                    | b'('
296                    | b')'
297                    | b'<'
298                    | b'>'
299                    | b'['
300                    | b']'
301                    | b'{'
302                    | b'}'
303                    | b'/'
304                    | b'%'
305            ) {
306                break;
307            }
308            token_bytes.push(b);
309            self.pos += 1;
310        }
311
312        let s = String::from_utf8_lossy(&token_bytes).into_owned();
313
314        // Check if integer
315        if let Ok(i) = s.parse::<i64>() {
316            return Ok(Token::Integer(i));
317        }
318
319        // Check if real float
320        if let Ok(f) = s.parse::<f64>() {
321            if s.contains('.') {
322                return Ok(Token::Real(f));
323            }
324        }
325
326        Ok(Token::Keyword(s))
327    }
328
329    /// Reads raw bytes for stream contents up to length or endstream keyword.
330    pub fn read_stream_data(&mut self, length: Option<usize>) -> PdfResult<Vec<u8>> {
331        // Skip whitespace after 'stream' keyword
332        if self.pos < self.data.len() && self.data[self.pos] == b'\r' {
333            self.pos += 1;
334        }
335        if self.pos < self.data.len() && self.data[self.pos] == b'\n' {
336            self.pos += 1;
337        }
338
339        if let Some(len) = length {
340            if self.pos + len <= self.data.len() {
341                let stream_bytes = self.data[self.pos..self.pos + len].to_vec();
342                self.pos += len;
343                return Ok(stream_bytes);
344            }
345        }
346
347        // Fallback: search for 'endstream'
348        let start = self.pos;
349        let pattern = b"endstream";
350        if let Some(idx) = self.data[start..]
351            .windows(pattern.len())
352            .position(|w| w == pattern)
353        {
354            let mut end = start + idx;
355            // Trim trailing \r and \n before endstream
356            if end > start && self.data[end - 1] == b'\n' {
357                end -= 1;
358            }
359            if end > start && self.data[end - 1] == b'\r' {
360                end -= 1;
361            }
362            let stream_bytes = self.data[start..end].to_vec();
363            self.pos = start + idx;
364            Ok(stream_bytes)
365        } else {
366            Err(PdfError::Parse {
367                offset: start,
368                message: "Missing endstream keyword".to_string(),
369            })
370        }
371    }
372}
rust_pdf_box/cos/lexer.rs

rust_pdf_box/cos/
lexer.rs
