nautilus_core/string/
urlencoding.rs

1// -------------------------------------------------------------------------------------------------
2//  Copyright (C) 2015-2026 Nautech Systems Pty Ltd. All rights reserved.
3//  https://nautechsystems.io
4//
5//  Licensed under the GNU Lesser General Public License Version 3.0 (the "License");
6//  You may not use this file except in compliance with the License.
7//  You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html
8//
9//  Unless required by applicable law or agreed to in writing, software
10//  distributed under the License is distributed on an "AS IS" BASIS,
11//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12//  See the License for the specific language governing permissions and
13//  limitations under the License.
14// -------------------------------------------------------------------------------------------------
15
16//! URL percent-encoding and decoding per [RFC 3986].
17//!
18//! The unreserved set is `ALPHA / DIGIT / "-" / "." / "_" / "~"`; every other
19//! byte is percent-encoded as `%HH` using uppercase hexadecimal as recommended
20//! by [RFC 3986 Section 2.1].
21//!
22//! Decoding accepts both uppercase and lowercase hex. A `%` that is not
23//! followed by two hex digits is passed through literally, matching the
24//! behavior of the `urlencoding` crate that this module replaces.
25//!
26//! [RFC 3986]: https://datatracker.ietf.org/doc/html/rfc3986
27//! [RFC 3986 Section 2.1]: https://datatracker.ietf.org/doc/html/rfc3986#section-2.1
28
29use std::{borrow::Cow, string::FromUtf8Error};
30
31use thiserror::Error;
32
33const UNRESERVED: [bool; 256] = {
34    let mut table = [false; 256];
35    let mut i = b'0';
36    while i <= b'9' {
37        table[i as usize] = true;
38        i += 1;
39    }
40    i = b'A';
41    while i <= b'Z' {
42        table[i as usize] = true;
43        i += 1;
44    }
45    i = b'a';
46    while i <= b'z' {
47        table[i as usize] = true;
48        i += 1;
49    }
50    table[b'-' as usize] = true;
51    table[b'.' as usize] = true;
52    table[b'_' as usize] = true;
53    table[b'~' as usize] = true;
54    table
55};
56
57const ENCODE_PAIR: [[u8; 2]; 256] = {
58    const NIBBLE: [u8; 16] = *b"0123456789ABCDEF";
59    let mut table = [[0u8; 2]; 256];
60    let mut i = 0u16;
61    while i < 256 {
62        table[i as usize] = [NIBBLE[(i >> 4) as usize], NIBBLE[(i & 0x0f) as usize]];
63        i += 1;
64    }
65    table
66};
67
68// 0xFF sentinel marks non-hex characters
69const DECODE_NIBBLE: [u8; 256] = {
70    let mut table = [0xFFu8; 256];
71    let mut i = 0u8;
72    while i < 10 {
73        table[(b'0' + i) as usize] = i;
74        i += 1;
75    }
76    i = 0;
77    while i < 6 {
78        table[(b'a' + i) as usize] = 10 + i;
79        table[(b'A' + i) as usize] = 10 + i;
80        i += 1;
81    }
82    table
83};
84
85/// Percent-encodes a string per RFC 3986.
86///
87/// Returns the input borrowed when every byte is already in the unreserved
88/// set, otherwise an owned encoded copy.
89///
90/// # Panics
91///
92/// Never panics in practice: [`encode_bytes`] only emits ASCII bytes
93/// (unreserved characters or `%HH` pairs), so [`String::from_utf8`] always
94/// succeeds.
95#[must_use]
96pub fn encode(input: &str) -> Cow<'_, str> {
97    match encode_bytes(input.as_bytes()) {
98        Cow::Borrowed(_) => Cow::Borrowed(input),
99        Cow::Owned(bytes) => Cow::Owned(String::from_utf8(bytes).expect("encoded output is ASCII")),
100    }
101}
102
103/// Percent-encodes a byte slice per RFC 3986.
104///
105/// Returns the input borrowed when every byte is already in the unreserved
106/// set, otherwise an owned encoded copy.
107#[must_use]
108pub fn encode_bytes(input: &[u8]) -> Cow<'_, [u8]> {
109    let Some(first) = input.iter().position(|&b| !UNRESERVED[b as usize]) else {
110        return Cow::Borrowed(input);
111    };
112
113    // Slack for payloads dominated by reserved chars without over-allocating
114    // on mostly-unreserved inputs; Vec's geometric growth covers the rest.
115    let mut out = Vec::with_capacity(input.len() + input.len() / 2 + 16);
116    out.extend_from_slice(&input[..first]);
117
118    let mut rest = &input[first..];
119    while let Some(&byte) = rest.first() {
120        if UNRESERVED[byte as usize] {
121            let run_end = rest
122                .iter()
123                .position(|&b| !UNRESERVED[b as usize])
124                .unwrap_or(rest.len());
125            out.extend_from_slice(&rest[..run_end]);
126            rest = &rest[run_end..];
127        } else {
128            out.push(b'%');
129            out.extend_from_slice(&ENCODE_PAIR[byte as usize]);
130            rest = &rest[1..];
131        }
132    }
133    Cow::Owned(out)
134}
135
136/// Percent-decodes a string per RFC 3986.
137///
138/// Returns the input borrowed when no `%` is present. Otherwise decodes
139/// `%HH` pairs (hex is case-insensitive) and leaves any `%` that is not
140/// followed by two hex digits in place.
141///
142/// # Errors
143///
144/// Returns [`DecodeError::InvalidUtf8`] if the decoded bytes are not valid
145/// UTF-8.
146pub fn decode(input: &str) -> Result<Cow<'_, str>, DecodeError> {
147    match decode_bytes(input.as_bytes()) {
148        Cow::Borrowed(_) => Ok(Cow::Borrowed(input)),
149        Cow::Owned(bytes) => String::from_utf8(bytes)
150            .map(Cow::Owned)
151            .map_err(DecodeError::InvalidUtf8),
152    }
153}
154
155/// Percent-decodes a byte slice.
156///
157/// Returns the input borrowed when no `%` is present. A `%` that is not
158/// followed by two hex digits is left in place.
159#[must_use]
160pub fn decode_bytes(input: &[u8]) -> Cow<'_, [u8]> {
161    let Some(first) = input.iter().position(|&b| b == b'%') else {
162        return Cow::Borrowed(input);
163    };
164
165    let mut out = Vec::with_capacity(input.len());
166    out.extend_from_slice(&input[..first]);
167
168    let mut i = first;
169    while i < input.len() {
170        if input[i] == b'%' {
171            if i + 2 < input.len() {
172                let hi = DECODE_NIBBLE[input[i + 1] as usize];
173                let lo = DECODE_NIBBLE[input[i + 2] as usize];
174                if (hi | lo) & 0xF0 == 0 {
175                    out.push((hi << 4) | lo);
176                    i += 3;
177                    continue;
178                }
179            }
180            // Malformed or trailing `%`: pass through literally.
181            out.push(b'%');
182            i += 1;
183        } else {
184            let run_start = i;
185            while i < input.len() && input[i] != b'%' {
186                i += 1;
187            }
188            out.extend_from_slice(&input[run_start..i]);
189        }
190    }
191    Cow::Owned(out)
192}
193
194/// Errors from URL percent-decoding.
195#[derive(Debug, Error)]
196pub enum DecodeError {
197    /// Decoded bytes are not valid UTF-8.
198    #[error("invalid UTF-8 in decoded bytes: {0}")]
199    InvalidUtf8(#[from] FromUtf8Error),
200}
201
202#[cfg(test)]
203mod tests {
204    use proptest::prelude::*;
205    use rstest::rstest;
206
207    use super::*;
208
209    // RFC 3986 Section 2.3: unreserved = ALPHA / DIGIT / "-" / "." / "_" / "~"
210    const UNRESERVED_CHARS: &str =
211        "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-._~";
212
213    // RFC 3986 Section 2.2: reserved chars (gen-delims + sub-delims) must be
214    // percent-encoded when used as data.
215    const RESERVED_CHARS: &str = ":/?#[]@!$&'()*+,;=";
216
217    #[rstest]
218    #[case("", "")]
219    #[case("abc", "abc")]
220    #[case("ABC-xyz_0.9~", "ABC-xyz_0.9~")]
221    #[case(" ", "%20")]
222    #[case("+", "%2B")]
223    #[case("/", "%2F")]
224    #[case("?", "%3F")]
225    #[case("#", "%23")]
226    #[case("&", "%26")]
227    #[case("=", "%3D")]
228    #[case("%", "%25")]
229    #[case("hello world", "hello%20world")]
230    #[case("a b+c/d", "a%20b%2Bc%2Fd")]
231    // Uppercase hex per RFC 3986 Section 2.1
232    #[case("\x7f", "%7F")]
233    fn test_encode_ascii_vectors(#[case] input: &str, #[case] expected: &str) {
234        assert_eq!(encode(input), expected);
235    }
236
237    #[rstest]
238    fn test_encode_all_unreserved_unchanged() {
239        // Every char in the unreserved set should pass through.
240        let out = encode(UNRESERVED_CHARS);
241        assert_eq!(out, UNRESERVED_CHARS);
242        // And the Cow should be Borrowed (zero-copy).
243        assert!(matches!(out, Cow::Borrowed(_)));
244    }
245
246    #[rstest]
247    fn test_encode_all_reserved_percent_encoded() {
248        let out = encode(RESERVED_CHARS);
249        // Each of the 18 chars becomes a 3-byte `%HH` sequence.
250        assert_eq!(out.len(), RESERVED_CHARS.len() * 3);
251        // None of the unreserved chars, `%`, or digits A-F should appear raw
252        // in the output except as part of a `%HH` triple.
253        for byte in out.bytes() {
254            assert!(
255                matches!(byte, b'%' | b'0'..=b'9' | b'A'..=b'F'),
256                "unexpected byte {byte:#04x} in encoded reserved output"
257            );
258        }
259    }
260
261    #[rstest]
262    fn test_encode_hex_is_uppercase() {
263        // Verify RFC 3986 Section 2.1: producers SHOULD emit uppercase hex.
264        let out = encode("/");
265        assert_eq!(out, "%2F");
266        assert!(!out.contains('f'));
267    }
268
269    #[rstest]
270    fn test_encode_every_byte_position() {
271        // For each byte 0x00..=0xFF, encode a one-byte slice and verify that
272        // the output matches the spec expectation.
273        for byte in 0u8..=255 {
274            let input = [byte];
275            let out = encode_bytes(&input);
276
277            if UNRESERVED[byte as usize] {
278                assert!(
279                    matches!(out, Cow::Borrowed(_)),
280                    "unreserved byte {byte:#04x} should not allocate"
281                );
282                assert_eq!(out.as_ref(), &[byte]);
283            } else {
284                let expected = format!("%{byte:02X}").into_bytes();
285                assert_eq!(out.as_ref(), expected.as_slice(), "byte {byte:#04x}");
286            }
287        }
288    }
289
290    #[rstest]
291    fn test_encode_utf8_multibyte() {
292        // U+00E9 encoded as UTF-8 is `0xC3 0xA9` (two bytes).
293        assert_eq!(encode("\u{00E9}"), "%C3%A9");
294        // U+4E2D encoded as UTF-8 is `0xE4 0xB8 0xAD` (three bytes).
295        assert_eq!(encode("\u{4E2D}"), "%E4%B8%AD");
296        // Grinning face emoji U+1F600 is `0xF0 0x9F 0x98 0x80` (four bytes).
297        assert_eq!(encode("\u{1F600}"), "%F0%9F%98%80");
298    }
299
300    #[rstest]
301    fn test_encode_mixed_ascii_and_utf8() {
302        assert_eq!(encode("a é/"), "a%20%C3%A9%2F");
303    }
304
305    #[rstest]
306    fn test_encode_returns_borrowed_when_no_work() {
307        let out = encode("safe-string_123.xyz~");
308        assert!(matches!(out, Cow::Borrowed(_)));
309    }
310
311    #[rstest]
312    fn test_encode_returns_owned_when_encoding_needed() {
313        let out = encode("needs encoding");
314        assert!(matches!(out, Cow::Owned(_)));
315    }
316
317    #[rstest]
318    #[case("", "")]
319    #[case("abc", "abc")]
320    #[case("%20", " ")]
321    #[case("%2F", "/")]
322    #[case("%2f", "/")] // lowercase hex must be accepted
323    #[case("%2b", "+")]
324    #[case("%25", "%")]
325    #[case("hello%20world", "hello world")]
326    #[case("a%20b%2Bc%2Fd", "a b+c/d")]
327    #[case("%C3%A9", "\u{00E9}")]
328    #[case("%E4%B8%AD", "\u{4E2D}")]
329    #[case("%F0%9F%98%80", "\u{1F600}")]
330    fn test_decode_ascii_and_utf8_vectors(#[case] input: &str, #[case] expected: &str) {
331        assert_eq!(decode(input).unwrap(), expected);
332    }
333
334    #[rstest]
335    #[case("%", "%")] // bare `%` at end passes through
336    #[case("%2", "%2")] // one hex digit at end
337    #[case("%GG", "%GG")] // non-hex digits
338    #[case("%2G", "%2G")] // second nibble invalid
339    #[case("%G2", "%G2")] // first nibble invalid
340    #[case("%%20", "% ")] // first `%` literal, then `%20` decodes
341    #[case("100%", "100%")] // `%` at end after ASCII
342    fn test_decode_malformed_percent_passes_through(#[case] input: &str, #[case] expected: &str) {
343        assert_eq!(decode(input).unwrap(), expected);
344    }
345
346    #[rstest]
347    fn test_decode_returns_borrowed_when_no_percent() {
348        let out = decode("no-percent-here").unwrap();
349        assert!(matches!(out, Cow::Borrowed(_)));
350    }
351
352    #[rstest]
353    fn test_decode_returns_owned_when_percent_present() {
354        let out = decode("a%20b").unwrap();
355        assert!(matches!(out, Cow::Owned(_)));
356    }
357
358    #[rstest]
359    #[case("this%2x%26that", "this%2x&that")]
360    #[case("%%25", "%%")]
361    #[case("%2%26", "%2&")]
362    #[case("a%2Zb%20c", "a%2Zb c")]
363    #[case("%:0", "%:0")]
364    fn test_decode_malformed_then_valid(#[case] input: &str, #[case] expected: &str) {
365        assert_eq!(decode(input).unwrap(), expected);
366    }
367
368    #[rstest]
369    fn test_decode_invalid_utf8_errors() {
370        // `0xFF` is not valid UTF-8 on its own.
371        let err = decode("%FF").unwrap_err();
372        assert!(matches!(err, DecodeError::InvalidUtf8(_)));
373    }
374
375    #[rstest]
376    fn test_decode_invalid_utf8_bytes_ok() {
377        // `decode_bytes` does not validate UTF-8.
378        let out = decode_bytes(b"%FF");
379        assert_eq!(out.as_ref(), &[0xFF]);
380    }
381
382    #[rstest]
383    fn test_decode_consecutive_percent_triples() {
384        // Three consecutive `%HH` sequences decoding multi-byte UTF-8.
385        assert_eq!(decode("%e2%98%83").unwrap(), "\u{2603}"); // snowman U+2603
386    }
387
388    #[rstest]
389    fn test_decode_nul_byte() {
390        // `%00` decodes to the NUL byte, which is valid UTF-8 (U+0000).
391        let decoded = decode("a%00b").unwrap();
392        assert_eq!(decoded.as_bytes(), &[b'a', 0x00, b'b']);
393    }
394
395    #[rstest]
396    fn test_roundtrip_every_byte() {
397        // For every byte 0x00..=0xFF, encoding then decoding must recover
398        // the original byte exactly.
399        for byte in 0u8..=255 {
400            let input = [byte];
401            let encoded = encode_bytes(&input);
402            let decoded = decode_bytes(encoded.as_ref());
403            assert_eq!(
404                decoded.as_ref(),
405                input.as_slice(),
406                "round-trip failed for byte {byte:#04x}"
407            );
408        }
409    }
410
411    #[rstest]
412    #[case("hello")]
413    #[case("a b c")]
414    #[case("https://example.com/path?q=1&x=2")]
415    #[case("\u{00E9}\u{00E0}\u{00FC}")]
416    #[case("\u{4E2D}\u{6587}\u{6D4B}\u{8BD5}")]
417    #[case("mix 123 !@# %^&*()")]
418    #[case("\u{1F600}\u{1F680}\u{1F3C6}")]
419    fn test_roundtrip_string(#[case] input: &str) {
420        let encoded = encode(input);
421        let decoded = decode(&encoded).unwrap();
422        assert_eq!(decoded, input);
423    }
424
425    #[rstest]
426    fn test_encoded_output_only_ascii() {
427        // Encoded output must always be pure ASCII (unreserved bytes + `%HH`).
428        let encoded = encode("\u{00E9}\u{4E2D}\u{1F600}");
429        assert!(encoded.is_ascii(), "encoded output must be ASCII-only");
430    }
431
432    #[rstest]
433    fn test_encode_bytes_arbitrary_binary() {
434        // Encoding arbitrary bytes (including non-UTF-8) yields a valid
435        // percent-encoded ASCII sequence.
436        let input: Vec<u8> = (0u8..=255).collect();
437        let encoded = encode_bytes(&input);
438        assert!(encoded.iter().all(u8::is_ascii));
439        let decoded = decode_bytes(encoded.as_ref());
440        assert_eq!(decoded.as_ref(), input.as_slice());
441    }
442
443    #[rstest]
444    fn test_decode_error_display_and_source() {
445        let err = decode("%FF").unwrap_err();
446        let msg = err.to_string();
447        assert!(msg.starts_with("invalid UTF-8"), "got: {msg}");
448        assert!(std::error::Error::source(&err).is_some());
449    }
450
451    // Independent reference implementation used to cross-check our tuned
452    // implementation on random inputs. Pure-Rust, loop-based, no table
453    // lookups: if both agree across thousands of random inputs we have
454    // strong evidence the tuned version is spec-correct.
455    fn reference_encode(input: &[u8]) -> Vec<u8> {
456        let mut out = Vec::with_capacity(input.len());
457        for &b in input {
458            let is_unreserved =
459                b.is_ascii_alphanumeric() || b == b'-' || b == b'.' || b == b'_' || b == b'~';
460            if is_unreserved {
461                out.push(b);
462            } else {
463                out.push(b'%');
464                out.extend_from_slice(format!("{b:02X}").as_bytes());
465            }
466        }
467        out
468    }
469
470    fn reference_decode(input: &[u8]) -> Vec<u8> {
471        let mut out = Vec::with_capacity(input.len());
472        let mut i = 0;
473        while i < input.len() {
474            if input[i] == b'%' && i + 2 < input.len() {
475                let a = input[i + 1];
476                let b = input[i + 2];
477                if a.is_ascii_hexdigit() && b.is_ascii_hexdigit() {
478                    let hi = if a.is_ascii_digit() {
479                        a - b'0'
480                    } else {
481                        (a | 0x20) - b'a' + 10
482                    };
483                    let lo = if b.is_ascii_digit() {
484                        b - b'0'
485                    } else {
486                        (b | 0x20) - b'a' + 10
487                    };
488                    out.push((hi << 4) | lo);
489                    i += 3;
490                    continue;
491                }
492            }
493            out.push(input[i]);
494            i += 1;
495        }
496        out
497    }
498
499    fn malformed_percent_sequence() -> impl Strategy<Value = Vec<u8>> {
500        prop_oneof![
501            Just(vec![b'%']),
502            (any::<u8>(), any::<u8>())
503                .prop_filter("contains a non-hex byte", |(hi, lo)| {
504                    !hi.is_ascii_hexdigit() || !lo.is_ascii_hexdigit()
505                })
506                .prop_map(|(hi, lo)| vec![b'%', hi, lo]),
507        ]
508    }
509
510    proptest::proptest! {
511        #[rstest]
512        fn prop_encode_matches_reference(input: Vec<u8>) {
513            let actual = encode_bytes(&input);
514            let expected = reference_encode(&input);
515            proptest::prop_assert_eq!(actual.as_ref(), expected.as_slice());
516        }
517
518        #[rstest]
519        fn prop_decode_matches_reference(input: Vec<u8>) {
520            let actual = decode_bytes(&input);
521            let expected = reference_decode(&input);
522            proptest::prop_assert_eq!(actual.as_ref(), expected.as_slice());
523        }
524
525        #[rstest]
526        fn prop_malformed_percent_sequences_match_reference(
527            prefix in proptest::collection::vec(any::<u8>(), 0..16),
528            malformed in malformed_percent_sequence(),
529            suffix in proptest::collection::vec(any::<u8>(), 0..16),
530        ) {
531            let mut input = prefix;
532            input.extend(malformed);
533            input.extend(suffix);
534
535            let actual = decode_bytes(&input);
536            let expected = reference_decode(&input);
537            proptest::prop_assert_eq!(actual.as_ref(), expected.as_slice());
538        }
539
540        #[rstest]
541        fn prop_bytes_roundtrip(input: Vec<u8>) {
542            let encoded = encode_bytes(&input);
543            let decoded = decode_bytes(encoded.as_ref());
544            proptest::prop_assert_eq!(decoded.as_ref(), input.as_slice());
545        }
546
547        #[rstest]
548        fn prop_string_roundtrip(input: String) {
549            let encoded = encode(&input);
550            let decoded = decode(&encoded).unwrap();
551            proptest::prop_assert_eq!(decoded.as_ref(), input.as_str());
552        }
553
554        #[rstest]
555        fn prop_encoded_output_ascii(input: Vec<u8>) {
556            let encoded = encode_bytes(&input);
557            proptest::prop_assert!(encoded.iter().all(u8::is_ascii));
558        }
559    }
560}
nautilus_core/string/urlencoding.rs

nautilus_core/string/
urlencoding.rs
