Parse surrogates in string literals properly

Author: coolreader18

Cargo.lock

@@ -536,8 +536,6 @@ def test_badandgoodxmlcharrefreplaceexceptions(self):
             ("".join("&#%d;" % c for c in cs), 1 + len(s))
         )
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     def test_badandgoodbackslashreplaceexceptions(self):
         # "backslashreplace" complains about a non-exception passed in
         self.assertRaises(
@@ -596,8 +594,6 @@ def test_badandgoodbackslashreplaceexceptions(self):
                     (r, 2)
                 )
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     def test_badandgoodnamereplaceexceptions(self):
         # "namereplace" complains about a non-exception passed in
         self.assertRaises(
@@ -644,8 +640,6 @@ def test_badandgoodnamereplaceexceptions(self):
                     (r, 1 + len(s))
                 )
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     def test_badandgoodsurrogateescapeexceptions(self):
         surrogateescape_errors = codecs.lookup_error('surrogateescape')
         # "surrogateescape" complains about a non-exception passed in

Lib/test/test_codeccallbacks.py

@@ -401,7 +401,7 @@ pub mod errors {
             let mut out = String::with_capacity(num_chars * 4);
             for c in err_str.code_points() {
                 let c_u32 = c.to_u32();
-                if let Some(c_name) = unicode_names2::name(c.to_char_lossy()) {
+                if let Some(c_name) = c.to_char().and_then(unicode_names2::name) {
                     write!(out, "\\N{{{c_name}}}").unwrap();
                 } else if c_u32 >= 0x10000 {
                     write!(out, "\\U{c_u32:08x}").unwrap();

common/src/encodings.rs

@@ -574,6 +574,12 @@ impl<W: AsRef<Wtf8>> FromIterator<W> for Wtf8Buf {
     }
 }
 
+impl Hash for Wtf8Buf {
+    fn hash<H: Hasher>(&self, state: &mut H) {
+        Wtf8::hash(self, state)
+    }
+}
+
 impl AsRef<Wtf8> for Wtf8Buf {
     fn as_ref(&self) -> &Wtf8 {
         self
@@ -692,6 +698,13 @@ impl Default for &Wtf8 {
     }
 }
 
+impl Hash for Wtf8 {
+    fn hash<H: Hasher>(&self, state: &mut H) {
+        state.write(self.as_bytes());
+        state.write_u8(0xff);
+    }
+}
+
 impl Wtf8 {
     /// Creates a WTF-8 slice from a UTF-8 `&str` slice.
     ///
@@ -722,6 +735,32 @@ impl Wtf8 {
         unsafe { &mut *(value as *mut [u8] as *mut Wtf8) }
     }
 
+    /// Create a WTF-8 slice from a WTF-8 byte slice.
+    //
+    // whooops! using WTF-8 for interchange!
+    #[inline]
+    pub fn from_bytes(b: &[u8]) -> Option<&Self> {
+        let mut rest = b;
+        while let Err(e) = std::str::from_utf8(rest) {
+            rest = &rest[e.valid_up_to()..];
+            Self::decode_surrogate(rest)?;
+            rest = &rest[3..];
+        }
+        Some(unsafe { Wtf8::from_bytes_unchecked(b) })
+    }
+
+    fn decode_surrogate(b: &[u8]) -> Option<CodePoint> {
+        let [a, b, c, ..] = *b else { return None };
+        if (a & 0xf0) == 0xe0 && (b & 0xc0) == 0x80 && (c & 0xc0) == 0x80 {
+            // it's a three-byte code
+            let c = ((a as u32 & 0x0f) << 12) + ((b as u32 & 0x3f) << 6) + (c as u32 & 0x3f);
+            let 0xD800..=0xDFFF = c else { return None };
+            Some(CodePoint { value: c })
+        } else {
+            None
+        }
+    }
+
     /// Returns the length, in WTF-8 bytes.
     #[inline]
     pub fn len(&self) -> usize {
@@ -875,6 +914,14 @@ impl Wtf8 {
         }
     }
 
+    #[inline]
+    fn final_lead_surrogate(&self) -> Option<u16> {
+        match self.bytes {
+            [.., 0xED, b2 @ 0xA0..=0xAF, b3] => Some(decode_surrogate(b2, b3)),
+            _ => None,
+        }
+    }
+
     pub fn is_code_point_boundary(&self, index: usize) -> bool {
         is_code_point_boundary(self, index)
     }
@@ -1481,6 +1528,12 @@ impl From<Wtf8Buf> for Box<Wtf8> {
     }
 }
 
+impl From<Box<Wtf8>> for Wtf8Buf {
+    fn from(w: Box<Wtf8>) -> Self {
+        Wtf8Buf::from_box(w)
+    }
+}
+
 impl From<String> for Box<Wtf8> {
     fn from(s: String) -> Self {
         s.into_boxed_str().into()

common/src/wtf8/mod.rs

@@ -30,6 +30,8 @@ num-complex = { workspace = true }
 num-traits = { workspace = true }
 thiserror = { workspace = true }
 malachite-bigint = { workspace = true }
+memchr = { workspace = true }
+unicode_names2 = { workspace = true }
 
 [dev-dependencies]
 # rustpython-parser = { workspace = true }