1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168
//! UTF-8 and UTF-16 decoding iterators
use crate::fmt;
use super::from_u32_unchecked;
/// An iterator that decodes UTF-16 encoded code points from an iterator of `u16`s.
///
/// This `struct` is created by the [`decode_utf16`] method on [`char`]. See its
/// documentation for more.
///
/// [`decode_utf16`]: char::decode_utf16
#[stable(feature = "decode_utf16", since = "1.9.0")]
#[derive(Clone, Debug)]
pub struct DecodeUtf16<I>
where
I: Iterator<Item = u16>,
{
iter: I,
buf: Option<u16>,
}
/// An error that can be returned when decoding UTF-16 code points.
///
/// This `struct` is created when using the [`DecodeUtf16`] type.
#[stable(feature = "decode_utf16", since = "1.9.0")]
#[derive(Debug, Clone, Eq, PartialEq)]
pub struct DecodeUtf16Error {
code: u16,
}
/// Creates an iterator over the UTF-16 encoded code points in `iter`,
/// returning unpaired surrogates as `Err`s.
///
/// # Examples
///
/// Basic usage:
///
/// ```
/// use std::char::decode_utf16;
///
/// // 𝄞mus<invalid>ic<invalid>
/// let v = [
/// 0xD834, 0xDD1E, 0x006d, 0x0075, 0x0073, 0xDD1E, 0x0069, 0x0063, 0xD834,
/// ];
///
/// assert_eq!(
/// decode_utf16(v.iter().cloned())
/// .map(|r| r.map_err(|e| e.unpaired_surrogate()))
/// .collect::<Vec<_>>(),
/// vec![
/// Ok('𝄞'),
/// Ok('m'), Ok('u'), Ok('s'),
/// Err(0xDD1E),
/// Ok('i'), Ok('c'),
/// Err(0xD834)
/// ]
/// );
/// ```
///
/// A lossy decoder can be obtained by replacing `Err` results with the replacement character:
///
/// ```
/// use std::char::{decode_utf16, REPLACEMENT_CHARACTER};
///
/// // 𝄞mus<invalid>ic<invalid>
/// let v = [
/// 0xD834, 0xDD1E, 0x006d, 0x0075, 0x0073, 0xDD1E, 0x0069, 0x0063, 0xD834,
/// ];
///
/// assert_eq!(
/// decode_utf16(v.iter().cloned())
/// .map(|r| r.unwrap_or(REPLACEMENT_CHARACTER))
/// .collect::<String>(),
/// "𝄞mus�ic�"
/// );
/// ```
#[stable(feature = "decode_utf16", since = "1.9.0")]
#[inline]
pub fn decode_utf16<I: IntoIterator<Item = u16>>(iter: I) -> DecodeUtf16<I::IntoIter> {
DecodeUtf16 { iter: iter.into_iter(), buf: None }
}
#[stable(feature = "decode_utf16", since = "1.9.0")]
impl<I: Iterator<Item = u16>> Iterator for DecodeUtf16<I> {
type Item = Result<char, DecodeUtf16Error>;
fn next(&mut self) -> Option<Result<char, DecodeUtf16Error>> {
let u = match self.buf.take() {
Some(buf) => buf,
None => self.iter.next()?,
};
if u < 0xD800 || 0xDFFF < u {
// SAFETY: not a surrogate
Some(Ok(unsafe { from_u32_unchecked(u as u32) }))
} else if u >= 0xDC00 {
// a trailing surrogate
Some(Err(DecodeUtf16Error { code: u }))
} else {
let u2 = match self.iter.next() {
Some(u2) => u2,
// eof
None => return Some(Err(DecodeUtf16Error { code: u })),
};
if u2 < 0xDC00 || u2 > 0xDFFF {
// not a trailing surrogate so we're not a valid
// surrogate pair, so rewind to redecode u2 next time.
self.buf = Some(u2);
return Some(Err(DecodeUtf16Error { code: u }));
}
// all ok, so lets decode it.
let c = (((u - 0xD800) as u32) << 10 | (u2 - 0xDC00) as u32) + 0x1_0000;
// SAFETY: we checked that it's a legal unicode value
Some(Ok(unsafe { from_u32_unchecked(c) }))
}
}
#[inline]
fn size_hint(&self) -> (usize, Option<usize>) {
let (low, high) = self.iter.size_hint();
let (low_buf, high_buf) = match self.buf {
// buf is empty, no additional elements from it.
None => (0, 0),
// `u` is a non surrogate, so it's always an additional character.
Some(u) if u < 0xD800 || 0xDFFF < u => (1, 1),
// `u` is a leading surrogate (it can never be a trailing surrogate and
// it's a surrogate due to the previous branch) and `self.iter` is empty.
//
// `u` can't be paired, since the `self.iter` is empty,
// so it will always become an additional element (error).
Some(_u) if high == Some(0) => (1, 1),
// `u` is a leading surrogate and `iter` may be non-empty.
//
// `u` can either pair with a trailing surrogate, in which case no additional elements
// are produced, or it can become an error, in which case it's an additional character (error).
Some(_u) => (0, 1),
};
// `self.iter` could contain entirely valid surrogates (2 elements per
// char), or entirely non-surrogates (1 element per char).
//
// On odd lower bound, at least one element must stay unpaired
// (with other elements from `self.iter`), so we round up.
let low = low.div_ceil(2) + low_buf;
let high = high.and_then(|h| h.checked_add(high_buf));
(low, high)
}
}
impl DecodeUtf16Error {
/// Returns the unpaired surrogate which caused this error.
#[must_use]
#[stable(feature = "decode_utf16", since = "1.9.0")]
pub fn unpaired_surrogate(&self) -> u16 {
self.code
}
}
#[stable(feature = "decode_utf16", since = "1.9.0")]
impl fmt::Display for DecodeUtf16Error {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "unpaired surrogate found: {:x}", self.code)
}
}