1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
|
// Copyright 2015 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.
//! Unmerged utf8 chars iterator vendored from std::io
//!
use std::io::{BufRead, ErrorKind, Error};
use std::fmt;
use std::error as std_error;
use std::result;
use std::char;
static UTF8_CHAR_WIDTH: [u8; 256] = [
1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, // 0x1F
1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, // 0x3F
1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, // 0x5F
1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, // 0x7F
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // 0x9F
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // 0xBF
0,0,2,2,2,2,2,2,2,2,2,2,2,2,2,2,
2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, // 0xDF
3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3, // 0xEF
4,4,4,4,4,0,0,0,0,0,0,0,0,0,0,0, // 0xFF
];
/// Given a first byte, determine how many bytes are in this UTF-8 character
#[inline]
pub fn utf8_char_width(b: u8) -> usize {
return UTF8_CHAR_WIDTH[b as usize] as usize;
}
/// An iterator over the `char`s of a reader.
///
/// This struct is generally created by calling [`utf8_chars()`][utf8_chars] on a reader.
/// Please see the documentation of `utf8_chars()` for more details.
///
/// [utf8_chars]: trait.BufRead.html#method.utf8_chars
pub struct Utf8Chars<R> {
inner: R,
}
impl<R> Utf8Chars<R> {
pub fn new(inner: R) -> Utf8Chars<R> {
Utf8Chars { inner: inner }
}
}
/// An enumeration of possible errors that can be generated from the `Utf8Chars`
/// adapter.
#[derive(Debug)]
pub enum Utf8CharsError {
/// Variant representing that the underlying stream was read successfully
/// but contains a byte sequence ill-formed in UTF-8.
InvalidUtf8,
/// Variant representing that the underlying stream contains the start
/// of a byte sequence well-formed in UTF-8, but ends prematurely.
///
/// Contains number of unused bytes
IncompleteUtf8(usize),
/// Variant representing that an I/O error occurred.
Io(Error),
}
impl<R: BufRead> Iterator for Utf8Chars<R> {
type Item = result::Result<char, Utf8CharsError>;
// allow(unused_assignments) because consumed += 1 is not recognized as being used
#[allow(unused_assignments)]
fn next(&mut self) -> Option<result::Result<char, Utf8CharsError>> {
macro_rules! read_byte {
(EOF => $on_eof: expr) => {
{
let byte;
loop {
match self.inner.fill_buf() {
Ok(buffer) => {
if let Some(&b) = buffer.first() {
byte = b;
break
} else {
$on_eof
}
}
Err(ref e) if e.kind() == ErrorKind::Interrupted => {}
Err(e) => return Some(Err(Utf8CharsError::Io(e))),
}
}
byte
}
}
}
let first = read_byte!(EOF => return None);
self.inner.consume(1);
let mut consumed = 1;
macro_rules! continuation_byte {
($range: pat) => {
{
match read_byte!(EOF => return Some(Err(Utf8CharsError::IncompleteUtf8(consumed)))) {
byte @ $range => {
self.inner.consume(1);
consumed += 1;
(byte & 0b0011_1111) as u32
}
_ => return Some(Err(Utf8CharsError::InvalidUtf8))
}
}
}
}
// Ranges can be checked against https://tools.ietf.org/html/rfc3629#section-4
let code_point = match utf8_char_width(first) {
1 => return Some(Ok(first as char)),
2 => {
let second = continuation_byte!(0x80...0xBF);
((first & 0b0001_1111) as u32) << 6 | second
}
3 => {
let second = match first {
0xE0 => continuation_byte!(0xA0...0xBF),
0xE1...0xEC => continuation_byte!(0x80...0xBF),
0xED => continuation_byte!(0x80...0x9F),
0xEE...0xEF => continuation_byte!(0x80...0xBF),
_ => unreachable!(),
};
let third = continuation_byte!(0x80...0xBF);
((first & 0b0000_1111) as u32) << 12 | second << 6 | third
}
4 => {
let second = match first {
0xF0 => continuation_byte!(0x90...0xBF),
0xF0...0xF3 => continuation_byte!(0x80...0xBF),
0xF4 => continuation_byte!(0x80...0x8F),
_ => unreachable!(),
};
let third = continuation_byte!(0x80...0xBF);
let fourth = continuation_byte!(0x80...0xBF);
((first & 0b0000_0111) as u32) << 18 | second << 12 | third << 6 | fourth
}
_ => return Some(Err(Utf8CharsError::InvalidUtf8))
};
unsafe {
Some(Ok(char::from_u32_unchecked(code_point)))
}
}
}
impl std_error::Error for Utf8CharsError {
fn description(&self) -> &str {
match *self {
Utf8CharsError::InvalidUtf8 => "invalid UTF-8 byte sequence",
Utf8CharsError::IncompleteUtf8(_) => {
"stream ended in the middle of an UTF-8 byte sequence"
}
Utf8CharsError::Io(ref e) => std_error::Error::description(e),
}
}
fn cause(&self) -> Option<&std_error::Error> {
match *self {
Utf8CharsError::InvalidUtf8 | Utf8CharsError::IncompleteUtf8(_) => None,
Utf8CharsError::Io(ref e) => e.cause(),
}
}
}
impl fmt::Display for Utf8CharsError {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match *self {
Utf8CharsError::InvalidUtf8 => {
"invalid UTF-8 byte sequence".fmt(f)
}
Utf8CharsError::IncompleteUtf8(_) => {
"stream ended in the middle of an UTF-8 byte sequence".fmt(f)
}
Utf8CharsError::Io(ref e) => e.fmt(f),
}
}
}
|
