diff --git a/rcdom/tests/xml-tokenizer.rs b/rcdom/tests/xml-tokenizer.rs index 80faaa7c..1e314a49 100644 --- a/rcdom/tests/xml-tokenizer.rs +++ b/rcdom/tests/xml-tokenizer.rs @@ -21,11 +21,10 @@ use util::runner::{run_all, Test}; use markup5ever::buffer_queue::BufferQueue; use xml5ever::tendril::{SliceExt, StrTendril}; -use xml5ever::tokenizer::{CharacterTokens, Token, TokenSink}; -use xml5ever::tokenizer::{CommentToken, EmptyTag, EndTag, ShortTag, StartTag, Tag}; -use xml5ever::tokenizer::{Doctype, DoctypeToken, PIToken, Pi}; -use xml5ever::tokenizer::{EOFToken, XmlTokenizer, XmlTokenizerOpts}; -use xml5ever::tokenizer::{NullCharacterToken, ParseError, TagToken}; +use xml5ever::tokenizer::{ + Doctype, EmptyTag, EndTag, Pi, ShortTag, StartTag, Tag, Token, TokenSink, XmlTokenizer, + XmlTokenizerOpts, +}; use xml5ever::{ns, Attribute, LocalName, QualName}; mod util { @@ -81,7 +80,7 @@ impl TokenLogger { fn finish_str(&self) { if !self.current_str.borrow().is_empty() { let s = self.current_str.take(); - self.tokens.borrow_mut().push(CharacterTokens(s)); + self.tokens.borrow_mut().push(Token::Characters(s)); } } @@ -96,21 +95,20 @@ impl TokenSink for TokenLogger { fn process_token(&self, token: Token) -> ProcessResult<()> { match token { - CharacterTokens(b) => { - self.current_str.borrow_mut().push_slice(&b); + Token::Characters(characters) => { + self.current_str.borrow_mut().push_slice(&characters); }, - NullCharacterToken => { + Token::NullCharacter => { self.current_str.borrow_mut().push_char('\0'); }, - ParseError(_) => { + Token::ParseError(_) => { if self.exact_errors { - self.push(ParseError(Borrowed(""))); + self.push(Token::ParseError(Borrowed(""))); } }, - - TagToken(mut t) => { + Token::Tag(mut t) => { // The spec seems to indicate that one can emit // erroneous end tags with attrs, but the test // cases don't contain them. @@ -120,11 +118,9 @@ impl TokenSink for TokenLogger { }, _ => t.attrs.sort_by(|a1, a2| a1.name.cmp(&a2.name)), } - self.push(TagToken(t)); + self.push(Token::Tag(t)); }, - - EOFToken => (), - + Token::EndOfFile => (), _ => self.push(token), }; ProcessResult::Continue @@ -211,7 +207,7 @@ fn json_to_token(js: &Value) -> Token { // Collect refs here so we don't have to use "ref" in all the patterns below. let args: Vec<&Value> = parts[1..].iter().collect(); match &*parts[0].get_str() { - "StartTag" => TagToken(Tag { + "StartTag" => Token::Tag(Tag { kind: StartTag, name: QualName::new(None, ns!(), LocalName::from(args[0].get_str())), attrs: args[1] @@ -224,19 +220,19 @@ fn json_to_token(js: &Value) -> Token { .collect(), }), - "EndTag" => TagToken(Tag { + "EndTag" => Token::Tag(Tag { kind: EndTag, name: QualName::new(None, ns!(), LocalName::from(args[0].get_str())), attrs: vec![], }), - "ShortTag" => TagToken(Tag { + "ShortTag" => Token::Tag(Tag { kind: ShortTag, name: QualName::new(None, ns!(), LocalName::from(args[0].get_str())), attrs: vec![], }), - "EmptyTag" => TagToken(Tag { + "EmptyTag" => Token::Tag(Tag { kind: EmptyTag, name: QualName::new(None, ns!(), LocalName::from(args[0].get_str())), attrs: args[1] @@ -249,16 +245,16 @@ fn json_to_token(js: &Value) -> Token { .collect(), }), - "Comment" => CommentToken(args[0].get_tendril()), + "Comment" => Token::Comment(args[0].get_tendril()), - "Character" => CharacterTokens(args[0].get_tendril()), + "Character" => Token::Characters(args[0].get_tendril()), - "PI" => PIToken(Pi { + "PI" => Token::ProcessingInstruction(Pi { target: args[0].get_tendril(), data: args[1].get_tendril(), }), - "DOCTYPE" => DoctypeToken(Doctype { + "DOCTYPE" => Token::Doctype(Doctype { name: args[0].get_nullable_tendril(), public_id: args[1].get_nullable_tendril(), system_id: args[2].get_nullable_tendril(), @@ -278,7 +274,7 @@ fn json_to_tokens(js: &Value, exact_errors: bool) -> Vec { for tok in js.as_array().unwrap().iter() { match *tok { Value::String(ref s) if &s[..] == "ParseError" => { - let _ = sink.process_token(ParseError(Borrowed(""))); + let _ = sink.process_token(Token::ParseError(Borrowed(""))); }, _ => { let _ = sink.process_token(json_to_token(tok)); diff --git a/xml5ever/examples/simple_xml_tokenizer.rs b/xml5ever/examples/simple_xml_tokenizer.rs index 662432bf..6dc06717 100644 --- a/xml5ever/examples/simple_xml_tokenizer.rs +++ b/xml5ever/examples/simple_xml_tokenizer.rs @@ -16,10 +16,7 @@ use std::io; use markup5ever::buffer_queue::BufferQueue; use xml5ever::tendril::{ByteTendril, ReadExt}; -use xml5ever::tokenizer::{CharacterTokens, NullCharacterToken, ProcessResult, TagToken}; -use xml5ever::tokenizer::{CommentToken, PIToken, Pi}; -use xml5ever::tokenizer::{Doctype, DoctypeToken, EOFToken}; -use xml5ever::tokenizer::{ParseError, Token, TokenSink, XmlTokenizer}; +use xml5ever::tokenizer::{Doctype, Pi, ProcessResult, Token, TokenSink, XmlTokenizer}; struct SimpleTokenPrinter; @@ -28,29 +25,29 @@ impl TokenSink for SimpleTokenPrinter { fn process_token(&self, token: Token) -> ProcessResult<()> { match token { - CharacterTokens(b) => { + Token::Characters(b) => { println!("TEXT: {}", &*b); }, - NullCharacterToken => print!("NULL"), - TagToken(tag) => { + Token::NullCharacter => print!("NULL"), + Token::Tag(tag) => { println!("{:?} {} ", tag.kind, &*tag.name.local); }, - ParseError(err) => { + Token::ParseError(err) => { println!("ERROR: {err}"); }, - PIToken(Pi { + Token::ProcessingInstruction(Pi { ref target, ref data, }) => { println!("PI : "); }, - CommentToken(ref comment) => { + Token::Comment(ref comment) => { println!(""); }, - EOFToken => { + Token::EndOfFile => { println!("EOF"); }, - DoctypeToken(Doctype { + Token::Doctype(Doctype { ref name, ref public_id, .. diff --git a/xml5ever/examples/xml_tokenizer.rs b/xml5ever/examples/xml_tokenizer.rs index 2ae52da0..3acaff74 100644 --- a/xml5ever/examples/xml_tokenizer.rs +++ b/xml5ever/examples/xml_tokenizer.rs @@ -17,10 +17,10 @@ use std::io; use markup5ever::buffer_queue::BufferQueue; use xml5ever::tendril::{ByteTendril, ReadExt}; -use xml5ever::tokenizer::{CharacterTokens, NullCharacterToken, ProcessResult, TagToken}; -use xml5ever::tokenizer::{EmptyTag, EndTag, ShortTag, StartTag}; -use xml5ever::tokenizer::{PIToken, Pi}; -use xml5ever::tokenizer::{ParseError, Token, TokenSink, XmlTokenizer, XmlTokenizerOpts}; +use xml5ever::tokenizer::{ + EmptyTag, EndTag, Pi, ProcessResult, ShortTag, StartTag, Token, TokenSink, XmlTokenizer, + XmlTokenizerOpts, +}; #[derive(Clone)] struct TokenPrinter { @@ -48,13 +48,13 @@ impl TokenSink for TokenPrinter { fn process_token(&self, token: Token) -> ProcessResult<()> { match token { - CharacterTokens(b) => { + Token::Characters(b) => { for c in b.chars() { self.do_char(c); } }, - NullCharacterToken => self.do_char('\0'), - TagToken(tag) => { + Token::NullCharacter => self.do_char('\0'), + Token::Tag(tag) => { self.is_char(false); // This is not proper HTML serialization, of course. match tag.kind { @@ -74,11 +74,11 @@ impl TokenSink for TokenPrinter { } println!(">"); }, - ParseError(err) => { + Token::ParseError(err) => { self.is_char(false); println!("ERROR: {err}"); }, - PIToken(Pi { target, data }) => { + Token::ProcessingInstruction(Pi { target, data }) => { self.is_char(false); println!("PI : "); }, diff --git a/xml5ever/src/tokenizer/interface.rs b/xml5ever/src/tokenizer/interface.rs index e4dda107..85606808 100644 --- a/xml5ever/src/tokenizer/interface.rs +++ b/xml5ever/src/tokenizer/interface.rs @@ -14,8 +14,6 @@ use crate::tokenizer::ProcessResult; use crate::{Attribute, QualName}; pub use self::TagKind::{EmptyTag, EndTag, ShortTag, StartTag}; -pub use self::Token::{CharacterTokens, EOFToken, NullCharacterToken, ParseError}; -pub use self::Token::{CommentToken, DoctypeToken, PIToken, TagToken}; /// Tag kind denotes which kind of tag did we encounter. #[derive(PartialEq, Eq, Hash, Copy, Clone, Debug)] @@ -87,20 +85,20 @@ pub struct Pi { #[derive(PartialEq, Eq, Debug)] pub enum Token { /// Doctype token - DoctypeToken(Doctype), + Doctype(Doctype), /// Token tag founds. This token applies to all /// possible kinds of tags (like start, end, empty tag, etc.). - TagToken(Tag), + Tag(Tag), /// Processing Instruction token - PIToken(Pi), + ProcessingInstruction(Pi), /// Comment token. - CommentToken(StrTendril), + Comment(StrTendril), /// Token that represents a series of characters. - CharacterTokens(StrTendril), + Characters(StrTendril), /// End of File found. - EOFToken, + EndOfFile, /// NullCharacter encountered. - NullCharacterToken, + NullCharacter, /// Error happened ParseError(Cow<'static, str>), } diff --git a/xml5ever/src/tokenizer/mod.rs b/xml5ever/src/tokenizer/mod.rs index 6e94bc62..22cacba5 100644 --- a/xml5ever/src/tokenizer/mod.rs +++ b/xml5ever/src/tokenizer/mod.rs @@ -12,11 +12,9 @@ mod interface; mod qname; pub mod states; -pub use self::interface::{CharacterTokens, EOFToken, NullCharacterToken}; -pub use self::interface::{CommentToken, DoctypeToken, PIToken, TagToken}; -pub use self::interface::{Doctype, Pi}; -pub use self::interface::{EmptyTag, EndTag, ShortTag, StartTag}; -pub use self::interface::{ParseError, Tag, TagKind, Token, TokenSink}; +pub use self::interface::{ + Doctype, EmptyTag, EndTag, Pi, ShortTag, StartTag, Tag, TagKind, Token, TokenSink, +}; pub use crate::{LocalName, Namespace, Prefix}; use crate::tendril::StrTendril; @@ -397,7 +395,7 @@ impl XmlTokenizer { } fn emit_char(&self, c: char) { - self.process_token(CharacterTokens(StrTendril::from_char(match c { + self.process_token(Token::Characters(StrTendril::from_char(match c { '\0' => '\u{FFFD}', c => c, }))); @@ -445,7 +443,7 @@ impl XmlTokenizer { }, } - let token = TagToken(Tag { + let token = Token::Tag(Tag { kind: self.current_tag_kind.get(), name: qname, attrs: self.current_tag_attrs.take(), @@ -456,12 +454,12 @@ impl XmlTokenizer { // The string must not contain '\0'! fn emit_chars(&self, b: StrTendril) { - self.process_token(CharacterTokens(b)); + self.process_token(Token::Characters(b)); } // Emits the current Processing Instruction fn emit_pi(&self) -> ProcessResult<::Handle> { - let token = PIToken(Pi { + let token = Token::ProcessingInstruction(Pi { target: replace(&mut *self.current_pi_target.borrow_mut(), StrTendril::new()), data: replace(&mut *self.current_pi_data.borrow_mut(), StrTendril::new()), }); @@ -476,21 +474,21 @@ impl XmlTokenizer { } fn emit_eof(&self) { - self.process_token(EOFToken); + self.process_token(Token::EndOfFile); } fn emit_error(&self, error: Cow<'static, str>) { - self.process_token(ParseError(error)); + self.process_token(Token::ParseError(error)); } fn emit_current_comment(&self) { let comment = self.current_comment.take(); - self.process_token(CommentToken(comment)); + self.process_token(Token::Comment(comment)); } fn emit_current_doctype(&self) { let doctype = self.current_doctype.take(); - self.process_token(DoctypeToken(doctype)); + self.process_token(Token::Doctype(doctype)); } fn doctype_id(&self, kind: DoctypeKind) -> RefMut<'_, Option> { diff --git a/xml5ever/src/tree_builder/mod.rs b/xml5ever/src/tree_builder/mod.rs index 469511dc..059956d2 100644 --- a/xml5ever/src/tree_builder/mod.rs +++ b/xml5ever/src/tree_builder/mod.rs @@ -410,18 +410,18 @@ where fn process_token(&self, token: tokenizer::Token) -> ProcessResult { // Handle `ParseError` and `DoctypeToken`; convert everything else to the local `Token` type. let token = match token { - tokenizer::ParseError(e) => { + tokenizer::Token::ParseError(e) => { self.sink.parse_error(e); return ProcessResult::Done; }, - tokenizer::DoctypeToken(d) => Token::Doctype(d), - tokenizer::PIToken(instruction) => Token::Pi(instruction), - tokenizer::TagToken(x) => Token::Tag(x), - tokenizer::CommentToken(x) => Token::Comment(x), - tokenizer::NullCharacterToken => Token::NullCharacter, - tokenizer::EOFToken => Token::Eof, - tokenizer::CharacterTokens(x) => Token::Characters(x), + tokenizer::Token::Doctype(d) => Token::Doctype(d), + tokenizer::Token::ProcessingInstruction(instruction) => Token::Pi(instruction), + tokenizer::Token::Tag(x) => Token::Tag(x), + tokenizer::Token::Comment(x) => Token::Comment(x), + tokenizer::Token::NullCharacter => Token::NullCharacter, + tokenizer::Token::EndOfFile => Token::Eof, + tokenizer::Token::Characters(x) => Token::Characters(x), }; self.process_to_completion(token) diff --git a/xml5ever/src/tree_builder/types.rs b/xml5ever/src/tree_builder/types.rs index 1081d3d1..cf221455 100644 --- a/xml5ever/src/tree_builder/types.rs +++ b/xml5ever/src/tree_builder/types.rs @@ -17,7 +17,7 @@ pub enum XmlPhase { End, } -/// A subset/refinement of `tokenizer::XToken`. Everything else is handled +/// A subset/refinement of `tokenizer::Token`. Everything else is handled /// specially at the beginning of `process_token`. #[derive(PartialEq, Eq, Clone, Debug)] pub enum Token {