|
|
// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package html
import ( "bytes" "io" "io/ioutil" "reflect" "runtime" "strings" "testing")
type tokenTest struct { // A short description of the test case.
desc string // The HTML to parse.
html string // The string representations of the expected tokens, joined by '$'.
golden string}
var tokenTests = []tokenTest{ { "empty", "", "", }, // A single text node. The tokenizer should not break text nodes on whitespace,
// nor should it normalize whitespace within a text node.
{ "text", "foo bar", "foo bar", }, // An entity.
{ "entity", "one < two", "one < two", }, // A start, self-closing and end tag. The tokenizer does not care if the start
// and end tokens don't match; that is the job of the parser.
{ "tags", "<a>b<c/>d</e>", "<a>$b$<c/>$d$</e>", }, // Angle brackets that aren't a tag.
{ "not a tag #0", "<", "<", }, { "not a tag #1", "</", "</", }, { "not a tag #2", "</>", "<!---->", }, { "not a tag #3", "a</>b", "a$<!---->$b", }, { "not a tag #4", "</ >", "<!-- -->", }, { "not a tag #5", "</.", "<!--.-->", }, { "not a tag #6", "</.>", "<!--.-->", }, { "not a tag #7", "a < b", "a < b", }, { "not a tag #8", "<.>", "<.>", }, { "not a tag #9", "a<<<b>>>c", "a<<$<b>$>>c", }, { "not a tag #10", "if x<0 and y < 0 then x*y>0", "if x<0 and y < 0 then x*y>0", }, { "not a tag #11", "<<p>", "<$<p>", }, // EOF in a tag name.
{ "tag name eof #0", "<a", "", }, { "tag name eof #1", "<a ", "", }, { "tag name eof #2", "a<b", "a", }, { "tag name eof #3", "<a><b", "<a>", }, { "tag name eof #4", `<a x`, ``, }, // Some malformed tags that are missing a '>'.
{ "malformed tag #0", `<p</p>`, `<p< p="">`, }, { "malformed tag #1", `<p </p>`, `<p <="" p="">`, }, { "malformed tag #2", `<p id`, ``, }, { "malformed tag #3", `<p id=`, ``, }, { "malformed tag #4", `<p id=>`, `<p id="">`, }, { "malformed tag #5", `<p id=0`, ``, }, { "malformed tag #6", `<p id=0</p>`, `<p id="0</p">`, }, { "malformed tag #7", `<p id="0</p>`, ``, }, { "malformed tag #8", `<p id="0"</p>`, `<p id="0" <="" p="">`, }, { "malformed tag #9", `<p></p id`, `<p>`, }, // Raw text and RCDATA.
{ "basic raw text", "<script><a></b></script>", "<script>$<a></b>$</script>", }, { "unfinished script end tag", "<SCRIPT>a</SCR", "<script>$a</SCR", }, { "broken script end tag", "<SCRIPT>a</SCR ipt>", "<script>$a</SCR ipt>", }, { "EOF in script end tag", "<SCRIPT>a</SCRipt", "<script>$a</SCRipt", }, { "scriptx end tag", "<SCRIPT>a</SCRiptx", "<script>$a</SCRiptx", }, { "' ' completes script end tag", "<SCRIPT>a</SCRipt ", "<script>$a", }, { "'>' completes script end tag", "<SCRIPT>a</SCRipt>", "<script>$a$</script>", }, { "self-closing script end tag", "<SCRIPT>a</SCRipt/>", "<script>$a$</script>", }, { "nested script tag", "<SCRIPT>a</SCRipt<script>", "<script>$a</SCRipt<script>", }, { "script end tag after unfinished", "<SCRIPT>a</SCRipt</script>", "<script>$a</SCRipt$</script>", }, { "script/style mismatched tags", "<script>a</style>", "<script>$a</style>", }, { "style element with entity", "<style>'", "<style>$&apos;", }, { "textarea with tag", "<textarea><div></textarea>", "<textarea>$<div>$</textarea>", }, { "title with tag and entity", "<title><b>K&R C</b></title>", "<title>$<b>K&R C</b>$</title>", }, // DOCTYPE tests.
{ "Proper DOCTYPE", "<!DOCTYPE html>", "<!DOCTYPE html>", }, { "DOCTYPE with no space", "<!doctypehtml>", "<!DOCTYPE html>", }, { "DOCTYPE with two spaces", "<!doctype html>", "<!DOCTYPE html>", }, { "looks like DOCTYPE but isn't", "<!DOCUMENT html>", "<!--DOCUMENT html-->", }, { "DOCTYPE at EOF", "<!DOCtype", "<!DOCTYPE >", }, // XML processing instructions.
{ "XML processing instruction", "<?xml?>", "<!--?xml?-->", }, // Comments.
{ "comment0", "abc<b><!-- skipme --></b>def", "abc$<b>$<!-- skipme -->$</b>$def", }, { "comment1", "a<!-->z", "a$<!---->$z", }, { "comment2", "a<!--->z", "a$<!---->$z", }, { "comment3", "a<!--x>-->z", "a$<!--x>-->$z", }, { "comment4", "a<!--x->-->z", "a$<!--x->-->$z", }, { "comment5", "a<!>z", "a$<!---->$z", }, { "comment6", "a<!->z", "a$<!----->$z", }, { "comment7", "a<!---<>z", "a$<!---<>z-->", }, { "comment8", "a<!--z", "a$<!--z-->", }, { "comment9", "a<!--z-", "a$<!--z-->", }, { "comment10", "a<!--z--", "a$<!--z-->", }, { "comment11", "a<!--z---", "a$<!--z--->", }, { "comment12", "a<!--z----", "a$<!--z---->", }, { "comment13", "a<!--x--!>z", "a$<!--x-->$z", }, // An attribute with a backslash.
{ "backslash", `<p id="a\"b">`, `<p id="a\" b"="">`, }, // Entities, tag name and attribute key lower-casing, and whitespace
// normalization within a tag.
{ "tricky", "<p \t\n iD=\"a"B\" foo=\"bar\"><EM>te<&;xt</em></p>", `<p id="a"B" foo="bar">$<em>$te<&;xt$</em>$</p>`, }, // A nonexistent entity. Tokenizing and converting back to a string should
// escape the "&" to become "&".
{ "noSuchEntity", `<a b="c&noSuchEntity;d"><&alsoDoesntExist;&`, `<a b="c&noSuchEntity;d">$<&alsoDoesntExist;&`, }, { "entity without semicolon", `¬it;∉<a b="q=z&=5¬ice=hello¬=world">`, `¬it;∉$<a b="q=z&amp=5&notice=hello¬=world">`, }, { "entity with digits", "½", "½", }, // Attribute tests:
// http://dev.w3.org/html5/pf-summary/Overview.html#attributes
{ "Empty attribute", `<input disabled FOO>`, `<input disabled="" foo="">`, }, { "Empty attribute, whitespace", `<input disabled FOO >`, `<input disabled="" foo="">`, }, { "Unquoted attribute value", `<input value=yes FOO=BAR>`, `<input value="yes" foo="BAR">`, }, { "Unquoted attribute value, spaces", `<input value = yes FOO = BAR>`, `<input value="yes" foo="BAR">`, }, { "Unquoted attribute value, trailing space", `<input value=yes FOO=BAR >`, `<input value="yes" foo="BAR">`, }, { "Single-quoted attribute value", `<input value='yes' FOO='BAR'>`, `<input value="yes" foo="BAR">`, }, { "Single-quoted attribute value, trailing space", `<input value='yes' FOO='BAR' >`, `<input value="yes" foo="BAR">`, }, { "Double-quoted attribute value", `<input value="I'm an attribute" FOO="BAR">`, `<input value="I'm an attribute" foo="BAR">`, }, { "Attribute name characters", `<meta http-equiv="content-type">`, `<meta http-equiv="content-type">`, }, { "Mixed attributes", `a<P V="0 1" w='2' X=3 y>z`, `a$<p v="0 1" w="2" x="3" y="">$z`, }, { "Attributes with a solitary single quote", `<p id=can't><p id=won't>`, `<p id="can't">$<p id="won't">`, },}
func TestTokenizer(t *testing.T) {loop: for _, tt := range tokenTests { z := NewTokenizer(strings.NewReader(tt.html)) if tt.golden != "" { for i, s := range strings.Split(tt.golden, "$") { if z.Next() == ErrorToken { t.Errorf("%s token %d: want %q got error %v", tt.desc, i, s, z.Err()) continue loop } actual := z.Token().String() if s != actual { t.Errorf("%s token %d: want %q got %q", tt.desc, i, s, actual) continue loop } } } z.Next() if z.Err() != io.EOF { t.Errorf("%s: want EOF got %q", tt.desc, z.Err()) } }}
func TestMaxBuffer(t *testing.T) { // Exceeding the maximum buffer size generates ErrBufferExceeded.
z := NewTokenizer(strings.NewReader("<" + strings.Repeat("t", 10))) z.SetMaxBuf(5) tt := z.Next() if got, want := tt, ErrorToken; got != want { t.Fatalf("token type: got: %v want: %v", got, want) } if got, want := z.Err(), ErrBufferExceeded; got != want { t.Errorf("error type: got: %v want: %v", got, want) } if got, want := string(z.Raw()), "<tttt"; got != want { t.Fatalf("buffered before overflow: got: %q want: %q", got, want) }}
func TestMaxBufferReconstruction(t *testing.T) { // Exceeding the maximum buffer size at any point while tokenizing permits
// reconstructing the original input.
tests: for _, test := range tokenTests { for maxBuf := 1; ; maxBuf++ { r := strings.NewReader(test.html) z := NewTokenizer(r) z.SetMaxBuf(maxBuf) var tokenized bytes.Buffer for { tt := z.Next() tokenized.Write(z.Raw()) if tt == ErrorToken { if err := z.Err(); err != io.EOF && err != ErrBufferExceeded { t.Errorf("%s: unexpected error: %v", test.desc, err) } break } } // Anything tokenized along with untokenized input or data left in the reader.
assembled, err := ioutil.ReadAll(io.MultiReader(&tokenized, bytes.NewReader(z.Buffered()), r)) if err != nil { t.Errorf("%s: ReadAll: %v", test.desc, err) continue tests } if got, want := string(assembled), test.html; got != want { t.Errorf("%s: reassembled html:\n got: %q\nwant: %q", test.desc, got, want) continue tests } // EOF indicates that we completed tokenization and hence found the max
// maxBuf that generates ErrBufferExceeded, so continue to the next test.
if z.Err() == io.EOF { break } } // buffer sizes
} // tests
}
func TestPassthrough(t *testing.T) { // Accumulating the raw output for each parse event should reconstruct the
// original input.
for _, test := range tokenTests { z := NewTokenizer(strings.NewReader(test.html)) var parsed bytes.Buffer for { tt := z.Next() parsed.Write(z.Raw()) if tt == ErrorToken { break } } if got, want := parsed.String(), test.html; got != want { t.Errorf("%s: parsed output:\n got: %q\nwant: %q", test.desc, got, want) } }}
func TestBufAPI(t *testing.T) { s := "0<a>1</a>2<b>3<a>4<a>5</a>6</b>7</a>8<a/>9" z := NewTokenizer(bytes.NewBufferString(s)) var result bytes.Buffer depth := 0loop: for { tt := z.Next() switch tt { case ErrorToken: if z.Err() != io.EOF { t.Error(z.Err()) } break loop case TextToken: if depth > 0 { result.Write(z.Text()) } case StartTagToken, EndTagToken: tn, _ := z.TagName() if len(tn) == 1 && tn[0] == 'a' { if tt == StartTagToken { depth++ } else { depth-- } } } } u := "14567" v := string(result.Bytes()) if u != v { t.Errorf("TestBufAPI: want %q got %q", u, v) }}
func TestConvertNewlines(t *testing.T) { testCases := map[string]string{ "Mac\rDOS\r\nUnix\n": "Mac\nDOS\nUnix\n", "Unix\nMac\rDOS\r\n": "Unix\nMac\nDOS\n", "DOS\r\nDOS\r\nDOS\r\n": "DOS\nDOS\nDOS\n", "": "", "\n": "\n", "\n\r": "\n\n", "\r": "\n", "\r\n": "\n", "\r\n\n": "\n\n", "\r\n\r": "\n\n", "\r\n\r\n": "\n\n", "\r\r": "\n\n", "\r\r\n": "\n\n", "\r\r\n\n": "\n\n\n", "\r\r\r\n": "\n\n\n", "\r \n": "\n \n", "xyz": "xyz", } for in, want := range testCases { if got := string(convertNewlines([]byte(in))); got != want { t.Errorf("input %q: got %q, want %q", in, got, want) } }}
func TestReaderEdgeCases(t *testing.T) { const s = "<p>An io.Reader can return (0, nil) or (n, io.EOF).</p>" testCases := []io.Reader{ &zeroOneByteReader{s: s}, &eofStringsReader{s: s}, &stuckReader{}, } for i, tc := range testCases { got := []TokenType{} z := NewTokenizer(tc) for { tt := z.Next() if tt == ErrorToken { break } got = append(got, tt) } if err := z.Err(); err != nil && err != io.EOF { if err != io.ErrNoProgress { t.Errorf("i=%d: %v", i, err) } continue } want := []TokenType{ StartTagToken, TextToken, EndTagToken, } if !reflect.DeepEqual(got, want) { t.Errorf("i=%d: got %v, want %v", i, got, want) continue } }}
// zeroOneByteReader is like a strings.Reader that alternates between
// returning 0 bytes and 1 byte at a time.
type zeroOneByteReader struct { s string n int}
func (r *zeroOneByteReader) Read(p []byte) (int, error) { if len(p) == 0 { return 0, nil } if len(r.s) == 0 { return 0, io.EOF } r.n++ if r.n%2 != 0 { return 0, nil } p[0], r.s = r.s[0], r.s[1:] return 1, nil}
// eofStringsReader is like a strings.Reader but can return an (n, err) where
// n > 0 && err != nil.
type eofStringsReader struct { s string}
func (r *eofStringsReader) Read(p []byte) (int, error) { n := copy(p, r.s) r.s = r.s[n:] if r.s != "" { return n, nil } return n, io.EOF}
// stuckReader is an io.Reader that always returns no data and no error.
type stuckReader struct{}
func (*stuckReader) Read(p []byte) (int, error) { return 0, nil}
const ( rawLevel = iota lowLevel highLevel)
func benchmarkTokenizer(b *testing.B, level int) { buf, err := ioutil.ReadFile("testdata/go1.html") if err != nil { b.Fatalf("could not read testdata/go1.html: %v", err) } b.SetBytes(int64(len(buf))) runtime.GC() b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { z := NewTokenizer(bytes.NewBuffer(buf)) for { tt := z.Next() if tt == ErrorToken { if err := z.Err(); err != nil && err != io.EOF { b.Fatalf("tokenizer error: %v", err) } break } switch level { case rawLevel: // Calling z.Raw just returns the raw bytes of the token. It does
// not unescape < to <, or lower-case tag names and attribute keys.
z.Raw() case lowLevel: // Caling z.Text, z.TagName and z.TagAttr returns []byte values
// whose contents may change on the next call to z.Next.
switch tt { case TextToken, CommentToken, DoctypeToken: z.Text() case StartTagToken, SelfClosingTagToken: _, more := z.TagName() for more { _, _, more = z.TagAttr() } case EndTagToken: z.TagName() } case highLevel: // Calling z.Token converts []byte values to strings whose validity
// extend beyond the next call to z.Next.
z.Token() } } }}
func BenchmarkRawLevelTokenizer(b *testing.B) { benchmarkTokenizer(b, rawLevel) }func BenchmarkLowLevelTokenizer(b *testing.B) { benchmarkTokenizer(b, lowLevel) }func BenchmarkHighLevelTokenizer(b *testing.B) { benchmarkTokenizer(b, highLevel) }
|
