windpress / vendor / masterminds / html5 / src / HTML5 / Parser / Tokenizer.php

Tokenizer.php in WindPress – Tailwind CSS integration for WordPress 3.0.13, at vendor/masterminds/html5/src/HTML5/Parser/Tokenizer.php

1,031 lines 35.3 KB

Raw Download Zip

1	<?php
2
3	namespace WindPressDeps\Masterminds\HTML5\Parser;
4
5	use WindPressDeps\Masterminds\HTML5\Elements;
6	/**
7	* The HTML5 tokenizer.
8	*
9	* The tokenizer's role is reading data from the scanner and gathering it into
10	* semantic units. From the tokenizer, data is emitted to an event handler,
11	* which may (for example) create a DOM tree.
12	*
13	* The HTML5 specification has a detailed explanation of tokenizing HTML5. We
14	* follow that specification to the maximum extent that we can. If you find
15	* a discrepancy that is not documented, please file a bug and/or submit a
16	* patch.
17	*
18	* This tokenizer is implemented as a recursive descent parser.
19	*
20	* Within the API documentation, you may see references to the specific section
21	* of the HTML5 spec that the code attempts to reproduce. Example: 8.2.4.1.
22	* This refers to section 8.2.4.1 of the HTML5 CR specification.
23	*
24	* @see http://www.w3.org/TR/2012/CR-html5-20121217/
25	*/
26	class Tokenizer
27	{
28	protected $scanner;
29	protected $events;
30	protected $tok;
31	/**
32	* Buffer for text.
33	*/
34	protected $text = '';
35	// When this goes to false, the parser stops.
36	protected $carryOn = \true;
37	protected $textMode = 0;
38	// TEXTMODE_NORMAL;
39	protected $untilTag = null;
40	const CONFORMANT_XML = 'xml';
41	const CONFORMANT_HTML = 'html';
42	protected $mode = self::CONFORMANT_HTML;
43	/**
44	* Create a new tokenizer.
45	*
46	* Typically, parsing a document involves creating a new tokenizer, giving
47	* it a scanner (input) and an event handler (output), and then calling
48	* the Tokenizer::parse() method.`
49	*
50	* @param Scanner $scanner A scanner initialized with an input stream.
51	* @param EventHandler $eventHandler An event handler, initialized and ready to receive events.
52	* @param string $mode
53	*/
54	public function __construct($scanner, $eventHandler, $mode = self::CONFORMANT_HTML)
55	{
56	$this->scanner = $scanner;
57	$this->events = $eventHandler;
58	$this->mode = $mode;
59	}
60	/**
61	* Begin parsing.
62	*
63	* This will begin scanning the document, tokenizing as it goes.
64	* Tokens are emitted into the event handler.
65	*
66	* Tokenizing will continue until the document is completely
67	* read. Errors are emitted into the event handler, but
68	* the parser will attempt to continue parsing until the
69	* entire input stream is read.
70	*/
71	public function parse()
72	{
73	do {
74	$this->consumeData();
75	// FIXME: Add infinite loop protection.
76	} while ($this->carryOn);
77	}
78	/**
79	* Set the text mode for the character data reader.
80	*
81	* HTML5 defines three different modes for reading text:
82	* - Normal: Read until a tag is encountered.
83	* - RCDATA: Read until a tag is encountered, but skip a few otherwise-
84	* special characters.
85	* - Raw: Read until a special closing tag is encountered (viz. pre, script)
86	*
87	* This allows those modes to be set.
88	*
89	* Normally, setting is done by the event handler via a special return code on
90	* startTag(), but it can also be set manually using this function.
91	*
92	* @param int $textmode One of Elements::TEXT_*.
93	* @param string $untilTag The tag that should stop RAW or RCDATA mode. Normal mode does not
94	* use this indicator.
95	*/
96	public function setTextMode($textmode, $untilTag = null)
97	{
98	$this->textMode = $textmode & (Elements::TEXT_RAW \| Elements::TEXT_RCDATA);
99	$this->untilTag = $untilTag;
100	}
101	/**
102	* Consume a character and make a move.
103	* HTML5 8.2.4.1.
104	*/
105	protected function consumeData()
106	{
107	$tok = $this->scanner->current();
108	if ('&' === $tok) {
109	// Character reference
110	$ref = $this->decodeCharacterReference();
111	$this->buffer($ref);
112	$tok = $this->scanner->current();
113	}
114	// Parse tag
115	if ('<' === $tok) {
116	// Any buffered text data can go out now.
117	$this->flushBuffer();
118	$tok = $this->scanner->next();
119	if (\false === $tok) {
120	// end of string
121	$this->parseError('Illegal tag opening');
122	} elseif ('!' === $tok) {
123	$this->markupDeclaration();
124	} elseif ('/' === $tok) {
125	$this->endTag();
126	} elseif ('?' === $tok) {
127	$this->processingInstruction();
128	} elseif ($this->is_alpha($tok)) {
129	$this->tagName();
130	} else {
131	$this->parseError('Illegal tag opening');
132	// TODO is this necessary ?
133	$this->characterData();
134	}
135	$tok = $this->scanner->current();
136	}
137	if (\false === $tok) {
138	// Handle end of document
139	$this->eof();
140	} else {
141	// Parse character
142	switch ($this->textMode) {
143	case Elements::TEXT_RAW:
144	$this->rawText($tok);
145	break;
146	case Elements::TEXT_RCDATA:
147	$this->rcdata($tok);
148	break;
149	default:
150	if ('<' === $tok \|\| '&' === $tok) {
151	break;
152	}
153	// NULL character
154	if ("\x00" === $tok) {
155	$this->parseError('Received null character.');
156	$this->text .= $tok;
157	$this->scanner->consume();
158	break;
159	}
160	$this->text .= $this->scanner->charsUntil("<&\x00");
161	}
162	}
163	return $this->carryOn;
164	}
165	/**
166	* Parse anything that looks like character data.
167	*
168	* Different rules apply based on the current text mode.
169	*
170	* @see Elements::TEXT_RAW Elements::TEXT_RCDATA.
171	*/
172	protected function characterData()
173	{
174	$tok = $this->scanner->current();
175	if (\false === $tok) {
176	return \false;
177	}
178	switch ($this->textMode) {
179	case Elements::TEXT_RAW:
180	return $this->rawText($tok);
181	case Elements::TEXT_RCDATA:
182	return $this->rcdata($tok);
183	default:
184	if ('<' === $tok \|\| '&' === $tok) {
185	return \false;
186	}
187	return $this->text($tok);
188	}
189	}
190	/**
191	* This buffers the current token as character data.
192	*
193	* @param string $tok The current token.
194	*
195	* @return bool
196	*/
197	protected function text($tok)
198	{
199	// This should never happen...
200	if (\false === $tok) {
201	return \false;
202	}
203	// NULL character
204	if ("\x00" === $tok) {
205	$this->parseError('Received null character.');
206	}
207	$this->buffer($tok);
208	$this->scanner->consume();
209	return \true;
210	}
211	/**
212	* Read text in RAW mode.
213	*
214	* @param string $tok The current token.
215	*
216	* @return bool
217	*/
218	protected function rawText($tok)
219	{
220	if (\is_null($this->untilTag)) {
221	return $this->text($tok);
222	}
223	$sequence = '</' . $this->untilTag . '>';
224	$txt = $this->readUntilSequence($sequence);
225	$this->events->text($txt);
226	$this->setTextMode(0);
227	return $this->endTag();
228	}
229	/**
230	* Read text in RCDATA mode.
231	*
232	* @param string $tok The current token.
233	*
234	* @return bool
235	*/
236	protected function rcdata($tok)
237	{
238	if (\is_null($this->untilTag)) {
239	return $this->text($tok);
240	}
241	$sequence = '</' . $this->untilTag;
242	$txt = '';
243	$caseSensitive = !Elements::isHtml5Element($this->untilTag);
244	while (\false !== $tok && !('<' == $tok && $this->scanner->sequenceMatches($sequence, $caseSensitive))) {
245	if ('&' == $tok) {
246	$txt .= $this->decodeCharacterReference();
247	$tok = $this->scanner->current();
248	} else {
249	$txt .= $tok;
250	$tok = $this->scanner->next();
251	}
252	}
253	$len = \strlen($sequence);
254	$this->scanner->consume($len);
255	$len += $this->scanner->whitespace();
256	if ('>' !== $this->scanner->current()) {
257	$this->parseError('Unclosed RCDATA end tag');
258	}
259	$this->scanner->unconsume($len);
260	$this->events->text($txt);
261	$this->setTextMode(0);
262	return $this->endTag();
263	}
264	/**
265	* If the document is read, emit an EOF event.
266	*/
267	protected function eof()
268	{
269	// fprintf(STDOUT, "EOF");
270	$this->flushBuffer();
271	$this->events->eof();
272	$this->carryOn = \false;
273	}
274	/**
275	* Look for markup.
276	*/
277	protected function markupDeclaration()
278	{
279	$tok = $this->scanner->next();
280	// Comment:
281	if ('-' == $tok && '-' == $this->scanner->peek()) {
282	$this->scanner->consume(2);
283	return $this->comment();
284	} elseif ('D' == $tok \|\| 'd' == $tok) {
285	// Doctype
286	return $this->doctype();
287	} elseif ('[' == $tok) {
288	// CDATA section
289	return $this->cdataSection();
290	}
291	// FINISH
292	$this->parseError('Expected <!--, <![CDATA[, or <!DOCTYPE. Got <!%s', $tok);
293	$this->bogusComment('<!');
294	return \true;
295	}
296	/**
297	* Consume an end tag. See section 8.2.4.9.
298	*/
299	protected function endTag()
300	{
301	if ('/' != $this->scanner->current()) {
302	return \false;
303	}
304	$tok = $this->scanner->next();
305	// a-zA-Z -> tagname
306	// > -> parse error
307	// EOF -> parse error
308	// -> parse error
309	if (!$this->is_alpha($tok)) {
310	$this->parseError("Expected tag name, got '%s'", $tok);
311	if ("\x00" == $tok \|\| \false === $tok) {
312	return \false;
313	}
314	return $this->bogusComment('</');
315	}
316	$name = $this->scanner->charsUntil("\n\f \t>");
317	$name = self::CONFORMANT_XML === $this->mode ? $name : \strtolower($name);
318	// Trash whitespace.
319	$this->scanner->whitespace();
320	$tok = $this->scanner->current();
321	if ('>' != $tok) {
322	$this->parseError("Expected >, got '%s'", $tok);
323	// We just trash stuff until we get to the next tag close.
324	$this->scanner->charsUntil('>');
325	}
326	$this->events->endTag($name);
327	$this->scanner->consume();
328	return \true;
329	}
330	/**
331	* Consume a tag name and body. See section 8.2.4.10.
332	*/
333	protected function tagName()
334	{
335	// We know this is at least one char.
336	$name = $this->scanner->charsWhile(':_-0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz');
337	$name = self::CONFORMANT_XML === $this->mode ? $name : \strtolower($name);
338	$attributes = array();
339	$selfClose = \false;
340	// Handle attribute parse exceptions here so that we can
341	// react by trying to build a sensible parse tree.
342	try {
343	do {
344	$this->scanner->whitespace();
345	$this->attribute($attributes);
346	} while (!$this->isTagEnd($selfClose));
347	} catch (ParseError $e) {
348	$selfClose = \false;
349	}
350	$mode = $this->events->startTag($name, $attributes, $selfClose);
351	if (\is_int($mode)) {
352	$this->setTextMode($mode, $name);
353	}
354	$this->scanner->consume();
355	return \true;
356	}
357	/**
358	* Check if the scanner has reached the end of a tag.
359	*/
360	protected function isTagEnd(&$selfClose)
361	{
362	$tok = $this->scanner->current();
363	if ('/' == $tok) {
364	$this->scanner->consume();
365	$this->scanner->whitespace();
366	$tok = $this->scanner->current();
367	if ('>' == $tok) {
368	$selfClose = \true;
369	return \true;
370	}
371	if (\false === $tok) {
372	$this->parseError('Unexpected EOF inside of tag.');
373	return \true;
374	}
375	// Basically, we skip the / token and go on.
376	// See 8.2.4.43.
377	$this->parseError("Unexpected '%s' inside of a tag.", $tok);
378	return \false;
379	}
380	if ('>' == $tok) {
381	return \true;
382	}
383	if (\false === $tok) {
384	$this->parseError('Unexpected EOF inside of tag.');
385	return \true;
386	}
387	return \false;
388	}
389	/**
390	* Parse attributes from inside of a tag.
391	*
392	* @param string[] $attributes
393	*
394	* @return bool
395	*
396	* @throws ParseError
397	*/
398	protected function attribute(&$attributes)
399	{
400	$tok = $this->scanner->current();
401	if ('/' == $tok \|\| '>' == $tok \|\| \false === $tok) {
402	return \false;
403	}
404	if ('<' == $tok) {
405	$this->parseError("Unexpected '<' inside of attributes list.");
406	// Push the < back onto the stack.
407	$this->scanner->unconsume();
408	// Let the caller figure out how to handle this.
409	throw new ParseError('Start tag inside of attribute.');
410	}
411	$name = \strtolower($this->scanner->charsUntil("/>=\n\f\t "));
412	if (0 == \strlen($name)) {
413	$tok = $this->scanner->current();
414	$this->parseError('Expected an attribute name, got %s.', $tok);
415	// Really, only '=' can be the char here. Everything else gets absorbed
416	// under one rule or another.
417	$name = $tok;
418	$this->scanner->consume();
419	}
420	$isValidAttribute = \true;
421	// Attribute names can contain most Unicode characters for HTML5.
422	// But method "DOMElement::setAttribute" is throwing exception
423	// because of it's own internal restriction so these have to be filtered.
424	// see issue #23: https://github.com/Masterminds/html5-php/issues/23
425	// and http://www.w3.org/TR/2011/WD-html5-20110525/syntax.html#syntax-attribute-name
426	if (\preg_match("/[\x01-,\\/;-@[-^`{-]/u", $name)) {
427	$this->parseError('Unexpected characters in attribute name: %s', $name);
428	$isValidAttribute = \false;
429	} elseif (\preg_match('/^[0-9.-]/u', $name)) {
430	$this->parseError('Unexpected character at the begining of attribute name: %s', $name);
431	$isValidAttribute = \false;
432	}
433	// 8.1.2.3
434	$this->scanner->whitespace();
435	$val = $this->attributeValue();
436	if ($isValidAttribute) {
437	$attributes[$name] = $val;
438	}
439	return \true;
440	}
441	/**
442	* Consume an attribute value. See section 8.2.4.37 and after.
443	*
444	* @return string\|null
445	*/
446	protected function attributeValue()
447	{
448	if ('=' != $this->scanner->current()) {
449	return null;
450	}
451	$this->scanner->consume();
452	// 8.1.2.3
453	$this->scanner->whitespace();
454	$tok = $this->scanner->current();
455	switch ($tok) {
456	case "\n":
457	case "\f":
458	case ' ':
459	case "\t":
460	// Whitespace here indicates an empty value.
461	return null;
462	case '"':
463	case "'":
464	$this->scanner->consume();
465	return $this->quotedAttributeValue($tok);
466	case '>':
467	// case '/': // 8.2.4.37 seems to allow foo=/ as a valid attr.
468	$this->parseError('Expected attribute value, got tag end.');
469	return null;
470	case '=':
471	case '`':
472	$this->parseError('Expecting quotes, got %s.', $tok);
473	return $this->unquotedAttributeValue();
474	default:
475	return $this->unquotedAttributeValue();
476	}
477	}
478	/**
479	* Get an attribute value string.
480	*
481	* @param string $quote IMPORTANT: This is a series of chars! Any one of which will be considered
482	* termination of an attribute's value. E.g. "\"'" will stop at either
483	* ' or ".
484	*
485	* @return string The attribute value.
486	*/
487	protected function quotedAttributeValue($quote)
488	{
489	$stoplist = "\f" . $quote;
490	$val = '';
491	while (\true) {
492	$tokens = $this->scanner->charsUntil($stoplist . '&');
493	if (\false !== $tokens) {
494	$val .= $tokens;
495	} else {
496	break;
497	}
498	$tok = $this->scanner->current();
499	if ('&' == $tok) {
500	$val .= $this->decodeCharacterReference(\true);
501	continue;
502	}
503	break;
504	}
505	$this->scanner->consume();
506	return $val;
507	}
508	protected function unquotedAttributeValue()
509	{
510	$val = '';
511	$tok = $this->scanner->current();
512	while (\false !== $tok) {
513	switch ($tok) {
514	case "\n":
515	case "\f":
516	case ' ':
517	case "\t":
518	case '>':
519	break 2;
520	case '&':
521	$val .= $this->decodeCharacterReference(\true);
522	$tok = $this->scanner->current();
523	break;
524	case "'":
525	case '"':
526	case '<':
527	case '=':
528	case '`':
529	$this->parseError('Unexpected chars in unquoted attribute value %s', $tok);
530	$val .= $tok;
531	$tok = $this->scanner->next();
532	break;
533	default:
534	$val .= $this->scanner->charsUntil("\t\n\f >&\"'<=`");
535	$tok = $this->scanner->current();
536	}
537	}
538	return $val;
539	}
540	/**
541	* Consume malformed markup as if it were a comment.
542	* 8.2.4.44.
543	*
544	* The spec requires that the ENTIRE tag-like thing be enclosed inside of
545	* the comment. So this will generate comments like:
546	*
547	* <!--&lt/+foo>-->
548	*
549	* @param string $leading Prepend any leading characters. This essentially
550	* negates the need to backtrack, but it's sort of a hack.
551	*
552	* @return bool
553	*/
554	protected function bogusComment($leading = '')
555	{
556	$comment = $leading;
557	$tokens = $this->scanner->charsUntil('>');
558	if (\false !== $tokens) {
559	$comment .= $tokens;
560	}
561	$tok = $this->scanner->current();
562	if (\false !== $tok) {
563	$comment .= $tok;
564	}
565	$this->flushBuffer();
566	$this->events->comment($comment);
567	$this->scanner->consume();
568	return \true;
569	}
570	/**
571	* Read a comment.
572	* Expects the first tok to be inside of the comment.
573	*
574	* @return bool
575	*/
576	protected function comment()
577	{
578	$tok = $this->scanner->current();
579	$comment = '';
580	// <!-->. Emit an empty comment because 8.2.4.46 says to.
581	if ('>' == $tok) {
582	// Parse error. Emit the comment token.
583	$this->parseError("Expected comment data, got '>'");
584	$this->events->comment('');
585	$this->scanner->consume();
586	return \true;
587	}
588	// Replace NULL with the replacement char.
589	if ("\x00" == $tok) {
590	$tok = UTF8Utils::FFFD;
591	}
592	while (!$this->isCommentEnd()) {
593	$comment .= $tok;
594	$tok = $this->scanner->next();
595	}
596	$this->events->comment($comment);
597	$this->scanner->consume();
598	return \true;
599	}
600	/**
601	* Check if the scanner has reached the end of a comment.
602	*
603	* @return bool
604	*/
605	protected function isCommentEnd()
606	{
607	$tok = $this->scanner->current();
608	// EOF
609	if (\false === $tok) {
610	// Hit the end.
611	$this->parseError('Unexpected EOF in a comment.');
612	return \true;
613	}
614	// If next two tokens are not '--', not the end.
615	if ('-' != $tok \|\| '-' != $this->scanner->peek()) {
616	return \false;
617	}
618	$this->scanner->consume(2);
619	// Consume '-' and one of '!' or '>'
620	// Test for '>'
621	if ('>' == $this->scanner->current()) {
622	return \true;
623	}
624	// Test for '!>'
625	if ('!' == $this->scanner->current() && '>' == $this->scanner->peek()) {
626	$this->scanner->consume();
627	// Consume the last '>'
628	return \true;
629	}
630	// Unread '-' and one of '!' or '>';
631	$this->scanner->unconsume(2);
632	return \false;
633	}
634	/**
635	* Parse a DOCTYPE.
636	*
637	* Parse a DOCTYPE declaration. This method has strong bearing on whether or
638	* not Quirksmode is enabled on the event handler.
639	*
640	* @todo This method is a little long. Should probably refactor.
641	*
642	* @return bool
643	*/
644	protected function doctype()
645	{
646	// Check that string is DOCTYPE.
647	if ($this->scanner->sequenceMatches('DOCTYPE', \false)) {
648	$this->scanner->consume(7);
649	} else {
650	$chars = $this->scanner->charsWhile('DOCTYPEdoctype');
651	$this->parseError('Expected DOCTYPE, got %s', $chars);
652	return $this->bogusComment('<!' . $chars);
653	}
654	$this->scanner->whitespace();
655	$tok = $this->scanner->current();
656	// EOF: die.
657	if (\false === $tok) {
658	$this->events->doctype('html5', EventHandler::DOCTYPE_NONE, '', \true);
659	$this->eof();
660	return \true;
661	}
662	// NULL char: convert.
663	if ("\x00" === $tok) {
664	$this->parseError('Unexpected null character in DOCTYPE.');
665	}
666	$stop = " \n\f>";
667	$doctypeName = $this->scanner->charsUntil($stop);
668	// Lowercase ASCII, replace \0 with FFFD
669	$doctypeName = \strtolower(\strtr($doctypeName, "\x00", UTF8Utils::FFFD));
670	$tok = $this->scanner->current();
671	// If false, emit a parse error, DOCTYPE, and return.
672	if (\false === $tok) {
673	$this->parseError('Unexpected EOF in DOCTYPE declaration.');
674	$this->events->doctype($doctypeName, EventHandler::DOCTYPE_NONE, null, \true);
675	return \true;
676	}
677	// Short DOCTYPE, like <!DOCTYPE html>
678	if ('>' == $tok) {
679	// DOCTYPE without a name.
680	if (0 == \strlen($doctypeName)) {
681	$this->parseError('Expected a DOCTYPE name. Got nothing.');
682	$this->events->doctype($doctypeName, 0, null, \true);
683	$this->scanner->consume();
684	return \true;
685	}
686	$this->events->doctype($doctypeName);
687	$this->scanner->consume();
688	return \true;
689	}
690	$this->scanner->whitespace();
691	$pub = \strtoupper($this->scanner->getAsciiAlpha());
692	$white = $this->scanner->whitespace();
693	// Get ID, and flag it as pub or system.
694	if (('PUBLIC' == $pub \|\| 'SYSTEM' == $pub) && $white > 0) {
695	// Get the sys ID.
696	$type = 'PUBLIC' == $pub ? EventHandler::DOCTYPE_PUBLIC : EventHandler::DOCTYPE_SYSTEM;
697	$id = $this->quotedString("\x00>");
698	if (\false === $id) {
699	$this->events->doctype($doctypeName, $type, $pub, \false);
700	return \true;
701	}
702	// Premature EOF.
703	if (\false === $this->scanner->current()) {
704	$this->parseError('Unexpected EOF in DOCTYPE');
705	$this->events->doctype($doctypeName, $type, $id, \true);
706	return \true;
707	}
708	// Well-formed complete DOCTYPE.
709	$this->scanner->whitespace();
710	if ('>' == $this->scanner->current()) {
711	$this->events->doctype($doctypeName, $type, $id, \false);
712	$this->scanner->consume();
713	return \true;
714	}
715	// If we get here, we have <!DOCTYPE foo PUBLIC "bar" SOME_JUNK
716	// Throw away the junk, parse error, quirks mode, return true.
717	$this->scanner->charsUntil('>');
718	$this->parseError('Malformed DOCTYPE.');
719	$this->events->doctype($doctypeName, $type, $id, \true);
720	$this->scanner->consume();
721	return \true;
722	}
723	// Else it's a bogus DOCTYPE.
724	// Consume to > and trash.
725	$this->scanner->charsUntil('>');
726	$this->parseError('Expected PUBLIC or SYSTEM. Got %s.', $pub);
727	$this->events->doctype($doctypeName, 0, null, \true);
728	$this->scanner->consume();
729	return \true;
730	}
731	/**
732	* Utility for reading a quoted string.
733	*
734	* @param string $stopchars Characters (in addition to a close-quote) that should stop the string.
735	* E.g. sometimes '>' is higher precedence than '"' or "'".
736	*
737	* @return mixed String if one is found (quotations omitted).
738	*/
739	protected function quotedString($stopchars)
740	{
741	$tok = $this->scanner->current();
742	if ('"' == $tok \|\| "'" == $tok) {
743	$this->scanner->consume();
744	$ret = $this->scanner->charsUntil($tok . $stopchars);
745	if ($this->scanner->current() == $tok) {
746	$this->scanner->consume();
747	} else {
748	// Parse error because no close quote.
749	$this->parseError('Expected %s, got %s', $tok, $this->scanner->current());
750	}
751	return $ret;
752	}
753	return \false;
754	}
755	/**
756	* Handle a CDATA section.
757	*
758	* @return bool
759	*/
760	protected function cdataSection()
761	{
762	$cdata = '';
763	$this->scanner->consume();
764	$chars = $this->scanner->charsWhile('CDAT');
765	if ('CDATA' != $chars \|\| '[' != $this->scanner->current()) {
766	$this->parseError('Expected [CDATA[, got %s', $chars);
767	return $this->bogusComment('<![' . $chars);
768	}
769	$tok = $this->scanner->next();
770	do {
771	if (\false === $tok) {
772	$this->parseError('Unexpected EOF inside CDATA.');
773	$this->bogusComment('<![CDATA[' . $cdata);
774	return \true;
775	}
776	$cdata .= $tok;
777	$tok = $this->scanner->next();
778	} while (!$this->scanner->sequenceMatches(']]>'));
779	// Consume ]]>
780	$this->scanner->consume(3);
781	$this->events->cdata($cdata);
782	return \true;
783	}
784	// ================================================================
785	// Non-HTML5
786	// ================================================================
787	/**
788	* Handle a processing instruction.
789	*
790	* XML processing instructions are supposed to be ignored in HTML5,
791	* treated as "bogus comments". However, since we're not a user
792	* agent, we allow them. We consume until ?> and then issue a
793	* EventListener::processingInstruction() event.
794	*
795	* @return bool
796	*/
797	protected function processingInstruction()
798	{
799	if ('?' != $this->scanner->current()) {
800	return \false;
801	}
802	$tok = $this->scanner->next();
803	$procName = $this->scanner->getAsciiAlpha();
804	$white = $this->scanner->whitespace();
805	// If not a PI, send to bogusComment.
806	if (0 == \strlen($procName) \|\| 0 == $white \|\| \false == $this->scanner->current()) {
807	$this->parseError("Expected processing instruction name, got {$tok}");
808	$this->bogusComment('<?' . $tok . $procName);
809	return \true;
810	}
811	$data = '';
812	// As long as it's not the case that the next two chars are ? and >.
813	while (!('?' == $this->scanner->current() && '>' == $this->scanner->peek())) {
814	$data .= $this->scanner->current();
815	$tok = $this->scanner->next();
816	if (\false === $tok) {
817	$this->parseError('Unexpected EOF in processing instruction.');
818	$this->events->processingInstruction($procName, $data);
819	return \true;
820	}
821	}
822	$this->scanner->consume(2);
823	// Consume the closing tag
824	$this->events->processingInstruction($procName, $data);
825	return \true;
826	}
827	// ================================================================
828	// UTILITY FUNCTIONS
829	// ================================================================
830	/**
831	* Read from the input stream until we get to the desired sequene
832	* or hit the end of the input stream.
833	*
834	* @param string $sequence
835	*
836	* @return string
837	*/
838	protected function readUntilSequence($sequence)
839	{
840	$buffer = '';
841	// Optimization for reading larger blocks faster.
842	$first = \substr($sequence, 0, 1);
843	while (\false !== $this->scanner->current()) {
844	$buffer .= $this->scanner->charsUntil($first);
845	// Stop as soon as we hit the stopping condition.
846	if ($this->scanner->sequenceMatches($sequence, \false)) {
847	return $buffer;
848	}
849	$buffer .= $this->scanner->current();
850	$this->scanner->consume();
851	}
852	// If we get here, we hit the EOF.
853	$this->parseError('Unexpected EOF during text read.');
854	return $buffer;
855	}
856	/**
857	* Check if upcomming chars match the given sequence.
858	*
859	* This will read the stream for the $sequence. If it's
860	* found, this will return true. If not, return false.
861	* Since this unconsumes any chars it reads, the caller
862	* will still need to read the next sequence, even if
863	* this returns true.
864	*
865	* Example: $this->scanner->sequenceMatches('</script>') will
866	* see if the input stream is at the start of a
867	* '</script>' string.
868	*
869	* @param string $sequence
870	* @param bool $caseSensitive
871	*
872	* @return bool
873	*/
874	protected function sequenceMatches($sequence, $caseSensitive = \true)
875	{
876	@\trigger_error(__METHOD__ . ' method is deprecated since version 2.4 and will be removed in 3.0. Use Scanner::sequenceMatches() instead.', \E_USER_DEPRECATED);
877	return $this->scanner->sequenceMatches($sequence, $caseSensitive);
878	}
879	/**
880	* Send a TEXT event with the contents of the text buffer.
881	*
882	* This emits an EventHandler::text() event with the current contents of the
883	* temporary text buffer. (The buffer is used to group as much PCDATA
884	* as we can instead of emitting lots and lots of TEXT events.)
885	*/
886	protected function flushBuffer()
887	{
888	if ('' === $this->text) {
889	return;
890	}
891	$this->events->text($this->text);
892	$this->text = '';
893	}
894	/**
895	* Add text to the temporary buffer.
896	*
897	* @see flushBuffer()
898	*
899	* @param string $str
900	*/
901	protected function buffer($str)
902	{
903	$this->text .= $str;
904	}
905	/**
906	* Emit a parse error.
907	*
908	* A parse error always returns false because it never consumes any
909	* characters.
910	*
911	* @param string $msg
912	*
913	* @return string
914	*/
915	protected function parseError($msg)
916	{
917	$args = \func_get_args();
918	if (\count($args) > 1) {
919	\array_shift($args);
920	$msg = \vsprintf($msg, $args);
921	}
922	$line = $this->scanner->currentLine();
923	$col = $this->scanner->columnOffset();
924	$this->events->parseError($msg, $line, $col);
925	return \false;
926	}
927	/**
928	* Decode a character reference and return the string.
929	*
930	* If $inAttribute is set to true, a bare & will be returned as-is.
931	*
932	* @param bool $inAttribute Set to true if the text is inside of an attribute value.
933	* false otherwise.
934	*
935	* @return string
936	*/
937	protected function decodeCharacterReference($inAttribute = \false)
938	{
939	// Next char after &.
940	$tok = $this->scanner->next();
941	$start = $this->scanner->position();
942	if (\false === $tok) {
943	return '&';
944	}
945	// These indicate not an entity. We return just
946	// the &.
947	if ("\t" === $tok \|\| "\n" === $tok \|\| "\f" === $tok \|\| ' ' === $tok \|\| '&' === $tok \|\| '<' === $tok) {
948	// $this->scanner->next();
949	return '&';
950	}
951	// Numeric entity
952	if ('#' === $tok) {
953	$tok = $this->scanner->next();
954	if (\false === $tok) {
955	$this->parseError('Expected &#DEC; &#HEX;, got EOF');
956	$this->scanner->unconsume(1);
957	return '&';
958	}
959	// Hexidecimal encoding.
960	// X[0-9a-fA-F]+;
961	// x[0-9a-fA-F]+;
962	if ('x' === $tok \|\| 'X' === $tok) {
963	$tok = $this->scanner->next();
964	// Consume x
965	// Convert from hex code to char.
966	$hex = $this->scanner->getHex();
967	if (empty($hex)) {
968	$this->parseError('Expected &#xHEX;, got &#x%s', $tok);
969	// We unconsume because we don't know what parser rules might
970	// be in effect for the remaining chars. For example. '&#>'
971	// might result in a specific parsing rule inside of tag
972	// contexts, while not inside of pcdata context.
973	$this->scanner->unconsume(2);
974	return '&';
975	}
976	$entity = CharacterReference::lookupHex($hex);
977	} else {
978	// Convert from decimal to char.
979	$numeric = $this->scanner->getNumeric();
980	if (\false === $numeric) {
981	$this->parseError('Expected &#DIGITS;, got &#%s', $tok);
982	$this->scanner->unconsume(2);
983	return '&';
984	}
985	$entity = CharacterReference::lookupDecimal($numeric);
986	}
987	} elseif ('=' === $tok && $inAttribute) {
988	return '&';
989	} else {
990	// String entity.
991	// Attempt to consume a string up to a ';'.
992	// [a-zA-Z0-9]+;
993	$cname = $this->scanner->getAsciiAlphaNum();
994	$entity = CharacterReference::lookupName($cname);
995	// When no entity is found provide the name of the unmatched string
996	// and continue on as the & is not part of an entity. The & will
997	// be converted to & elsewhere.
998	if (null === $entity) {
999	if (!$inAttribute \|\| '' === $cname) {
1000	$this->parseError("No match in entity table for '%s'", $cname);
1001	}
1002	$this->scanner->unconsume($this->scanner->position() - $start);
1003	return '&';
1004	}
1005	}
1006	// The scanner has advanced the cursor for us.
1007	$tok = $this->scanner->current();
1008	// We have an entity. We're done here.
1009	if (';' === $tok) {
1010	$this->scanner->consume();
1011	return $entity;
1012	}
1013	// Failing to match ; means unconsume the entire string.
1014	$this->scanner->unconsume($this->scanner->position() - $start);
1015	$this->parseError('Expected &ENTITY;, got &ENTITY%s (no trailing ;) ', $tok);
1016	return '&';
1017	}
1018	/**
1019	* Checks whether a (single-byte) character is an ASCII letter or not.
1020	*
1021	* @param string $input A single-byte string
1022	*
1023	* @return bool True if it is a letter, False otherwise
1024	*/
1025	protected function is_alpha($input)
1026	{
1027	$code = \ord($input);
1028	return $code >= 97 && $code <= 122 \|\| $code >= 65 && $code <= 90;
1029	}
1030	}
1031