wp-parsely / vendor / masterminds / html5 / src / HTML5 / Parser / Tokenizer.php

Tokenizer.php in Parse.ly 3.24.2, at vendor/masterminds/html5/src/HTML5/Parser/Tokenizer.php

1,220 lines 35.8 KB

Raw Download Zip

1	<?php
2
3	namespace Masterminds\HTML5\Parser;
4
5	use Masterminds\HTML5\Elements;
6
7	/**
8	* The HTML5 tokenizer.
9	*
10	* The tokenizer's role is reading data from the scanner and gathering it into
11	* semantic units. From the tokenizer, data is emitted to an event handler,
12	* which may (for example) create a DOM tree.
13	*
14	* The HTML5 specification has a detailed explanation of tokenizing HTML5. We
15	* follow that specification to the maximum extent that we can. If you find
16	* a discrepancy that is not documented, please file a bug and/or submit a
17	* patch.
18	*
19	* This tokenizer is implemented as a recursive descent parser.
20	*
21	* Within the API documentation, you may see references to the specific section
22	* of the HTML5 spec that the code attempts to reproduce. Example: 8.2.4.1.
23	* This refers to section 8.2.4.1 of the HTML5 CR specification.
24	*
25	* @see http://www.w3.org/TR/2012/CR-html5-20121217/
26	*/
27	class Tokenizer
28	{
29	protected $scanner;
30
31	protected $events;
32
33	protected $tok;
34
35	/**
36	* Buffer for text.
37	*/
38	protected $text = '';
39
40	// When this goes to false, the parser stops.
41	protected $carryOn = true;
42
43	protected $textMode = 0; // TEXTMODE_NORMAL;
44	protected $untilTag = null;
45
46	const CONFORMANT_XML = 'xml';
47	const CONFORMANT_HTML = 'html';
48	protected $mode = self::CONFORMANT_HTML;
49
50	/**
51	* Create a new tokenizer.
52	*
53	* Typically, parsing a document involves creating a new tokenizer, giving
54	* it a scanner (input) and an event handler (output), and then calling
55	* the Tokenizer::parse() method.`
56	*
57	* @param Scanner $scanner A scanner initialized with an input stream.
58	* @param EventHandler $eventHandler An event handler, initialized and ready to receive events.
59	* @param string $mode
60	*/
61	public function __construct($scanner, $eventHandler, $mode = self::CONFORMANT_HTML)
62	{
63	$this->scanner = $scanner;
64	$this->events = $eventHandler;
65	$this->mode = $mode;
66	}
67
68	/**
69	* Begin parsing.
70	*
71	* This will begin scanning the document, tokenizing as it goes.
72	* Tokens are emitted into the event handler.
73	*
74	* Tokenizing will continue until the document is completely
75	* read. Errors are emitted into the event handler, but
76	* the parser will attempt to continue parsing until the
77	* entire input stream is read.
78	*/
79	public function parse()
80	{
81	do {
82	$this->consumeData();
83	// FIXME: Add infinite loop protection.
84	} while ($this->carryOn);
85	}
86
87	/**
88	* Set the text mode for the character data reader.
89	*
90	* HTML5 defines three different modes for reading text:
91	* - Normal: Read until a tag is encountered.
92	* - RCDATA: Read until a tag is encountered, but skip a few otherwise-
93	* special characters.
94	* - Raw: Read until a special closing tag is encountered (viz. pre, script)
95	*
96	* This allows those modes to be set.
97	*
98	* Normally, setting is done by the event handler via a special return code on
99	* startTag(), but it can also be set manually using this function.
100	*
101	* @param int $textmode One of Elements::TEXT_*.
102	* @param string $untilTag The tag that should stop RAW or RCDATA mode. Normal mode does not
103	* use this indicator.
104	*/
105	public function setTextMode($textmode, $untilTag = null)
106	{
107	$this->textMode = $textmode & (Elements::TEXT_RAW \| Elements::TEXT_RCDATA);
108	$this->untilTag = $untilTag;
109	}
110
111	/**
112	* Consume a character and make a move.
113	* HTML5 8.2.4.1.
114	*/
115	protected function consumeData()
116	{
117	$tok = $this->scanner->current();
118
119	if ('&' === $tok) {
120	// Character reference
121	$ref = $this->decodeCharacterReference();
122	$this->buffer($ref);
123
124	$tok = $this->scanner->current();
125	}
126
127	// Parse tag
128	if ('<' === $tok) {
129	// Any buffered text data can go out now.
130	$this->flushBuffer();
131
132	$tok = $this->scanner->next();
133
134	if (false === $tok) {
135	// end of string
136	$this->parseError('Illegal tag opening');
137	} elseif ('!' === $tok) {
138	$this->markupDeclaration();
139	} elseif ('/' === $tok) {
140	$this->endTag();
141	} elseif ('?' === $tok) {
142	$this->processingInstruction();
143	} elseif ($this->is_alpha($tok)) {
144	$this->tagName();
145	} else {
146	$this->parseError('Illegal tag opening');
147	// TODO is this necessary ?
148	$this->characterData();
149	}
150
151	$tok = $this->scanner->current();
152	}
153
154	if (false === $tok) {
155	// Handle end of document
156	$this->eof();
157	} else {
158	// Parse character
159	switch ($this->textMode) {
160	case Elements::TEXT_RAW:
161	$this->rawText($tok);
162	break;
163
164	case Elements::TEXT_RCDATA:
165	$this->rcdata($tok);
166	break;
167
168	default:
169	if ('<' === $tok \|\| '&' === $tok) {
170	break;
171	}
172
173	// NULL character
174	if ("\00" === $tok) {
175	$this->parseError('Received null character.');
176
177	$this->text .= $tok;
178	$this->scanner->consume();
179
180	break;
181	}
182
183	$this->text .= $this->scanner->charsUntil("<&\0");
184	}
185	}
186
187	return $this->carryOn;
188	}
189
190	/**
191	* Parse anything that looks like character data.
192	*
193	* Different rules apply based on the current text mode.
194	*
195	* @see Elements::TEXT_RAW Elements::TEXT_RCDATA.
196	*/
197	protected function characterData()
198	{
199	$tok = $this->scanner->current();
200	if (false === $tok) {
201	return false;
202	}
203	switch ($this->textMode) {
204	case Elements::TEXT_RAW:
205	return $this->rawText($tok);
206	case Elements::TEXT_RCDATA:
207	return $this->rcdata($tok);
208	default:
209	if ('<' === $tok \|\| '&' === $tok) {
210	return false;
211	}
212
213	return $this->text($tok);
214	}
215	}
216
217	/**
218	* This buffers the current token as character data.
219	*
220	* @param string $tok The current token.
221	*
222	* @return bool
223	*/
224	protected function text($tok)
225	{
226	// This should never happen...
227	if (false === $tok) {
228	return false;
229	}
230
231	// NULL character
232	if ("\00" === $tok) {
233	$this->parseError('Received null character.');
234	}
235
236	$this->buffer($tok);
237	$this->scanner->consume();
238
239	return true;
240	}
241
242	/**
243	* Read text in RAW mode.
244	*
245	* @param string $tok The current token.
246	*
247	* @return bool
248	*/
249	protected function rawText($tok)
250	{
251	if (is_null($this->untilTag)) {
252	return $this->text($tok);
253	}
254
255	$sequence = '</' . $this->untilTag . '>';
256	$txt = $this->readUntilSequence($sequence);
257	$this->events->text($txt);
258	$this->setTextMode(0);
259
260	return $this->endTag();
261	}
262
263	/**
264	* Read text in RCDATA mode.
265	*
266	* @param string $tok The current token.
267	*
268	* @return bool
269	*/
270	protected function rcdata($tok)
271	{
272	if (is_null($this->untilTag)) {
273	return $this->text($tok);
274	}
275
276	$sequence = '</' . $this->untilTag;
277	$txt = '';
278
279	$caseSensitive = !Elements::isHtml5Element($this->untilTag);
280	while (false !== $tok && !('<' == $tok && ($this->scanner->sequenceMatches($sequence, $caseSensitive)))) {
281	if ('&' == $tok) {
282	$txt .= $this->decodeCharacterReference();
283	$tok = $this->scanner->current();
284	} else {
285	$txt .= $tok;
286	$tok = $this->scanner->next();
287	}
288	}
289	$len = strlen($sequence);
290	$this->scanner->consume($len);
291	$len += $this->scanner->whitespace();
292	if ('>' !== $this->scanner->current()) {
293	$this->parseError('Unclosed RCDATA end tag');
294	}
295
296	$this->scanner->unconsume($len);
297	$this->events->text($txt);
298	$this->setTextMode(0);
299
300	return $this->endTag();
301	}
302
303	/**
304	* If the document is read, emit an EOF event.
305	*/
306	protected function eof()
307	{
308	// fprintf(STDOUT, "EOF");
309	$this->flushBuffer();
310	$this->events->eof();
311	$this->carryOn = false;
312	}
313
314	/**
315	* Look for markup.
316	*/
317	protected function markupDeclaration()
318	{
319	$tok = $this->scanner->next();
320
321	// Comment:
322	if ('-' == $tok && '-' == $this->scanner->peek()) {
323	$this->scanner->consume(2);
324
325	return $this->comment();
326	} elseif ('D' == $tok \|\| 'd' == $tok) { // Doctype
327	return $this->doctype();
328	} elseif ('[' == $tok) { // CDATA section
329	return $this->cdataSection();
330	}
331
332	// FINISH
333	$this->parseError('Expected <!--, <![CDATA[, or <!DOCTYPE. Got <!%s', $tok);
334	$this->bogusComment('<!');
335
336	return true;
337	}
338
339	/**
340	* Consume an end tag. See section 8.2.4.9.
341	*/
342	protected function endTag()
343	{
344	if ('/' != $this->scanner->current()) {
345	return false;
346	}
347	$tok = $this->scanner->next();
348
349	// a-zA-Z -> tagname
350	// > -> parse error
351	// EOF -> parse error
352	// -> parse error
353	if (!$this->is_alpha($tok)) {
354	$this->parseError("Expected tag name, got '%s'", $tok);
355	if ("\0" == $tok \|\| false === $tok) {
356	return false;
357	}
358
359	return $this->bogusComment('</');
360	}
361
362	$name = $this->scanner->charsUntil("\n\f \t>");
363	$name = self::CONFORMANT_XML === $this->mode ? $name : strtolower($name);
364	// Trash whitespace.
365	$this->scanner->whitespace();
366
367	$tok = $this->scanner->current();
368	if ('>' != $tok) {
369	$this->parseError("Expected >, got '%s'", $tok);
370	// We just trash stuff until we get to the next tag close.
371	$this->scanner->charsUntil('>');
372	}
373
374	$this->events->endTag($name);
375	$this->scanner->consume();
376
377	return true;
378	}
379
380	/**
381	* Consume a tag name and body. See section 8.2.4.10.
382	*/
383	protected function tagName()
384	{
385	// We know this is at least one char.
386	$name = $this->scanner->charsWhile(':_-0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz');
387	$name = self::CONFORMANT_XML === $this->mode ? $name : strtolower($name);
388	$attributes = array();
389	$selfClose = false;
390
391	// Handle attribute parse exceptions here so that we can
392	// react by trying to build a sensible parse tree.
393	try {
394	do {
395	$this->scanner->whitespace();
396	$this->attribute($attributes);
397	} while (!$this->isTagEnd($selfClose));
398	} catch (ParseError $e) {
399	$selfClose = false;
400	}
401
402	$mode = $this->events->startTag($name, $attributes, $selfClose);
403
404	if (is_int($mode)) {
405	$this->setTextMode($mode, $name);
406	}
407
408	$this->scanner->consume();
409
410	return true;
411	}
412
413	/**
414	* Check if the scanner has reached the end of a tag.
415	*/
416	protected function isTagEnd(&$selfClose)
417	{
418	$tok = $this->scanner->current();
419	if ('/' == $tok) {
420	$this->scanner->consume();
421	$this->scanner->whitespace();
422	$tok = $this->scanner->current();
423
424	if ('>' == $tok) {
425	$selfClose = true;
426
427	return true;
428	}
429	if (false === $tok) {
430	$this->parseError('Unexpected EOF inside of tag.');
431
432	return true;
433	}
434	// Basically, we skip the / token and go on.
435	// See 8.2.4.43.
436	$this->parseError("Unexpected '%s' inside of a tag.", $tok);
437
438	return false;
439	}
440
441	if ('>' == $tok) {
442	return true;
443	}
444	if (false === $tok) {
445	$this->parseError('Unexpected EOF inside of tag.');
446
447	return true;
448	}
449
450	return false;
451	}
452
453	/**
454	* Parse attributes from inside of a tag.
455	*
456	* @param string[] $attributes
457	*
458	* @return bool
459	*
460	* @throws ParseError
461	*/
462	protected function attribute(&$attributes)
463	{
464	$tok = $this->scanner->current();
465	if ('/' == $tok \|\| '>' == $tok \|\| false === $tok) {
466	return false;
467	}
468
469	if ('<' == $tok) {
470	$this->parseError("Unexpected '<' inside of attributes list.");
471	// Push the < back onto the stack.
472	$this->scanner->unconsume();
473	// Let the caller figure out how to handle this.
474	throw new ParseError('Start tag inside of attribute.');
475	}
476
477	$name = strtolower($this->scanner->charsUntil("/>=\n\f\t "));
478
479	if (0 == strlen($name)) {
480	$tok = $this->scanner->current();
481	$this->parseError('Expected an attribute name, got %s.', $tok);
482	// Really, only '=' can be the char here. Everything else gets absorbed
483	// under one rule or another.
484	$name = $tok;
485	$this->scanner->consume();
486	}
487
488	$isValidAttribute = true;
489	// Attribute names can contain most Unicode characters for HTML5.
490	// But method "DOMElement::setAttribute" is throwing exception
491	// because of it's own internal restriction so these have to be filtered.
492	// see issue #23: https://github.com/Masterminds/html5-php/issues/23
493	// and http://www.w3.org/TR/2011/WD-html5-20110525/syntax.html#syntax-attribute-name
494	if (preg_match("/[\x1-\x2C\\/\x3B-\x40\x5B-\x5E\x60\x7B-\x7F]/u", $name)) {
495	$this->parseError('Unexpected characters in attribute name: %s', $name);
496	$isValidAttribute = false;
497	} // There is no limitation for 1st character in HTML5.
498	// But method "DOMElement::setAttribute" is throwing exception for the
499	// characters below so they have to be filtered.
500	// see issue #23: https://github.com/Masterminds/html5-php/issues/23
501	// and http://www.w3.org/TR/2011/WD-html5-20110525/syntax.html#syntax-attribute-name
502	elseif (preg_match('/^[0-9.-]/u', $name)) {
503	$this->parseError('Unexpected character at the begining of attribute name: %s', $name);
504	$isValidAttribute = false;
505	}
506	// 8.1.2.3
507	$this->scanner->whitespace();
508
509	$val = $this->attributeValue();
510	if ($isValidAttribute && !array_key_exists($name, $attributes)) {
511	$attributes[$name] = $val;
512	}
513
514	return true;
515	}
516
517	/**
518	* Consume an attribute value. See section 8.2.4.37 and after.
519	*
520	* @return string\|null
521	*/
522	protected function attributeValue()
523	{
524	if ('=' != $this->scanner->current()) {
525	return null;
526	}
527	$this->scanner->consume();
528	// 8.1.2.3
529	$this->scanner->whitespace();
530
531	$tok = $this->scanner->current();
532	switch ($tok) {
533	case "\n":
534	case "\f":
535	case ' ':
536	case "\t":
537	// Whitespace here indicates an empty value.
538	return null;
539	case '"':
540	case "'":
541	$this->scanner->consume();
542
543	return $this->quotedAttributeValue($tok);
544	case '>':
545	// case '/': // 8.2.4.37 seems to allow foo=/ as a valid attr.
546	$this->parseError('Expected attribute value, got tag end.');
547
548	return null;
549	case '=':
550	case '`':
551	$this->parseError('Expecting quotes, got %s.', $tok);
552
553	return $this->unquotedAttributeValue();
554	default:
555	return $this->unquotedAttributeValue();
556	}
557	}
558
559	/**
560	* Get an attribute value string.
561	*
562	* @param string $quote IMPORTANT: This is a series of chars! Any one of which will be considered
563	* termination of an attribute's value. E.g. "\"'" will stop at either
564	* ' or ".
565	*
566	* @return string The attribute value.
567	*/
568	protected function quotedAttributeValue($quote)
569	{
570	$stoplist = "\f" . $quote;
571	$val = '';
572
573	while (true) {
574	$tokens = $this->scanner->charsUntil($stoplist . '&');
575	if (false !== $tokens) {
576	$val .= $tokens;
577	} else {
578	break;
579	}
580
581	$tok = $this->scanner->current();
582	if ('&' == $tok) {
583	$val .= $this->decodeCharacterReference(true);
584	continue;
585	}
586	break;
587	}
588	$this->scanner->consume();
589
590	return $val;
591	}
592
593	protected function unquotedAttributeValue()
594	{
595	$val = '';
596	$tok = $this->scanner->current();
597	while (false !== $tok) {
598	switch ($tok) {
599	case "\n":
600	case "\f":
601	case ' ':
602	case "\t":
603	case '>':
604	break 2;
605
606	case '&':
607	$val .= $this->decodeCharacterReference(true);
608	$tok = $this->scanner->current();
609
610	break;
611
612	case "'":
613	case '"':
614	case '<':
615	case '=':
616	case '`':
617	$this->parseError('Unexpected chars in unquoted attribute value %s', $tok);
618	$val .= $tok;
619	$tok = $this->scanner->next();
620	break;
621
622	default:
623	$val .= $this->scanner->charsUntil("\t\n\f >&\"'<=`");
624
625	$tok = $this->scanner->current();
626	}
627	}
628
629	return $val;
630	}
631
632	/**
633	* Consume malformed markup as if it were a comment.
634	* 8.2.4.44.
635	*
636	* The spec requires that the ENTIRE tag-like thing be enclosed inside of
637	* the comment. So this will generate comments like:
638	*
639	* <!--&lt/+foo>-->
640	*
641	* @param string $leading Prepend any leading characters. This essentially
642	* negates the need to backtrack, but it's sort of a hack.
643	*
644	* @return bool
645	*/
646	protected function bogusComment($leading = '')
647	{
648	$comment = $leading;
649	$tokens = $this->scanner->charsUntil('>');
650	if (false !== $tokens) {
651	$comment .= $tokens;
652	}
653	$tok = $this->scanner->current();
654	if (false !== $tok) {
655	$comment .= $tok;
656	}
657
658	$this->flushBuffer();
659	$this->events->comment($comment);
660	$this->scanner->consume();
661
662	return true;
663	}
664
665	/**
666	* Read a comment.
667	* Expects the first tok to be inside of the comment.
668	*
669	* @return bool
670	*/
671	protected function comment()
672	{
673	$tok = $this->scanner->current();
674	$comment = '';
675
676	// <!-->. Emit an empty comment because 8.2.4.46 says to.
677	if ('>' == $tok) {
678	// Parse error. Emit the comment token.
679	$this->parseError("Expected comment data, got '>'");
680	$this->events->comment('');
681	$this->scanner->consume();
682
683	return true;
684	}
685
686	// Replace NULL with the replacement char.
687	if ("\0" == $tok) {
688	$tok = UTF8Utils::FFFD;
689	}
690	while (!$this->isCommentEnd()) {
691	$comment .= $tok;
692	$tok = $this->scanner->next();
693	}
694
695	$this->events->comment($comment);
696	$this->scanner->consume();
697
698	return true;
699	}
700
701	/**
702	* Check if the scanner has reached the end of a comment.
703	*
704	* @return bool
705	*/
706	protected function isCommentEnd()
707	{
708	$tok = $this->scanner->current();
709
710	// EOF
711	if (false === $tok) {
712	// Hit the end.
713	$this->parseError('Unexpected EOF in a comment.');
714
715	return true;
716	}
717
718	// If next two tokens are not '--', not the end.
719	if ('-' != $tok \|\| '-' != $this->scanner->peek()) {
720	return false;
721	}
722
723	$this->scanner->consume(2); // Consume '-' and one of '!' or '>'
724
725	// Test for '>'
726	if ('>' == $this->scanner->current()) {
727	return true;
728	}
729	// Test for '!>'
730	if ('!' == $this->scanner->current() && '>' == $this->scanner->peek()) {
731	$this->scanner->consume(); // Consume the last '>'
732
733	return true;
734	}
735	// Unread '-' and one of '!' or '>';
736	$this->scanner->unconsume(2);
737
738	return false;
739	}
740
741	/**
742	* Parse a DOCTYPE.
743	*
744	* Parse a DOCTYPE declaration. This method has strong bearing on whether or
745	* not Quirksmode is enabled on the event handler.
746	*
747	* @todo This method is a little long. Should probably refactor.
748	*
749	* @return bool
750	*/
751	protected function doctype()
752	{
753	// Check that string is DOCTYPE.
754	if ($this->scanner->sequenceMatches('DOCTYPE', false)) {
755	$this->scanner->consume(7);
756	} else {
757	$chars = $this->scanner->charsWhile('DOCTYPEdoctype');
758	$this->parseError('Expected DOCTYPE, got %s', $chars);
759
760	return $this->bogusComment('<!' . $chars);
761	}
762
763	$this->scanner->whitespace();
764	$tok = $this->scanner->current();
765
766	// EOF: die.
767	if (false === $tok) {
768	$this->events->doctype('html5', EventHandler::DOCTYPE_NONE, '', true);
769	$this->eof();
770
771	return true;
772	}
773
774	// NULL char: convert.
775	if ("\0" === $tok) {
776	$this->parseError('Unexpected null character in DOCTYPE.');
777	}
778
779	$stop = " \n\f>";
780	$doctypeName = $this->scanner->charsUntil($stop);
781	// Lowercase ASCII, replace \0 with FFFD
782	$doctypeName = strtolower(strtr($doctypeName, "\0", UTF8Utils::FFFD));
783
784	$tok = $this->scanner->current();
785
786	// If false, emit a parse error, DOCTYPE, and return.
787	if (false === $tok) {
788	$this->parseError('Unexpected EOF in DOCTYPE declaration.');
789	$this->events->doctype($doctypeName, EventHandler::DOCTYPE_NONE, null, true);
790
791	return true;
792	}
793
794	// Short DOCTYPE, like <!DOCTYPE html>
795	if ('>' == $tok) {
796	// DOCTYPE without a name.
797	if (0 == strlen($doctypeName)) {
798	$this->parseError('Expected a DOCTYPE name. Got nothing.');
799	$this->events->doctype($doctypeName, 0, null, true);
800	$this->scanner->consume();
801
802	return true;
803	}
804	$this->events->doctype($doctypeName);
805	$this->scanner->consume();
806
807	return true;
808	}
809	$this->scanner->whitespace();
810
811	$pub = strtoupper($this->scanner->getAsciiAlpha());
812	$white = $this->scanner->whitespace();
813
814	// Get ID, and flag it as pub or system.
815	if (('PUBLIC' == $pub \|\| 'SYSTEM' == $pub) && $white > 0) {
816	// Get the sys ID.
817	$type = 'PUBLIC' == $pub ? EventHandler::DOCTYPE_PUBLIC : EventHandler::DOCTYPE_SYSTEM;
818	$id = $this->quotedString("\0>");
819	if (false === $id) {
820	$this->events->doctype($doctypeName, $type, $pub, false);
821
822	return true;
823	}
824
825	// Premature EOF.
826	if (false === $this->scanner->current()) {
827	$this->parseError('Unexpected EOF in DOCTYPE');
828	$this->events->doctype($doctypeName, $type, $id, true);
829
830	return true;
831	}
832
833	// Well-formed complete DOCTYPE.
834	$this->scanner->whitespace();
835	if ('>' == $this->scanner->current()) {
836	$this->events->doctype($doctypeName, $type, $id, false);
837	$this->scanner->consume();
838
839	return true;
840	}
841
842	// If we get here, we have <!DOCTYPE foo PUBLIC "bar" SOME_JUNK
843	// Throw away the junk, parse error, quirks mode, return true.
844	$this->scanner->charsUntil('>');
845	$this->parseError('Malformed DOCTYPE.');
846	$this->events->doctype($doctypeName, $type, $id, true);
847	$this->scanner->consume();
848
849	return true;
850	}
851
852	// Else it's a bogus DOCTYPE.
853	// Consume to > and trash.
854	$this->scanner->charsUntil('>');
855
856	$this->parseError('Expected PUBLIC or SYSTEM. Got %s.', $pub);
857	$this->events->doctype($doctypeName, 0, null, true);
858	$this->scanner->consume();
859
860	return true;
861	}
862
863	/**
864	* Utility for reading a quoted string.
865	*
866	* @param string $stopchars Characters (in addition to a close-quote) that should stop the string.
867	* E.g. sometimes '>' is higher precedence than '"' or "'".
868	*
869	* @return mixed String if one is found (quotations omitted).
870	*/
871	protected function quotedString($stopchars)
872	{
873	$tok = $this->scanner->current();
874	if ('"' == $tok \|\| "'" == $tok) {
875	$this->scanner->consume();
876	$ret = $this->scanner->charsUntil($tok . $stopchars);
877	if ($this->scanner->current() == $tok) {
878	$this->scanner->consume();
879	} else {
880	// Parse error because no close quote.
881	$this->parseError('Expected %s, got %s', $tok, $this->scanner->current());
882	}
883
884	return $ret;
885	}
886
887	return false;
888	}
889
890	/**
891	* Handle a CDATA section.
892	*
893	* @return bool
894	*/
895	protected function cdataSection()
896	{
897	$cdata = '';
898	$this->scanner->consume();
899
900	$chars = $this->scanner->charsWhile('CDAT');
901	if ('CDATA' != $chars \|\| '[' != $this->scanner->current()) {
902	$this->parseError('Expected [CDATA[, got %s', $chars);
903
904	return $this->bogusComment('<![' . $chars);
905	}
906
907	$tok = $this->scanner->next();
908	do {
909	if (false === $tok) {
910	$this->parseError('Unexpected EOF inside CDATA.');
911	$this->bogusComment('<![CDATA[' . $cdata);
912
913	return true;
914	}
915	$cdata .= $tok;
916	$tok = $this->scanner->next();
917	} while (!$this->scanner->sequenceMatches(']]>'));
918
919	// Consume ]]>
920	$this->scanner->consume(3);
921
922	$this->events->cdata($cdata);
923
924	return true;
925	}
926
927	// ================================================================
928	// Non-HTML5
929	// ================================================================
930
931	/**
932	* Handle a processing instruction.
933	*
934	* XML processing instructions are supposed to be ignored in HTML5,
935	* treated as "bogus comments". However, since we're not a user
936	* agent, we allow them. We consume until ?> and then issue a
937	* EventListener::processingInstruction() event.
938	*
939	* @return bool
940	*/
941	protected function processingInstruction()
942	{
943	if ('?' != $this->scanner->current()) {
944	return false;
945	}
946
947	$tok = $this->scanner->next();
948	$procName = $this->scanner->getAsciiAlpha();
949	$white = $this->scanner->whitespace();
950
951	// If not a PI, send to bogusComment.
952	if (0 == strlen($procName) \|\| 0 == $white \|\| false == $this->scanner->current()) {
953	$this->parseError("Expected processing instruction name, got $tok");
954	$this->bogusComment('<?' . $tok . $procName);
955
956	return true;
957	}
958
959	$data = '';
960	// As long as it's not the case that the next two chars are ? and >.
961	while (!('?' == $this->scanner->current() && '>' == $this->scanner->peek())) {
962	$data .= $this->scanner->current();
963
964	$tok = $this->scanner->next();
965	if (false === $tok) {
966	$this->parseError('Unexpected EOF in processing instruction.');
967	$this->events->processingInstruction($procName, $data);
968
969	return true;
970	}
971	}
972
973	$this->scanner->consume(2); // Consume the closing tag
974	$this->events->processingInstruction($procName, $data);
975
976	return true;
977	}
978
979	// ================================================================
980	// UTILITY FUNCTIONS
981	// ================================================================
982
983	/**
984	* Read from the input stream until we get to the desired sequene
985	* or hit the end of the input stream.
986	*
987	* @param string $sequence
988	*
989	* @return string
990	*/
991	protected function readUntilSequence($sequence)
992	{
993	$buffer = '';
994
995	// Optimization for reading larger blocks faster.
996	$first = substr($sequence, 0, 1);
997	while (false !== $this->scanner->current()) {
998	$buffer .= $this->scanner->charsUntil($first);
999
1000	// Stop as soon as we hit the stopping condition.
1001	if ($this->scanner->sequenceMatches($sequence, false)) {
1002	return $buffer;
1003	}
1004	$buffer .= $this->scanner->current();
1005	$this->scanner->consume();
1006	}
1007
1008	// If we get here, we hit the EOF.
1009	$this->parseError('Unexpected EOF during text read.');
1010
1011	return $buffer;
1012	}
1013
1014	/**
1015	* Check if upcomming chars match the given sequence.
1016	*
1017	* This will read the stream for the $sequence. If it's
1018	* found, this will return true. If not, return false.
1019	* Since this unconsumes any chars it reads, the caller
1020	* will still need to read the next sequence, even if
1021	* this returns true.
1022	*
1023	* Example: $this->scanner->sequenceMatches('</script>') will
1024	* see if the input stream is at the start of a
1025	* '</script>' string.
1026	*
1027	* @param string $sequence
1028	* @param bool $caseSensitive
1029	*
1030	* @return bool
1031	*/
1032	protected function sequenceMatches($sequence, $caseSensitive = true)
1033	{
1034	@trigger_error(__METHOD__ . ' method is deprecated since version 2.4 and will be removed in 3.0. Use Scanner::sequenceMatches() instead.', E_USER_DEPRECATED);
1035
1036	return $this->scanner->sequenceMatches($sequence, $caseSensitive);
1037	}
1038
1039	/**
1040	* Send a TEXT event with the contents of the text buffer.
1041	*
1042	* This emits an EventHandler::text() event with the current contents of the
1043	* temporary text buffer. (The buffer is used to group as much PCDATA
1044	* as we can instead of emitting lots and lots of TEXT events.)
1045	*/
1046	protected function flushBuffer()
1047	{
1048	if ('' === $this->text) {
1049	return;
1050	}
1051	$this->events->text($this->text);
1052	$this->text = '';
1053	}
1054
1055	/**
1056	* Add text to the temporary buffer.
1057	*
1058	* @see flushBuffer()
1059	*
1060	* @param string $str
1061	*/
1062	protected function buffer($str)
1063	{
1064	$this->text .= $str;
1065	}
1066
1067	/**
1068	* Emit a parse error.
1069	*
1070	* A parse error always returns false because it never consumes any
1071	* characters.
1072	*
1073	* @param string $msg
1074	*
1075	* @return string
1076	*/
1077	protected function parseError($msg)
1078	{
1079	$args = func_get_args();
1080
1081	if (count($args) > 1) {
1082	array_shift($args);
1083	$msg = vsprintf($msg, $args);
1084	}
1085
1086	$line = $this->scanner->currentLine();
1087	$col = $this->scanner->columnOffset();
1088	$this->events->parseError($msg, $line, $col);
1089
1090	return false;
1091	}
1092
1093	/**
1094	* Decode a character reference and return the string.
1095	*
1096	* If $inAttribute is set to true, a bare & will be returned as-is.
1097	*
1098	* @param bool $inAttribute Set to true if the text is inside of an attribute value.
1099	* false otherwise.
1100	*
1101	* @return string
1102	*/
1103	protected function decodeCharacterReference($inAttribute = false)
1104	{
1105	// Next char after &.
1106	$tok = $this->scanner->next();
1107	$start = $this->scanner->position();
1108
1109	if (false === $tok) {
1110	return '&';
1111	}
1112
1113	// These indicate not an entity. We return just
1114	// the &.
1115	if ("\t" === $tok \|\| "\n" === $tok \|\| "\f" === $tok \|\| ' ' === $tok \|\| '&' === $tok \|\| '<' === $tok) {
1116	// $this->scanner->next();
1117	return '&';
1118	}
1119
1120	// Numeric entity
1121	if ('#' === $tok) {
1122	$tok = $this->scanner->next();
1123
1124	if (false === $tok) {
1125	$this->parseError('Expected &#DEC; &#HEX;, got EOF');
1126	$this->scanner->unconsume(1);
1127
1128	return '&';
1129	}
1130
1131	// Hexadecimal encoding.
1132	// X[0-9a-fA-F]+;
1133	// x[0-9a-fA-F]+;
1134	if ('x' === $tok \|\| 'X' === $tok) {
1135	$tok = $this->scanner->next(); // Consume x
1136
1137	// Convert from hex code to char.
1138	$hex = $this->scanner->getHex();
1139	if (empty($hex)) {
1140	$this->parseError('Expected &#xHEX;, got &#x%s', $tok);
1141	// We unconsume because we don't know what parser rules might
1142	// be in effect for the remaining chars. For example. '&#>'
1143	// might result in a specific parsing rule inside of tag
1144	// contexts, while not inside of pcdata context.
1145	$this->scanner->unconsume(2);
1146
1147	return '&';
1148	}
1149	$entity = CharacterReference::lookupHex($hex);
1150	} // Decimal encoding.
1151	// [0-9]+;
1152	else {
1153	// Convert from decimal to char.
1154	$numeric = $this->scanner->getNumeric();
1155	if (false === $numeric) {
1156	$this->parseError('Expected &#DIGITS;, got &#%s', $tok);
1157	$this->scanner->unconsume(2);
1158
1159	return '&';
1160	}
1161	$entity = CharacterReference::lookupDecimal($numeric);
1162	}
1163	} elseif ('=' === $tok && $inAttribute) {
1164	return '&';
1165	} else { // String entity.
1166	// Attempt to consume a string up to a ';'.
1167	// [a-zA-Z0-9]+;
1168	$cname = $this->scanner->getAsciiAlphaNum();
1169	$entity = CharacterReference::lookupName($cname);
1170
1171	// When no entity is found provide the name of the unmatched string
1172	// and continue on as the & is not part of an entity. The & will
1173	// be converted to & elsewhere.
1174	if (null === $entity) {
1175	if (!$inAttribute \|\| '' === $cname) {
1176	$this->parseError("No match in entity table for '%s'", $cname);
1177	}
1178	$this->scanner->unconsume($this->scanner->position() - $start);
1179
1180	return '&';
1181	}
1182	}
1183
1184	// The scanner has advanced the cursor for us.
1185	$tok = $this->scanner->current();
1186
1187	// We have an entity. We're done here.
1188	if (';' === $tok) {
1189	$this->scanner->consume();
1190
1191	return $entity;
1192	}
1193
1194	// Failing to match ; means unconsume the entire string.
1195	$this->scanner->unconsume($this->scanner->position() - $start);
1196
1197	$this->parseError('Expected &ENTITY;, got &ENTITY%s (no trailing ;) ', $tok);
1198
1199	return '&';
1200	}
1201
1202	/**
1203	* Checks whether a (single-byte) character is an ASCII letter or not.
1204	*
1205	* @param string $input A single-byte string
1206	*
1207	* @return bool True if it is a letter, False otherwise
1208	*/
1209	protected function is_alpha($input)
1210	{
1211	if (!is_string($input) \|\| 1 !== strlen($input)) {
1212	return false;
1213	}
1214
1215	$code = ord($input);
1216
1217	return ($code >= 97 && $code <= 122) \|\| ($code >= 65 && $code <= 90);
1218	}
1219	}
1220