← All changes | assets/lib/Mustache/Tokenizer.php +174 -74 2.0.2 → 4.0.3 View file →

		@@ -2,9 +2,9 @@
2	2
3	3	/*
4	4	* This file is part of Mustache.php.
5	5	*
6		- * (c) 2012 Justin Hileman
	6	+ * (c) 2010-2017 Justin Hileman
7	7	*
8	8	* For the full copyright and license information, please view the LICENSE
9	9	* file that was distributed with this source code.
10	10	*/
		@@ -15,9 +15,8 @@
15	15	* This class is responsible for turning raw template source into a set of Mustache tokens.
16	16	*/
17	17	class Mustache_Tokenizer
18	18	{
19		-
20	19	// Finite state machine states
21	20	const IN_TEXT = 0;
22	21	const IN_TAG_TYPE = 1;
23	22	const IN_TAG = 2;
		@@ -27,9 +26,9 @@
27	26	const T_INVERTED = '^';
28	27	const T_END_SECTION = '/';
29	28	const T_COMMENT = '!';
30	29	const T_PARTIAL = '>';
31		- const T_PARTIAL_2 = '<';
	30	+ const T_PARENT = '<';
32	31	const T_DELIM_CHANGE = '=';
33	32	const T_ESCAPED = '_v';
34	33	const T_UNESCAPED = '{';
35	34	const T_UNESCAPED_2 = '&';
		@@ -34,8 +33,10 @@
34	33	const T_UNESCAPED = '{';
35	34	const T_UNESCAPED_2 = '&';
36	35	const T_TEXT = '_t';
37	36	const T_PRAGMA = '%';
	37	+ const T_BLOCK_VAR = '$';
	38	+ const T_BLOCK_ARG = '$arg';
38	39
39	40	// Valid token types
40	41	private static $tagTypes = array(
41	42	self::T_SECTION => true,
		@@ -42,61 +43,78 @@
42	43	self::T_INVERTED => true,
43	44	self::T_END_SECTION => true,
44	45	self::T_COMMENT => true,
45	46	self::T_PARTIAL => true,
46		- self::T_PARTIAL_2 => true,
	47	+ self::T_PARENT => true,
47	48	self::T_DELIM_CHANGE => true,
48	49	self::T_ESCAPED => true,
49	50	self::T_UNESCAPED => true,
50	51	self::T_UNESCAPED_2 => true,
51	52	self::T_PRAGMA => true,
	53	+ self::T_BLOCK_VAR => true,
52	54	);
53	55
54		- // Interpolated tags
55		- private static $interpolatedTags = array(
56		- self::T_ESCAPED => true,
57		- self::T_UNESCAPED => true,
58		- self::T_UNESCAPED_2 => true,
59		- );
60		-
61	56	// Token properties
62		- const TYPE = 'type';
63		- const NAME = 'name';
64		- const OTAG = 'otag';
65		- const CTAG = 'ctag';
66		- const LINE = 'line';
67		- const INDEX = 'index';
68		- const END = 'end';
69		- const INDENT = 'indent';
70		- const NODES = 'nodes';
71		- const VALUE = 'value';
	57	+ const TYPE = 'type';
	58	+ const NAME = 'name';
	59	+ const OTAG = 'otag';
	60	+ const CTAG = 'ctag';
	61	+ const LINE = 'line';
	62	+ const INDEX = 'index';
	63	+ const END = 'end';
	64	+ const INDENT = 'indent';
	65	+ const NODES = 'nodes';
	66	+ const VALUE = 'value';
	67	+ const FILTERS = 'filters';
72	68
73	69	private $state;
74	70	private $tagType;
75		- private $tag;
76	71	private $buffer;
77	72	private $tokens;
78	73	private $seenTag;
79	74	private $line;
	75	+
80	76	private $otag;
	77	+ private $otagChar;
	78	+ private $otagLen;
	79	+
81	80	private $ctag;
	81	+ private $ctagChar;
	82	+ private $ctagLen;
82	83
83	84	/**
84	85	* Scan and tokenize template source.
85	86	*
	87	+ * @throws Mustache_Exception_SyntaxException when mismatched section tags are encountered
	88	+ * @throws Mustache_Exception_InvalidArgumentException when $delimiters string is invalid
	89	+ *
86	90	* @param string $text Mustache template source to tokenize
87		- * @param string $delimiters Optionally, pass initial opening and closing delimiters (default: null)
	91	+ * @param string $delimiters Optionally, pass initial opening and closing delimiters (default: empty string)
88	92	*
89	93	* @return array Set of Mustache tokens
90	94	*/
91		- public function scan($text, $delimiters = null)
	95	+ public function scan($text, $delimiters = '')
92	96	{
	97	+ // Setting mbstring.func_overload makes things *really* slow.
	98	+ // Let's do everyone a favor and scan this string as ASCII instead.
	99	+ //
	100	+ // The INI directive was removed in PHP 8.0 so we don't need to check there (and can drop it
	101	+ // when we remove support for older versions of PHP).
	102	+ //
	103	+ // @codeCoverageIgnoreStart
	104	+ $encoding = null;
	105	+ if (version_compare(PHP_VERSION, '8.0.0', '<')) {
	106	+ if (function_exists('mb_internal_encoding') && ini_get('mbstring.func_overload') & 2) {
	107	+ $encoding = mb_internal_encoding();
	108	+ mb_internal_encoding('ASCII');
	109	+ }
	110	+ }
	111	+ // @codeCoverageIgnoreEnd
	112	+
93	113	$this->reset();
94	114
95		- if ($delimiters = trim($delimiters)) {
96		- list($otag, $ctag) = explode(' ', $delimiters);
97		- $this->otag = $otag;
98		- $this->ctag = $ctag;
	115	+ if (is_string($delimiters) && $delimiters = trim($delimiters)) {
	116	+ $this->setDelimiters($delimiters);
99	117	}
100	118
101	119	$len = strlen($text);
102	120	for ($i = 0; $i < $len; $i++) {
		@@ -101,16 +119,17 @@
101	119	$len = strlen($text);
102	120	for ($i = 0; $i < $len; $i++) {
103	121	switch ($this->state) {
104	122	case self::IN_TEXT:
105		- if ($this->tagChange($this->otag, $text, $i)) {
	123	+ $char = $text[$i];
	124	+ // Test whether it's time to change tags.
	125	+ if ($char === $this->otagChar && substr($text, $i, $this->otagLen) === $this->otag) {
106	126	$i--;
107	127	$this->flushBuffer();
108	128	$this->state = self::IN_TAG_TYPE;
109	129	} else {
110		- $char = substr($text, $i, 1);
111	130	$this->buffer .= $char;
112		- if ($char == "\n") {
	131	+ if ($char === "\n") {
113	132	$this->flushBuffer();
114	133	$this->line++;
115	134	}
116	135	}
		@@ -116,10 +135,10 @@
116	135	}
117	136	break;
118	137
119	138	case self::IN_TAG_TYPE:
120		- $i += strlen($this->otag) - 1;
121		- $char = substr($text, $i + 1, 1);
	139	+ $i += $this->otagLen - 1;
	140	+ $char = $text[$i + 1];
122	141	if (isset(self::$tagTypes[$char])) {
123	142	$tag = $char;
124	143	$this->tagType = $tag;
125	144	} else {
		@@ -142,41 +161,74 @@
142	161	$this->seenTag = $i;
143	162	break;
144	163
145	164	default:
146		- if ($this->tagChange($this->ctag, $text, $i)) {
147		- $this->tokens[] = array(
	165	+ $char = $text[$i];
	166	+ // Test whether it's time to change tags.
	167	+ if ($char === $this->ctagChar && substr($text, $i, $this->ctagLen) === $this->ctag) {
	168	+ $token = array(
148	169	self::TYPE => $this->tagType,
149	170	self::NAME => trim($this->buffer),
150	171	self::OTAG => $this->otag,
151	172	self::CTAG => $this->ctag,
152	173	self::LINE => $this->line,
153		- self::INDEX => ($this->tagType == self::T_END_SECTION) ? $this->seenTag - strlen($this->otag) : $i + strlen($this->ctag)
	174	+ self::INDEX => ($this->tagType === self::T_END_SECTION) ? $this->seenTag - $this->otagLen : $i + $this->ctagLen,
154	175	);
155	176
156		- $this->buffer = '';
157		- $i += strlen($this->ctag) - 1;
158		- $this->state = self::IN_TEXT;
159		- if ($this->tagType == self::T_UNESCAPED) {
160		- if ($this->ctag == '}}') {
161		- $i++;
	177	+ if ($this->tagType === self::T_UNESCAPED) {
	178	+ // Clean up `{{{ tripleStache }}}` style tokens.
	179	+ if ($this->ctag === '}}') {
	180	+ if (($i + 2 < $len) && $text[$i + 2] === '}') {
	181	+ $i++;
	182	+ } else {
	183	+ $msg = sprintf(
	184	+ 'Mismatched tag delimiters: %s on line %d',
	185	+ $token[self::NAME],
	186	+ $token[self::LINE]
	187	+ );
	188	+
	189	+ throw new Mustache_Exception_SyntaxException($msg, $token);
	190	+ }
162	191	} else {
163		- // Clean up `{{{ tripleStache }}}` style tokens.
164		- $lastName = $this->tokens[count($this->tokens) - 1][self::NAME];
	192	+ $lastName = $token[self::NAME];
165	193	if (substr($lastName, -1) === '}') {
166		- $this->tokens[count($this->tokens) - 1][self::NAME] = trim(substr($lastName, 0, -1));
	194	+ $token[self::NAME] = trim(substr($lastName, 0, -1));
	195	+ } else {
	196	+ $msg = sprintf(
	197	+ 'Mismatched tag delimiters: %s on line %d',
	198	+ $token[self::NAME],
	199	+ $token[self::LINE]
	200	+ );
	201	+
	202	+ throw new Mustache_Exception_SyntaxException($msg, $token);
167	203	}
168	204	}
169	205	}
	206	+
	207	+ $this->buffer = '';
	208	+ $i += $this->ctagLen - 1;
	209	+ $this->state = self::IN_TEXT;
	210	+ $this->tokens[] = $token;
170	211	} else {
171		- $this->buffer .= substr($text, $i, 1);
	212	+ $this->buffer .= $char;
172	213	}
173	214	break;
174	215	}
175	216	}
176	217
	218	+ if ($this->state !== self::IN_TEXT) {
	219	+ $this->throwUnclosedTagException();
	220	+ }
	221	+
177	222	$this->flushBuffer();
178	223
	224	+ // Restore the user's encoding...
	225	+ // @codeCoverageIgnoreStart
	226	+ if ($encoding) {
	227	+ mb_internal_encoding($encoding);
	228	+ }
	229	+ // @codeCoverageIgnoreEnd
	230	+
179	231	return $this->tokens;
180	232	}
181	233
182	234	/**
		@@ -183,17 +235,22 @@
183	235	* Helper function to reset tokenizer internal state.
184	236	*/
185	237	private function reset()
186	238	{
187		- $this->state = self::IN_TEXT;
188		- $this->tagType = null;
189		- $this->tag = null;
190		- $this->buffer = '';
191		- $this->tokens = array();
192		- $this->seenTag = false;
193		- $this->line = 0;
194		- $this->otag = '{{';
195		- $this->ctag = '}}';
	239	+ $this->state = self::IN_TEXT;
	240	+ $this->tagType = null;
	241	+ $this->buffer = '';
	242	+ $this->tokens = array();
	243	+ $this->seenTag = false;
	244	+ $this->line = 0;
	245	+
	246	+ $this->otag = '{{';
	247	+ $this->otagChar = '{';
	248	+ $this->otagLen = 2;
	249	+
	250	+ $this->ctag = '}}';
	251	+ $this->ctagChar = '}';
	252	+ $this->ctagLen = 2;
196	253	}
197	254
198	255	/**
199	256	* Flush the current buffer to a token.
		@@ -199,13 +256,13 @@
199	256	* Flush the current buffer to a token.
200	257	*/
201	258	private function flushBuffer()
202	259	{
203		- if (!empty($this->buffer)) {
	260	+ if (strlen($this->buffer) > 0) {
204	261	$this->tokens[] = array(
205	262	self::TYPE => self::T_TEXT,
206	263	self::LINE => $this->line,
207		- self::VALUE => $this->buffer
	264	+ self::VALUE => $this->buffer,
208	265	);
209	266	$this->buffer = '';
210	267	}
211	268	}
		@@ -212,8 +269,10 @@
212	269
213	270	/**
214	271	* Change the current Mustache delimiters. Set new `otag` and `ctag` values.
215	272	*
	273	+ * @throws Mustache_Exception_SyntaxException when delimiter string is invalid
	274	+ *
216	275	* @param string $text Mustache template source
217	276	* @param int $index Current tokenizer index
218	277	*
219	278	* @return int New index value
		@@ -220,24 +279,56 @@
220	279	*/
221	280	private function changeDelimiters($text, $index)
222	281	{
223	282	$startIndex = strpos($text, '=', $index) + 1;
224		- $close = '='.$this->ctag;
	283	+ $close = '=' . $this->ctag;
225	284	$closeIndex = strpos($text, $close, $index);
226	285
227		- list($otag, $ctag) = explode(' ', trim(substr($text, $startIndex, $closeIndex - $startIndex)));
228		- $this->otag = $otag;
229		- $this->ctag = $ctag;
	286	+ if ($closeIndex === false) {
	287	+ $this->throwUnclosedTagException();
	288	+ }
230	289
231		- $this->tokens[] = array(
	290	+ $token = array(
232	291	self::TYPE => self::T_DELIM_CHANGE,
233	292	self::LINE => $this->line,
234	293	);
235	294
	295	+ try {
	296	+ $this->setDelimiters(trim(substr($text, $startIndex, $closeIndex - $startIndex)));
	297	+ } catch (Mustache_Exception_InvalidArgumentException $e) {
	298	+ throw new Mustache_Exception_SyntaxException($e->getMessage(), $token);
	299	+ }
	300	+
	301	+ $this->tokens[] = $token;
	302	+
236	303	return $closeIndex + strlen($close) - 1;
237	304	}
238	305
239	306	/**
	307	+ * Set the current Mustache `otag` and `ctag` delimiters.
	308	+ *
	309	+ * @throws Mustache_Exception_InvalidArgumentException when delimiter string is invalid
	310	+ *
	311	+ * @param string $delimiters
	312	+ */
	313	+ private function setDelimiters($delimiters)
	314	+ {
	315	+ if (!preg_match('/^\s*(\S+)\s+(\S+)\s*$/', $delimiters, $matches)) {
	316	+ throw new Mustache_Exception_InvalidArgumentException(sprintf('Invalid delimiters: %s', $delimiters));
	317	+ }
	318	+
	319	+ list($_, $otag, $ctag) = $matches;
	320	+
	321	+ $this->otag = $otag;
	322	+ $this->otagChar = $otag[0];
	323	+ $this->otagLen = strlen($otag);
	324	+
	325	+ $this->ctag = $ctag;
	326	+ $this->ctagChar = $ctag[0];
	327	+ $this->ctagLen = strlen($ctag);
	328	+ }
	329	+
	330	+ /**
240	331	* Add pragma token.
241	332	*
242	333	* Pragmas are hoisted to the front of the template, so all pragma tokens
243	334	* will appear at the front of the token list.
		@@ -249,8 +340,12 @@
249	340	*/
250	341	private function addPragma($text, $index)
251	342	{
252	343	$end = strpos($text, $this->ctag, $index);
	344	+ if ($end === false) {
	345	+ $this->throwUnclosedTagException();
	346	+ }
	347	+
253	348	$pragma = trim(substr($text, $index + 2, $end - $index - 2));
254	349
255	350	// Pragmas are hoisted to the front of the template.
256	351	array_unshift($this->tokens, array(
		@@ -258,21 +353,26 @@
258	353	self::NAME => $pragma,
259	354	self::LINE => 0,
260	355	));
261	356
262		- return $end + strlen($this->ctag) - 1;
	357	+ return $end + $this->ctagLen - 1;
263	358	}
264	359
265		- /**
266		- * Test whether it's time to change tags.
267		- *
268		- * @param string $tag Current tag name
269		- * @param string $text Mustache template source
270		- * @param int $index Current tokenizer index
271		- *
272		- * @return boolean True if this is a closing section tag
273		- */
274		- private function tagChange($tag, $text, $index)
	360	+ private function throwUnclosedTagException()
275	361	{
276		- return substr($text, $index, strlen($tag)) === $tag;
	362	+ $name = trim($this->buffer);
	363	+ if ($name !== '') {
	364	+ $msg = sprintf('Unclosed tag: %s on line %d', $name, $this->line);
	365	+ } else {
	366	+ $msg = sprintf('Unclosed tag on line %d', $this->line);
	367	+ }
	368	+
	369	+ throw new Mustache_Exception_SyntaxException($msg, array(
	370	+ self::TYPE => $this->tagType,
	371	+ self::NAME => $name,
	372	+ self::OTAG => $this->otag,
	373	+ self::CTAG => $this->ctag,
	374	+ self::LINE => $this->line,
	375	+ self::INDEX => $this->seenTag - $this->otagLen,
	376	+ ));
277	377	}
278	378	}