defender-security / src / extra / php-codesniffer-php-token / Tokenizers / JS.php

JS.php in Defender Security – Malware Scanner, Login Security & Firewall 3.0.1, at src/extra/php-codesniffer-php-token/Tokenizers/JS.php

1,257 lines 49.0 KB

Raw Download Zip

1	<?php
2	/**
3	* Tokenizes JS code.
4	*
5	* @author Greg Sherwood <[email protected]>
6	* @copyright 2006-2015 Squiz Pty Ltd (ABN 77 084 670 600)
7	* @license https://github.com/squizlabs/PHP_CodeSniffer/blob/master/licence.txt BSD Licence
8	*/
9
10	namespace PHP_CodeSniffer\Tokenizers;
11
12	use PHP_CodeSniffer\Config;
13	use PHP_CodeSniffer\Exceptions\TokenizerException;
14	use PHP_CodeSniffer\Util;
15
16	class JS extends Tokenizer
17	{
18
19	/**
20	* A list of tokens that are allowed to open a scope.
21	*
22	* This array also contains information about what kind of token the scope
23	* opener uses to open and close the scope, if the token strictly requires
24	* an opener, if the token can share a scope closer, and who it can be shared
25	* with. An example of a token that shares a scope closer is a CASE scope.
26	*
27	* @var array
28	*/
29	public $scopeOpeners = [
30	T_IF => [
31	'start' => [T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET],
32	'end' => [T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET],
33	'strict' => false,
34	'shared' => false,
35	'with' => [],
36	],
37	T_TRY => [
38	'start' => [T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET],
39	'end' => [T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET],
40	'strict' => true,
41	'shared' => false,
42	'with' => [],
43	],
44	T_CATCH => [
45	'start' => [T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET],
46	'end' => [T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET],
47	'strict' => true,
48	'shared' => false,
49	'with' => [],
50	],
51	T_ELSE => [
52	'start' => [T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET],
53	'end' => [T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET],
54	'strict' => false,
55	'shared' => false,
56	'with' => [],
57	],
58	T_FOR => [
59	'start' => [T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET],
60	'end' => [T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET],
61	'strict' => false,
62	'shared' => false,
63	'with' => [],
64	],
65	T_CLASS => [
66	'start' => [T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET],
67	'end' => [T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET],
68	'strict' => true,
69	'shared' => false,
70	'with' => [],
71	],
72	T_FUNCTION => [
73	'start' => [T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET],
74	'end' => [T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET],
75	'strict' => false,
76	'shared' => false,
77	'with' => [],
78	],
79	T_WHILE => [
80	'start' => [T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET],
81	'end' => [T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET],
82	'strict' => false,
83	'shared' => false,
84	'with' => [],
85	],
86	T_DO => [
87	'start' => [T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET],
88	'end' => [T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET],
89	'strict' => true,
90	'shared' => false,
91	'with' => [],
92	],
93	T_SWITCH => [
94	'start' => [T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET],
95	'end' => [T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET],
96	'strict' => true,
97	'shared' => false,
98	'with' => [],
99	],
100	T_CASE => [
101	'start' => [T_COLON => T_COLON],
102	'end' => [
103	T_BREAK => T_BREAK,
104	T_RETURN => T_RETURN,
105	T_CONTINUE => T_CONTINUE,
106	T_THROW => T_THROW,
107	],
108	'strict' => true,
109	'shared' => true,
110	'with' => [
111	T_DEFAULT => T_DEFAULT,
112	T_CASE => T_CASE,
113	T_SWITCH => T_SWITCH,
114	],
115	],
116	T_DEFAULT => [
117	'start' => [T_COLON => T_COLON],
118	'end' => [
119	T_BREAK => T_BREAK,
120	T_RETURN => T_RETURN,
121	T_CONTINUE => T_CONTINUE,
122	T_THROW => T_THROW,
123	],
124	'strict' => true,
125	'shared' => true,
126	'with' => [
127	T_CASE => T_CASE,
128	T_SWITCH => T_SWITCH,
129	],
130	],
131	];
132
133	/**
134	* A list of tokens that end the scope.
135	*
136	* This array is just a unique collection of the end tokens
137	* from the _scopeOpeners array. The data is duplicated here to
138	* save time during parsing of the file.
139	*
140	* @var array
141	*/
142	public $endScopeTokens = [
143	T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET,
144	T_BREAK => T_BREAK,
145	];
146
147	/**
148	* A list of special JS tokens and their types.
149	*
150	* @var array
151	*/
152	protected $tokenValues = [
153	'class' => 'T_CLASS',
154	'function' => 'T_FUNCTION',
155	'prototype' => 'T_PROTOTYPE',
156	'try' => 'T_TRY',
157	'catch' => 'T_CATCH',
158	'return' => 'T_RETURN',
159	'throw' => 'T_THROW',
160	'break' => 'T_BREAK',
161	'switch' => 'T_SWITCH',
162	'continue' => 'T_CONTINUE',
163	'if' => 'T_IF',
164	'else' => 'T_ELSE',
165	'do' => 'T_DO',
166	'while' => 'T_WHILE',
167	'for' => 'T_FOR',
168	'var' => 'T_VAR',
169	'case' => 'T_CASE',
170	'default' => 'T_DEFAULT',
171	'true' => 'T_TRUE',
172	'false' => 'T_FALSE',
173	'null' => 'T_NULL',
174	'this' => 'T_THIS',
175	'typeof' => 'T_TYPEOF',
176	'(' => 'T_OPEN_PARENTHESIS',
177	')' => 'T_CLOSE_PARENTHESIS',
178	'{' => 'T_OPEN_CURLY_BRACKET',
179	'}' => 'T_CLOSE_CURLY_BRACKET',
180	'[' => 'T_OPEN_SQUARE_BRACKET',
181	']' => 'T_CLOSE_SQUARE_BRACKET',
182	'?' => 'T_INLINE_THEN',
183	'.' => 'T_OBJECT_OPERATOR',
184	'+' => 'T_PLUS',
185	'-' => 'T_MINUS',
186	'*' => 'T_MULTIPLY',
187	'%' => 'T_MODULUS',
188	'/' => 'T_DIVIDE',
189	'^' => 'T_LOGICAL_XOR',
190	',' => 'T_COMMA',
191	';' => 'T_SEMICOLON',
192	':' => 'T_COLON',
193	'<' => 'T_LESS_THAN',
194	'>' => 'T_GREATER_THAN',
195	'<<' => 'T_SL',
196	'>>' => 'T_SR',
197	'>>>' => 'T_ZSR',
198	'<<=' => 'T_SL_EQUAL',
199	'>>=' => 'T_SR_EQUAL',
200	'>>>=' => 'T_ZSR_EQUAL',
201	'<=' => 'T_IS_SMALLER_OR_EQUAL',
202	'>=' => 'T_IS_GREATER_OR_EQUAL',
203	'=>' => 'T_DOUBLE_ARROW',
204	'!' => 'T_BOOLEAN_NOT',
205	'\|\|' => 'T_BOOLEAN_OR',
206	'&&' => 'T_BOOLEAN_AND',
207	'\|' => 'T_BITWISE_OR',
208	'&' => 'T_BITWISE_AND',
209	'!=' => 'T_IS_NOT_EQUAL',
210	'!==' => 'T_IS_NOT_IDENTICAL',
211	'=' => 'T_EQUAL',
212	'==' => 'T_IS_EQUAL',
213	'===' => 'T_IS_IDENTICAL',
214	'-=' => 'T_MINUS_EQUAL',
215	'+=' => 'T_PLUS_EQUAL',
216	'*=' => 'T_MUL_EQUAL',
217	'/=' => 'T_DIV_EQUAL',
218	'%=' => 'T_MOD_EQUAL',
219	'++' => 'T_INC',
220	'--' => 'T_DEC',
221	'//' => 'T_COMMENT',
222	'/*' => 'T_COMMENT',
223	'/**' => 'T_DOC_COMMENT',
224	'*/' => 'T_COMMENT',
225	];
226
227	/**
228	* A list string delimiters.
229	*
230	* @var array
231	*/
232	protected $stringTokens = [
233	'\'' => '\'',
234	'"' => '"',
235	];
236
237	/**
238	* A list tokens that start and end comments.
239	*
240	* @var array
241	*/
242	protected $commentTokens = [
243	'//' => null,
244	'/' => '/',
245	'/*' => '/',
246	];
247
248
249	/**
250	* Initialise the tokenizer.
251	*
252	* Pre-checks the content to see if it looks minified.
253	*
254	* @param string $content The content to tokenize,
255	* @param \PHP_CodeSniffer\Config $config The config data for the run.
256	* @param string $eolChar The EOL char used in the content.
257	*
258	* @return void
259	* @throws \PHP_CodeSniffer\Exceptions\TokenizerException If the file appears to be minified.
260	*/
261	public function __construct($content, Config $config, $eolChar='\n')
262	{
263	if ($this->isMinifiedContent($content, $eolChar) === true) {
264	throw new TokenizerException('File appears to be minified and cannot be processed');
265	}
266
267	parent::__construct($content, $config, $eolChar);
268
269	}//end __construct()
270
271
272	/**
273	* Creates an array of tokens when given some JS code.
274	*
275	* @param string $string The string to tokenize.
276	*
277	* @return array
278	*/
279	public function tokenize($string)
280	{
281	if (PHP_CODESNIFFER_VERBOSITY > 1) {
282	echo "\t* START JS TOKENIZING *".PHP_EOL;
283	}
284
285	$maxTokenLength = 0;
286	foreach ($this->tokenValues as $token => $values) {
287	if (strlen($token) > $maxTokenLength) {
288	$maxTokenLength = strlen($token);
289	}
290	}
291
292	$tokens = [];
293	$inString = '';
294	$stringChar = null;
295	$inComment = '';
296	$buffer = '';
297	$preStringBuffer = '';
298	$cleanBuffer = false;
299
300	$commentTokenizer = new Comment();
301
302	$tokens[] = [
303	'code' => T_OPEN_TAG,
304	'type' => 'T_OPEN_TAG',
305	'content' => '',
306	];
307
308	// Convert newlines to single characters for ease of
309	// processing. We will change them back later.
310	$string = str_replace($this->eolChar, "\n", $string);
311
312	$chars = str_split($string);
313	$numChars = count($chars);
314	for ($i = 0; $i < $numChars; $i++) {
315	$char = $chars[$i];
316
317	if (PHP_CODESNIFFER_VERBOSITY > 1) {
318	$content = Util\Common::prepareForOutput($char);
319	$bufferContent = Util\Common::prepareForOutput($buffer);
320
321	if ($inString !== '') {
322	echo "\t";
323	}
324
325	if ($inComment !== '') {
326	echo "\t";
327	}
328
329	echo "\tProcess char $i => $content (buffer: $bufferContent)".PHP_EOL;
330	}//end if
331
332	if ($inString === '' && $inComment === '' && $buffer !== '') {
333	// If the buffer only has whitespace and we are about to
334	// add a character, store the whitespace first.
335	if (trim($char) !== '' && trim($buffer) === '') {
336	$tokens[] = [
337	'code' => T_WHITESPACE,
338	'type' => 'T_WHITESPACE',
339	'content' => str_replace("\n", $this->eolChar, $buffer),
340	];
341
342	if (PHP_CODESNIFFER_VERBOSITY > 1) {
343	$content = Util\Common::prepareForOutput($buffer);
344	echo "\t=> Added token T_WHITESPACE ($content)".PHP_EOL;
345	}
346
347	$buffer = '';
348	}
349
350	// If the buffer is not whitespace and we are about to
351	// add a whitespace character, store the content first.
352	if ($inString === ''
353	&& $inComment === ''
354	&& trim($char) === ''
355	&& trim($buffer) !== ''
356	) {
357	$tokens[] = [
358	'code' => T_STRING,
359	'type' => 'T_STRING',
360	'content' => str_replace("\n", $this->eolChar, $buffer),
361	];
362
363	if (PHP_CODESNIFFER_VERBOSITY > 1) {
364	$content = Util\Common::prepareForOutput($buffer);
365	echo "\t=> Added token T_STRING ($content)".PHP_EOL;
366	}
367
368	$buffer = '';
369	}
370	}//end if
371
372	// Process strings.
373	if ($inComment === '' && isset($this->stringTokens[$char]) === true) {
374	if ($inString === $char) {
375	// This could be the end of the string, but make sure it
376	// is not escaped first.
377	$escapes = 0;
378	for ($x = ($i - 1); $x >= 0; $x--) {
379	if ($chars[$x] !== '\\') {
380	break;
381	}
382
383	$escapes++;
384	}
385
386	if ($escapes === 0 \|\| ($escapes % 2) === 0) {
387	// There is an even number escape chars,
388	// so this is not escaped, it is the end of the string.
389	$tokens[] = [
390	'code' => T_CONSTANT_ENCAPSED_STRING,
391	'type' => 'T_CONSTANT_ENCAPSED_STRING',
392	'content' => str_replace("\n", $this->eolChar, $buffer).$char,
393	];
394
395	if (PHP_CODESNIFFER_VERBOSITY > 1) {
396	echo "\t\t* found end of string *".PHP_EOL;
397	$content = Util\Common::prepareForOutput($buffer.$char);
398	echo "\t=> Added token T_CONSTANT_ENCAPSED_STRING ($content)".PHP_EOL;
399	}
400
401	$buffer = '';
402	$preStringBuffer = '';
403	$inString = '';
404	$stringChar = null;
405	continue;
406	}//end if
407	} else if ($inString === '') {
408	$inString = $char;
409	$stringChar = $i;
410	$preStringBuffer = $buffer;
411
412	if (PHP_CODESNIFFER_VERBOSITY > 1) {
413	echo "\t\t* looking for string closer *".PHP_EOL;
414	}
415	}//end if
416	}//end if
417
418	if ($inString !== '' && $char === "\n") {
419	// Unless this newline character is escaped, the string did not
420	// end before the end of the line, which means it probably
421	// wasn't a string at all (maybe a regex).
422	if ($chars[($i - 1)] !== '\\') {
423	$i = $stringChar;
424	$buffer = $preStringBuffer;
425	$preStringBuffer = '';
426	$inString = '';
427	$stringChar = null;
428	$char = $chars[$i];
429
430	if (PHP_CODESNIFFER_VERBOSITY > 1) {
431	echo "\t\t* found newline before end of string, bailing *".PHP_EOL;
432	}
433	}
434	}
435
436	$buffer .= $char;
437
438	// We don't look for special tokens inside strings,
439	// so if we are in a string, we can continue here now
440	// that the current char is in the buffer.
441	if ($inString !== '') {
442	continue;
443	}
444
445	// Special case for T_DIVIDE which can actually be
446	// the start of a regular expression.
447	if ($buffer === $char && $char === '/' && $chars[($i + 1)] !== '*') {
448	$regex = $this->getRegexToken($i, $string, $chars, $tokens);
449	if ($regex !== null) {
450	$tokens[] = [
451	'code' => T_REGULAR_EXPRESSION,
452	'type' => 'T_REGULAR_EXPRESSION',
453	'content' => $regex['content'],
454	];
455
456	if (PHP_CODESNIFFER_VERBOSITY > 1) {
457	$content = Util\Common::prepareForOutput($regex['content']);
458	echo "\t=> Added token T_REGULAR_EXPRESSION ($content)".PHP_EOL;
459	}
460
461	$i = $regex['end'];
462	$buffer = '';
463	$cleanBuffer = false;
464	continue;
465	}//end if
466	}//end if
467
468	// Check for known tokens, but ignore tokens found that are not at
469	// the end of a string, like FOR and this.FORmat.
470	if (isset($this->tokenValues[strtolower($buffer)]) === true
471	&& (preg_match('\|[a-zA-z0-9_]\|', $char) === 0
472	\|\| isset($chars[($i + 1)]) === false
473	\|\| preg_match('\|[a-zA-z0-9_]\|', $chars[($i + 1)]) === 0)
474	) {
475	$matchedToken = false;
476	$lookAheadLength = ($maxTokenLength - strlen($buffer));
477
478	if ($lookAheadLength > 0) {
479	// The buffer contains a token type, but we need
480	// to look ahead at the next chars to see if this is
481	// actually part of a larger token. For example,
482	// FOR and FOREACH.
483	if (PHP_CODESNIFFER_VERBOSITY > 1) {
484	echo "\t\t* buffer possibly contains token, looking ahead $lookAheadLength chars *".PHP_EOL;
485	}
486
487	$charBuffer = $buffer;
488	for ($x = 1; $x <= $lookAheadLength; $x++) {
489	if (isset($chars[($i + $x)]) === false) {
490	break;
491	}
492
493	$charBuffer .= $chars[($i + $x)];
494
495	if (PHP_CODESNIFFER_VERBOSITY > 1) {
496	$content = Util\Common::prepareForOutput($charBuffer);
497	echo "\t\t=> Looking ahead $x chars => $content".PHP_EOL;
498	}
499
500	if (isset($this->tokenValues[strtolower($charBuffer)]) === true) {
501	// We've found something larger that matches
502	// so we can ignore this char. Except for 1 very specific
503	// case where a comment like /**/ needs to tokenize as
504	// T_COMMENT and not T_DOC_COMMENT.
505	$oldType = $this->tokenValues[strtolower($buffer)];
506	$newType = $this->tokenValues[strtolower($charBuffer)];
507	if ($oldType === 'T_COMMENT'
508	&& $newType === 'T_DOC_COMMENT'
509	&& $chars[($i + $x + 1)] === '/'
510	) {
511	if (PHP_CODESNIFFER_VERBOSITY > 1) {
512	echo "\t\t* look ahead ignored T_DOC_COMMENT, continuing *".PHP_EOL;
513	}
514	} else {
515	if (PHP_CODESNIFFER_VERBOSITY > 1) {
516	echo "\t\t* look ahead found more specific token ($newType), ignoring $i *".PHP_EOL;
517	}
518
519	$matchedToken = true;
520	break;
521	}
522	}//end if
523	}//end for
524	}//end if
525
526	if ($matchedToken === false) {
527	if (PHP_CODESNIFFER_VERBOSITY > 1 && $lookAheadLength > 0) {
528	echo "\t\t* look ahead found nothing *".PHP_EOL;
529	}
530
531	$value = $this->tokenValues[strtolower($buffer)];
532
533	if ($value === 'T_FUNCTION' && $buffer !== 'function') {
534	// The function keyword needs to be all lowercase or else
535	// it is just a function called "Function".
536	$value = 'T_STRING';
537	}
538
539	$tokens[] = [
540	'code' => constant($value),
541	'type' => $value,
542	'content' => $buffer,
543	];
544
545	if (PHP_CODESNIFFER_VERBOSITY > 1) {
546	$content = Util\Common::prepareForOutput($buffer);
547	echo "\t=> Added token $value ($content)".PHP_EOL;
548	}
549
550	$cleanBuffer = true;
551	}//end if
552	} else if (isset($this->tokenValues[strtolower($char)]) === true) {
553	// No matter what token we end up using, we don't
554	// need the content in the buffer any more because we have
555	// found a valid token.
556	$newContent = substr(str_replace("\n", $this->eolChar, $buffer), 0, -1);
557	if ($newContent !== '') {
558	$tokens[] = [
559	'code' => T_STRING,
560	'type' => 'T_STRING',
561	'content' => $newContent,
562	];
563
564	if (PHP_CODESNIFFER_VERBOSITY > 1) {
565	$content = Util\Common::prepareForOutput(substr($buffer, 0, -1));
566	echo "\t=> Added token T_STRING ($content)".PHP_EOL;
567	}
568	}
569
570	if (PHP_CODESNIFFER_VERBOSITY > 1) {
571	echo "\t\t* char is token, looking ahead ".($maxTokenLength - 1).' chars *'.PHP_EOL;
572	}
573
574	// The char is a token type, but we need to look ahead at the
575	// next chars to see if this is actually part of a larger token.
576	// For example, = and ===.
577	$charBuffer = $char;
578	$matchedToken = false;
579	for ($x = 1; $x <= $maxTokenLength; $x++) {
580	if (isset($chars[($i + $x)]) === false) {
581	break;
582	}
583
584	$charBuffer .= $chars[($i + $x)];
585
586	if (PHP_CODESNIFFER_VERBOSITY > 1) {
587	$content = Util\Common::prepareForOutput($charBuffer);
588	echo "\t\t=> Looking ahead $x chars => $content".PHP_EOL;
589	}
590
591	if (isset($this->tokenValues[strtolower($charBuffer)]) === true) {
592	// We've found something larger that matches
593	// so we can ignore this char.
594	if (PHP_CODESNIFFER_VERBOSITY > 1) {
595	$type = $this->tokenValues[strtolower($charBuffer)];
596	echo "\t\t* look ahead found more specific token ($type), ignoring $i *".PHP_EOL;
597	}
598
599	$matchedToken = true;
600	break;
601	}
602	}//end for
603
604	if ($matchedToken === false) {
605	$value = $this->tokenValues[strtolower($char)];
606	$tokens[] = [
607	'code' => constant($value),
608	'type' => $value,
609	'content' => $char,
610	];
611
612	if (PHP_CODESNIFFER_VERBOSITY > 1) {
613	echo "\t\t* look ahead found nothing *".PHP_EOL;
614	$content = Util\Common::prepareForOutput($char);
615	echo "\t=> Added token $value ($content)".PHP_EOL;
616	}
617
618	$cleanBuffer = true;
619	} else {
620	$buffer = $char;
621	}//end if
622	}//end if
623
624	// Keep track of content inside comments.
625	if ($inComment === ''
626	&& array_key_exists($buffer, $this->commentTokens) === true
627	) {
628	// This is not really a comment if the content
629	// looks like \// (i.e., it is escaped).
630	if (isset($chars[($i - 2)]) === true && $chars[($i - 2)] === '\\') {
631	$lastToken = array_pop($tokens);
632	$lastContent = $lastToken['content'];
633	if (PHP_CODESNIFFER_VERBOSITY > 1) {
634	$value = $this->tokenValues[strtolower($lastContent)];
635	$content = Util\Common::prepareForOutput($lastContent);
636	echo "\t=> Removed token $value ($content)".PHP_EOL;
637	}
638
639	$lastChars = str_split($lastContent);
640	$lastNumChars = count($lastChars);
641	for ($x = 0; $x < $lastNumChars; $x++) {
642	$lastChar = $lastChars[$x];
643	$value = $this->tokenValues[strtolower($lastChar)];
644	$tokens[] = [
645	'code' => constant($value),
646	'type' => $value,
647	'content' => $lastChar,
648	];
649
650	if (PHP_CODESNIFFER_VERBOSITY > 1) {
651	$content = Util\Common::prepareForOutput($lastChar);
652	echo "\t=> Added token $value ($content)".PHP_EOL;
653	}
654	}
655	} else {
656	// We have started a comment.
657	$inComment = $buffer;
658
659	if (PHP_CODESNIFFER_VERBOSITY > 1) {
660	echo "\t\t* looking for end of comment *".PHP_EOL;
661	}
662	}//end if
663	} else if ($inComment !== '') {
664	if ($this->commentTokens[$inComment] === null) {
665	// Comment ends at the next newline.
666	if (strpos($buffer, "\n") !== false) {
667	$inComment = '';
668	}
669	} else {
670	if ($this->commentTokens[$inComment] === $buffer) {
671	$inComment = '';
672	}
673	}
674
675	if (PHP_CODESNIFFER_VERBOSITY > 1) {
676	if ($inComment === '') {
677	echo "\t\t* found end of comment *".PHP_EOL;
678	}
679	}
680
681	if ($inComment === '' && $cleanBuffer === false) {
682	$tokens[] = [
683	'code' => T_STRING,
684	'type' => 'T_STRING',
685	'content' => str_replace("\n", $this->eolChar, $buffer),
686	];
687
688	if (PHP_CODESNIFFER_VERBOSITY > 1) {
689	$content = Util\Common::prepareForOutput($buffer);
690	echo "\t=> Added token T_STRING ($content)".PHP_EOL;
691	}
692
693	$buffer = '';
694	}
695	}//end if
696
697	if ($cleanBuffer === true) {
698	$buffer = '';
699	$cleanBuffer = false;
700	}
701	}//end for
702
703	if (empty($buffer) === false) {
704	if ($inString !== '') {
705	// The string did not end before the end of the file,
706	// which means there was probably a syntax error somewhere.
707	$tokens[] = [
708	'code' => T_STRING,
709	'type' => 'T_STRING',
710	'content' => str_replace("\n", $this->eolChar, $buffer),
711	];
712
713	if (PHP_CODESNIFFER_VERBOSITY > 1) {
714	$content = Util\Common::prepareForOutput($buffer);
715	echo "\t=> Added token T_STRING ($content)".PHP_EOL;
716	}
717	} else {
718	// Buffer contains whitespace from the end of the file.
719	$tokens[] = [
720	'code' => T_WHITESPACE,
721	'type' => 'T_WHITESPACE',
722	'content' => str_replace("\n", $this->eolChar, $buffer),
723	];
724
725	if (PHP_CODESNIFFER_VERBOSITY > 1) {
726	$content = Util\Common::prepareForOutput($buffer);
727	echo "\t=> Added token T_WHITESPACE ($content)".PHP_EOL;
728	}
729	}//end if
730	}//end if
731
732	$tokens[] = [
733	'code' => T_CLOSE_TAG,
734	'type' => 'T_CLOSE_TAG',
735	'content' => '',
736	];
737
738	/*
739	Now that we have done some basic tokenizing, we need to
740	modify the tokens to join some together and split some apart
741	so they match what the PHP tokenizer does.
742	*/
743
744	$finalTokens = [];
745	$newStackPtr = 0;
746	$numTokens = count($tokens);
747	for ($stackPtr = 0; $stackPtr < $numTokens; $stackPtr++) {
748	$token = $tokens[$stackPtr];
749
750	/*
751	Look for comments and join the tokens together.
752	*/
753
754	if ($token['code'] === T_COMMENT \|\| $token['code'] === T_DOC_COMMENT) {
755	$newContent = '';
756	$tokenContent = $token['content'];
757
758	$endContent = null;
759	if (isset($this->commentTokens[$tokenContent]) === true) {
760	$endContent = $this->commentTokens[$tokenContent];
761	}
762
763	while ($tokenContent !== $endContent) {
764	if ($endContent === null
765	&& strpos($tokenContent, $this->eolChar) !== false
766	) {
767	// A null end token means the comment ends at the end of
768	// the line so we look for newlines and split the token.
769	$tokens[$stackPtr]['content'] = substr(
770	$tokenContent,
771	(strpos($tokenContent, $this->eolChar) + strlen($this->eolChar))
772	);
773
774	$tokenContent = substr(
775	$tokenContent,
776	0,
777	(strpos($tokenContent, $this->eolChar) + strlen($this->eolChar))
778	);
779
780	// If the substr failed, skip the token as the content
781	// will now be blank.
782	if ($tokens[$stackPtr]['content'] !== false
783	&& $tokens[$stackPtr]['content'] !== ''
784	) {
785	$stackPtr--;
786	}
787
788	break;
789	}//end if
790
791	$stackPtr++;
792	$newContent .= $tokenContent;
793	if (isset($tokens[$stackPtr]) === false) {
794	break;
795	}
796
797	$tokenContent = $tokens[$stackPtr]['content'];
798	}//end while
799
800	if ($token['code'] === T_DOC_COMMENT) {
801	$commentTokens = $commentTokenizer->tokenizeString($newContent.$tokenContent, $this->eolChar, $newStackPtr);
802	foreach ($commentTokens as $commentToken) {
803	$finalTokens[$newStackPtr] = $commentToken;
804	$newStackPtr++;
805	}
806
807	continue;
808	} else {
809	// Save the new content in the current token so
810	// the code below can chop it up on newlines.
811	$token['content'] = $newContent.$tokenContent;
812	}
813	}//end if
814
815	/*
816	If this token has newlines in its content, split each line up
817	and create a new token for each line. We do this so it's easier
818	to ascertain where errors occur on a line.
819	Note that $token[1] is the token's content.
820	*/
821
822	if (strpos($token['content'], $this->eolChar) !== false) {
823	$tokenLines = explode($this->eolChar, $token['content']);
824	$numLines = count($tokenLines);
825
826	for ($i = 0; $i < $numLines; $i++) {
827	$newToken = ['content' => $tokenLines[$i]];
828	if ($i === ($numLines - 1)) {
829	if ($tokenLines[$i] === '') {
830	break;
831	}
832	} else {
833	$newToken['content'] .= $this->eolChar;
834	}
835
836	$newToken['type'] = $token['type'];
837	$newToken['code'] = $token['code'];
838	$finalTokens[$newStackPtr] = $newToken;
839	$newStackPtr++;
840	}
841	} else {
842	$finalTokens[$newStackPtr] = $token;
843	$newStackPtr++;
844	}//end if
845
846	// Convert numbers, including decimals.
847	if ($token['code'] === T_STRING
848	\|\| $token['code'] === T_OBJECT_OPERATOR
849	) {
850	$newContent = '';
851	$oldStackPtr = $stackPtr;
852	while (preg_match('\|^[0-9\.]+$\|', $tokens[$stackPtr]['content']) !== 0) {
853	$newContent .= $tokens[$stackPtr]['content'];
854	$stackPtr++;
855	}
856
857	if ($newContent !== '' && $newContent !== '.') {
858	$finalTokens[($newStackPtr - 1)]['content'] = $newContent;
859	if (ctype_digit($newContent) === true) {
860	$finalTokens[($newStackPtr - 1)]['code'] = constant('T_LNUMBER');
861	$finalTokens[($newStackPtr - 1)]['type'] = 'T_LNUMBER';
862	} else {
863	$finalTokens[($newStackPtr - 1)]['code'] = constant('T_DNUMBER');
864	$finalTokens[($newStackPtr - 1)]['type'] = 'T_DNUMBER';
865	}
866
867	$stackPtr--;
868	continue;
869	} else {
870	$stackPtr = $oldStackPtr;
871	}
872	}//end if
873
874	// Convert the token after an object operator into a string, in most cases.
875	if ($token['code'] === T_OBJECT_OPERATOR) {
876	for ($i = ($stackPtr + 1); $i < $numTokens; $i++) {
877	if (isset(Util\Tokens::$emptyTokens[$tokens[$i]['code']]) === true) {
878	continue;
879	}
880
881	if ($tokens[$i]['code'] !== T_PROTOTYPE
882	&& $tokens[$i]['code'] !== T_LNUMBER
883	&& $tokens[$i]['code'] !== T_DNUMBER
884	) {
885	$tokens[$i]['code'] = T_STRING;
886	$tokens[$i]['type'] = 'T_STRING';
887	}
888
889	break;
890	}
891	}
892	}//end for
893
894	if (PHP_CODESNIFFER_VERBOSITY > 1) {
895	echo "\t* END TOKENIZING *".PHP_EOL;
896	}
897
898	return $finalTokens;
899
900	}//end tokenize()
901
902
903	/**
904	* Tokenizes a regular expression if one is found.
905	*
906	* If a regular expression is not found, NULL is returned.
907	*
908	* @param string $char The index of the possible regex start character.
909	* @param string $string The complete content of the string being tokenized.
910	* @param string $chars An array of characters being tokenized.
911	* @param string $tokens The current array of tokens found in the string.
912	*
913	* @return array<string, string>\|null
914	*/
915	public function getRegexToken($char, $string, $chars, $tokens)
916	{
917	$beforeTokens = [
918	T_EQUAL => true,
919	T_IS_NOT_EQUAL => true,
920	T_IS_IDENTICAL => true,
921	T_IS_NOT_IDENTICAL => true,
922	T_OPEN_PARENTHESIS => true,
923	T_OPEN_SQUARE_BRACKET => true,
924	T_RETURN => true,
925	T_BOOLEAN_OR => true,
926	T_BOOLEAN_AND => true,
927	T_BOOLEAN_NOT => true,
928	T_BITWISE_OR => true,
929	T_BITWISE_AND => true,
930	T_COMMA => true,
931	T_COLON => true,
932	T_TYPEOF => true,
933	T_INLINE_THEN => true,
934	T_INLINE_ELSE => true,
935	];
936
937	$afterTokens = [
938	',' => true,
939	')' => true,
940	']' => true,
941	';' => true,
942	' ' => true,
943	'.' => true,
944	':' => true,
945	$this->eolChar => true,
946	];
947
948	// Find the last non-whitespace token that was added
949	// to the tokens array.
950	$numTokens = count($tokens);
951	for ($prev = ($numTokens - 1); $prev >= 0; $prev--) {
952	if (isset(Util\Tokens::$emptyTokens[$tokens[$prev]['code']]) === false) {
953	break;
954	}
955	}
956
957	if (isset($beforeTokens[$tokens[$prev]['code']]) === false) {
958	return null;
959	}
960
961	// This is probably a regular expression, so look for the end of it.
962	if (PHP_CODESNIFFER_VERBOSITY > 1) {
963	echo "\t* token possibly starts a regular expression *".PHP_EOL;
964	}
965
966	$numChars = count($chars);
967	for ($next = ($char + 1); $next < $numChars; $next++) {
968	if ($chars[$next] === '/') {
969	// Just make sure this is not escaped first.
970	if ($chars[($next - 1)] !== '\\') {
971	// In the simple form: /.../ so we found the end.
972	break;
973	} else if ($chars[($next - 2)] === '\\') {
974	// In the form: /...\\/ so we found the end.
975	break;
976	}
977	} else {
978	$possibleEolChar = substr($string, $next, strlen($this->eolChar));
979	if ($possibleEolChar === $this->eolChar) {
980	// This is the last token on the line and regular
981	// expressions need to be defined on a single line,
982	// so this is not a regular expression.
983	break;
984	}
985	}
986	}
987
988	if ($chars[$next] !== '/') {
989	if (PHP_CODESNIFFER_VERBOSITY > 1) {
990	echo "\t* could not find end of regular expression *".PHP_EOL;
991	}
992
993	return null;
994	}
995
996	while (preg_match('\|[a-zA-Z]\|', $chars[($next + 1)]) !== 0) {
997	// The token directly after the end of the regex can
998	// be modifiers like global and case insensitive
999	// (.e.g, /pattern/gi).
1000	$next++;
1001	}
1002
1003	$regexEnd = $next;
1004	if (PHP_CODESNIFFER_VERBOSITY > 1) {
1005	echo "\t* found end of regular expression at token $regexEnd *".PHP_EOL;
1006	}
1007
1008	for ($next += 1; $next < $numChars; $next++) {
1009	if ($chars[$next] !== ' ') {
1010	break;
1011	} else {
1012	$possibleEolChar = substr($string, $next, strlen($this->eolChar));
1013	if ($possibleEolChar === $this->eolChar) {
1014	// This is the last token on the line.
1015	break;
1016	}
1017	}
1018	}
1019
1020	if (isset($afterTokens[$chars[$next]]) === false) {
1021	if (PHP_CODESNIFFER_VERBOSITY > 1) {
1022	echo "\t* tokens after regular expression do not look correct *".PHP_EOL;
1023	}
1024
1025	return null;
1026	}
1027
1028	// This is a regular expression, so join all the tokens together.
1029	$content = '';
1030	for ($x = $char; $x <= $regexEnd; $x++) {
1031	$content .= $chars[$x];
1032	}
1033
1034	$token = [
1035	'start' => $char,
1036	'end' => $regexEnd,
1037	'content' => $content,
1038	];
1039
1040	return $token;
1041
1042	}//end getRegexToken()
1043
1044
1045	/**
1046	* Performs additional processing after main tokenizing.
1047	*
1048	* This additional processing looks for properties, closures, labels and objects.
1049	*
1050	* @return void
1051	*/
1052	public function processAdditional()
1053	{
1054	if (PHP_CODESNIFFER_VERBOSITY > 1) {
1055	echo "\t* START ADDITIONAL JS PROCESSING *".PHP_EOL;
1056	}
1057
1058	$numTokens = count($this->tokens);
1059	$classStack = [];
1060
1061	for ($i = 0; $i < $numTokens; $i++) {
1062	if (PHP_CODESNIFFER_VERBOSITY > 1) {
1063	$type = $this->tokens[$i]['type'];
1064	$content = Util\Common::prepareForOutput($this->tokens[$i]['content']);
1065
1066	echo str_repeat("\t", count($classStack));
1067	echo "\tProcess token $i: $type => $content".PHP_EOL;
1068	}
1069
1070	// Looking for functions that are actually closures.
1071	if ($this->tokens[$i]['code'] === T_FUNCTION && isset($this->tokens[$i]['scope_opener']) === true) {
1072	for ($x = ($i + 1); $x < $numTokens; $x++) {
1073	if (isset(Util\Tokens::$emptyTokens[$this->tokens[$x]['code']]) === false) {
1074	break;
1075	}
1076	}
1077
1078	if ($this->tokens[$x]['code'] === T_OPEN_PARENTHESIS) {
1079	$this->tokens[$i]['code'] = T_CLOSURE;
1080	$this->tokens[$i]['type'] = 'T_CLOSURE';
1081	if (PHP_CODESNIFFER_VERBOSITY > 1) {
1082	$line = $this->tokens[$i]['line'];
1083	echo str_repeat("\t", count($classStack));
1084	echo "\t* token $i on line $line changed from T_FUNCTION to T_CLOSURE *".PHP_EOL;
1085	}
1086
1087	for ($x = ($this->tokens[$i]['scope_opener'] + 1); $x < $this->tokens[$i]['scope_closer']; $x++) {
1088	if (isset($this->tokens[$x]['conditions'][$i]) === false) {
1089	continue;
1090	}
1091
1092	$this->tokens[$x]['conditions'][$i] = T_CLOSURE;
1093	if (PHP_CODESNIFFER_VERBOSITY > 1) {
1094	$type = $this->tokens[$x]['type'];
1095	echo str_repeat("\t", count($classStack));
1096	echo "\t\t* cleaned $x ($type) *".PHP_EOL;
1097	}
1098	}
1099	}//end if
1100
1101	continue;
1102	} else if ($this->tokens[$i]['code'] === T_OPEN_CURLY_BRACKET
1103	&& isset($this->tokens[$i]['scope_condition']) === false
1104	&& isset($this->tokens[$i]['bracket_closer']) === true
1105	) {
1106	$condition = $this->tokens[$i]['conditions'];
1107	$condition = end($condition);
1108	if ($condition === T_CLASS) {
1109	// Possibly an ES6 method. To be classified as one, the previous
1110	// non-empty tokens need to be a set of parenthesis, and then a string
1111	// (the method name).
1112	for ($parenCloser = ($i - 1); $parenCloser > 0; $parenCloser--) {
1113	if (isset(Util\Tokens::$emptyTokens[$this->tokens[$parenCloser]['code']]) === false) {
1114	break;
1115	}
1116	}
1117
1118	if ($this->tokens[$parenCloser]['code'] === T_CLOSE_PARENTHESIS) {
1119	$parenOpener = $this->tokens[$parenCloser]['parenthesis_opener'];
1120	for ($name = ($parenOpener - 1); $name > 0; $name--) {
1121	if (isset(Util\Tokens::$emptyTokens[$this->tokens[$name]['code']]) === false) {
1122	break;
1123	}
1124	}
1125
1126	if ($this->tokens[$name]['code'] === T_STRING) {
1127	// We found a method name.
1128	if (PHP_CODESNIFFER_VERBOSITY > 1) {
1129	$line = $this->tokens[$name]['line'];
1130	echo str_repeat("\t", count($classStack));
1131	echo "\t* token $name on line $line changed from T_STRING to T_FUNCTION *".PHP_EOL;
1132	}
1133
1134	$closer = $this->tokens[$i]['bracket_closer'];
1135
1136	$this->tokens[$name]['code'] = T_FUNCTION;
1137	$this->tokens[$name]['type'] = 'T_FUNCTION';
1138
1139	foreach ([$name, $i, $closer] as $token) {
1140	$this->tokens[$token]['scope_condition'] = $name;
1141	$this->tokens[$token]['scope_opener'] = $i;
1142	$this->tokens[$token]['scope_closer'] = $closer;
1143	$this->tokens[$token]['parenthesis_opener'] = $parenOpener;
1144	$this->tokens[$token]['parenthesis_closer'] = $parenCloser;
1145	$this->tokens[$token]['parenthesis_owner'] = $name;
1146	}
1147
1148	$this->tokens[$parenOpener]['parenthesis_owner'] = $name;
1149	$this->tokens[$parenCloser]['parenthesis_owner'] = $name;
1150
1151	for ($x = ($i + 1); $x < $closer; $x++) {
1152	$this->tokens[$x]['conditions'][$name] = T_FUNCTION;
1153	ksort($this->tokens[$x]['conditions'], SORT_NUMERIC);
1154	if (PHP_CODESNIFFER_VERBOSITY > 1) {
1155	$type = $this->tokens[$x]['type'];
1156	echo str_repeat("\t", count($classStack));
1157	echo "\t\t* added T_FUNCTION condition to $x ($type) *".PHP_EOL;
1158	}
1159	}
1160
1161	continue;
1162	}//end if
1163	}//end if
1164	}//end if
1165
1166	$classStack[] = $i;
1167
1168	$closer = $this->tokens[$i]['bracket_closer'];
1169	$this->tokens[$i]['code'] = T_OBJECT;
1170	$this->tokens[$i]['type'] = 'T_OBJECT';
1171	$this->tokens[$closer]['code'] = T_CLOSE_OBJECT;
1172	$this->tokens[$closer]['type'] = 'T_CLOSE_OBJECT';
1173
1174	if (PHP_CODESNIFFER_VERBOSITY > 1) {
1175	echo str_repeat("\t", count($classStack));
1176	echo "\t* token $i converted from T_OPEN_CURLY_BRACKET to T_OBJECT *".PHP_EOL;
1177	echo str_repeat("\t", count($classStack));
1178	echo "\t* token $closer converted from T_CLOSE_CURLY_BRACKET to T_CLOSE_OBJECT *".PHP_EOL;
1179	}
1180
1181	for ($x = ($i + 1); $x < $closer; $x++) {
1182	$this->tokens[$x]['conditions'][$i] = T_OBJECT;
1183	ksort($this->tokens[$x]['conditions'], SORT_NUMERIC);
1184	if (PHP_CODESNIFFER_VERBOSITY > 1) {
1185	$type = $this->tokens[$x]['type'];
1186	echo str_repeat("\t", count($classStack));
1187	echo "\t\t* added T_OBJECT condition to $x ($type) *".PHP_EOL;
1188	}
1189	}
1190	} else if ($this->tokens[$i]['code'] === T_CLOSE_OBJECT) {
1191	$opener = array_pop($classStack);
1192	} else if ($this->tokens[$i]['code'] === T_COLON) {
1193	// If it is a scope opener, it belongs to a
1194	// DEFAULT or CASE statement.
1195	if (isset($this->tokens[$i]['scope_condition']) === true) {
1196	continue;
1197	}
1198
1199	// Make sure this is not part of an inline IF statement.
1200	for ($x = ($i - 1); $x >= 0; $x--) {
1201	if ($this->tokens[$x]['code'] === T_INLINE_THEN) {
1202	$this->tokens[$i]['code'] = T_INLINE_ELSE;
1203	$this->tokens[$i]['type'] = 'T_INLINE_ELSE';
1204
1205	if (PHP_CODESNIFFER_VERBOSITY > 1) {
1206	echo str_repeat("\t", count($classStack));
1207	echo "\t* token $i converted from T_COLON to T_INLINE_THEN *".PHP_EOL;
1208	}
1209
1210	continue(2);
1211	} else if ($this->tokens[$x]['line'] < $this->tokens[$i]['line']) {
1212	break;
1213	}
1214	}
1215
1216	// The string to the left of the colon is either a property or label.
1217	for ($label = ($i - 1); $label >= 0; $label--) {
1218	if (isset(Util\Tokens::$emptyTokens[$this->tokens[$label]['code']]) === false) {
1219	break;
1220	}
1221	}
1222
1223	if ($this->tokens[$label]['code'] !== T_STRING
1224	&& $this->tokens[$label]['code'] !== T_CONSTANT_ENCAPSED_STRING
1225	) {
1226	continue;
1227	}
1228
1229	if (empty($classStack) === false) {
1230	$this->tokens[$label]['code'] = T_PROPERTY;
1231	$this->tokens[$label]['type'] = 'T_PROPERTY';
1232
1233	if (PHP_CODESNIFFER_VERBOSITY > 1) {
1234	echo str_repeat("\t", count($classStack));
1235	echo "\t* token $label converted from T_STRING to T_PROPERTY *".PHP_EOL;
1236	}
1237	} else {
1238	$this->tokens[$label]['code'] = T_LABEL;
1239	$this->tokens[$label]['type'] = 'T_LABEL';
1240
1241	if (PHP_CODESNIFFER_VERBOSITY > 1) {
1242	echo str_repeat("\t", count($classStack));
1243	echo "\t* token $label converted from T_STRING to T_LABEL *".PHP_EOL;
1244	}
1245	}//end if
1246	}//end if
1247	}//end for
1248
1249	if (PHP_CODESNIFFER_VERBOSITY > 1) {
1250	echo "\t* END ADDITIONAL JS PROCESSING *".PHP_EOL;
1251	}
1252
1253	}//end processAdditional()
1254
1255
1256	}//end class
1257