defender-security / src / extra / php-codesniffer-php-token / Tokenizers / JS.php

JS.php in Defender Security – Malware Scanner, Login Security & Firewall 6.2.1, at src/extra/php-codesniffer-php-token/Tokenizers/JS.php

1,252 lines 37.7 KB

Raw Download Zip

1	<?php
2	/**
3	* Tokenizes JS code.
4	*
5	* @author Greg Sherwood <[email protected]>
6	* @copyright 2006-2015 Squiz Pty Ltd (ABN 77 084 670 600)
7	* @license https://github.com/squizlabs/PHP_CodeSniffer/blob/master/licence.txt BSD Licence
8	*/
9
10	namespace PHP_CodeSniffer\Tokenizers;
11
12	use PHP_CodeSniffer\Config;
13	use PHP_CodeSniffer\Exceptions\TokenizerException;
14	use PHP_CodeSniffer\Util;
15
16	class JS extends Tokenizer {
17
18	/**
19	* A list of tokens that are allowed to open a scope.
20	*
21	* This array also contains information about what kind of token the scope
22	* opener uses to open and close the scope, if the token strictly requires
23	* an opener, if the token can share a scope closer, and who it can be shared
24	* with. An example of a token that shares a scope closer is a CASE scope.
25	*
26	* @var array
27	*/
28	public $scopeOpeners = [
29	T_IF => [
30	'start' => [ T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET ],
31	'end' => [ T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET ],
32	'strict' => false,
33	'shared' => false,
34	'with' => [],
35	],
36	T_TRY => [
37	'start' => [ T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET ],
38	'end' => [ T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET ],
39	'strict' => true,
40	'shared' => false,
41	'with' => [],
42	],
43	T_CATCH => [
44	'start' => [ T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET ],
45	'end' => [ T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET ],
46	'strict' => true,
47	'shared' => false,
48	'with' => [],
49	],
50	T_ELSE => [
51	'start' => [ T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET ],
52	'end' => [ T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET ],
53	'strict' => false,
54	'shared' => false,
55	'with' => [],
56	],
57	T_FOR => [
58	'start' => [ T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET ],
59	'end' => [ T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET ],
60	'strict' => false,
61	'shared' => false,
62	'with' => [],
63	],
64	T_CLASS => [
65	'start' => [ T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET ],
66	'end' => [ T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET ],
67	'strict' => true,
68	'shared' => false,
69	'with' => [],
70	],
71	T_FUNCTION => [
72	'start' => [ T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET ],
73	'end' => [ T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET ],
74	'strict' => false,
75	'shared' => false,
76	'with' => [],
77	],
78	T_WHILE => [
79	'start' => [ T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET ],
80	'end' => [ T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET ],
81	'strict' => false,
82	'shared' => false,
83	'with' => [],
84	],
85	T_DO => [
86	'start' => [ T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET ],
87	'end' => [ T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET ],
88	'strict' => true,
89	'shared' => false,
90	'with' => [],
91	],
92	T_SWITCH => [
93	'start' => [ T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET ],
94	'end' => [ T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET ],
95	'strict' => true,
96	'shared' => false,
97	'with' => [],
98	],
99	T_CASE => [
100	'start' => [ T_COLON => T_COLON ],
101	'end' => [
102	T_BREAK => T_BREAK,
103	T_RETURN => T_RETURN,
104	T_CONTINUE => T_CONTINUE,
105	T_THROW => T_THROW,
106	],
107	'strict' => true,
108	'shared' => true,
109	'with' => [
110	T_DEFAULT => T_DEFAULT,
111	T_CASE => T_CASE,
112	T_SWITCH => T_SWITCH,
113	],
114	],
115	T_DEFAULT => [
116	'start' => [ T_COLON => T_COLON ],
117	'end' => [
118	T_BREAK => T_BREAK,
119	T_RETURN => T_RETURN,
120	T_CONTINUE => T_CONTINUE,
121	T_THROW => T_THROW,
122	],
123	'strict' => true,
124	'shared' => true,
125	'with' => [
126	T_CASE => T_CASE,
127	T_SWITCH => T_SWITCH,
128	],
129	],
130	];
131
132	/**
133	* A list of tokens that end the scope.
134	*
135	* This array is just a unique collection of the end tokens
136	* from the _scopeOpeners array. The data is duplicated here to
137	* save time during parsing of the file.
138	*
139	* @var array
140	*/
141	public $endScopeTokens = [
142	T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET,
143	T_BREAK => T_BREAK,
144	];
145
146	/**
147	* A list of special JS tokens and their types.
148	*
149	* @var array
150	*/
151	protected $tokenValues = [
152	'class' => 'T_CLASS',
153	'function' => 'T_FUNCTION',
154	'prototype' => 'T_PROTOTYPE',
155	'try' => 'T_TRY',
156	'catch' => 'T_CATCH',
157	'return' => 'T_RETURN',
158	'throw' => 'T_THROW',
159	'break' => 'T_BREAK',
160	'switch' => 'T_SWITCH',
161	'continue' => 'T_CONTINUE',
162	'if' => 'T_IF',
163	'else' => 'T_ELSE',
164	'do' => 'T_DO',
165	'while' => 'T_WHILE',
166	'for' => 'T_FOR',
167	'var' => 'T_VAR',
168	'case' => 'T_CASE',
169	'default' => 'T_DEFAULT',
170	'true' => 'T_TRUE',
171	'false' => 'T_FALSE',
172	'null' => 'T_NULL',
173	'this' => 'T_THIS',
174	'typeof' => 'T_TYPEOF',
175	'(' => 'T_OPEN_PARENTHESIS',
176	')' => 'T_CLOSE_PARENTHESIS',
177	'{' => 'T_OPEN_CURLY_BRACKET',
178	'}' => 'T_CLOSE_CURLY_BRACKET',
179	'[' => 'T_OPEN_SQUARE_BRACKET',
180	']' => 'T_CLOSE_SQUARE_BRACKET',
181	'?' => 'T_INLINE_THEN',
182	'.' => 'T_OBJECT_OPERATOR',
183	'+' => 'T_PLUS',
184	'-' => 'T_MINUS',
185	'*' => 'T_MULTIPLY',
186	'%' => 'T_MODULUS',
187	'/' => 'T_DIVIDE',
188	'^' => 'T_LOGICAL_XOR',
189	',' => 'T_COMMA',
190	';' => 'T_SEMICOLON',
191	':' => 'T_COLON',
192	'<' => 'T_LESS_THAN',
193	'>' => 'T_GREATER_THAN',
194	'<<' => 'T_SL',
195	'>>' => 'T_SR',
196	'>>>' => 'T_ZSR',
197	'<<=' => 'T_SL_EQUAL',
198	'>>=' => 'T_SR_EQUAL',
199	'>>>=' => 'T_ZSR_EQUAL',
200	'<=' => 'T_IS_SMALLER_OR_EQUAL',
201	'>=' => 'T_IS_GREATER_OR_EQUAL',
202	'=>' => 'T_DOUBLE_ARROW',
203	'!' => 'T_BOOLEAN_NOT',
204	'\|\|' => 'T_BOOLEAN_OR',
205	'&&' => 'T_BOOLEAN_AND',
206	'\|' => 'T_BITWISE_OR',
207	'&' => 'T_BITWISE_AND',
208	'!=' => 'T_IS_NOT_EQUAL',
209	'!==' => 'T_IS_NOT_IDENTICAL',
210	'=' => 'T_EQUAL',
211	'==' => 'T_IS_EQUAL',
212	'===' => 'T_IS_IDENTICAL',
213	'-=' => 'T_MINUS_EQUAL',
214	'+=' => 'T_PLUS_EQUAL',
215	'*=' => 'T_MUL_EQUAL',
216	'/=' => 'T_DIV_EQUAL',
217	'%=' => 'T_MOD_EQUAL',
218	'++' => 'T_INC',
219	'--' => 'T_DEC',
220	'//' => 'T_COMMENT',
221	'/*' => 'T_COMMENT',
222	'/**' => 'T_DOC_COMMENT',
223	'*/' => 'T_COMMENT',
224	];
225
226	/**
227	* A list string delimiters.
228	*
229	* @var array
230	*/
231	protected $stringTokens = [
232	'\'' => '\'',
233	'"' => '"',
234	];
235
236	/**
237	* A list tokens that start and end comments.
238	*
239	* @var array
240	*/
241	protected $commentTokens = [
242	'//' => null,
243	'/' => '/',
244	'/*' => '/',
245	];
246
247
248	/**
249	* Initialise the tokenizer.
250	*
251	* Pre-checks the content to see if it looks minified.
252	*
253	* @param string $content The content to tokenize,
254	* @param \PHP_CodeSniffer\Config $config The config data for the run.
255	* @param string $eolChar The EOL char used in the content.
256	*
257	* @return void
258	* @throws \PHP_CodeSniffer\Exceptions\TokenizerException If the file appears to be minified.
259	*/
260	public function __construct( $content, Config $config, $eolChar = '\n' ) {
261	if ( $this->isMinifiedContent( $content, $eolChar ) === true ) {
262	throw new TokenizerException( 'File appears to be minified and cannot be processed' );
263	}
264
265	parent::__construct( $content, $config, $eolChar );
266
267	}//end __construct()
268
269
270	/**
271	* Creates an array of tokens when given some JS code.
272	*
273	* @param string $string The string to tokenize.
274	*
275	* @return array
276	*/
277	public function tokenize( $string ) {
278	if ( PHP_CODESNIFFER_VERBOSITY > 1 ) {
279	echo "\t* START JS TOKENIZING *" . PHP_EOL;
280	}
281
282	$maxTokenLength = 0;
283	foreach ( $this->tokenValues as $token => $values ) {
284	if ( strlen( $token ) > $maxTokenLength ) {
285	$maxTokenLength = strlen( $token );
286	}
287	}
288
289	$tokens = [];
290	$inString = '';
291	$stringChar = null;
292	$inComment = '';
293	$buffer = '';
294	$preStringBuffer = '';
295	$cleanBuffer = false;
296
297	$commentTokenizer = new Comment();
298
299	$tokens[] = [
300	'code' => T_OPEN_TAG,
301	'type' => 'T_OPEN_TAG',
302	'content' => '',
303	];
304
305	// Convert newlines to single characters for ease of
306	// processing. We will change them back later.
307	$string = str_replace( $this->eolChar, "\n", $string );
308
309	$chars = str_split( $string );
310	$numChars = count( $chars );
311	for ( $i = 0; $i < $numChars; $i ++ ) {
312	$char = $chars[ $i ];
313
314	if ( PHP_CODESNIFFER_VERBOSITY > 1 ) {
315	$content = Util\Common::prepareForOutput( $char );
316	$bufferContent = Util\Common::prepareForOutput( $buffer );
317
318	if ( $inString !== '' ) {
319	echo "\t";
320	}
321
322	if ( $inComment !== '' ) {
323	echo "\t";
324	}
325
326	echo "\tProcess char $i => $content (buffer: $bufferContent)" . PHP_EOL;
327	}//end if
328
329	if ( $inString === '' && $inComment === '' && $buffer !== '' ) {
330	// If the buffer only has whitespace and we are about to
331	// add a character, store the whitespace first.
332	if ( trim( $char ) !== '' && trim( $buffer ) === '' ) {
333	$tokens[] = [
334	'code' => T_WHITESPACE,
335	'type' => 'T_WHITESPACE',
336	'content' => str_replace( "\n", $this->eolChar, $buffer ),
337	];
338
339	if ( PHP_CODESNIFFER_VERBOSITY > 1 ) {
340	$content = Util\Common::prepareForOutput( $buffer );
341	echo "\t=> Added token T_WHITESPACE ($content)" . PHP_EOL;
342	}
343
344	$buffer = '';
345	}
346
347	// If the buffer is not whitespace and we are about to
348	// add a whitespace character, store the content first.
349	if ( $inString === ''
350	&& $inComment === ''
351	&& trim( $char ) === ''
352	&& trim( $buffer ) !== ''
353	) {
354	$tokens[] = [
355	'code' => T_STRING,
356	'type' => 'T_STRING',
357	'content' => str_replace( "\n", $this->eolChar, $buffer ),
358	];
359
360	if ( PHP_CODESNIFFER_VERBOSITY > 1 ) {
361	$content = Util\Common::prepareForOutput( $buffer );
362	echo "\t=> Added token T_STRING ($content)" . PHP_EOL;
363	}
364
365	$buffer = '';
366	}
367	}//end if
368
369	// Process strings.
370	if ( $inComment === '' && isset( $this->stringTokens[ $char ] ) === true ) {
371	if ( $inString === $char ) {
372	// This could be the end of the string, but make sure it
373	// is not escaped first.
374	$escapes = 0;
375	for ( $x = ( $i - 1 ); $x >= 0; $x -- ) {
376	if ( $chars[ $x ] !== '\\' ) {
377	break;
378	}
379
380	$escapes ++;
381	}
382
383	if ( $escapes === 0 \|\| ( $escapes % 2 ) === 0 ) {
384	// There is an even number escape chars,
385	// so this is not escaped, it is the end of the string.
386	$tokens[] = [
387	'code' => T_CONSTANT_ENCAPSED_STRING,
388	'type' => 'T_CONSTANT_ENCAPSED_STRING',
389	'content' => str_replace( "\n", $this->eolChar, $buffer ) . $char,
390	];
391
392	if ( PHP_CODESNIFFER_VERBOSITY > 1 ) {
393	echo "\t\t* found end of string *" . PHP_EOL;
394	$content = Util\Common::prepareForOutput( $buffer . $char );
395	echo "\t=> Added token T_CONSTANT_ENCAPSED_STRING ($content)" . PHP_EOL;
396	}
397
398	$buffer = '';
399	$preStringBuffer = '';
400	$inString = '';
401	$stringChar = null;
402	continue;
403	}//end if
404	} else if ( $inString === '' ) {
405	$inString = $char;
406	$stringChar = $i;
407	$preStringBuffer = $buffer;
408
409	if ( PHP_CODESNIFFER_VERBOSITY > 1 ) {
410	echo "\t\t* looking for string closer *" . PHP_EOL;
411	}
412	}//end if
413	}//end if
414
415	if ( $inString !== '' && $char === "\n" ) {
416	// Unless this newline character is escaped, the string did not
417	// end before the end of the line, which means it probably
418	// wasn't a string at all (maybe a regex).
419	if ( $chars[ ( $i - 1 ) ] !== '\\' ) {
420	$i = $stringChar;
421	$buffer = $preStringBuffer;
422	$preStringBuffer = '';
423	$inString = '';
424	$stringChar = null;
425	$char = $chars[ $i ];
426
427	if ( PHP_CODESNIFFER_VERBOSITY > 1 ) {
428	echo "\t\t* found newline before end of string, bailing *" . PHP_EOL;
429	}
430	}
431	}
432
433	$buffer .= $char;
434
435	// We don't look for special tokens inside strings,
436	// so if we are in a string, we can continue here now
437	// that the current char is in the buffer.
438	if ( $inString !== '' ) {
439	continue;
440	}
441
442	// Special case for T_DIVIDE which can actually be
443	// the start of a regular expression.
444	if ( $buffer === $char && $char === '/' && $chars[ ( $i + 1 ) ] !== '*' ) {
445	$regex = $this->getRegexToken( $i, $string, $chars, $tokens );
446	if ( $regex !== null ) {
447	$tokens[] = [
448	'code' => T_REGULAR_EXPRESSION,
449	'type' => 'T_REGULAR_EXPRESSION',
450	'content' => $regex['content'],
451	];
452
453	if ( PHP_CODESNIFFER_VERBOSITY > 1 ) {
454	$content = Util\Common::prepareForOutput( $regex['content'] );
455	echo "\t=> Added token T_REGULAR_EXPRESSION ($content)" . PHP_EOL;
456	}
457
458	$i = $regex['end'];
459	$buffer = '';
460	$cleanBuffer = false;
461	continue;
462	}//end if
463	}//end if
464
465	// Check for known tokens, but ignore tokens found that are not at
466	// the end of a string, like FOR and this.FORmat.
467	if ( isset( $this->tokenValues[ strtolower( $buffer ) ] ) === true
468	&& ( preg_match( '\|[a-zA-z0-9_]\|', $char ) === 0
469	\|\| isset( $chars[ ( $i + 1 ) ] ) === false
470	\|\| preg_match( '\|[a-zA-z0-9_]\|', $chars[ ( $i + 1 ) ] ) === 0 )
471	) {
472	$matchedToken = false;
473	$lookAheadLength = ( $maxTokenLength - strlen( $buffer ) );
474
475	if ( $lookAheadLength > 0 ) {
476	// The buffer contains a token type, but we need
477	// to look ahead at the next chars to see if this is
478	// actually part of a larger token. For example,
479	// FOR and FOREACH.
480	if ( PHP_CODESNIFFER_VERBOSITY > 1 ) {
481	echo "\t\t* buffer possibly contains token, looking ahead $lookAheadLength chars *" . PHP_EOL;
482	}
483
484	$charBuffer = $buffer;
485	for ( $x = 1; $x <= $lookAheadLength; $x ++ ) {
486	if ( isset( $chars[ ( $i + $x ) ] ) === false ) {
487	break;
488	}
489
490	$charBuffer .= $chars[ ( $i + $x ) ];
491
492	if ( PHP_CODESNIFFER_VERBOSITY > 1 ) {
493	$content = Util\Common::prepareForOutput( $charBuffer );
494	echo "\t\t=> Looking ahead $x chars => $content" . PHP_EOL;
495	}
496
497	if ( isset( $this->tokenValues[ strtolower( $charBuffer ) ] ) === true ) {
498	// We've found something larger that matches
499	// so we can ignore this char. Except for 1 very specific
500	// case where a comment like /**/ needs to tokenize as
501	// T_COMMENT and not T_DOC_COMMENT.
502	$oldType = $this->tokenValues[ strtolower( $buffer ) ];
503	$newType = $this->tokenValues[ strtolower( $charBuffer ) ];
504	if ( $oldType === 'T_COMMENT'
505	&& $newType === 'T_DOC_COMMENT'
506	&& $chars[ ( $i + $x + 1 ) ] === '/'
507	) {
508	if ( PHP_CODESNIFFER_VERBOSITY > 1 ) {
509	echo "\t\t* look ahead ignored T_DOC_COMMENT, continuing *" . PHP_EOL;
510	}
511	} else {
512	if ( PHP_CODESNIFFER_VERBOSITY > 1 ) {
513	echo "\t\t* look ahead found more specific token ($newType), ignoring $i *" . PHP_EOL;
514	}
515
516	$matchedToken = true;
517	break;
518	}
519	}//end if
520	}//end for
521	}//end if
522
523	if ( $matchedToken === false ) {
524	if ( PHP_CODESNIFFER_VERBOSITY > 1 && $lookAheadLength > 0 ) {
525	echo "\t\t* look ahead found nothing *" . PHP_EOL;
526	}
527
528	$value = $this->tokenValues[ strtolower( $buffer ) ];
529
530	if ( $value === 'T_FUNCTION' && $buffer !== 'function' ) {
531	// The function keyword needs to be all lowercase or else
532	// it is just a function called "Function".
533	$value = 'T_STRING';
534	}
535
536	$tokens[] = [
537	'code' => constant( $value ),
538	'type' => $value,
539	'content' => $buffer,
540	];
541
542	if ( PHP_CODESNIFFER_VERBOSITY > 1 ) {
543	$content = Util\Common::prepareForOutput( $buffer );
544	echo "\t=> Added token $value ($content)" . PHP_EOL;
545	}
546
547	$cleanBuffer = true;
548	}//end if
549	} else if ( isset( $this->tokenValues[ strtolower( $char ) ] ) === true ) {
550	// No matter what token we end up using, we don't
551	// need the content in the buffer any more because we have
552	// found a valid token.
553	$newContent = substr( str_replace( "\n", $this->eolChar, $buffer ), 0, - 1 );
554	if ( $newContent !== '' ) {
555	$tokens[] = [
556	'code' => T_STRING,
557	'type' => 'T_STRING',
558	'content' => $newContent,
559	];
560
561	if ( PHP_CODESNIFFER_VERBOSITY > 1 ) {
562	$content = Util\Common::prepareForOutput( substr( $buffer, 0, - 1 ) );
563	echo "\t=> Added token T_STRING ($content)" . PHP_EOL;
564	}
565	}
566
567	if ( PHP_CODESNIFFER_VERBOSITY > 1 ) {
568	echo "\t\t* char is token, looking ahead " . ( $maxTokenLength - 1 ) . ' chars *' . PHP_EOL;
569	}
570
571	// The char is a token type, but we need to look ahead at the
572	// next chars to see if this is actually part of a larger token.
573	// For example, = and ===.
574	$charBuffer = $char;
575	$matchedToken = false;
576	for ( $x = 1; $x <= $maxTokenLength; $x ++ ) {
577	if ( isset( $chars[ ( $i + $x ) ] ) === false ) {
578	break;
579	}
580
581	$charBuffer .= $chars[ ( $i + $x ) ];
582
583	if ( PHP_CODESNIFFER_VERBOSITY > 1 ) {
584	$content = Util\Common::prepareForOutput( $charBuffer );
585	echo "\t\t=> Looking ahead $x chars => $content" . PHP_EOL;
586	}
587
588	if ( isset( $this->tokenValues[ strtolower( $charBuffer ) ] ) === true ) {
589	// We've found something larger that matches
590	// so we can ignore this char.
591	if ( PHP_CODESNIFFER_VERBOSITY > 1 ) {
592	$type = $this->tokenValues[ strtolower( $charBuffer ) ];
593	echo "\t\t* look ahead found more specific token ($type), ignoring $i *" . PHP_EOL;
594	}
595
596	$matchedToken = true;
597	break;
598	}
599	}//end for
600
601	if ( $matchedToken === false ) {
602	$value = $this->tokenValues[ strtolower( $char ) ];
603	$tokens[] = [
604	'code' => constant( $value ),
605	'type' => $value,
606	'content' => $char,
607	];
608
609	if ( PHP_CODESNIFFER_VERBOSITY > 1 ) {
610	echo "\t\t* look ahead found nothing *" . PHP_EOL;
611	$content = Util\Common::prepareForOutput( $char );
612	echo "\t=> Added token $value ($content)" . PHP_EOL;
613	}
614
615	$cleanBuffer = true;
616	} else {
617	$buffer = $char;
618	}//end if
619	}//end if
620
621	// Keep track of content inside comments.
622	if ( $inComment === ''
623	&& array_key_exists( $buffer, $this->commentTokens ) === true
624	) {
625	// This is not really a comment if the content
626	// looks like \// (i.e., it is escaped).
627	if ( isset( $chars[ ( $i - 2 ) ] ) === true && $chars[ ( $i - 2 ) ] === '\\' ) {
628	$lastToken = array_pop( $tokens );
629	$lastContent = $lastToken['content'];
630	if ( PHP_CODESNIFFER_VERBOSITY > 1 ) {
631	$value = $this->tokenValues[ strtolower( $lastContent ) ];
632	$content = Util\Common::prepareForOutput( $lastContent );
633	echo "\t=> Removed token $value ($content)" . PHP_EOL;
634	}
635
636	$lastChars = str_split( $lastContent );
637	$lastNumChars = count( $lastChars );
638	for ( $x = 0; $x < $lastNumChars; $x ++ ) {
639	$lastChar = $lastChars[ $x ];
640	$value = $this->tokenValues[ strtolower( $lastChar ) ];
641	$tokens[] = [
642	'code' => constant( $value ),
643	'type' => $value,
644	'content' => $lastChar,
645	];
646
647	if ( PHP_CODESNIFFER_VERBOSITY > 1 ) {
648	$content = Util\Common::prepareForOutput( $lastChar );
649	echo "\t=> Added token $value ($content)" . PHP_EOL;
650	}
651	}
652	} else {
653	// We have started a comment.
654	$inComment = $buffer;
655
656	if ( PHP_CODESNIFFER_VERBOSITY > 1 ) {
657	echo "\t\t* looking for end of comment *" . PHP_EOL;
658	}
659	}//end if
660	} else if ( $inComment !== '' ) {
661	if ( $this->commentTokens[ $inComment ] === null ) {
662	// Comment ends at the next newline.
663	if ( strpos( $buffer, "\n" ) !== false ) {
664	$inComment = '';
665	}
666	} else {
667	if ( $this->commentTokens[ $inComment ] === $buffer ) {
668	$inComment = '';
669	}
670	}
671
672	if ( PHP_CODESNIFFER_VERBOSITY > 1 ) {
673	if ( $inComment === '' ) {
674	echo "\t\t* found end of comment *" . PHP_EOL;
675	}
676	}
677
678	if ( $inComment === '' && $cleanBuffer === false ) {
679	$tokens[] = [
680	'code' => T_STRING,
681	'type' => 'T_STRING',
682	'content' => str_replace( "\n", $this->eolChar, $buffer ),
683	];
684
685	if ( PHP_CODESNIFFER_VERBOSITY > 1 ) {
686	$content = Util\Common::prepareForOutput( $buffer );
687	echo "\t=> Added token T_STRING ($content)" . PHP_EOL;
688	}
689
690	$buffer = '';
691	}
692	}//end if
693
694	if ( $cleanBuffer === true ) {
695	$buffer = '';
696	$cleanBuffer = false;
697	}
698	}//end for
699
700	if ( empty( $buffer ) === false ) {
701	if ( $inString !== '' ) {
702	// The string did not end before the end of the file,
703	// which means there was probably a syntax error somewhere.
704	$tokens[] = [
705	'code' => T_STRING,
706	'type' => 'T_STRING',
707	'content' => str_replace( "\n", $this->eolChar, $buffer ),
708	];
709
710	if ( PHP_CODESNIFFER_VERBOSITY > 1 ) {
711	$content = Util\Common::prepareForOutput( $buffer );
712	echo "\t=> Added token T_STRING ($content)" . PHP_EOL;
713	}
714	} else {
715	// Buffer contains whitespace from the end of the file.
716	$tokens[] = [
717	'code' => T_WHITESPACE,
718	'type' => 'T_WHITESPACE',
719	'content' => str_replace( "\n", $this->eolChar, $buffer ),
720	];
721
722	if ( PHP_CODESNIFFER_VERBOSITY > 1 ) {
723	$content = Util\Common::prepareForOutput( $buffer );
724	echo "\t=> Added token T_WHITESPACE ($content)" . PHP_EOL;
725	}
726	}//end if
727	}//end if
728
729	$tokens[] = [
730	'code' => T_CLOSE_TAG,
731	'type' => 'T_CLOSE_TAG',
732	'content' => '',
733	];
734
735	/*
736	Now that we have done some basic tokenizing, we need to
737	modify the tokens to join some together and split some apart
738	so they match what the PHP tokenizer does.
739	*/
740
741	$finalTokens = [];
742	$newStackPtr = 0;
743	$numTokens = count( $tokens );
744	for ( $stackPtr = 0; $stackPtr < $numTokens; $stackPtr ++ ) {
745	$token = $tokens[ $stackPtr ];
746
747	/*
748	Look for comments and join the tokens together.
749	*/
750
751	if ( $token['code'] === T_COMMENT \|\| $token['code'] === T_DOC_COMMENT ) {
752	$newContent = '';
753	$tokenContent = $token['content'];
754
755	$endContent = null;
756	if ( isset( $this->commentTokens[ $tokenContent ] ) === true ) {
757	$endContent = $this->commentTokens[ $tokenContent ];
758	}
759
760	while ( $tokenContent !== $endContent ) {
761	if ( $endContent === null
762	&& strpos( $tokenContent, $this->eolChar ) !== false
763	) {
764	// A null end token means the comment ends at the end of
765	// the line so we look for newlines and split the token.
766	$tokens[ $stackPtr ]['content'] = substr(
767	$tokenContent,
768	( strpos( $tokenContent, $this->eolChar ) + strlen( $this->eolChar ) )
769	);
770
771	$tokenContent = substr(
772	$tokenContent,
773	0,
774	( strpos( $tokenContent, $this->eolChar ) + strlen( $this->eolChar ) )
775	);
776
777	// If the substr failed, skip the token as the content
778	// will now be blank.
779	if ( $tokens[ $stackPtr ]['content'] !== false
780	&& $tokens[ $stackPtr ]['content'] !== ''
781	) {
782	$stackPtr --;
783	}
784
785	break;
786	}//end if
787
788	$stackPtr ++;
789	$newContent .= $tokenContent;
790	if ( isset( $tokens[ $stackPtr ] ) === false ) {
791	break;
792	}
793
794	$tokenContent = $tokens[ $stackPtr ]['content'];
795	}//end while
796
797	if ( $token['code'] === T_DOC_COMMENT ) {
798	$commentTokens = $commentTokenizer->tokenizeString( $newContent . $tokenContent, $this->eolChar, $newStackPtr );
799	foreach ( $commentTokens as $commentToken ) {
800	$finalTokens[ $newStackPtr ] = $commentToken;
801	$newStackPtr ++;
802	}
803
804	continue;
805	} else {
806	// Save the new content in the current token so
807	// the code below can chop it up on newlines.
808	$token['content'] = $newContent . $tokenContent;
809	}
810	}//end if
811
812	/*
813	If this token has newlines in its content, split each line up
814	and create a new token for each line. We do this so it's easier
815	to ascertain where errors occur on a line.
816	Note that $token[1] is the token's content.
817	*/
818
819	if ( strpos( $token['content'], $this->eolChar ) !== false ) {
820	$tokenLines = explode( $this->eolChar, $token['content'] );
821	$numLines = count( $tokenLines );
822
823	for ( $i = 0; $i < $numLines; $i ++ ) {
824	$newToken = [ 'content' => $tokenLines[ $i ] ];
825	if ( $i === ( $numLines - 1 ) ) {
826	if ( $tokenLines[ $i ] === '' ) {
827	break;
828	}
829	} else {
830	$newToken['content'] .= $this->eolChar;
831	}
832
833	$newToken['type'] = $token['type'];
834	$newToken['code'] = $token['code'];
835	$finalTokens[ $newStackPtr ] = $newToken;
836	$newStackPtr ++;
837	}
838	} else {
839	$finalTokens[ $newStackPtr ] = $token;
840	$newStackPtr ++;
841	}//end if
842
843	// Convert numbers, including decimals.
844	if ( $token['code'] === T_STRING
845	\|\| $token['code'] === T_OBJECT_OPERATOR
846	) {
847	$newContent = '';
848	$oldStackPtr = $stackPtr;
849	while ( preg_match( '\|^[0-9\.]+$\|', $tokens[ $stackPtr ]['content'] ) !== 0 ) {
850	$newContent .= $tokens[ $stackPtr ]['content'];
851	$stackPtr ++;
852	}
853
854	if ( $newContent !== '' && $newContent !== '.' ) {
855	$finalTokens[ ( $newStackPtr - 1 ) ]['content'] = $newContent;
856	if ( ctype_digit( $newContent ) === true ) {
857	$finalTokens[ ( $newStackPtr - 1 ) ]['code'] = constant( 'T_LNUMBER' );
858	$finalTokens[ ( $newStackPtr - 1 ) ]['type'] = 'T_LNUMBER';
859	} else {
860	$finalTokens[ ( $newStackPtr - 1 ) ]['code'] = constant( 'T_DNUMBER' );
861	$finalTokens[ ( $newStackPtr - 1 ) ]['type'] = 'T_DNUMBER';
862	}
863
864	$stackPtr --;
865	continue;
866	} else {
867	$stackPtr = $oldStackPtr;
868	}
869	}//end if
870
871	// Convert the token after an object operator into a string, in most cases.
872	if ( $token['code'] === T_OBJECT_OPERATOR ) {
873	for ( $i = ( $stackPtr + 1 ); $i < $numTokens; $i ++ ) {
874	if ( isset( Util\Tokens::$emptyTokens[ $tokens[ $i ]['code'] ] ) === true ) {
875	continue;
876	}
877
878	if ( $tokens[ $i ]['code'] !== T_PROTOTYPE
879	&& $tokens[ $i ]['code'] !== T_LNUMBER
880	&& $tokens[ $i ]['code'] !== T_DNUMBER
881	) {
882	$tokens[ $i ]['code'] = T_STRING;
883	$tokens[ $i ]['type'] = 'T_STRING';
884	}
885
886	break;
887	}
888	}
889	}//end for
890
891	if ( PHP_CODESNIFFER_VERBOSITY > 1 ) {
892	echo "\t* END TOKENIZING *" . PHP_EOL;
893	}
894
895	return $finalTokens;
896
897	}//end tokenize()
898
899
900	/**
901	* Tokenizes a regular expression if one is found.
902	*
903	* If a regular expression is not found, NULL is returned.
904	*
905	* @param string $char The index of the possible regex start character.
906	* @param string $string The complete content of the string being tokenized.
907	* @param string $chars An array of characters being tokenized.
908	* @param string $tokens The current array of tokens found in the string.
909	*
910	* @return array<string, string>\|null
911	*/
912	public function getRegexToken( $char, $string, $chars, $tokens ) {
913	$beforeTokens = [
914	T_EQUAL => true,
915	T_IS_NOT_EQUAL => true,
916	T_IS_IDENTICAL => true,
917	T_IS_NOT_IDENTICAL => true,
918	T_OPEN_PARENTHESIS => true,
919	T_OPEN_SQUARE_BRACKET => true,
920	T_RETURN => true,
921	T_BOOLEAN_OR => true,
922	T_BOOLEAN_AND => true,
923	T_BOOLEAN_NOT => true,
924	T_BITWISE_OR => true,
925	T_BITWISE_AND => true,
926	T_COMMA => true,
927	T_COLON => true,
928	T_TYPEOF => true,
929	T_INLINE_THEN => true,
930	T_INLINE_ELSE => true,
931	];
932
933	$afterTokens = [
934	',' => true,
935	')' => true,
936	']' => true,
937	';' => true,
938	' ' => true,
939	'.' => true,
940	':' => true,
941	$this->eolChar => true,
942	];
943
944	// Find the last non-whitespace token that was added
945	// to the tokens array.
946	$numTokens = count( $tokens );
947	for ( $prev = ( $numTokens - 1 ); $prev >= 0; $prev -- ) {
948	if ( isset( Util\Tokens::$emptyTokens[ $tokens[ $prev ]['code'] ] ) === false ) {
949	break;
950	}
951	}
952
953	if ( isset( $beforeTokens[ $tokens[ $prev ]['code'] ] ) === false ) {
954	return null;
955	}
956
957	// This is probably a regular expression, so look for the end of it.
958	if ( PHP_CODESNIFFER_VERBOSITY > 1 ) {
959	echo "\t* token possibly starts a regular expression *" . PHP_EOL;
960	}
961
962	$numChars = count( $chars );
963	for ( $next = ( $char + 1 ); $next < $numChars; $next ++ ) {
964	if ( $chars[ $next ] === '/' ) {
965	// Just make sure this is not escaped first.
966	if ( $chars[ ( $next - 1 ) ] !== '\\' ) {
967	// In the simple form: /.../ so we found the end.
968	break;
969	} else if ( $chars[ ( $next - 2 ) ] === '\\' ) {
970	// In the form: /...\\/ so we found the end.
971	break;
972	}
973	} else {
974	$possibleEolChar = substr( $string, $next, strlen( $this->eolChar ) );
975	if ( $possibleEolChar === $this->eolChar ) {
976	// This is the last token on the line and regular
977	// expressions need to be defined on a single line,
978	// so this is not a regular expression.
979	break;
980	}
981	}
982	}
983
984	if ( $chars[ $next ] !== '/' ) {
985	if ( PHP_CODESNIFFER_VERBOSITY > 1 ) {
986	echo "\t* could not find end of regular expression *" . PHP_EOL;
987	}
988
989	return null;
990	}
991
992	while ( preg_match( '\|[a-zA-Z]\|', $chars[ ( $next + 1 ) ] ) !== 0 ) {
993	// The token directly after the end of the regex can
994	// be modifiers like global and case insensitive
995	// (.e.g, /pattern/gi).
996	$next ++;
997	}
998
999	$regexEnd = $next;
1000	if ( PHP_CODESNIFFER_VERBOSITY > 1 ) {
1001	echo "\t* found end of regular expression at token $regexEnd *" . PHP_EOL;
1002	}
1003
1004	for ( $next += 1; $next < $numChars; $next ++ ) {
1005	if ( $chars[ $next ] !== ' ' ) {
1006	break;
1007	} else {
1008	$possibleEolChar = substr( $string, $next, strlen( $this->eolChar ) );
1009	if ( $possibleEolChar === $this->eolChar ) {
1010	// This is the last token on the line.
1011	break;
1012	}
1013	}
1014	}
1015
1016	if ( isset( $afterTokens[ $chars[ $next ] ] ) === false ) {
1017	if ( PHP_CODESNIFFER_VERBOSITY > 1 ) {
1018	echo "\t* tokens after regular expression do not look correct *" . PHP_EOL;
1019	}
1020
1021	return null;
1022	}
1023
1024	// This is a regular expression, so join all the tokens together.
1025	$content = '';
1026	for ( $x = $char; $x <= $regexEnd; $x ++ ) {
1027	$content .= $chars[ $x ];
1028	}
1029
1030	$token = [
1031	'start' => $char,
1032	'end' => $regexEnd,
1033	'content' => $content,
1034	];
1035
1036	return $token;
1037
1038	}//end getRegexToken()
1039
1040
1041	/**
1042	* Performs additional processing after main tokenizing.
1043	*
1044	* This additional processing looks for properties, closures, labels and objects.
1045	*
1046	* @return void
1047	*/
1048	public function processAdditional() {
1049	if ( PHP_CODESNIFFER_VERBOSITY > 1 ) {
1050	echo "\t* START ADDITIONAL JS PROCESSING *" . PHP_EOL;
1051	}
1052
1053	$numTokens = count( $this->tokens );
1054	$classStack = [];
1055
1056	for ( $i = 0; $i < $numTokens; $i ++ ) {
1057	if ( PHP_CODESNIFFER_VERBOSITY > 1 ) {
1058	$type = $this->tokens[ $i ]['type'];
1059	$content = Util\Common::prepareForOutput( $this->tokens[ $i ]['content'] );
1060
1061	echo str_repeat( "\t", count( $classStack ) );
1062	echo "\tProcess token $i: $type => $content" . PHP_EOL;
1063	}
1064
1065	// Looking for functions that are actually closures.
1066	if ( $this->tokens[ $i ]['code'] === T_FUNCTION && isset( $this->tokens[ $i ]['scope_opener'] ) === true ) {
1067	for ( $x = ( $i + 1 ); $x < $numTokens; $x ++ ) {
1068	if ( isset( Util\Tokens::$emptyTokens[ $this->tokens[ $x ]['code'] ] ) === false ) {
1069	break;
1070	}
1071	}
1072
1073	if ( $this->tokens[ $x ]['code'] === T_OPEN_PARENTHESIS ) {
1074	$this->tokens[ $i ]['code'] = T_CLOSURE;
1075	$this->tokens[ $i ]['type'] = 'T_CLOSURE';
1076	if ( PHP_CODESNIFFER_VERBOSITY > 1 ) {
1077	$line = $this->tokens[ $i ]['line'];
1078	echo str_repeat( "\t", count( $classStack ) );
1079	echo "\t* token $i on line $line changed from T_FUNCTION to T_CLOSURE *" . PHP_EOL;
1080	}
1081
1082	for ( $x = ( $this->tokens[ $i ]['scope_opener'] + 1 ); $x < $this->tokens[ $i ]['scope_closer']; $x ++ ) {
1083	if ( isset( $this->tokens[ $x ]['conditions'][ $i ] ) === false ) {
1084	continue;
1085	}
1086
1087	$this->tokens[ $x ]['conditions'][ $i ] = T_CLOSURE;
1088	if ( PHP_CODESNIFFER_VERBOSITY > 1 ) {
1089	$type = $this->tokens[ $x ]['type'];
1090	echo str_repeat( "\t", count( $classStack ) );
1091	echo "\t\t* cleaned $x ($type) *" . PHP_EOL;
1092	}
1093	}
1094	}//end if
1095
1096	continue;
1097	} else if ( $this->tokens[ $i ]['code'] === T_OPEN_CURLY_BRACKET
1098	&& isset( $this->tokens[ $i ]['scope_condition'] ) === false
1099	&& isset( $this->tokens[ $i ]['bracket_closer'] ) === true
1100	) {
1101	$condition = $this->tokens[ $i ]['conditions'];
1102	$condition = end( $condition );
1103	if ( $condition === T_CLASS ) {
1104	// Possibly an ES6 method. To be classified as one, the previous
1105	// non-empty tokens need to be a set of parenthesis, and then a string
1106	// (the method name).
1107	for ( $parenCloser = ( $i - 1 ); $parenCloser > 0; $parenCloser -- ) {
1108	if ( isset( Util\Tokens::$emptyTokens[ $this->tokens[ $parenCloser ]['code'] ] ) === false ) {
1109	break;
1110	}
1111	}
1112
1113	if ( $this->tokens[ $parenCloser ]['code'] === T_CLOSE_PARENTHESIS ) {
1114	$parenOpener = $this->tokens[ $parenCloser ]['parenthesis_opener'];
1115	for ( $name = ( $parenOpener - 1 ); $name > 0; $name -- ) {
1116	if ( isset( Util\Tokens::$emptyTokens[ $this->tokens[ $name ]['code'] ] ) === false ) {
1117	break;
1118	}
1119	}
1120
1121	if ( $this->tokens[ $name ]['code'] === T_STRING ) {
1122	// We found a method name.
1123	if ( PHP_CODESNIFFER_VERBOSITY > 1 ) {
1124	$line = $this->tokens[ $name ]['line'];
1125	echo str_repeat( "\t", count( $classStack ) );
1126	echo "\t* token $name on line $line changed from T_STRING to T_FUNCTION *" . PHP_EOL;
1127	}
1128
1129	$closer = $this->tokens[ $i ]['bracket_closer'];
1130
1131	$this->tokens[ $name ]['code'] = T_FUNCTION;
1132	$this->tokens[ $name ]['type'] = 'T_FUNCTION';
1133
1134	foreach ( [ $name, $i, $closer ] as $token ) {
1135	$this->tokens[ $token ]['scope_condition'] = $name;
1136	$this->tokens[ $token ]['scope_opener'] = $i;
1137	$this->tokens[ $token ]['scope_closer'] = $closer;
1138	$this->tokens[ $token ]['parenthesis_opener'] = $parenOpener;
1139	$this->tokens[ $token ]['parenthesis_closer'] = $parenCloser;
1140	$this->tokens[ $token ]['parenthesis_owner'] = $name;
1141	}
1142
1143	$this->tokens[ $parenOpener ]['parenthesis_owner'] = $name;
1144	$this->tokens[ $parenCloser ]['parenthesis_owner'] = $name;
1145
1146	for ( $x = ( $i + 1 ); $x < $closer; $x ++ ) {
1147	$this->tokens[ $x ]['conditions'][ $name ] = T_FUNCTION;
1148	ksort( $this->tokens[ $x ]['conditions'], SORT_NUMERIC );
1149	if ( PHP_CODESNIFFER_VERBOSITY > 1 ) {
1150	$type = $this->tokens[ $x ]['type'];
1151	echo str_repeat( "\t", count( $classStack ) );
1152	echo "\t\t* added T_FUNCTION condition to $x ($type) *" . PHP_EOL;
1153	}
1154	}
1155
1156	continue;
1157	}//end if
1158	}//end if
1159	}//end if
1160
1161	$classStack[] = $i;
1162
1163	$closer = $this->tokens[ $i ]['bracket_closer'];
1164	$this->tokens[ $i ]['code'] = T_OBJECT;
1165	$this->tokens[ $i ]['type'] = 'T_OBJECT';
1166	$this->tokens[ $closer ]['code'] = T_CLOSE_OBJECT;
1167	$this->tokens[ $closer ]['type'] = 'T_CLOSE_OBJECT';
1168
1169	if ( PHP_CODESNIFFER_VERBOSITY > 1 ) {
1170	echo str_repeat( "\t", count( $classStack ) );
1171	echo "\t* token $i converted from T_OPEN_CURLY_BRACKET to T_OBJECT *" . PHP_EOL;
1172	echo str_repeat( "\t", count( $classStack ) );
1173	echo "\t* token $closer converted from T_CLOSE_CURLY_BRACKET to T_CLOSE_OBJECT *" . PHP_EOL;
1174	}
1175
1176	for ( $x = ( $i + 1 ); $x < $closer; $x ++ ) {
1177	$this->tokens[ $x ]['conditions'][ $i ] = T_OBJECT;
1178	ksort( $this->tokens[ $x ]['conditions'], SORT_NUMERIC );
1179	if ( PHP_CODESNIFFER_VERBOSITY > 1 ) {
1180	$type = $this->tokens[ $x ]['type'];
1181	echo str_repeat( "\t", count( $classStack ) );
1182	echo "\t\t* added T_OBJECT condition to $x ($type) *" . PHP_EOL;
1183	}
1184	}
1185	} else if ( $this->tokens[ $i ]['code'] === T_CLOSE_OBJECT ) {
1186	$opener = array_pop( $classStack );
1187	} else if ( $this->tokens[ $i ]['code'] === T_COLON ) {
1188	// If it is a scope opener, it belongs to a
1189	// DEFAULT or CASE statement.
1190	if ( isset( $this->tokens[ $i ]['scope_condition'] ) === true ) {
1191	continue;
1192	}
1193
1194	// Make sure this is not part of an inline IF statement.
1195	for ( $x = ( $i - 1 ); $x >= 0; $x -- ) {
1196	if ( $this->tokens[ $x ]['code'] === T_INLINE_THEN ) {
1197	$this->tokens[ $i ]['code'] = T_INLINE_ELSE;
1198	$this->tokens[ $i ]['type'] = 'T_INLINE_ELSE';
1199
1200	if ( PHP_CODESNIFFER_VERBOSITY > 1 ) {
1201	echo str_repeat( "\t", count( $classStack ) );
1202	echo "\t* token $i converted from T_COLON to T_INLINE_THEN *" . PHP_EOL;
1203	}
1204
1205	continue( 2 );
1206	} else if ( $this->tokens[ $x ]['line'] < $this->tokens[ $i ]['line'] ) {
1207	break;
1208	}
1209	}
1210
1211	// The string to the left of the colon is either a property or label.
1212	for ( $label = ( $i - 1 ); $label >= 0; $label -- ) {
1213	if ( isset( Util\Tokens::$emptyTokens[ $this->tokens[ $label ]['code'] ] ) === false ) {
1214	break;
1215	}
1216	}
1217
1218	if ( $this->tokens[ $label ]['code'] !== T_STRING
1219	&& $this->tokens[ $label ]['code'] !== T_CONSTANT_ENCAPSED_STRING
1220	) {
1221	continue;
1222	}
1223
1224	if ( empty( $classStack ) === false ) {
1225	$this->tokens[ $label ]['code'] = T_PROPERTY;
1226	$this->tokens[ $label ]['type'] = 'T_PROPERTY';
1227
1228	if ( PHP_CODESNIFFER_VERBOSITY > 1 ) {
1229	echo str_repeat( "\t", count( $classStack ) );
1230	echo "\t* token $label converted from T_STRING to T_PROPERTY *" . PHP_EOL;
1231	}
1232	} else {
1233	$this->tokens[ $label ]['code'] = T_LABEL;
1234	$this->tokens[ $label ]['type'] = 'T_LABEL';
1235
1236	if ( PHP_CODESNIFFER_VERBOSITY > 1 ) {
1237	echo str_repeat( "\t", count( $classStack ) );
1238	echo "\t* token $label converted from T_STRING to T_LABEL *" . PHP_EOL;
1239	}
1240	}//end if
1241	}//end if
1242	}//end for
1243
1244	if ( PHP_CODESNIFFER_VERBOSITY > 1 ) {
1245	echo "\t* END ADDITIONAL JS PROCESSING *" . PHP_EOL;
1246	}
1247
1248	}//end processAdditional()
1249
1250
1251	}//end class
1252