defender-security / src / extra / php-codesniffer-php-token / Tokenizers / PHP.php

PHP.php in Defender Security – Malware Scanner, Login Security & Firewall 6.2.2, at src/extra/php-codesniffer-php-token/Tokenizers/PHP.php

2,379 lines 74.0 KB

Raw Download Zip

1	<?php
2	/**
3	* Tokenizes PHP code.
4	*
5	* @author Greg Sherwood <[email protected]>
6	* @copyright 2006-2015 Squiz Pty Ltd (ABN 77 084 670 600)
7	* @license https://github.com/squizlabs/PHP_CodeSniffer/blob/master/licence.txt BSD Licence
8	*/
9
10	namespace PHP_CodeSniffer\Tokenizers;
11
12	use PHP_CodeSniffer\Util;
13
14	class PHP extends Tokenizer {
15
16	/**
17	* A list of tokens that are allowed to open a scope.
18	*
19	* This array also contains information about what kind of token the scope
20	* opener uses to open and close the scope, if the token strictly requires
21	* an opener, if the token can share a scope closer, and who it can be shared
22	* with. An example of a token that shares a scope closer is a CASE scope.
23	*
24	* @var array
25	*/
26	public $scopeOpeners = [
27	T_IF => [
28	'start' => [
29	T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET,
30	T_COLON => T_COLON,
31	],
32	'end' => [
33	T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET,
34	T_ENDIF => T_ENDIF,
35	T_ELSE => T_ELSE,
36	T_ELSEIF => T_ELSEIF,
37	],
38	'strict' => false,
39	'shared' => false,
40	'with' => [
41	T_ELSE => T_ELSE,
42	T_ELSEIF => T_ELSEIF,
43	],
44	],
45	T_TRY => [
46	'start' => [ T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET ],
47	'end' => [ T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET ],
48	'strict' => true,
49	'shared' => false,
50	'with' => [],
51	],
52	T_CATCH => [
53	'start' => [ T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET ],
54	'end' => [ T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET ],
55	'strict' => true,
56	'shared' => false,
57	'with' => [],
58	],
59	T_FINALLY => [
60	'start' => [ T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET ],
61	'end' => [ T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET ],
62	'strict' => true,
63	'shared' => false,
64	'with' => [],
65	],
66	T_ELSE => [
67	'start' => [
68	T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET,
69	T_COLON => T_COLON,
70	],
71	'end' => [
72	T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET,
73	T_ENDIF => T_ENDIF,
74	],
75	'strict' => false,
76	'shared' => false,
77	'with' => [
78	T_IF => T_IF,
79	T_ELSEIF => T_ELSEIF,
80	],
81	],
82	T_ELSEIF => [
83	'start' => [
84	T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET,
85	T_COLON => T_COLON,
86	],
87	'end' => [
88	T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET,
89	T_ENDIF => T_ENDIF,
90	T_ELSE => T_ELSE,
91	T_ELSEIF => T_ELSEIF,
92	],
93	'strict' => false,
94	'shared' => false,
95	'with' => [
96	T_IF => T_IF,
97	T_ELSE => T_ELSE,
98	],
99	],
100	T_FOR => [
101	'start' => [
102	T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET,
103	T_COLON => T_COLON,
104	],
105	'end' => [
106	T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET,
107	T_ENDFOR => T_ENDFOR,
108	],
109	'strict' => false,
110	'shared' => false,
111	'with' => [],
112	],
113	T_FOREACH => [
114	'start' => [
115	T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET,
116	T_COLON => T_COLON,
117	],
118	'end' => [
119	T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET,
120	T_ENDFOREACH => T_ENDFOREACH,
121	],
122	'strict' => false,
123	'shared' => false,
124	'with' => [],
125	],
126	T_INTERFACE => [
127	'start' => [ T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET ],
128	'end' => [ T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET ],
129	'strict' => true,
130	'shared' => false,
131	'with' => [],
132	],
133	T_FUNCTION => [
134	'start' => [ T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET ],
135	'end' => [ T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET ],
136	'strict' => true,
137	'shared' => false,
138	'with' => [],
139	],
140	T_CLASS => [
141	'start' => [ T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET ],
142	'end' => [ T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET ],
143	'strict' => true,
144	'shared' => false,
145	'with' => [],
146	],
147	T_TRAIT => [
148	'start' => [ T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET ],
149	'end' => [ T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET ],
150	'strict' => true,
151	'shared' => false,
152	'with' => [],
153	],
154	T_USE => [
155	'start' => [ T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET ],
156	'end' => [ T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET ],
157	'strict' => false,
158	'shared' => false,
159	'with' => [],
160	],
161	T_DECLARE => [
162	'start' => [
163	T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET,
164	T_COLON => T_COLON,
165	],
166	'end' => [
167	T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET,
168	T_ENDDECLARE => T_ENDDECLARE,
169	],
170	'strict' => false,
171	'shared' => false,
172	'with' => [],
173	],
174	T_NAMESPACE => [
175	'start' => [ T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET ],
176	'end' => [ T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET ],
177	'strict' => false,
178	'shared' => false,
179	'with' => [],
180	],
181	T_WHILE => [
182	'start' => [
183	T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET,
184	T_COLON => T_COLON,
185	],
186	'end' => [
187	T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET,
188	T_ENDWHILE => T_ENDWHILE,
189	],
190	'strict' => false,
191	'shared' => false,
192	'with' => [],
193	],
194	T_DO => [
195	'start' => [ T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET ],
196	'end' => [ T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET ],
197	'strict' => true,
198	'shared' => false,
199	'with' => [],
200	],
201	T_SWITCH => [
202	'start' => [
203	T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET,
204	T_COLON => T_COLON,
205	],
206	'end' => [
207	T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET,
208	T_ENDSWITCH => T_ENDSWITCH,
209	],
210	'strict' => true,
211	'shared' => false,
212	'with' => [],
213	],
214	T_CASE => [
215	'start' => [
216	T_COLON => T_COLON,
217	T_SEMICOLON => T_SEMICOLON,
218	],
219	'end' => [
220	T_BREAK => T_BREAK,
221	T_RETURN => T_RETURN,
222	T_CONTINUE => T_CONTINUE,
223	T_THROW => T_THROW,
224	T_EXIT => T_EXIT,
225	],
226	'strict' => true,
227	'shared' => true,
228	'with' => [
229	T_DEFAULT => T_DEFAULT,
230	T_CASE => T_CASE,
231	T_SWITCH => T_SWITCH,
232	],
233	],
234	T_DEFAULT => [
235	'start' => [
236	T_COLON => T_COLON,
237	T_SEMICOLON => T_SEMICOLON,
238	],
239	'end' => [
240	T_BREAK => T_BREAK,
241	T_RETURN => T_RETURN,
242	T_CONTINUE => T_CONTINUE,
243	T_THROW => T_THROW,
244	T_EXIT => T_EXIT,
245	],
246	'strict' => true,
247	'shared' => true,
248	'with' => [
249	T_CASE => T_CASE,
250	T_SWITCH => T_SWITCH,
251	],
252	],
253	T_START_HEREDOC => [
254	'start' => [ T_START_HEREDOC => T_START_HEREDOC ],
255	'end' => [ T_END_HEREDOC => T_END_HEREDOC ],
256	'strict' => true,
257	'shared' => false,
258	'with' => [],
259	],
260	T_START_NOWDOC => [
261	'start' => [ T_START_NOWDOC => T_START_NOWDOC ],
262	'end' => [ T_END_NOWDOC => T_END_NOWDOC ],
263	'strict' => true,
264	'shared' => false,
265	'with' => [],
266	],
267	];
268
269	/**
270	* A list of tokens that end the scope.
271	*
272	* This array is just a unique collection of the end tokens
273	* from the scopeOpeners array. The data is duplicated here to
274	* save time during parsing of the file.
275	*
276	* @var array
277	*/
278	public $endScopeTokens = [
279	T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET,
280	T_ENDIF => T_ENDIF,
281	T_ENDFOR => T_ENDFOR,
282	T_ENDFOREACH => T_ENDFOREACH,
283	T_ENDWHILE => T_ENDWHILE,
284	T_ENDSWITCH => T_ENDSWITCH,
285	T_BREAK => T_BREAK,
286	T_END_HEREDOC => T_END_HEREDOC,
287	];
288
289	/**
290	* Known lengths of tokens.
291	*
292	* @var array<int, int>
293	*/
294	public $knownLengths = [
295	T_ABSTRACT => 8,
296	T_AND_EQUAL => 2,
297	T_ARRAY => 5,
298	T_AS => 2,
299	T_BOOLEAN_AND => 2,
300	T_BOOLEAN_OR => 2,
301	T_BREAK => 5,
302	T_CALLABLE => 8,
303	T_CASE => 4,
304	T_CATCH => 5,
305	T_CLASS => 5,
306	T_CLASS_C => 9,
307	T_CLONE => 5,
308	T_CONCAT_EQUAL => 2,
309	T_CONST => 5,
310	T_CONTINUE => 8,
311	T_CURLY_OPEN => 2,
312	T_DEC => 2,
313	T_DECLARE => 7,
314	T_DEFAULT => 7,
315	T_DIR => 7,
316	T_DIV_EQUAL => 2,
317	T_DO => 2,
318	T_DOLLAR_OPEN_CURLY_BRACES => 2,
319	T_DOUBLE_ARROW => 2,
320	T_DOUBLE_COLON => 2,
321	T_ECHO => 4,
322	T_ELSE => 4,
323	T_ELSEIF => 6,
324	T_EMPTY => 5,
325	T_ENDDECLARE => 10,
326	T_ENDFOR => 6,
327	T_ENDFOREACH => 10,
328	T_ENDIF => 5,
329	T_ENDSWITCH => 9,
330	T_ENDWHILE => 8,
331	T_EVAL => 4,
332	T_EXTENDS => 7,
333	T_FILE => 8,
334	T_FINAL => 5,
335	T_FINALLY => 7,
336	T_FOR => 3,
337	T_FOREACH => 7,
338	T_FUNCTION => 8,
339	T_FUNC_C => 12,
340	T_GLOBAL => 6,
341	T_GOTO => 4,
342	T_HALT_COMPILER => 15,
343	T_IF => 2,
344	T_IMPLEMENTS => 10,
345	T_INC => 2,
346	T_INCLUDE => 7,
347	T_INCLUDE_ONCE => 12,
348	T_INSTANCEOF => 10,
349	T_INSTEADOF => 9,
350	T_INTERFACE => 9,
351	T_ISSET => 5,
352	T_IS_EQUAL => 2,
353	T_IS_GREATER_OR_EQUAL => 2,
354	T_IS_IDENTICAL => 3,
355	T_IS_NOT_EQUAL => 2,
356	T_IS_NOT_IDENTICAL => 3,
357	T_IS_SMALLER_OR_EQUAL => 2,
358	T_LINE => 8,
359	T_LIST => 4,
360	T_LOGICAL_AND => 3,
361	T_LOGICAL_OR => 2,
362	T_LOGICAL_XOR => 3,
363	T_METHOD_C => 10,
364	T_MINUS_EQUAL => 2,
365	T_POW_EQUAL => 3,
366	T_MOD_EQUAL => 2,
367	T_MUL_EQUAL => 2,
368	T_NAMESPACE => 9,
369	T_NS_C => 13,
370	T_NS_SEPARATOR => 1,
371	T_NEW => 3,
372	T_OBJECT_OPERATOR => 2,
373	T_OPEN_TAG_WITH_ECHO => 3,
374	T_OR_EQUAL => 2,
375	T_PLUS_EQUAL => 2,
376	T_PRINT => 5,
377	T_PRIVATE => 7,
378	T_PUBLIC => 6,
379	T_PROTECTED => 9,
380	T_REQUIRE => 7,
381	T_REQUIRE_ONCE => 12,
382	T_RETURN => 6,
383	T_STATIC => 6,
384	T_SWITCH => 6,
385	T_THROW => 5,
386	T_TRAIT => 5,
387	T_TRAIT_C => 9,
388	T_TRY => 3,
389	T_UNSET => 5,
390	T_USE => 3,
391	T_VAR => 3,
392	T_WHILE => 5,
393	T_XOR_EQUAL => 2,
394	T_YIELD => 5,
395	T_OPEN_CURLY_BRACKET => 1,
396	T_CLOSE_CURLY_BRACKET => 1,
397	T_OPEN_SQUARE_BRACKET => 1,
398	T_CLOSE_SQUARE_BRACKET => 1,
399	T_OPEN_PARENTHESIS => 1,
400	T_CLOSE_PARENTHESIS => 1,
401	T_COLON => 1,
402	T_STRING_CONCAT => 1,
403	T_INLINE_THEN => 1,
404	T_INLINE_ELSE => 1,
405	T_NULLABLE => 1,
406	T_NULL => 4,
407	T_FALSE => 5,
408	T_TRUE => 4,
409	T_SEMICOLON => 1,
410	T_EQUAL => 1,
411	T_MULTIPLY => 1,
412	T_DIVIDE => 1,
413	T_PLUS => 1,
414	T_MINUS => 1,
415	T_MODULUS => 1,
416	T_POW => 2,
417	T_SPACESHIP => 3,
418	T_COALESCE => 2,
419	T_COALESCE_EQUAL => 3,
420	T_BITWISE_AND => 1,
421	T_BITWISE_OR => 1,
422	T_BITWISE_XOR => 1,
423	T_SL => 2,
424	T_SR => 2,
425	T_SL_EQUAL => 3,
426	T_SR_EQUAL => 3,
427	T_GREATER_THAN => 1,
428	T_LESS_THAN => 1,
429	T_BOOLEAN_NOT => 1,
430	T_SELF => 4,
431	T_PARENT => 6,
432	T_COMMA => 1,
433	T_THIS => 4,
434	T_CLOSURE => 8,
435	T_BACKTICK => 1,
436	T_OPEN_SHORT_ARRAY => 1,
437	T_CLOSE_SHORT_ARRAY => 1,
438	];
439
440	/**
441	* A cache of different token types, resolved into arrays.
442	*
443	* @var array
444	* @see standardiseToken()
445	*/
446	private static $resolveTokenCache = [];
447
448
449	/**
450	* Creates an array of tokens when given some PHP code.
451	*
452	* Starts by using token_get_all() but does a lot of extra processing
453	* to insert information about the context of the token.
454	*
455	* @param string $string The string to tokenize.
456	*
457	* @return array
458	*/
459	protected function tokenize( $string ) {
460	if ( PHP_CODESNIFFER_VERBOSITY > 1 ) {
461	echo "\t* START PHP TOKENIZING *" . PHP_EOL;
462	$isWin = false;
463	if ( strtoupper( substr( PHP_OS, 0, 3 ) ) === 'WIN' ) {
464	$isWin = true;
465	}
466	}
467
468	$tokens = @token_get_all( $string );
469	$finalTokens = [];
470
471	$newStackPtr = 0;
472	$numTokens = count( $tokens );
473	$lastNotEmptyToken = 0;
474
475	$insideInlineIf = [];
476	$insideUseGroup = false;
477
478	$commentTokenizer = new Comment();
479
480	for ( $stackPtr = 0; $stackPtr < $numTokens; $stackPtr ++ ) {
481	// Special case for tokens we have needed to blank out.
482	if ( $tokens[ $stackPtr ] === null ) {
483	continue;
484	}
485
486	$token = (array) $tokens[ $stackPtr ];
487	$tokenIsArray = isset( $token[1] );
488
489	if ( PHP_CODESNIFFER_VERBOSITY > 1 ) {
490	if ( $tokenIsArray === true ) {
491	$type = Util\Tokens::tokenName( $token[0] );
492	$content = Util\Common::prepareForOutput( $token[1] );
493	} else {
494	$newToken = self::resolveSimpleToken( $token[0] );
495	$type = $newToken['type'];
496	$content = Util\Common::prepareForOutput( $token[0] );
497	}
498
499	echo "\tProcess token ";
500	if ( $tokenIsArray === true ) {
501	echo "[$stackPtr]";
502	} else {
503	echo " $stackPtr ";
504	}
505
506	echo ": $type => $content";
507	}//end if
508
509	if ( $newStackPtr > 0
510	&& isset( Util\Tokens::$emptyTokens[ $finalTokens[ ( $newStackPtr - 1 ) ]['code'] ] ) === false
511	) {
512	$lastNotEmptyToken = ( $newStackPtr - 1 );
513	}
514
515	/*
516	If we are using \r\n newline characters, the \r and \n are sometimes
517	split over two tokens. This normally occurs after comments. We need
518	to merge these two characters together so that our line endings are
519	consistent for all lines.
520	*/
521
522	if ( $tokenIsArray === true && substr( $token[1], - 1 ) === "\r" ) {
523	if ( isset( $tokens[ ( $stackPtr + 1 ) ] ) === true
524	&& is_array( $tokens[ ( $stackPtr + 1 ) ] ) === true
525	&& $tokens[ ( $stackPtr + 1 ) ][1][0] === "\n"
526	) {
527	$token[1] .= "\n";
528	if ( PHP_CODESNIFFER_VERBOSITY > 1 ) {
529	if ( $isWin === true ) {
530	echo '\n';
531	} else {
532	echo "\033[30;1m\\n\033[0m";
533	}
534	}
535
536	if ( $tokens[ ( $stackPtr + 1 ) ][1] === "\n" ) {
537	// This token's content has been merged into the previous,
538	// so we can skip it.
539	$tokens[ ( $stackPtr + 1 ) ] = '';
540	} else {
541	$tokens[ ( $stackPtr + 1 ) ][1] = substr( $tokens[ ( $stackPtr + 1 ) ][1], 1 );
542	}
543	}
544	}//end if
545
546	if ( PHP_CODESNIFFER_VERBOSITY > 1 ) {
547	echo PHP_EOL;
548	}
549
550	/*
551	Parse doc blocks into something that can be easily iterated over.
552	*/
553
554	if ( $tokenIsArray === true
555	&& ( $token[0] === T_DOC_COMMENT
556	\|\| ( $token[0] === T_COMMENT && strpos( $token[1], '/**' ) === 0 ) )
557	) {
558	$commentTokens = $commentTokenizer->tokenizeString( $token[1], $this->eolChar, $newStackPtr );
559	foreach ( $commentTokens as $commentToken ) {
560	$finalTokens[ $newStackPtr ] = $commentToken;
561	$newStackPtr ++;
562	}
563
564	continue;
565	}
566
567	/*
568	If this is a double quoted string, PHP will tokenize the whole
569	thing which causes problems with the scope map when braces are
570	within the string. So we need to merge the tokens together to
571	provide a single string.
572	*/
573
574	if ( $tokenIsArray === false && ( $token[0] === '"' \|\| $token[0] === 'b"' ) ) {
575	// Binary casts need a special token.
576	if ( $token[0] === 'b"' ) {
577	$finalTokens[ $newStackPtr ] = [
578	'code' => T_BINARY_CAST,
579	'type' => 'T_BINARY_CAST',
580	'content' => 'b',
581	];
582	$newStackPtr ++;
583	}
584
585	$tokenContent = '"';
586	$nestedVars = [];
587	for ( $i = ( $stackPtr + 1 ); $i < $numTokens; $i ++ ) {
588	$subToken = (array) $tokens[ $i ];
589	$subTokenIsArray = isset( $subToken[1] );
590
591	if ( $subTokenIsArray === true ) {
592	$tokenContent .= $subToken[1];
593	if ( $subToken[1] === '{'
594	&& $subToken[0] !== T_ENCAPSED_AND_WHITESPACE
595	) {
596	$nestedVars[] = $i;
597	}
598	} else {
599	$tokenContent .= $subToken[0];
600	if ( $subToken[0] === '}' ) {
601	array_pop( $nestedVars );
602	}
603	}
604
605	if ( $subTokenIsArray === false
606	&& $subToken[0] === '"'
607	&& empty( $nestedVars ) === true
608	) {
609	// We found the other end of the double quoted string.
610	break;
611	}
612	}//end for
613
614	$stackPtr = $i;
615
616	// Convert each line within the double quoted string to a
617	// new token, so it conforms with other multiple line tokens.
618	$tokenLines = explode( $this->eolChar, $tokenContent );
619	$numLines = count( $tokenLines );
620	$newToken = [];
621
622	for ( $j = 0; $j < $numLines; $j ++ ) {
623	$newToken['content'] = $tokenLines[ $j ];
624	if ( $j === ( $numLines - 1 ) ) {
625	if ( $tokenLines[ $j ] === '' ) {
626	break;
627	}
628	} else {
629	$newToken['content'] .= $this->eolChar;
630	}
631
632	$newToken['code'] = T_DOUBLE_QUOTED_STRING;
633	$newToken['type'] = 'T_DOUBLE_QUOTED_STRING';
634	$finalTokens[ $newStackPtr ] = $newToken;
635	$newStackPtr ++;
636	}
637
638	// Continue, as we're done with this token.
639	continue;
640	}//end if
641
642	/*
643	Detect binary casting and assign the casts their own token.
644	*/
645
646	if ( $tokenIsArray === true
647	&& $token[0] === T_CONSTANT_ENCAPSED_STRING
648	&& ( substr( $token[1], 0, 2 ) === 'b"'
649	\|\| substr( $token[1], 0, 2 ) === "b'" )
650	) {
651	$finalTokens[ $newStackPtr ] = [
652	'code' => T_BINARY_CAST,
653	'type' => 'T_BINARY_CAST',
654	'content' => 'b',
655	];
656	$newStackPtr ++;
657	$token[1] = substr( $token[1], 1 );
658	}
659
660	if ( $tokenIsArray === true
661	&& $token[0] === T_STRING_CAST
662	&& preg_match( '`^$\sbinary\s$$`i', $token[1] ) === 1
663	) {
664	$finalTokens[ $newStackPtr ] = [
665	'code' => T_BINARY_CAST,
666	'type' => 'T_BINARY_CAST',
667	'content' => $token[1],
668	];
669	$newStackPtr ++;
670	continue;
671	}
672
673	/*
674	If this is a heredoc, PHP will tokenize the whole
675	thing which causes problems when heredocs don't
676	contain real PHP code, which is almost never.
677	We want to leave the start and end heredoc tokens
678	alone though.
679	*/
680
681	if ( $tokenIsArray === true && $token[0] === T_START_HEREDOC ) {
682	// Add the start heredoc token to the final array.
683	$finalTokens[ $newStackPtr ] = self::standardiseToken( $token );
684
685	// Check if this is actually a nowdoc and use a different token
686	// to help the sniffs.
687	$nowdoc = false;
688	if ( strpos( $token[1], "'" ) !== false ) {
689	$finalTokens[ $newStackPtr ]['code'] = T_START_NOWDOC;
690	$finalTokens[ $newStackPtr ]['type'] = 'T_START_NOWDOC';
691	$nowdoc = true;
692	}
693
694	$tokenContent = '';
695	for ( $i = ( $stackPtr + 1 ); $i < $numTokens; $i ++ ) {
696	$subTokenIsArray = is_array( $tokens[ $i ] );
697	if ( $subTokenIsArray === true
698	&& $tokens[ $i ][0] === T_END_HEREDOC
699	) {
700	// We found the other end of the heredoc.
701	break;
702	}
703
704	if ( $subTokenIsArray === true ) {
705	$tokenContent .= $tokens[ $i ][1];
706	} else {
707	$tokenContent .= $tokens[ $i ];
708	}
709	}
710
711	if ( $i === $numTokens ) {
712	// We got to the end of the file and never
713	// found the closing token, so this probably wasn't
714	// a heredoc.
715	if ( PHP_CODESNIFFER_VERBOSITY > 1 ) {
716	$type = $finalTokens[ $newStackPtr ]['type'];
717	echo "\t\t* failed to find the end of the here/nowdoc" . PHP_EOL;
718	echo "\t\t* token $stackPtr changed from $type to T_STRING" . PHP_EOL;
719	}
720
721	$finalTokens[ $newStackPtr ]['code'] = T_STRING;
722	$finalTokens[ $newStackPtr ]['type'] = 'T_STRING';
723	$newStackPtr ++;
724	continue;
725	}
726
727	$stackPtr = $i;
728	$newStackPtr ++;
729
730	// Convert each line within the heredoc to a
731	// new token, so it conforms with other multiple line tokens.
732	$tokenLines = explode( $this->eolChar, $tokenContent );
733	$numLines = count( $tokenLines );
734	$newToken = [];
735
736	for ( $j = 0; $j < $numLines; $j ++ ) {
737	$newToken['content'] = $tokenLines[ $j ];
738	if ( $j === ( $numLines - 1 ) ) {
739	if ( $tokenLines[ $j ] === '' ) {
740	break;
741	}
742	} else {
743	$newToken['content'] .= $this->eolChar;
744	}
745
746	if ( $nowdoc === true ) {
747	$newToken['code'] = T_NOWDOC;
748	$newToken['type'] = 'T_NOWDOC';
749	} else {
750	$newToken['code'] = T_HEREDOC;
751	$newToken['type'] = 'T_HEREDOC';
752	}
753
754	$finalTokens[ $newStackPtr ] = $newToken;
755	$newStackPtr ++;
756	}//end for
757
758	// Add the end heredoc token to the final array.
759	$finalTokens[ $newStackPtr ] = self::standardiseToken( $tokens[ $stackPtr ] );
760
761	if ( $nowdoc === true ) {
762	$finalTokens[ $newStackPtr ]['code'] = T_END_NOWDOC;
763	$finalTokens[ $newStackPtr ]['type'] = 'T_END_NOWDOC';
764	}
765
766	$newStackPtr ++;
767
768	// Continue, as we're done with this token.
769	continue;
770	}//end if
771
772	/*
773	Before PHP 7.0, the "yield from" was tokenized as
774	T_YIELD, T_WHITESPACE and T_STRING. So look for
775	and change this token in earlier versions.
776	*/
777
778	if ( PHP_VERSION_ID < 70000
779	&& PHP_VERSION_ID >= 50500
780	&& $tokenIsArray === true
781	&& $token[0] === T_YIELD
782	&& isset( $tokens[ ( $stackPtr + 1 ) ] ) === true
783	&& isset( $tokens[ ( $stackPtr + 2 ) ] ) === true
784	&& $tokens[ ( $stackPtr + 1 ) ][0] === T_WHITESPACE
785	&& $tokens[ ( $stackPtr + 2 ) ][0] === T_STRING
786	&& strtolower( $tokens[ ( $stackPtr + 2 ) ][1] ) === 'from'
787	) {
788	// Could be multi-line, so just the token stack.
789	$token[0] = T_YIELD_FROM;
790	$token[1] .= $tokens[ ( $stackPtr + 1 ) ][1] . $tokens[ ( $stackPtr + 2 ) ][1];
791
792	if ( PHP_CODESNIFFER_VERBOSITY > 1 ) {
793	for ( $i = ( $stackPtr + 1 ); $i <= ( $stackPtr + 2 ); $i ++ ) {
794	$type = Util\Tokens::tokenName( $tokens[ $i ][0] );
795	$content = Util\Common::prepareForOutput( $tokens[ $i ][1] );
796	echo "\t\t* token $i merged into T_YIELD_FROM; was: $type => $content" . PHP_EOL;
797	}
798	}
799
800	$tokens[ ( $stackPtr + 1 ) ] = null;
801	$tokens[ ( $stackPtr + 2 ) ] = null;
802	}
803
804	/*
805	Before PHP 5.5, the yield keyword was tokenized as
806	T_STRING. So look for and change this token in
807	earlier versions.
808	Checks also if it is just "yield" or "yield from".
809	*/
810
811	if ( PHP_VERSION_ID < 50500
812	&& $tokenIsArray === true
813	&& $token[0] === T_STRING
814	&& strtolower( $token[1] ) === 'yield'
815	) {
816	if ( isset( $tokens[ ( $stackPtr + 1 ) ] ) === true
817	&& isset( $tokens[ ( $stackPtr + 2 ) ] ) === true
818	&& $tokens[ ( $stackPtr + 1 ) ][0] === T_WHITESPACE
819	&& $tokens[ ( $stackPtr + 2 ) ][0] === T_STRING
820	&& strtolower( $tokens[ ( $stackPtr + 2 ) ][1] ) === 'from'
821	) {
822	// Could be multi-line, so just just the token stack.
823	$token[0] = T_YIELD_FROM;
824	$token[1] .= $tokens[ ( $stackPtr + 1 ) ][1] . $tokens[ ( $stackPtr + 2 ) ][1];
825
826	if ( PHP_CODESNIFFER_VERBOSITY > 1 ) {
827	for ( $i = ( $stackPtr + 1 ); $i <= ( $stackPtr + 2 ); $i ++ ) {
828	$type = Util\Tokens::tokenName( $tokens[ $i ][0] );
829	$content = Util\Common::prepareForOutput( $tokens[ $i ][1] );
830	echo "\t\t* token $i merged into T_YIELD_FROM; was: $type => $content" . PHP_EOL;
831	}
832	}
833
834	$tokens[ ( $stackPtr + 1 ) ] = null;
835	$tokens[ ( $stackPtr + 2 ) ] = null;
836	} else {
837	$newToken = [];
838	$newToken['code'] = T_YIELD;
839	$newToken['type'] = 'T_YIELD';
840	$newToken['content'] = $token[1];
841	$finalTokens[ $newStackPtr ] = $newToken;
842
843	$newStackPtr ++;
844	continue;
845	}//end if
846	}//end if
847
848	/*
849	Before PHP 5.6, the ... operator was tokenized as three
850	T_STRING_CONCAT tokens in a row. So look for and combine
851	these tokens in earlier versions.
852	*/
853
854	if ( $tokenIsArray === false
855	&& $token[0] === '.'
856	&& isset( $tokens[ ( $stackPtr + 1 ) ] ) === true
857	&& isset( $tokens[ ( $stackPtr + 2 ) ] ) === true
858	&& $tokens[ ( $stackPtr + 1 ) ] === '.'
859	&& $tokens[ ( $stackPtr + 2 ) ] === '.'
860	) {
861	$newToken = [];
862	$newToken['code'] = T_ELLIPSIS;
863	$newToken['type'] = 'T_ELLIPSIS';
864	$newToken['content'] = '...';
865	$finalTokens[ $newStackPtr ] = $newToken;
866
867	$newStackPtr ++;
868	$stackPtr += 2;
869	continue;
870	}
871
872	/*
873	Before PHP 5.6, the ** operator was tokenized as two
874	T_MULTIPLY tokens in a row. So look for and combine
875	these tokens in earlier versions.
876	*/
877
878	if ( $tokenIsArray === false
879	&& $token[0] === '*'
880	&& isset( $tokens[ ( $stackPtr + 1 ) ] ) === true
881	&& $tokens[ ( $stackPtr + 1 ) ] === '*'
882	) {
883	$newToken = [];
884	$newToken['code'] = T_POW;
885	$newToken['type'] = 'T_POW';
886	$newToken['content'] = '**';
887	$finalTokens[ $newStackPtr ] = $newToken;
888
889	$newStackPtr ++;
890	$stackPtr ++;
891	continue;
892	}
893
894	/*
895	Before PHP 5.6, the **= operator was tokenized as
896	T_MULTIPLY followed by T_MUL_EQUAL. So look for and combine
897	these tokens in earlier versions.
898	*/
899
900	if ( $tokenIsArray === false
901	&& $token[0] === '*'
902	&& isset( $tokens[ ( $stackPtr + 1 ) ] ) === true
903	&& is_array( $tokens[ ( $stackPtr + 1 ) ] ) === true
904	&& $tokens[ ( $stackPtr + 1 ) ][1] === '*='
905	) {
906	$newToken = [];
907	$newToken['code'] = T_POW_EQUAL;
908	$newToken['type'] = 'T_POW_EQUAL';
909	$newToken['content'] = '**=';
910	$finalTokens[ $newStackPtr ] = $newToken;
911
912	$newStackPtr ++;
913	$stackPtr ++;
914	continue;
915	}
916
917	/*
918	Before PHP 7, the ??= operator was tokenized as
919	T_INLINE_THEN, T_INLINE_THEN, T_EQUAL.
920	Between PHP 7.0 and 7.2, the ??= operator was tokenized as
921	T_COALESCE, T_EQUAL.
922	So look for and combine these tokens in earlier versions.
923	*/
924
925	if ( ( $tokenIsArray === false
926	&& $token[0] === '?'
927	&& isset( $tokens[ ( $stackPtr + 1 ) ] ) === true
928	&& $tokens[ ( $stackPtr + 1 ) ][0] === '?'
929	&& isset( $tokens[ ( $stackPtr + 2 ) ] ) === true
930	&& $tokens[ ( $stackPtr + 2 ) ][0] === '=' )
931	\|\| ( $tokenIsArray === true
932	&& $token[0] === T_COALESCE
933	&& isset( $tokens[ ( $stackPtr + 1 ) ] ) === true
934	&& $tokens[ ( $stackPtr + 1 ) ][0] === '=' )
935	) {
936	$newToken = [];
937	$newToken['code'] = T_COALESCE_EQUAL;
938	$newToken['type'] = 'T_COALESCE_EQUAL';
939	$newToken['content'] = '??=';
940	$finalTokens[ $newStackPtr ] = $newToken;
941
942	$newStackPtr ++;
943	$stackPtr ++;
944
945	if ( $tokenIsArray === false ) {
946	// Pre PHP 7.
947	$stackPtr ++;
948	}
949
950	continue;
951	}
952
953	/*
954	Before PHP 7, the ?? operator was tokenized as
955	T_INLINE_THEN followed by T_INLINE_THEN.
956	So look for and combine these tokens in earlier versions.
957	*/
958
959	if ( $tokenIsArray === false
960	&& $token[0] === '?'
961	&& isset( $tokens[ ( $stackPtr + 1 ) ] ) === true
962	&& $tokens[ ( $stackPtr + 1 ) ][0] === '?'
963	) {
964	$newToken = [];
965	$newToken['code'] = T_COALESCE;
966	$newToken['type'] = 'T_COALESCE';
967	$newToken['content'] = '??';
968	$finalTokens[ $newStackPtr ] = $newToken;
969
970	$newStackPtr ++;
971	$stackPtr ++;
972	continue;
973	}
974
975	/*
976	Before PHP 7.4, underscores inside T_LNUMBER and T_DNUMBER
977	tokens split the token with a T_STRING. So look for
978	and change these tokens in earlier versions.
979	*/
980
981	if ( PHP_VERSION_ID < 70400
982	&& ( $tokenIsArray === true
983	&& ( $token[0] === T_LNUMBER
984	\|\| $token[0] === T_DNUMBER )
985	&& isset( $tokens[ ( $stackPtr + 1 ) ] ) === true
986	&& is_array( $tokens[ ( $stackPtr + 1 ) ] ) === true
987	&& $tokens[ ( $stackPtr + 1 ) ][0] === T_STRING
988	&& $tokens[ ( $stackPtr + 1 ) ][1][0] === '_' )
989	) {
990	$newContent = $token[1];
991	$newType = $token[0];
992	for ( $i = ( $stackPtr + 1 ); $i < $numTokens; $i ++ ) {
993	if ( is_array( $tokens[ $i ] ) === false ) {
994	break;
995	}
996
997	if ( $tokens[ $i ][0] === T_LNUMBER
998	\|\| $tokens[ $i ][0] === T_DNUMBER
999	) {
1000	$newContent .= $tokens[ $i ][1];
1001	continue;
1002	}
1003
1004	if ( $tokens[ $i ][0] === T_STRING
1005	&& $tokens[ $i ][1][0] === '_'
1006	&& ( ( strpos( $newContent, '0x' ) === 0
1007	&& preg_match( '`^((?<!\.)_[0-9A-F][0-9A-F\.]*)+$`iD', $tokens[ $i ][1] ) === 1 )
1008	\|\| ( strpos( $newContent, '0x' ) !== 0
1009	&& substr( $newContent, - 1 ) !== '.'
1010	&& substr( strtolower( $newContent ), - 1 ) !== 'e'
1011	&& preg_match( '`^(?:(?<![\.e])_[0-9][0-9e\.]*)+$`iD', $tokens[ $i ][1] ) === 1 ) )
1012	) {
1013	$newContent .= $tokens[ $i ][1];
1014
1015	// Support floats.
1016	if ( substr( strtolower( $tokens[ $i ][1] ), - 1 ) === 'e'
1017	&& ( $tokens[ ( $i + 1 ) ] === '-'
1018	\|\| $tokens[ ( $i + 1 ) ] === '+' )
1019	) {
1020	$newContent .= $tokens[ ( $i + 1 ) ];
1021	$i ++;
1022	}
1023
1024	continue;
1025	}//end if
1026
1027	break;
1028	}//end for
1029
1030	if ( $newType === T_LNUMBER
1031	&& ( ( stripos( $newContent, '0x' ) === 0 && hexdec( str_replace( '_', '', $newContent ) ) > PHP_INT_MAX )
1032	\|\| ( stripos( $newContent, '0b' ) === 0 && bindec( str_replace( '_', '', $newContent ) ) > PHP_INT_MAX )
1033	\|\| ( stripos( $newContent, '0x' ) !== 0
1034	&& stripos( $newContent, 'e' ) !== false \|\| strpos( $newContent, '.' ) !== false )
1035	\|\| ( strpos( $newContent, '0' ) === 0 && stripos( $newContent, '0x' ) !== 0
1036	&& stripos( $newContent, '0b' ) !== 0 && octdec( str_replace( '_', '', $newContent ) ) > PHP_INT_MAX )
1037	\|\| ( strpos( $newContent, '0' ) !== 0 && str_replace( '_', '', $newContent ) > PHP_INT_MAX ) )
1038	) {
1039	$newType = T_DNUMBER;
1040	}
1041
1042	$newToken = [];
1043	$newToken['code'] = $newType;
1044	$newToken['type'] = Util\Tokens::tokenName( $newType );
1045	$newToken['content'] = $newContent;
1046	$finalTokens[ $newStackPtr ] = $newToken;
1047
1048	$newStackPtr ++;
1049	$stackPtr = ( $i - 1 );
1050	continue;
1051	}//end if
1052
1053	/*
1054	Convert ? to T_NULLABLE OR T_INLINE_THEN
1055	*/
1056
1057	if ( $tokenIsArray === false && $token[0] === '?' ) {
1058	$newToken = [];
1059	$newToken['content'] = '?';
1060
1061	/*
1062	* Check if the next non-empty token is one of the tokens which can be used
1063	* in type declarations. If not, it's definitely a ternary.
1064	* At this point, the only token types which need to be taken into consideration
1065	* as potential type declarations are T_STRING, T_ARRAY, T_CALLABLE and T_NS_SEPARATOR.
1066	*/
1067
1068	$lastRelevantNonEmpty = null;
1069
1070	for ( $i = ( $stackPtr + 1 ); $i < $numTokens; $i ++ ) {
1071	if ( is_array( $tokens[ $i ] ) === true ) {
1072	$tokenType = $tokens[ $i ][0];
1073	} else {
1074	$tokenType = $tokens[ $i ];
1075	}
1076
1077	if ( isset( Util\Tokens::$emptyTokens[ $tokenType ] ) === true ) {
1078	continue;
1079	}
1080
1081	if ( $tokenType === T_STRING
1082	\|\| $tokenType === T_ARRAY
1083	\|\| $tokenType === T_NS_SEPARATOR
1084	) {
1085	$lastRelevantNonEmpty = $tokenType;
1086	continue;
1087	}
1088
1089	if ( ( $tokenType !== T_CALLABLE
1090	&& isset( $lastRelevantNonEmpty ) === false )
1091	\|\| ( $lastRelevantNonEmpty === T_ARRAY
1092	&& $tokenType === '(' )
1093	\|\| ( $lastRelevantNonEmpty === T_STRING
1094	&& ( $tokenType === T_DOUBLE_COLON
1095	\|\| $tokenType === '('
1096	\|\| $tokenType === ':' ) )
1097	) {
1098	if ( PHP_CODESNIFFER_VERBOSITY > 1 ) {
1099	echo "\t\t* token $stackPtr changed from ? to T_INLINE_THEN" . PHP_EOL;
1100	}
1101
1102	$newToken['code'] = T_INLINE_THEN;
1103	$newToken['type'] = 'T_INLINE_THEN';
1104
1105	$insideInlineIf[] = $stackPtr;
1106
1107	$finalTokens[ $newStackPtr ] = $newToken;
1108	$newStackPtr ++;
1109	continue 2;
1110	}
1111
1112	break;
1113	}//end for
1114
1115	/*
1116	* This can still be a nullable type or a ternary.
1117	* Do additional checking.
1118	*/
1119
1120	$prevNonEmpty = null;
1121	$lastSeenNonEmpty = null;
1122
1123	for ( $i = ( $stackPtr - 1 ); $i >= 0; $i -- ) {
1124	if ( is_array( $tokens[ $i ] ) === true ) {
1125	$tokenType = $tokens[ $i ][0];
1126	} else {
1127	$tokenType = $tokens[ $i ];
1128	}
1129
1130	if ( $tokenType === T_STATIC
1131	&& ( $lastSeenNonEmpty === T_DOUBLE_COLON
1132	\|\| $lastSeenNonEmpty === '(' )
1133	) {
1134	$lastSeenNonEmpty = $tokenType;
1135	continue;
1136	}
1137
1138	if ( $prevNonEmpty === null
1139	&& isset( Util\Tokens::$emptyTokens[ $tokenType ] ) === false
1140	) {
1141	// Found the previous non-empty token.
1142	if ( $tokenType === ':' \|\| $tokenType === ',' ) {
1143	$newToken['code'] = T_NULLABLE;
1144	$newToken['type'] = 'T_NULLABLE';
1145
1146	if ( PHP_CODESNIFFER_VERBOSITY > 1 ) {
1147	echo "\t\t* token $stackPtr changed from ? to T_NULLABLE" . PHP_EOL;
1148	}
1149
1150	break;
1151	}
1152
1153	$prevNonEmpty = $tokenType;
1154	}
1155
1156	if ( $tokenType === T_FUNCTION
1157	\|\| $tokenType === T_FN
1158	\|\| isset( Util\Tokens::$methodPrefixes[ $tokenType ] ) === true
1159	\|\| $tokenType === T_VAR
1160	) {
1161	if ( PHP_CODESNIFFER_VERBOSITY > 1 ) {
1162	echo "\t\t* token $stackPtr changed from ? to T_NULLABLE" . PHP_EOL;
1163	}
1164
1165	$newToken['code'] = T_NULLABLE;
1166	$newToken['type'] = 'T_NULLABLE';
1167	break;
1168	} else if ( in_array( $tokenType, [
1169	T_DOUBLE_ARROW,
1170	T_OPEN_TAG,
1171	T_OPEN_TAG_WITH_ECHO,
1172	'=',
1173	'{',
1174	';'
1175	], true ) === true ) {
1176	if ( PHP_CODESNIFFER_VERBOSITY > 1 ) {
1177	echo "\t\t* token $stackPtr changed from ? to T_INLINE_THEN" . PHP_EOL;
1178	}
1179
1180	$newToken['code'] = T_INLINE_THEN;
1181	$newToken['type'] = 'T_INLINE_THEN';
1182
1183	$insideInlineIf[] = $stackPtr;
1184	break;
1185	}
1186
1187	if ( isset( Util\Tokens::$emptyTokens[ $tokenType ] ) === false ) {
1188	$lastSeenNonEmpty = $tokenType;
1189	}
1190	}//end for
1191
1192	$finalTokens[ $newStackPtr ] = $newToken;
1193	$newStackPtr ++;
1194	continue;
1195	}//end if
1196
1197	/*
1198	Tokens after a double colon may look like scope openers,
1199	such as when writing code like Foo::NAMESPACE, but they are
1200	only ever variables or strings.
1201	*/
1202
1203	if ( $stackPtr > 1
1204	&& ( is_array( $tokens[ ( $stackPtr - 1 ) ] ) === true
1205	&& $tokens[ ( $stackPtr - 1 ) ][0] === T_PAAMAYIM_NEKUDOTAYIM )
1206	&& $tokenIsArray === true
1207	&& $token[0] !== T_STRING
1208	&& $token[0] !== T_VARIABLE
1209	&& $token[0] !== T_DOLLAR
1210	&& isset( Util\Tokens::$emptyTokens[ $token[0] ] ) === false
1211	) {
1212	$newToken = [];
1213	$newToken['code'] = T_STRING;
1214	$newToken['type'] = 'T_STRING';
1215	$newToken['content'] = $token[1];
1216	$finalTokens[ $newStackPtr ] = $newToken;
1217
1218	$newStackPtr ++;
1219	continue;
1220	}
1221
1222	/*
1223	Backfill the T_FN token for PHP versions < 7.4.
1224	*/
1225
1226	if ( $tokenIsArray === true
1227	&& $token[0] === T_STRING
1228	&& strtolower( $token[1] ) === 'fn'
1229	) {
1230	// Modify the original token stack so that
1231	// future checks (like looking for T_NULLABLE) can
1232	// detect the T_FN token more easily.
1233	$tokens[ $stackPtr ][0] = T_FN;
1234	$token[0] = T_FN;
1235	}
1236
1237	/*
1238	The string-like token after a function keyword should always be
1239	tokenized as T_STRING even if it appears to be a different token,
1240	such as when writing code like: function default(): foo
1241	so go forward and change the token type before it is processed.
1242	*/
1243
1244	if ( $tokenIsArray === true
1245	&& ( $token[0] === T_FUNCTION
1246	\|\| $token[0] === T_FN )
1247	&& $finalTokens[ $lastNotEmptyToken ]['code'] !== T_USE
1248	) {
1249	for ( $x = ( $stackPtr + 1 ); $x < $numTokens; $x ++ ) {
1250	if ( is_array( $tokens[ $x ] ) === false
1251	\|\| isset( Util\Tokens::$emptyTokens[ $tokens[ $x ][0] ] ) === false
1252	) {
1253	// Non-empty content.
1254	break;
1255	}
1256	}
1257
1258	if ( $x < $numTokens && is_array( $tokens[ $x ] ) === true ) {
1259	$tokens[ $x ][0] = T_STRING;
1260	}
1261
1262	/*
1263	This is a special condition for T_ARRAY tokens used for
1264	function return types. We want to keep the parenthesis map clean,
1265	so let's tag these tokens as T_STRING.
1266	*/
1267
1268	// Go looking for the colon to start the return type hint.
1269	// Start by finding the closing parenthesis of the function.
1270	$parenthesisStack = [];
1271	$parenthesisCloser = false;
1272	for ( $x = ( $stackPtr + 1 ); $x < $numTokens; $x ++ ) {
1273	if ( is_array( $tokens[ $x ] ) === false && $tokens[ $x ] === '(' ) {
1274	$parenthesisStack[] = $x;
1275	} else if ( is_array( $tokens[ $x ] ) === false && $tokens[ $x ] === ')' ) {
1276	array_pop( $parenthesisStack );
1277	if ( empty( $parenthesisStack ) === true ) {
1278	$parenthesisCloser = $x;
1279	break;
1280	}
1281	}
1282	}
1283
1284	if ( $parenthesisCloser !== false ) {
1285	for ( $x = ( $parenthesisCloser + 1 ); $x < $numTokens; $x ++ ) {
1286	if ( is_array( $tokens[ $x ] ) === false
1287	\|\| isset( Util\Tokens::$emptyTokens[ $tokens[ $x ][0] ] ) === false
1288	) {
1289	// Non-empty content.
1290	if ( is_array( $tokens[ $x ] ) === true && $tokens[ $x ][0] === T_USE ) {
1291	// Found a use statements, so search ahead for the closing parenthesis.
1292	for ( $x += 1; $x < $numTokens; $x ++ ) {
1293	if ( is_array( $tokens[ $x ] ) === false && $tokens[ $x ] === ')' ) {
1294	continue( 2 );
1295	}
1296	}
1297	}
1298
1299	break;
1300	}
1301	}
1302
1303	if ( isset( $tokens[ $x ] ) === true
1304	&& is_array( $tokens[ $x ] ) === false
1305	&& $tokens[ $x ] === ':'
1306	) {
1307	$allowed = [
1308	T_STRING => T_STRING,
1309	T_ARRAY => T_ARRAY,
1310	T_CALLABLE => T_CALLABLE,
1311	T_SELF => T_SELF,
1312	T_PARENT => T_PARENT,
1313	T_NS_SEPARATOR => T_NS_SEPARATOR,
1314	];
1315
1316	$allowed += Util\Tokens::$emptyTokens;
1317
1318	// Find the start of the return type.
1319	for ( $x += 1; $x < $numTokens; $x ++ ) {
1320	if ( is_array( $tokens[ $x ] ) === true
1321	&& isset( Util\Tokens::$emptyTokens[ $tokens[ $x ][0] ] ) === true
1322	) {
1323	// Whitespace or comments before the return type.
1324	continue;
1325	}
1326
1327	if ( is_array( $tokens[ $x ] ) === false && $tokens[ $x ] === '?' ) {
1328	// Found a nullable operator, so skip it.
1329	// But also covert the token to save the tokenizer
1330	// a bit of time later on.
1331	$tokens[ $x ] = [
1332	T_NULLABLE,
1333	'?',
1334	];
1335
1336	if ( PHP_CODESNIFFER_VERBOSITY > 1 ) {
1337	echo "\t\t* token $x changed from ? to T_NULLABLE" . PHP_EOL;
1338	}
1339
1340	continue;
1341	}
1342
1343	break;
1344	}//end for
1345
1346	// Any T_ARRAY tokens we find between here and the next
1347	// token that can't be part of the return type need to be
1348	// converted to T_STRING tokens.
1349	for ( $x; $x < $numTokens; $x ++ ) {
1350	if ( is_array( $tokens[ $x ] ) === false \|\| isset( $allowed[ $tokens[ $x ][0] ] ) === false ) {
1351	break;
1352	} else if ( $tokens[ $x ][0] === T_ARRAY ) {
1353	$tokens[ $x ][0] = T_STRING;
1354
1355	if ( PHP_CODESNIFFER_VERBOSITY > 1 ) {
1356	echo "\t\t* token $x changed from T_ARRAY to T_STRING" . PHP_EOL;
1357	}
1358	}
1359	}
1360	}//end if
1361	}//end if
1362	}//end if
1363
1364	/*
1365	Before PHP 7, the <=> operator was tokenized as
1366	T_IS_SMALLER_OR_EQUAL followed by T_GREATER_THAN.
1367	So look for and combine these tokens in earlier versions.
1368	*/
1369
1370	if ( $tokenIsArray === true
1371	&& $token[0] === T_IS_SMALLER_OR_EQUAL
1372	&& isset( $tokens[ ( $stackPtr + 1 ) ] ) === true
1373	&& $tokens[ ( $stackPtr + 1 ) ][0] === '>'
1374	) {
1375	$newToken = [];
1376	$newToken['code'] = T_SPACESHIP;
1377	$newToken['type'] = 'T_SPACESHIP';
1378	$newToken['content'] = '<=>';
1379	$finalTokens[ $newStackPtr ] = $newToken;
1380
1381	$newStackPtr ++;
1382	$stackPtr ++;
1383	continue;
1384	}
1385
1386	/*
1387	PHP doesn't assign a token to goto labels, so we have to.
1388	These are just string tokens with a single colon after them. Double
1389	colons are already tokenized and so don't interfere with this check.
1390	But we do have to account for CASE statements, that look just like
1391	goto labels.
1392	*/
1393
1394	if ( $tokenIsArray === true
1395	&& $token[0] === T_STRING
1396	&& isset( $tokens[ ( $stackPtr + 1 ) ] ) === true
1397	&& $tokens[ ( $stackPtr + 1 ) ] === ':'
1398	&& $tokens[ ( $stackPtr - 1 ) ][0] !== T_PAAMAYIM_NEKUDOTAYIM
1399	) {
1400	$stopTokens = [
1401	T_CASE => true,
1402	T_SEMICOLON => true,
1403	T_OPEN_CURLY_BRACKET => true,
1404	T_INLINE_THEN => true,
1405	];
1406
1407	for ( $x = ( $newStackPtr - 1 ); $x > 0; $x -- ) {
1408	if ( isset( $stopTokens[ $finalTokens[ $x ]['code'] ] ) === true ) {
1409	break;
1410	}
1411	}
1412
1413	if ( $finalTokens[ $x ]['code'] !== T_CASE
1414	&& $finalTokens[ $x ]['code'] !== T_INLINE_THEN
1415	) {
1416	$finalTokens[ $newStackPtr ] = [
1417	'content' => $token[1] . ':',
1418	'code' => T_GOTO_LABEL,
1419	'type' => 'T_GOTO_LABEL',
1420	];
1421
1422	if ( PHP_CODESNIFFER_VERBOSITY > 1 ) {
1423	echo "\t\t* token $stackPtr changed from T_STRING to T_GOTO_LABEL" . PHP_EOL;
1424	echo "\t\t* skipping T_COLON token " . ( $stackPtr + 1 ) . PHP_EOL;
1425	}
1426
1427	$newStackPtr ++;
1428	$stackPtr ++;
1429	continue;
1430	}
1431	}//end if
1432
1433	/*
1434	If this token has newlines in its content, split each line up
1435	and create a new token for each line. We do this so it's easier
1436	to ascertain where errors occur on a line.
1437	Note that $token[1] is the token's content.
1438	*/
1439
1440	if ( $tokenIsArray === true && strpos( $token[1], $this->eolChar ) !== false ) {
1441	$tokenLines = explode( $this->eolChar, $token[1] );
1442	$numLines = count( $tokenLines );
1443	$newToken = [
1444	'type' => Util\Tokens::tokenName( $token[0] ),
1445	'code' => $token[0],
1446	'content' => '',
1447	];
1448
1449	for ( $i = 0; $i < $numLines; $i ++ ) {
1450	$newToken['content'] = $tokenLines[ $i ];
1451	if ( $i === ( $numLines - 1 ) ) {
1452	if ( $tokenLines[ $i ] === '' ) {
1453	break;
1454	}
1455	} else {
1456	$newToken['content'] .= $this->eolChar;
1457	}
1458
1459	$finalTokens[ $newStackPtr ] = $newToken;
1460	$newStackPtr ++;
1461	}
1462	} else {
1463	if ( $tokenIsArray === true && $token[0] === T_STRING ) {
1464	// Some T_STRING tokens should remain that way
1465	// due to their context.
1466	$context = [
1467	T_OBJECT_OPERATOR => true,
1468	T_FUNCTION => true,
1469	T_CLASS => true,
1470	T_EXTENDS => true,
1471	T_IMPLEMENTS => true,
1472	T_NEW => true,
1473	T_CONST => true,
1474	T_NS_SEPARATOR => true,
1475	T_USE => true,
1476	T_NAMESPACE => true,
1477	T_PAAMAYIM_NEKUDOTAYIM => true,
1478	];
1479
1480	if ( isset( $context[ $finalTokens[ $lastNotEmptyToken ]['code'] ] ) === true ) {
1481	// Special case for syntax like: return new self
1482	// where self should not be a string.
1483	if ( $finalTokens[ $lastNotEmptyToken ]['code'] === T_NEW
1484	&& strtolower( $token[1] ) === 'self'
1485	) {
1486	$finalTokens[ $newStackPtr ] = [
1487	'content' => $token[1],
1488	'code' => T_SELF,
1489	'type' => 'T_SELF',
1490	];
1491	} else {
1492	$finalTokens[ $newStackPtr ] = [
1493	'content' => $token[1],
1494	'code' => T_STRING,
1495	'type' => 'T_STRING',
1496	];
1497	}
1498
1499	$newStackPtr ++;
1500	continue;
1501	}//end if
1502	}//end if
1503
1504	$newToken = null;
1505	if ( $tokenIsArray === false ) {
1506	if ( isset( self::$resolveTokenCache[ $token[0] ] ) === true ) {
1507	$newToken = self::$resolveTokenCache[ $token[0] ];
1508	}
1509	} else {
1510	$cacheKey = null;
1511	if ( $token[0] === T_STRING ) {
1512	$cacheKey = strtolower( $token[1] );
1513	} else if ( $token[0] !== T_CURLY_OPEN ) {
1514	$cacheKey = $token[0];
1515	}
1516
1517	if ( $cacheKey !== null && isset( self::$resolveTokenCache[ $cacheKey ] ) === true ) {
1518	$newToken = self::$resolveTokenCache[ $cacheKey ];
1519	$newToken['content'] = $token[1];
1520	}
1521	}
1522
1523	if ( $newToken === null ) {
1524	$newToken = self::standardiseToken( $token );
1525	}
1526
1527	// Convert colons that are actually the ELSE component of an
1528	// inline IF statement.
1529	if ( empty( $insideInlineIf ) === false && $newToken['code'] === T_COLON ) {
1530	// Make sure this isn't the return type separator of a closure.
1531	$isInlineIf = true;
1532	for ( $i = ( $stackPtr - 1 ); $i > 0; $i -- ) {
1533	if ( is_array( $tokens[ $i ] ) === false
1534	\|\| ( $tokens[ $i ][0] !== T_DOC_COMMENT
1535	&& $tokens[ $i ][0] !== T_COMMENT
1536	&& $tokens[ $i ][0] !== T_WHITESPACE )
1537	) {
1538	break;
1539	}
1540	}
1541
1542	if ( $tokens[ $i ] === ')' ) {
1543	$parenCount = 1;
1544	for ( $i --; $i > 0; $i -- ) {
1545	if ( $tokens[ $i ] === '(' ) {
1546	$parenCount --;
1547	if ( $parenCount === 0 ) {
1548	break;
1549	}
1550	} else if ( $tokens[ $i ] === ')' ) {
1551	$parenCount ++;
1552	}
1553	}
1554
1555	// We've found the open parenthesis, so if the previous
1556	// non-empty token is FUNCTION or USE, this is a closure.
1557	for ( $i --; $i > 0; $i -- ) {
1558	if ( is_array( $tokens[ $i ] ) === false
1559	\|\| ( $tokens[ $i ][0] !== T_DOC_COMMENT
1560	&& $tokens[ $i ][0] !== T_COMMENT
1561	&& $tokens[ $i ][0] !== T_WHITESPACE )
1562	) {
1563	break;
1564	}
1565	}
1566
1567	if ( $tokens[ $i ][0] === T_FUNCTION \|\| $tokens[ $i ][0] === T_FN \|\| $tokens[ $i ][0] === T_USE ) {
1568	$isInlineIf = false;
1569	if ( PHP_CODESNIFFER_VERBOSITY > 1 ) {
1570	echo "\t\t* token is function return type, not T_INLINE_ELSE" . PHP_EOL;
1571	}
1572	}
1573	}//end if
1574
1575	// Check to see if this is a CASE or DEFAULT opener.
1576	$inlineIfToken = $insideInlineIf[ ( count( $insideInlineIf ) - 1 ) ];
1577	for ( $i = $stackPtr; $i > $inlineIfToken; $i -- ) {
1578	if ( is_array( $tokens[ $i ] ) === true
1579	&& ( $tokens[ $i ][0] === T_CASE
1580	\|\| $tokens[ $i ][0] === T_DEFAULT )
1581	) {
1582	$isInlineIf = false;
1583	if ( PHP_CODESNIFFER_VERBOSITY > 1 ) {
1584	echo "\t\t* token is T_CASE or T_DEFAULT opener, not T_INLINE_ELSE" . PHP_EOL;
1585	}
1586
1587	break;
1588	}
1589
1590	if ( is_array( $tokens[ $i ] ) === false
1591	&& ( $tokens[ $i ] === ';'
1592	\|\| $tokens[ $i ] === '{' )
1593	) {
1594	break;
1595	}
1596	}
1597
1598	if ( $isInlineIf === true ) {
1599	array_pop( $insideInlineIf );
1600	$newToken['code'] = T_INLINE_ELSE;
1601	$newToken['type'] = 'T_INLINE_ELSE';
1602
1603	if ( PHP_CODESNIFFER_VERBOSITY > 1 ) {
1604	echo "\t\t* token changed from T_COLON to T_INLINE_ELSE" . PHP_EOL;
1605	}
1606	}
1607	}//end if
1608
1609	// This is a special condition for T_ARRAY tokens used for
1610	// type hinting function arguments as being arrays. We want to keep
1611	// the parenthesis map clean, so let's tag these tokens as
1612	// T_STRING.
1613	if ( $newToken['code'] === T_ARRAY ) {
1614	for ( $i = $stackPtr; $i < $numTokens; $i ++ ) {
1615	if ( $tokens[ $i ] === '(' ) {
1616	break;
1617	} else if ( $tokens[ $i ][0] === T_VARIABLE ) {
1618	$newToken['code'] = T_STRING;
1619	$newToken['type'] = 'T_STRING';
1620	break;
1621	}
1622	}
1623	}
1624
1625	// This is a special case when checking PHP 5.5+ code in PHP < 5.5
1626	// where "finally" should be T_FINALLY instead of T_STRING.
1627	if ( $newToken['code'] === T_STRING
1628	&& strtolower( $newToken['content'] ) === 'finally'
1629	) {
1630	$newToken['code'] = T_FINALLY;
1631	$newToken['type'] = 'T_FINALLY';
1632	}
1633
1634	// This is a special case for the PHP 5.5 classname::class syntax
1635	// where "class" should be T_STRING instead of T_CLASS.
1636	if ( ( $newToken['code'] === T_CLASS
1637	\|\| $newToken['code'] === T_FUNCTION )
1638	&& $finalTokens[ $lastNotEmptyToken ]['code'] === T_DOUBLE_COLON
1639	) {
1640	$newToken['code'] = T_STRING;
1641	$newToken['type'] = 'T_STRING';
1642	}
1643
1644	// This is a special case for PHP 5.6 use function and use const
1645	// where "function" and "const" should be T_STRING instead of T_FUNCTION
1646	// and T_CONST.
1647	if ( ( $newToken['code'] === T_FUNCTION
1648	\|\| $newToken['code'] === T_CONST )
1649	&& ( $finalTokens[ $lastNotEmptyToken ]['code'] === T_USE \|\| $insideUseGroup === true )
1650	) {
1651	$newToken['code'] = T_STRING;
1652	$newToken['type'] = 'T_STRING';
1653	}
1654
1655	// This is a special case for use groups in PHP 7+ where leaving
1656	// the curly braces as their normal tokens would confuse
1657	// the scope map and sniffs.
1658	if ( $newToken['code'] === T_OPEN_CURLY_BRACKET
1659	&& $finalTokens[ $lastNotEmptyToken ]['code'] === T_NS_SEPARATOR
1660	) {
1661	$newToken['code'] = T_OPEN_USE_GROUP;
1662	$newToken['type'] = 'T_OPEN_USE_GROUP';
1663	$insideUseGroup = true;
1664	}
1665
1666	if ( $insideUseGroup === true && $newToken['code'] === T_CLOSE_CURLY_BRACKET ) {
1667	$newToken['code'] = T_CLOSE_USE_GROUP;
1668	$newToken['type'] = 'T_CLOSE_USE_GROUP';
1669	$insideUseGroup = false;
1670	}
1671
1672	$finalTokens[ $newStackPtr ] = $newToken;
1673	$newStackPtr ++;
1674	}//end if
1675	}//end for
1676
1677	if ( PHP_CODESNIFFER_VERBOSITY > 1 ) {
1678	echo "\t* END PHP TOKENIZING *" . PHP_EOL;
1679	}
1680
1681	return $finalTokens;
1682
1683	}//end tokenize()
1684
1685
1686	/**
1687	* Performs additional processing after main tokenizing.
1688	*
1689	* This additional processing checks for CASE statements that are using curly
1690	* braces for scope openers and closers. It also turns some T_FUNCTION tokens
1691	* into T_CLOSURE when they are not standard function definitions. It also
1692	* detects short array syntax and converts those square brackets into new tokens.
1693	* It also corrects some usage of the static and class keywords. It also
1694	* assigns tokens to function return types.
1695	*
1696	* @return void
1697	*/
1698	protected function processAdditional() {
1699	if ( PHP_CODESNIFFER_VERBOSITY > 1 ) {
1700	echo "\t* START ADDITIONAL PHP PROCESSING *" . PHP_EOL;
1701	}
1702
1703	$numTokens = count( $this->tokens );
1704	for ( $i = ( $numTokens - 1 ); $i >= 0; $i -- ) {
1705	// Check for any unset scope conditions due to alternate IF/ENDIF syntax.
1706	if ( isset( $this->tokens[ $i ]['scope_opener'] ) === true
1707	&& isset( $this->tokens[ $i ]['scope_condition'] ) === false
1708	) {
1709	$this->tokens[ $i ]['scope_condition'] = $this->tokens[ $this->tokens[ $i ]['scope_opener'] ]['scope_condition'];
1710	}
1711
1712	if ( $this->tokens[ $i ]['code'] === T_FUNCTION ) {
1713	/*
1714	Detect functions that are actually closures and
1715	assign them a different token.
1716	*/
1717
1718	if ( isset( $this->tokens[ $i ]['scope_opener'] ) === true ) {
1719	for ( $x = ( $i + 1 ); $x < $numTokens; $x ++ ) {
1720	if ( isset( Util\Tokens::$emptyTokens[ $this->tokens[ $x ]['code'] ] ) === false
1721	&& $this->tokens[ $x ]['code'] !== T_BITWISE_AND
1722	) {
1723	break;
1724	}
1725	}
1726
1727	if ( $this->tokens[ $x ]['code'] === T_OPEN_PARENTHESIS ) {
1728	$this->tokens[ $i ]['code'] = T_CLOSURE;
1729	$this->tokens[ $i ]['type'] = 'T_CLOSURE';
1730	if ( PHP_CODESNIFFER_VERBOSITY > 1 ) {
1731	$line = $this->tokens[ $i ]['line'];
1732	echo "\t* token $i on line $line changed from T_FUNCTION to T_CLOSURE" . PHP_EOL;
1733	}
1734
1735	for ( $x = ( $this->tokens[ $i ]['scope_opener'] + 1 ); $x < $this->tokens[ $i ]['scope_closer']; $x ++ ) {
1736	if ( isset( $this->tokens[ $x ]['conditions'][ $i ] ) === false ) {
1737	continue;
1738	}
1739
1740	$this->tokens[ $x ]['conditions'][ $i ] = T_CLOSURE;
1741	if ( PHP_CODESNIFFER_VERBOSITY > 1 ) {
1742	$type = $this->tokens[ $x ]['type'];
1743	echo "\t\t* cleaned $x ($type) *" . PHP_EOL;
1744	}
1745	}
1746	}
1747	}//end if
1748
1749	continue;
1750	} else if ( $this->tokens[ $i ]['code'] === T_CLASS && isset( $this->tokens[ $i ]['scope_opener'] ) === true ) {
1751	/*
1752	Detect anonymous classes and assign them a different token.
1753	*/
1754
1755	for ( $x = ( $i + 1 ); $x < $numTokens; $x ++ ) {
1756	if ( isset( Util\Tokens::$emptyTokens[ $this->tokens[ $x ]['code'] ] ) === false ) {
1757	break;
1758	}
1759	}
1760
1761	if ( $this->tokens[ $x ]['code'] === T_OPEN_PARENTHESIS
1762	\|\| $this->tokens[ $x ]['code'] === T_OPEN_CURLY_BRACKET
1763	\|\| $this->tokens[ $x ]['code'] === T_EXTENDS
1764	\|\| $this->tokens[ $x ]['code'] === T_IMPLEMENTS
1765	) {
1766	$this->tokens[ $i ]['code'] = T_ANON_CLASS;
1767	$this->tokens[ $i ]['type'] = 'T_ANON_CLASS';
1768	if ( PHP_CODESNIFFER_VERBOSITY > 1 ) {
1769	$line = $this->tokens[ $i ]['line'];
1770	echo "\t* token $i on line $line changed from T_CLASS to T_ANON_CLASS" . PHP_EOL;
1771	}
1772
1773	if ( $this->tokens[ $x ]['code'] === T_OPEN_PARENTHESIS
1774	&& isset( $this->tokens[ $x ]['parenthesis_closer'] ) === true
1775	) {
1776	$closer = $this->tokens[ $x ]['parenthesis_closer'];
1777
1778	$this->tokens[ $i ]['parenthesis_opener'] = $x;
1779	$this->tokens[ $i ]['parenthesis_closer'] = $closer;
1780	$this->tokens[ $i ]['parenthesis_owner'] = $i;
1781	$this->tokens[ $x ]['parenthesis_owner'] = $i;
1782	$this->tokens[ $closer ]['parenthesis_owner'] = $i;
1783
1784	if ( PHP_CODESNIFFER_VERBOSITY > 1 ) {
1785	$line = $this->tokens[ $i ]['line'];
1786	echo "\t\t* added parenthesis keys to T_ANON_CLASS token $i on line $line" . PHP_EOL;
1787	}
1788	}
1789
1790	for ( $x = ( $this->tokens[ $i ]['scope_opener'] + 1 ); $x < $this->tokens[ $i ]['scope_closer']; $x ++ ) {
1791	if ( isset( $this->tokens[ $x ]['conditions'][ $i ] ) === false ) {
1792	continue;
1793	}
1794
1795	$this->tokens[ $x ]['conditions'][ $i ] = T_ANON_CLASS;
1796	if ( PHP_CODESNIFFER_VERBOSITY > 1 ) {
1797	$type = $this->tokens[ $x ]['type'];
1798	echo "\t\t* cleaned $x ($type) *" . PHP_EOL;
1799	}
1800	}
1801	}//end if
1802
1803	continue;
1804	} else if ( $this->tokens[ $i ]['code'] === T_FN && isset( $this->tokens[ ( $i + 1 ) ] ) === true ) {
1805	// Possible arrow function.
1806	for ( $x = ( $i + 1 ); $x < $numTokens; $x ++ ) {
1807	if ( isset( Util\Tokens::$emptyTokens[ $this->tokens[ $x ]['code'] ] ) === false
1808	&& $this->tokens[ $x ]['code'] !== T_BITWISE_AND
1809	) {
1810	// Non-whitespace content.
1811	break;
1812	}
1813	}
1814
1815	if ( isset( $this->tokens[ $x ] ) === true && $this->tokens[ $x ]['code'] === T_OPEN_PARENTHESIS ) {
1816	$ignore = Util\Tokens::$emptyTokens;
1817	$ignore += [
1818	T_STRING => T_STRING,
1819	T_ARRAY => T_ARRAY,
1820	T_COLON => T_COLON,
1821	T_NS_SEPARATOR => T_NS_SEPARATOR,
1822	T_NULLABLE => T_NULLABLE,
1823	T_CALLABLE => T_CALLABLE,
1824	T_PARENT => T_PARENT,
1825	T_SELF => T_SELF,
1826	];
1827
1828	$closer = $this->tokens[ $x ]['parenthesis_closer'];
1829	for ( $arrow = ( $closer + 1 ); $arrow < $numTokens; $arrow ++ ) {
1830	if ( isset( $ignore[ $this->tokens[ $arrow ]['code'] ] ) === false ) {
1831	break;
1832	}
1833	}
1834
1835	if ( $this->tokens[ $arrow ]['code'] === T_DOUBLE_ARROW ) {
1836	$endTokens = [
1837	T_COLON => true,
1838	T_COMMA => true,
1839	T_SEMICOLON => true,
1840	T_CLOSE_PARENTHESIS => true,
1841	T_CLOSE_SQUARE_BRACKET => true,
1842	T_CLOSE_CURLY_BRACKET => true,
1843	T_CLOSE_SHORT_ARRAY => true,
1844	T_OPEN_TAG => true,
1845	T_CLOSE_TAG => true,
1846	];
1847
1848	$inTernary = false;
1849
1850	for ( $scopeCloser = ( $arrow + 1 ); $scopeCloser < $numTokens; $scopeCloser ++ ) {
1851	if ( isset( $endTokens[ $this->tokens[ $scopeCloser ]['code'] ] ) === true ) {
1852	break;
1853	}
1854
1855	if ( isset( $this->tokens[ $scopeCloser ]['scope_closer'] ) === true
1856	&& $this->tokens[ $scopeCloser ]['code'] !== T_INLINE_ELSE
1857	) {
1858	// We minus 1 here in case the closer can be shared with us.
1859	$scopeCloser = ( $this->tokens[ $scopeCloser ]['scope_closer'] - 1 );
1860	continue;
1861	}
1862
1863	if ( isset( $this->tokens[ $scopeCloser ]['parenthesis_closer'] ) === true ) {
1864	$scopeCloser = $this->tokens[ $scopeCloser ]['parenthesis_closer'];
1865	continue;
1866	}
1867
1868	if ( isset( $this->tokens[ $scopeCloser ]['bracket_closer'] ) === true ) {
1869	$scopeCloser = $this->tokens[ $scopeCloser ]['bracket_closer'];
1870	continue;
1871	}
1872
1873	if ( $this->tokens[ $scopeCloser ]['code'] === T_INLINE_THEN ) {
1874	$inTernary = true;
1875	continue;
1876	}
1877
1878	if ( $this->tokens[ $scopeCloser ]['code'] === T_INLINE_ELSE ) {
1879	if ( $inTernary === false ) {
1880	break;
1881	}
1882
1883	$inTernary = false;
1884	continue;
1885	}
1886	}//end for
1887
1888	if ( $scopeCloser !== $numTokens ) {
1889	if ( PHP_CODESNIFFER_VERBOSITY > 1 ) {
1890	$line = $this->tokens[ $i ]['line'];
1891	echo "\t=> token $i on line $line processed as arrow function" . PHP_EOL;
1892	}
1893
1894	$this->tokens[ $i ]['code'] = T_FN;
1895	$this->tokens[ $i ]['type'] = 'T_FN';
1896	$this->tokens[ $i ]['scope_condition'] = $i;
1897	$this->tokens[ $i ]['scope_opener'] = $arrow;
1898	$this->tokens[ $i ]['scope_closer'] = $scopeCloser;
1899	$this->tokens[ $i ]['parenthesis_owner'] = $i;
1900	$this->tokens[ $i ]['parenthesis_opener'] = $x;
1901	$this->tokens[ $i ]['parenthesis_closer'] = $this->tokens[ $x ]['parenthesis_closer'];
1902
1903	$this->tokens[ $arrow ]['code'] = T_FN_ARROW;
1904	$this->tokens[ $arrow ]['type'] = 'T_FN_ARROW';
1905
1906	$this->tokens[ $arrow ]['scope_condition'] = $i;
1907	$this->tokens[ $arrow ]['scope_opener'] = $arrow;
1908	$this->tokens[ $arrow ]['scope_closer'] = $scopeCloser;
1909	$this->tokens[ $scopeCloser ]['scope_condition'] = $i;
1910	$this->tokens[ $scopeCloser ]['scope_opener'] = $arrow;
1911	$this->tokens[ $scopeCloser ]['scope_closer'] = $scopeCloser;
1912
1913	$opener = $this->tokens[ $i ]['parenthesis_opener'];
1914	$closer = $this->tokens[ $i ]['parenthesis_closer'];
1915	$this->tokens[ $opener ]['parenthesis_owner'] = $i;
1916	$this->tokens[ $closer ]['parenthesis_owner'] = $i;
1917
1918	if ( PHP_CODESNIFFER_VERBOSITY > 1 ) {
1919	$line = $this->tokens[ $arrow ]['line'];
1920	echo "\t\t* token $arrow on line $line changed from T_DOUBLE_ARROW to T_FN_ARROW" . PHP_EOL;
1921	}
1922	}//end if
1923	}//end if
1924	}//end if
1925	} else if ( $this->tokens[ $i ]['code'] === T_OPEN_SQUARE_BRACKET ) {
1926	if ( isset( $this->tokens[ $i ]['bracket_closer'] ) === false ) {
1927	continue;
1928	}
1929
1930	// Unless there is a variable or a bracket before this token,
1931	// it is the start of an array being defined using the short syntax.
1932	$isShortArray = false;
1933	$allowed = [
1934	T_CLOSE_SQUARE_BRACKET => T_CLOSE_SQUARE_BRACKET,
1935	T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET,
1936	T_CLOSE_PARENTHESIS => T_CLOSE_PARENTHESIS,
1937	T_VARIABLE => T_VARIABLE,
1938	T_OBJECT_OPERATOR => T_OBJECT_OPERATOR,
1939	T_STRING => T_STRING,
1940	T_CONSTANT_ENCAPSED_STRING => T_CONSTANT_ENCAPSED_STRING,
1941	];
1942
1943	for ( $x = ( $i - 1 ); $x >= 0; $x -- ) {
1944	// If we hit a scope opener, the statement has ended
1945	// without finding anything, so it's probably an array
1946	// using PHP 7.1 short list syntax.
1947	if ( isset( $this->tokens[ $x ]['scope_opener'] ) === true ) {
1948	$isShortArray = true;
1949	break;
1950	}
1951
1952	if ( isset( Util\Tokens::$emptyTokens[ $this->tokens[ $x ]['code'] ] ) === false ) {
1953	if ( isset( $allowed[ $this->tokens[ $x ]['code'] ] ) === false ) {
1954	$isShortArray = true;
1955	}
1956
1957	break;
1958	}
1959	}
1960
1961	if ( $isShortArray === true ) {
1962	$this->tokens[ $i ]['code'] = T_OPEN_SHORT_ARRAY;
1963	$this->tokens[ $i ]['type'] = 'T_OPEN_SHORT_ARRAY';
1964
1965	$closer = $this->tokens[ $i ]['bracket_closer'];
1966	$this->tokens[ $closer ]['code'] = T_CLOSE_SHORT_ARRAY;
1967	$this->tokens[ $closer ]['type'] = 'T_CLOSE_SHORT_ARRAY';
1968	if ( PHP_CODESNIFFER_VERBOSITY > 1 ) {
1969	$line = $this->tokens[ $i ]['line'];
1970	echo "\t* token $i on line $line changed from T_OPEN_SQUARE_BRACKET to T_OPEN_SHORT_ARRAY" . PHP_EOL;
1971	$line = $this->tokens[ $closer ]['line'];
1972	echo "\t* token $closer on line $line changed from T_CLOSE_SQUARE_BRACKET to T_CLOSE_SHORT_ARRAY" . PHP_EOL;
1973	}
1974	}
1975
1976	continue;
1977	} else if ( $this->tokens[ $i ]['code'] === T_STATIC ) {
1978	for ( $x = ( $i - 1 ); $x > 0; $x -- ) {
1979	if ( isset( Util\Tokens::$emptyTokens[ $this->tokens[ $x ]['code'] ] ) === false ) {
1980	break;
1981	}
1982	}
1983
1984	if ( $this->tokens[ $x ]['code'] === T_INSTANCEOF ) {
1985	$this->tokens[ $i ]['code'] = T_STRING;
1986	$this->tokens[ $i ]['type'] = 'T_STRING';
1987
1988	if ( PHP_CODESNIFFER_VERBOSITY > 1 ) {
1989	$line = $this->tokens[ $i ]['line'];
1990	echo "\t* token $i on line $line changed from T_STATIC to T_STRING" . PHP_EOL;
1991	}
1992	}
1993
1994	continue;
1995	} else if ( $this->tokens[ $i ]['code'] === T_TRUE
1996	\|\| $this->tokens[ $i ]['code'] === T_FALSE
1997	\|\| $this->tokens[ $i ]['code'] === T_NULL
1998	) {
1999	for ( $x = ( $i + 1 ); $i < $numTokens; $x ++ ) {
2000	if ( isset( Util\Tokens::$emptyTokens[ $this->tokens[ $x ]['code'] ] ) === false ) {
2001	// Non-whitespace content.
2002	break;
2003	}
2004	}
2005
2006	$context = [
2007	T_OBJECT_OPERATOR => true,
2008	T_NS_SEPARATOR => true,
2009	T_PAAMAYIM_NEKUDOTAYIM => true,
2010	];
2011	if ( isset( $context[ $this->tokens[ $x ]['code'] ] ) === true ) {
2012	if ( PHP_CODESNIFFER_VERBOSITY > 1 ) {
2013	$line = $this->tokens[ $i ]['line'];
2014	$type = $this->tokens[ $i ]['type'];
2015	echo "\t* token $i on line $line changed from $type to T_STRING" . PHP_EOL;
2016	}
2017
2018	$this->tokens[ $i ]['code'] = T_STRING;
2019	$this->tokens[ $i ]['type'] = 'T_STRING';
2020	}
2021	} else if ( $this->tokens[ $i ]['code'] === T_CONST ) {
2022	// Context sensitive keywords support.
2023	for ( $x = ( $i + 1 ); $i < $numTokens; $x ++ ) {
2024	if ( isset( Util\Tokens::$emptyTokens[ $this->tokens[ $x ]['code'] ] ) === false ) {
2025	// Non-whitespace content.
2026	break;
2027	}
2028	}
2029
2030	if ( $this->tokens[ $x ]['code'] !== T_STRING ) {
2031	if ( PHP_CODESNIFFER_VERBOSITY > 1 ) {
2032	$line = $this->tokens[ $x ]['line'];
2033	$type = $this->tokens[ $x ]['type'];
2034	echo "\t* token $x on line $line changed from $type to T_STRING" . PHP_EOL;
2035	}
2036
2037	$this->tokens[ $x ]['code'] = T_STRING;
2038	$this->tokens[ $x ]['type'] = 'T_STRING';
2039	}
2040	}//end if
2041
2042	if ( ( $this->tokens[ $i ]['code'] !== T_CASE
2043	&& $this->tokens[ $i ]['code'] !== T_DEFAULT )
2044	\|\| isset( $this->tokens[ $i ]['scope_opener'] ) === false
2045	) {
2046	// Only interested in CASE and DEFAULT statements from here on in.
2047	continue;
2048	}
2049
2050	$scopeOpener = $this->tokens[ $i ]['scope_opener'];
2051	$scopeCloser = $this->tokens[ $i ]['scope_closer'];
2052
2053	// If the first char after the opener is a curly brace
2054	// and that brace has been ignored, it is actually
2055	// opening this case statement and the opener and closer are
2056	// probably set incorrectly.
2057	for ( $x = ( $scopeOpener + 1 ); $x < $numTokens; $x ++ ) {
2058	if ( isset( Util\Tokens::$emptyTokens[ $this->tokens[ $x ]['code'] ] ) === false ) {
2059	// Non-whitespace content.
2060	break;
2061	}
2062	}
2063
2064	if ( $this->tokens[ $x ]['code'] === T_CASE \|\| $this->tokens[ $x ]['code'] === T_DEFAULT ) {
2065	// Special case for multiple CASE statements that share the same
2066	// closer. Because we are going backwards through the file, this next
2067	// CASE statement is already fixed, so just use its closer and don't
2068	// worry about fixing anything.
2069	$newCloser = $this->tokens[ $x ]['scope_closer'];
2070	$this->tokens[ $i ]['scope_closer'] = $newCloser;
2071	if ( PHP_CODESNIFFER_VERBOSITY > 1 ) {
2072	$oldType = $this->tokens[ $scopeCloser ]['type'];
2073	$newType = $this->tokens[ $newCloser ]['type'];
2074	$line = $this->tokens[ $i ]['line'];
2075	echo "\t* token $i (T_CASE) on line $line closer changed from $scopeCloser ($oldType) to $newCloser ($newType)" . PHP_EOL;
2076	}
2077
2078	continue;
2079	}
2080
2081	if ( $this->tokens[ $x ]['code'] !== T_OPEN_CURLY_BRACKET
2082	\|\| isset( $this->tokens[ $x ]['scope_condition'] ) === true
2083	) {
2084	// Not a CASE/DEFAULT with a curly brace opener.
2085	continue;
2086	}
2087
2088	// The closer for this CASE/DEFAULT should be the closing curly brace and
2089	// not whatever it already is. The opener needs to be the opening curly
2090	// brace so everything matches up.
2091	$newCloser = $this->tokens[ $x ]['bracket_closer'];
2092	foreach ( [ $i, $x, $newCloser ] as $index ) {
2093	$this->tokens[ $index ]['scope_condition'] = $i;
2094	$this->tokens[ $index ]['scope_opener'] = $x;
2095	$this->tokens[ $index ]['scope_closer'] = $newCloser;
2096	}
2097
2098	if ( PHP_CODESNIFFER_VERBOSITY > 1 ) {
2099	$line = $this->tokens[ $i ]['line'];
2100	$tokenType = $this->tokens[ $i ]['type'];
2101
2102	$oldType = $this->tokens[ $scopeOpener ]['type'];
2103	$newType = $this->tokens[ $x ]['type'];
2104	echo "\t* token $i ($tokenType) on line $line opener changed from $scopeOpener ($oldType) to $x ($newType)" . PHP_EOL;
2105
2106	$oldType = $this->tokens[ $scopeCloser ]['type'];
2107	$newType = $this->tokens[ $newCloser ]['type'];
2108	echo "\t* token $i ($tokenType) on line $line closer changed from $scopeCloser ($oldType) to $newCloser ($newType)" . PHP_EOL;
2109	}
2110
2111	if ( $this->tokens[ $scopeOpener ]['scope_condition'] === $i ) {
2112	unset( $this->tokens[ $scopeOpener ]['scope_condition'] );
2113	unset( $this->tokens[ $scopeOpener ]['scope_opener'] );
2114	unset( $this->tokens[ $scopeOpener ]['scope_closer'] );
2115	}
2116
2117	if ( $this->tokens[ $scopeCloser ]['scope_condition'] === $i ) {
2118	unset( $this->tokens[ $scopeCloser ]['scope_condition'] );
2119	unset( $this->tokens[ $scopeCloser ]['scope_opener'] );
2120	unset( $this->tokens[ $scopeCloser ]['scope_closer'] );
2121	} else {
2122	// We were using a shared closer. All tokens that were
2123	// sharing this closer with us, except for the scope condition
2124	// and it's opener, need to now point to the new closer.
2125	$condition = $this->tokens[ $scopeCloser ]['scope_condition'];
2126	$start = ( $this->tokens[ $condition ]['scope_opener'] + 1 );
2127	for ( $y = $start; $y < $scopeCloser; $y ++ ) {
2128	if ( isset( $this->tokens[ $y ]['scope_closer'] ) === true
2129	&& $this->tokens[ $y ]['scope_closer'] === $scopeCloser
2130	) {
2131	$this->tokens[ $y ]['scope_closer'] = $newCloser;
2132
2133	if ( PHP_CODESNIFFER_VERBOSITY > 1 ) {
2134	$line = $this->tokens[ $y ]['line'];
2135	$tokenType = $this->tokens[ $y ]['type'];
2136	$oldType = $this->tokens[ $scopeCloser ]['type'];
2137	$newType = $this->tokens[ $newCloser ]['type'];
2138	echo "\t\t* token $y ($tokenType) on line $line closer changed from $scopeCloser ($oldType) to $newCloser ($newType)" . PHP_EOL;
2139	}
2140	}
2141	}
2142	}//end if
2143
2144	unset( $this->tokens[ $x ]['bracket_opener'] );
2145	unset( $this->tokens[ $x ]['bracket_closer'] );
2146	unset( $this->tokens[ $newCloser ]['bracket_opener'] );
2147	unset( $this->tokens[ $newCloser ]['bracket_closer'] );
2148	$this->tokens[ $scopeCloser ]['conditions'][] = $i;
2149
2150	// Now fix up all the tokens that think they are
2151	// inside the CASE/DEFAULT statement when they are really outside.
2152	for ( $x = $newCloser; $x < $scopeCloser; $x ++ ) {
2153	foreach ( $this->tokens[ $x ]['conditions'] as $num => $oldCond ) {
2154	if ( $oldCond === $this->tokens[ $i ]['code'] ) {
2155	$oldConditions = $this->tokens[ $x ]['conditions'];
2156	unset( $this->tokens[ $x ]['conditions'][ $num ] );
2157
2158	if ( PHP_CODESNIFFER_VERBOSITY > 1 ) {
2159	$type = $this->tokens[ $x ]['type'];
2160	$oldConds = '';
2161	foreach ( $oldConditions as $condition ) {
2162	$oldConds .= Util\Tokens::tokenName( $condition ) . ',';
2163	}
2164
2165	$oldConds = rtrim( $oldConds, ',' );
2166
2167	$newConds = '';
2168	foreach ( $this->tokens[ $x ]['conditions'] as $condition ) {
2169	$newConds .= Util\Tokens::tokenName( $condition ) . ',';
2170	}
2171
2172	$newConds = rtrim( $newConds, ',' );
2173
2174	echo "\t\t* cleaned $x ($type) *" . PHP_EOL;
2175	echo "\t\t\t=> conditions changed from $oldConds to $newConds" . PHP_EOL;
2176	}
2177
2178	break;
2179	}//end if
2180	}//end foreach
2181	}//end for
2182	}//end for
2183
2184	if ( PHP_CODESNIFFER_VERBOSITY > 1 ) {
2185	echo "\t* END ADDITIONAL PHP PROCESSING *" . PHP_EOL;
2186	}
2187
2188	}//end processAdditional()
2189
2190
2191	/**
2192	* Takes a token produced from <code>token_get_all()</code> and produces a
2193	* more uniform token.
2194	*
2195	* @param string\|array $token The token to convert.
2196	*
2197	* @return array The new token.
2198	*/
2199	public static function standardiseToken( $token ) {
2200	if ( isset( $token[1] ) === false ) {
2201	if ( isset( self::$resolveTokenCache[ $token[0] ] ) === true ) {
2202	return self::$resolveTokenCache[ $token[0] ];
2203	}
2204	} else {
2205	$cacheKey = null;
2206	if ( $token[0] === T_STRING ) {
2207	$cacheKey = strtolower( $token[1] );
2208	} else if ( $token[0] !== T_CURLY_OPEN ) {
2209	$cacheKey = $token[0];
2210	}
2211
2212	if ( $cacheKey !== null && isset( self::$resolveTokenCache[ $cacheKey ] ) === true ) {
2213	$newToken = self::$resolveTokenCache[ $cacheKey ];
2214	$newToken['content'] = $token[1];
2215
2216	return $newToken;
2217	}
2218	}
2219
2220	if ( isset( $token[1] ) === false ) {
2221	return self::resolveSimpleToken( $token[0] );
2222	}
2223
2224	if ( $token[0] === T_STRING ) {
2225	switch ( $cacheKey ) {
2226	case 'false':
2227	$newToken['type'] = 'T_FALSE';
2228	break;
2229	case 'true':
2230	$newToken['type'] = 'T_TRUE';
2231	break;
2232	case 'null':
2233	$newToken['type'] = 'T_NULL';
2234	break;
2235	case 'self':
2236	$newToken['type'] = 'T_SELF';
2237	break;
2238	case 'parent':
2239	$newToken['type'] = 'T_PARENT';
2240	break;
2241	default:
2242	$newToken['type'] = 'T_STRING';
2243	break;
2244	}
2245
2246	$newToken['code'] = constant( $newToken['type'] );
2247
2248	self::$resolveTokenCache[ $cacheKey ] = $newToken;
2249	} else if ( $token[0] === T_CURLY_OPEN ) {
2250	$newToken = [
2251	'code' => T_OPEN_CURLY_BRACKET,
2252	'type' => 'T_OPEN_CURLY_BRACKET',
2253	];
2254	} else {
2255	$newToken = [
2256	'code' => $token[0],
2257	'type' => Util\Tokens::tokenName( $token[0] ),
2258	];
2259
2260	self::$resolveTokenCache[ $token[0] ] = $newToken;
2261	}//end if
2262
2263	$newToken['content'] = $token[1];
2264
2265	return $newToken;
2266
2267	}//end standardiseToken()
2268
2269
2270	/**
2271	* Converts simple tokens into a format that conforms to complex tokens
2272	* produced by token_get_all().
2273	*
2274	* Simple tokens are tokens that are not in array form when produced from
2275	* token_get_all().
2276	*
2277	* @param string $token The simple token to convert.
2278	*
2279	* @return array The new token in array format.
2280	*/
2281	public static function resolveSimpleToken( $token ) {
2282	$newToken = [];
2283
2284	switch ( $token ) {
2285	case '{':
2286	$newToken['type'] = 'T_OPEN_CURLY_BRACKET';
2287	break;
2288	case '}':
2289	$newToken['type'] = 'T_CLOSE_CURLY_BRACKET';
2290	break;
2291	case '[':
2292	$newToken['type'] = 'T_OPEN_SQUARE_BRACKET';
2293	break;
2294	case ']':
2295	$newToken['type'] = 'T_CLOSE_SQUARE_BRACKET';
2296	break;
2297	case '(':
2298	$newToken['type'] = 'T_OPEN_PARENTHESIS';
2299	break;
2300	case ')':
2301	$newToken['type'] = 'T_CLOSE_PARENTHESIS';
2302	break;
2303	case ':':
2304	$newToken['type'] = 'T_COLON';
2305	break;
2306	case '.':
2307	$newToken['type'] = 'T_STRING_CONCAT';
2308	break;
2309	case ';':
2310	$newToken['type'] = 'T_SEMICOLON';
2311	break;
2312	case '=':
2313	$newToken['type'] = 'T_EQUAL';
2314	break;
2315	case '*':
2316	$newToken['type'] = 'T_MULTIPLY';
2317	break;
2318	case '/':
2319	$newToken['type'] = 'T_DIVIDE';
2320	break;
2321	case '+':
2322	$newToken['type'] = 'T_PLUS';
2323	break;
2324	case '-':
2325	$newToken['type'] = 'T_MINUS';
2326	break;
2327	case '%':
2328	$newToken['type'] = 'T_MODULUS';
2329	break;
2330	case '^':
2331	$newToken['type'] = 'T_BITWISE_XOR';
2332	break;
2333	case '&':
2334	$newToken['type'] = 'T_BITWISE_AND';
2335	break;
2336	case '\|':
2337	$newToken['type'] = 'T_BITWISE_OR';
2338	break;
2339	case '~':
2340	$newToken['type'] = 'T_BITWISE_NOT';
2341	break;
2342	case '<':
2343	$newToken['type'] = 'T_LESS_THAN';
2344	break;
2345	case '>':
2346	$newToken['type'] = 'T_GREATER_THAN';
2347	break;
2348	case '!':
2349	$newToken['type'] = 'T_BOOLEAN_NOT';
2350	break;
2351	case ',':
2352	$newToken['type'] = 'T_COMMA';
2353	break;
2354	case '@':
2355	$newToken['type'] = 'T_ASPERAND';
2356	break;
2357	case '$':
2358	$newToken['type'] = 'T_DOLLAR';
2359	break;
2360	case '`':
2361	$newToken['type'] = 'T_BACKTICK';
2362	break;
2363	default:
2364	$newToken['type'] = 'T_NONE';
2365	break;
2366	}//end switch
2367
2368	$newToken['code'] = constant( $newToken['type'] );
2369	$newToken['content'] = $token;
2370
2371	self::$resolveTokenCache[ $token ] = $newToken;
2372
2373	return $newToken;
2374
2375	}//end resolveSimpleToken()
2376
2377
2378	}//end class
2379