| @@ -50,13 +50,13 @@ | ||
| 50 | 50 | } |
| 51 | 51 | |
| 52 | 52 | public function getTokenName(): ?string |
| 53 | 53 | { |
| 54 | - if ('UNKNOWN' === $name = token_name($this->id)) { | |
| 55 | - $name = \strlen($this->text) > 1 || \ord($this->text) < 32 ? null : $this->text; | |
| 54 | + if ($this->id < 256) { | |
| 55 | + return \chr($this->id & 0xFF); | |
| 56 | 56 | } |
| 57 | 57 | |
| 58 | - return $name; | |
| 58 | + return 'UNKNOWN' === ($name = token_name($this->id)) ? null : $name; | |
| 59 | 59 | } |
| 60 | 60 | |
| 61 | 61 | /** |
| 62 | 62 | * @param int|string|array $kind |
| @@ -91,9 +91,10 @@ | ||
| 91 | 91 | $position = 0; |
| 92 | 92 | $tokens = token_get_all($code, $flags); |
| 93 | 93 | foreach ($tokens as $index => $token) { |
| 94 | 94 | if (\is_string($token)) { |
| 95 | - $id = \ord($token); | |
| 95 | + // b" is the only such token longer than one char | |
| 96 | + $id = \ord($token[-1]); | |
| 96 | 97 | $text = $token; |
| 97 | 98 | } else { |
| 98 | 99 | [$id, $text, $line] = $token; |
| 99 | 100 | } |
| @@ -98,8 +99,9 @@ | ||
| 98 | 99 | [$id, $text, $line] = $token; |
| 99 | 100 | } |
| 100 | 101 | $tokens[$index] = new static($id, $text, $line, $position); |
| 101 | 102 | $position += \strlen($text); |
| 103 | + $line += preg_match_all('/\r\n?|\n/', $text); | |
| 102 | 104 | } |
| 103 | 105 | |
| 104 | 106 | return $tokens; |
| 105 | 107 | } |