Lexer.php — WooCommerce 10.9.3

woocommerce / lib / packages / GraphQL / Language / Lexer.php

woocommerce / lib / packages / GraphQL / Language Last commit date

AST 2 months ago BlockString.php 2 months ago DirectiveLocation.php 2 months ago Lexer.php 2 months ago Parser.php 2 months ago Printer.php 2 months ago Source.php 2 months ago SourceLocation.php 2 months ago Token.php 2 months ago Visitor.php 2 months ago VisitorOperation.php 2 months ago VisitorRemoveNode.php 2 months ago VisitorSkipNode.php 2 months ago VisitorStop.php 2 months ago

Lexer.php

744 lines

1	<?php declare(strict_types=1);
2
3	namespace Automattic\WooCommerce\Vendor\GraphQL\Language;
4
5	use Automattic\WooCommerce\Vendor\GraphQL\Error\SyntaxError;
6	use Automattic\WooCommerce\Vendor\GraphQL\Utils\Utils;
7
8	/**
9	* A lexer is a stateful stream generator, it returns the next token in the Source when advanced.
10	* Assuming the source is valid, the final returned token will be EOF,
11	* after which the lexer will repeatedly return the same EOF token whenever called.
12	*
13	* Algorithm is O(N) both on memory and time.
14	*
15	* @phpstan-import-type ParserOptions from Parser
16	*
17	* @see \Automattic\WooCommerce\Vendor\GraphQL\Tests\Language\LexerTest
18	*/
19	class Lexer
20	{
21	// https://spec.graphql.org/October2021/#sec-Punctuators
22	private const TOKEN_BANG = 33;
23	private const TOKEN_DOLLAR = 36;
24	private const TOKEN_AMP = 38;
25	private const TOKEN_PAREN_L = 40;
26	private const TOKEN_PAREN_R = 41;
27	private const TOKEN_DOT = 46;
28	private const TOKEN_COLON = 58;
29	private const TOKEN_EQUALS = 61;
30	private const TOKEN_AT = 64;
31	private const TOKEN_BRACKET_L = 91;
32	private const TOKEN_BRACKET_R = 93;
33	private const TOKEN_BRACE_L = 123;
34	private const TOKEN_PIPE = 124;
35	private const TOKEN_BRACE_R = 125;
36
37	public Source $source;
38
39	/** @phpstan-var ParserOptions */
40	public array $options;
41
42	/** The previously focused non-ignored token. */
43	public Token $lastToken;
44
45	/** The currently focused non-ignored token. */
46	public Token $token;
47
48	/** The (1-indexed) line containing the current token. */
49	public int $line = 1;
50
51	/** The character offset at which the current line begins. */
52	public int $lineStart = 0;
53
54	/** Current cursor position for UTF8 encoding of the source. */
55	private int $position = 0;
56
57	/** Current cursor position for ASCII representation of the source. */
58	private int $byteStreamPosition = 0;
59
60	/** @phpstan-param ParserOptions $options */
61	public function __construct(Source $source, array $options = [])
62	{
63	$startOfFileToken = new Token(Token::SOF, 0, 0, 0, 0);
64
65	$this->source = $source;
66	$this->options = $options;
67	$this->lastToken = $startOfFileToken;
68	$this->token = $startOfFileToken;
69	}
70
71	/**
72	* @throws \JsonException
73	* @throws SyntaxError
74	*/
75	public function advance(): Token
76	{
77	$this->lastToken = $this->token;
78
79	return $this->token = $this->lookahead();
80	}
81
82	/**
83	* @throws \JsonException
84	* @throws SyntaxError
85	*/
86	public function lookahead(): Token
87	{
88	$token = $this->token;
89	if ($token->kind !== Token::EOF) {
90	do {
91	$token = $token->next ?? ($token->next = $this->readToken($token));
92	} while ($token->kind === Token::COMMENT);
93	}
94
95	return $token;
96	}
97
98	/**
99	* @throws \JsonException
100	* @throws SyntaxError
101	*/
102	private function readToken(Token $prev): Token
103	{
104	$bodyLength = $this->source->length;
105
106	$this->positionAfterWhitespace();
107	$position = $this->position;
108
109	$line = $this->line;
110	$col = 1 + $position - $this->lineStart;
111
112	if ($position >= $bodyLength) {
113	return new Token(Token::EOF, $bodyLength, $bodyLength, $line, $col, $prev);
114	}
115
116	// Read next char and advance string cursor:
117	[, $code, $bytes] = $this->readChar(true);
118
119	switch ($code) {
120	case self::TOKEN_BANG: // !
121	return new Token(Token::BANG, $position, $position + 1, $line, $col, $prev);
122	case 35: // #
123	$this->moveStringCursor(-1, -1 * $bytes);
124
125	return $this->readComment($line, $col, $prev);
126	case self::TOKEN_DOLLAR: // $
127	return new Token(Token::DOLLAR, $position, $position + 1, $line, $col, $prev);
128	case self::TOKEN_AMP: // &
129	return new Token(Token::AMP, $position, $position + 1, $line, $col, $prev);
130	case self::TOKEN_PAREN_L: // (
131	return new Token(Token::PAREN_L, $position, $position + 1, $line, $col, $prev);
132	case self::TOKEN_PAREN_R: // )
133	return new Token(Token::PAREN_R, $position, $position + 1, $line, $col, $prev);
134	case self::TOKEN_DOT: // .
135	[, $charCode1] = $this->readChar(true);
136	[, $charCode2] = $this->readChar(true);
137
138	if ($charCode1 === self::TOKEN_DOT && $charCode2 === self::TOKEN_DOT) {
139	return new Token(Token::SPREAD, $position, $position + 3, $line, $col, $prev);
140	}
141
142	break;
143	case self::TOKEN_COLON: // :
144	return new Token(Token::COLON, $position, $position + 1, $line, $col, $prev);
145	case self::TOKEN_EQUALS: // =
146	return new Token(Token::EQUALS, $position, $position + 1, $line, $col, $prev);
147	case self::TOKEN_AT: // @
148	return new Token(Token::AT, $position, $position + 1, $line, $col, $prev);
149	case self::TOKEN_BRACKET_L: // [
150	return new Token(Token::BRACKET_L, $position, $position + 1, $line, $col, $prev);
151	case self::TOKEN_BRACKET_R: // ]
152	return new Token(Token::BRACKET_R, $position, $position + 1, $line, $col, $prev);
153	case self::TOKEN_BRACE_L: // {
154	return new Token(Token::BRACE_L, $position, $position + 1, $line, $col, $prev);
155	case self::TOKEN_PIPE: // \|
156	return new Token(Token::PIPE, $position, $position + 1, $line, $col, $prev);
157	case self::TOKEN_BRACE_R: // }
158	return new Token(Token::BRACE_R, $position, $position + 1, $line, $col, $prev);
159	// A-Z
160	case 65:
161	case 66:
162	case 67:
163	case 68:
164	case 69:
165	case 70:
166	case 71:
167	case 72:
168	case 73:
169	case 74:
170	case 75:
171	case 76:
172	case 77:
173	case 78:
174	case 79:
175	case 80:
176	case 81:
177	case 82:
178	case 83:
179	case 84:
180	case 85:
181	case 86:
182	case 87:
183	case 88:
184	case 89:
185	case 90:
186	// _
187	case 95:
188	// a-z
189	case 97:
190	case 98:
191	case 99:
192	case 100:
193	case 101:
194	case 102:
195	case 103:
196	case 104:
197	case 105:
198	case 106:
199	case 107:
200	case 108:
201	case 109:
202	case 110:
203	case 111:
204	case 112:
205	case 113:
206	case 114:
207	case 115:
208	case 116:
209	case 117:
210	case 118:
211	case 119:
212	case 120:
213	case 121:
214	case 122:
215	return $this->moveStringCursor(-1, -1 * $bytes)
216	->readName($line, $col, $prev);
217	// -
218	case 45:
219	// 0-9
220	case 48:
221	case 49:
222	case 50:
223	case 51:
224	case 52:
225	case 53:
226	case 54:
227	case 55:
228	case 56:
229	case 57:
230	return $this->moveStringCursor(-1, -1 * $bytes)
231	->readNumber($line, $col, $prev);
232	// "
233	case 34:
234	[, $nextCode] = $this->readChar();
235	[, $nextNextCode] = $this->moveStringCursor(1, 1)
236	->readChar();
237
238	if ($nextCode === 34 && $nextNextCode === 34) {
239	return $this->moveStringCursor(-2, (-1 * $bytes) - 1)
240	->readBlockString($line, $col, $prev);
241	}
242
243	return $this->moveStringCursor(-2, (-1 * $bytes) - 1)
244	->readString($line, $col, $prev);
245	}
246
247	throw new SyntaxError($this->source, $position, $this->unexpectedCharacterMessage($code));
248	}
249
250	/** @throws \JsonException */
251	private function unexpectedCharacterMessage(?int $code): string
252	{
253	// SourceCharacter
254	if ($code < 0x0020 && $code !== 0x0009 && $code !== 0x000A && $code !== 0x000D) {
255	return 'Cannot contain the invalid character ' . Utils::printCharCode($code);
256	}
257
258	if ($code === 39) {
259	return 'Unexpected single quote character (\'), did you mean to use a double quote (")?';
260	}
261
262	return 'Cannot parse the unexpected character ' . Utils::printCharCode($code) . '.';
263	}
264
265	/**
266	* Reads an alphanumeric + underscore name from the source.
267	*
268	* [_A-Za-z][_0-9A-Za-z]*
269	*/
270	private function readName(int $line, int $col, Token $prev): Token
271	{
272	$start = $this->position;
273	$body = $this->source->body;
274	$length = strspn($body, 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_', $this->byteStreamPosition);
275	$value = substr($body, $this->byteStreamPosition, $length);
276	$this->moveStringCursor($length, $length);
277
278	return new Token(
279	Token::NAME,
280	$start,
281	$this->position,
282	$line,
283	$col,
284	$prev,
285	$value
286	);
287	}
288
289	/**
290	* Reads a number token from the source file, either a float
291	* or an int depending on whether a decimal point appears.
292	*
293	* Int: -?(0\|[1-9][0-9]*)
294	* Float: -?(0\|[1-9][0-9]*)(\.[0-9]+)?((E\|e)(+\|-)?[0-9]+)?
295	*
296	* @throws \JsonException
297	* @throws SyntaxError
298	*/
299	private function readNumber(int $line, int $col, Token $prev): Token
300	{
301	$value = '';
302	$start = $this->position;
303	[$char, $code] = $this->readChar();
304
305	$isFloat = false;
306
307	if ($code === 45) { // -
308	$value .= $char;
309	[$char, $code] = $this->moveStringCursor(1, 1)->readChar();
310	}
311
312	// guard against leading zero's
313	if ($code === 48) { // 0
314	$value .= $char;
315	[$char, $code] = $this->moveStringCursor(1, 1)->readChar();
316
317	if ($code >= 48 && $code <= 57) {
318	throw new SyntaxError($this->source, $this->position, 'Invalid number, unexpected digit after 0: ' . Utils::printCharCode($code));
319	}
320	} else {
321	$value .= $this->readDigits();
322	[$char, $code] = $this->readChar();
323	}
324
325	if ($code === 46) { // .
326	$isFloat = true;
327	$this->moveStringCursor(1, 1);
328
329	$value .= $char;
330	$value .= $this->readDigits();
331	[$char, $code] = $this->readChar();
332	}
333
334	if ($code === 69 \|\| $code === 101) { // E e
335	$isFloat = true;
336	$value .= $char;
337	[$char, $code] = $this->moveStringCursor(1, 1)->readChar();
338
339	if ($code === 43 \|\| $code === 45) { // + -
340	$value .= $char;
341	$this->moveStringCursor(1, 1);
342	}
343
344	$value .= $this->readDigits();
345	}
346
347	return new Token(
348	$isFloat ? Token::FLOAT : Token::INT,
349	$start,
350	$this->position,
351	$line,
352	$col,
353	$prev,
354	$value
355	);
356	}
357
358	/**
359	* Returns string with all digits + changes current string cursor position to point to the first char after digits.
360	*
361	* @throws \JsonException
362	* @throws SyntaxError
363	*/
364	private function readDigits(): string
365	{
366	[$char, $code] = $this->readChar();
367
368	if ($code >= 48 && $code <= 57) { // 0 - 9
369	$value = '';
370
371	do {
372	$value .= $char;
373	[$char, $code] = $this->moveStringCursor(1, 1)->readChar();
374	} while ($code >= 48 && $code <= 57); // 0 - 9
375
376	return $value;
377	}
378
379	if ($this->position > $this->source->length - 1) {
380	$code = null;
381	}
382
383	throw new SyntaxError($this->source, $this->position, 'Invalid number, expected digit but got: ' . Utils::printCharCode($code));
384	}
385
386	/**
387	* @throws \JsonException
388	* @throws SyntaxError
389	*/
390	private function readString(int $line, int $col, Token $prev): Token
391	{
392	$start = $this->position;
393
394	// Skip leading quote and read first string char:
395	[$char, $code, $bytes] = $this->moveStringCursor(1, 1)
396	->readChar();
397
398	$chunk = '';
399	$value = '';
400
401	while (! in_array($code, [null, 10, 13], true)) { // not LineTerminator
402	if ($code === 34) { // Closing Quote (")
403	$value .= $chunk;
404
405	// Skip quote
406	$this->moveStringCursor(1, 1);
407
408	return new Token(
409	Token::STRING,
410	$start,
411	$this->position,
412	$line,
413	$col,
414	$prev,
415	$value
416	);
417	}
418
419	$this->assertValidStringCharacterCode($code, $this->position);
420	$this->moveStringCursor(1, $bytes);
421
422	if ($code === 92) { // \
423	$value .= $chunk;
424	[, $code] = $this->readChar(true);
425
426	switch ($code) {
427	case 34:
428	$value .= '"';
429	break;
430	case 47:
431	$value .= '/';
432	break;
433	case 92:
434	$value .= '\\';
435	break;
436	case 98:
437	$value .= chr(8); // \b (backspace)
438	break;
439	case 102:
440	$value .= "\f";
441	break;
442	case 110:
443	$value .= "\n";
444	break;
445	case 114:
446	$value .= "\r";
447	break;
448	case 116:
449	$value .= "\t";
450	break;
451	case 117:
452	$position = $this->position;
453	[$hex] = $this->readChars(4);
454	if (preg_match('/[0-9a-fA-F]{4}/', $hex) !== 1) {
455	throw new SyntaxError($this->source, $position - 1, "Invalid character escape sequence: \\u{$hex}");
456	}
457
458	$code = hexdec($hex);
459	assert(is_int($code), 'Since only a single char is read');
460
461	// UTF-16 surrogate pair detection and handling.
462	$highOrderByte = $code >> 8;
463	if ($highOrderByte >= 0xD8 && $highOrderByte <= 0xDF) {
464	[$utf16Continuation] = $this->readChars(6);
465	if (preg_match('/^\\\u[0-9a-fA-F]{4}$/', $utf16Continuation) !== 1) {
466	throw new SyntaxError($this->source, $this->position - 5, 'Invalid UTF-16 trailing surrogate: ' . $utf16Continuation);
467	}
468
469	$surrogatePairHex = $hex . substr($utf16Continuation, 2, 4);
470	$value .= mb_convert_encoding(pack('H*', $surrogatePairHex), 'UTF-8', 'UTF-16');
471	break;
472	}
473
474	$this->assertValidStringCharacterCode($code, $position - 2);
475
476	$value .= Utils::chr($code);
477	break;
478	// null means EOF, will delegate to general handling of unterminated strings
479	case null:
480	continue 2;
481	default:
482	$chr = Utils::chr($code);
483	throw new SyntaxError($this->source, $this->position - 1, "Invalid character escape sequence: \\{$chr}");
484	}
485
486	$chunk = '';
487	} else {
488	$chunk .= $char;
489	}
490
491	[$char, $code, $bytes] = $this->readChar();
492	}
493
494	throw new SyntaxError($this->source, $this->position, 'Unterminated string.');
495	}
496
497	/**
498	* Reads a block string token from the source file.
499	*
500	* """("?"?(\\"""\|\\(?!=""")\|[^"\\]))*"""
501	*
502	* @throws \JsonException
503	* @throws SyntaxError
504	*/
505	private function readBlockString(int $line, int $col, Token $prev): Token
506	{
507	$start = $this->position;
508
509	// Skip leading quotes and read first string char:
510	[$char, $code, $bytes] = $this->moveStringCursor(3, 3)->readChar();
511
512	$chunk = '';
513	$value = '';
514
515	while ($code !== null) {
516	// Closing Triple-Quote (""")
517	if ($code === 34) {
518	// Move 2 quotes
519	[, $nextCode] = $this->moveStringCursor(1, 1)->readChar();
520	[, $nextNextCode] = $this->moveStringCursor(1, 1)->readChar();
521
522	if ($nextCode === 34 && $nextNextCode === 34) {
523	$value .= $chunk;
524
525	$this->moveStringCursor(1, 1);
526
527	return new Token(
528	Token::BLOCK_STRING,
529	$start,
530	$this->position,
531	$line,
532	$col,
533	$prev,
534	BlockString::dedentBlockStringLines($value)
535	);
536	}
537
538	// move cursor back to before the first quote
539	$this->moveStringCursor(-2, -2);
540	}
541
542	$this->assertValidBlockStringCharacterCode($code, $this->position);
543	$this->moveStringCursor(1, $bytes);
544
545	[, $nextCode] = $this->readChar();
546	[, $nextNextCode] = $this->moveStringCursor(1, 1)->readChar();
547	[, $nextNextNextCode] = $this->moveStringCursor(1, 1)->readChar();
548
549	// Escape Triple-Quote (\""")
550	if (
551	$code === 92
552	&& $nextCode === 34
553	&& $nextNextCode === 34
554	&& $nextNextNextCode === 34
555	) {
556	$this->moveStringCursor(1, 1);
557	$value .= $chunk . '"""';
558	$chunk = '';
559	} else {
560	// move cursor back to before the first quote
561	$this->moveStringCursor(-2, -2);
562
563	if ($code === 10) { // new line
564	++$this->line;
565	$this->lineStart = $this->position;
566	}
567
568	$chunk .= $char;
569	}
570
571	[$char, $code, $bytes] = $this->readChar();
572	}
573
574	throw new SyntaxError($this->source, $this->position, 'Unterminated string.');
575	}
576
577	/**
578	* @throws \JsonException
579	* @throws SyntaxError
580	*/
581	private function assertValidStringCharacterCode(int $code, int $position): void
582	{
583	// SourceCharacter
584	if ($code < 0x0020 && $code !== 0x0009) {
585	$char = Utils::printCharCode($code);
586	throw new SyntaxError($this->source, $position, "Invalid character within String: {$char}");
587	}
588	}
589
590	/**
591	* @throws \JsonException
592	* @throws SyntaxError
593	*/
594	private function assertValidBlockStringCharacterCode(int $code, int $position): void
595	{
596	// SourceCharacter
597	if ($code < 0x0020 && $code !== 0x0009 && $code !== 0x000A && $code !== 0x000D) {
598	$char = Utils::printCharCode($code);
599	throw new SyntaxError($this->source, $position, "Invalid character within String: {$char}");
600	}
601	}
602
603	/**
604	* Reads from body starting at startPosition until it finds a non-whitespace
605	* or commented character, then places cursor to the position of that character.
606	*/
607	private function positionAfterWhitespace(): void
608	{
609	while ($this->position < $this->source->length) {
610	[, $code, $bytes] = $this->readChar();
611
612	// Skip whitespace
613	// tab \| space \| comma \| BOM
614	if (in_array($code, [9, 32, 44, 0xFEFF], true)) {
615	$this->moveStringCursor(1, $bytes);
616	} elseif ($code === 10) { // new line
617	$this->moveStringCursor(1, $bytes);
618	++$this->line;
619	$this->lineStart = $this->position;
620	} elseif ($code === 13) { // carriage return
621	[, $nextCode, $nextBytes] = $this->moveStringCursor(1, $bytes)->readChar();
622
623	if ($nextCode === 10) { // lf after cr
624	$this->moveStringCursor(1, $nextBytes);
625	}
626
627	++$this->line;
628	$this->lineStart = $this->position;
629	} else {
630	break;
631	}
632	}
633	}
634
635	/**
636	* Reads a comment token from the source file.
637	*
638	* #[\u0009\u0020-\uFFFF]*
639	*/
640	private function readComment(int $line, int $col, Token $prev): Token
641	{
642	$start = $this->position;
643	$value = '';
644	$bytes = 1;
645
646	do {
647	[$char, $code, $bytes] = $this->moveStringCursor(1, $bytes)->readChar();
648	$value .= $char;
649	} while (
650	$code !== null
651	// SourceCharacter but not LineTerminator
652	&& ($code > 0x001F \|\| $code === 0x0009)
653	);
654
655	return new Token(
656	Token::COMMENT,
657	$start,
658	$this->position,
659	$line,
660	$col,
661	$prev,
662	$value
663	);
664	}
665
666	/**
667	* Reads next UTF8Character from the byte stream, starting from $byteStreamPosition.
668	*
669	* @return array{string, int\|null, int}
670	*/
671	private function readChar(bool $advance = false, ?int $byteStreamPosition = null): array
672	{
673	if ($byteStreamPosition === null) {
674	$byteStreamPosition = $this->byteStreamPosition;
675	}
676
677	$code = null;
678	$utf8char = '';
679	$bytes = 0;
680	$positionOffset = 0;
681
682	if (isset($this->source->body[$byteStreamPosition])) {
683	$ord = ord($this->source->body[$byteStreamPosition]);
684
685	if ($ord < 128) {
686	$bytes = 1;
687	} elseif ($ord < 224) {
688	$bytes = 2;
689	} elseif ($ord < 240) {
690	$bytes = 3;
691	} else {
692	$bytes = 4;
693	}
694
695	for ($pos = $byteStreamPosition; $pos < $byteStreamPosition + $bytes; ++$pos) {
696	$utf8char .= $this->source->body[$pos];
697	}
698
699	$positionOffset = 1;
700	$code = $bytes === 1
701	? $ord
702	: Utils::ord($utf8char);
703	}
704
705	if ($advance) {
706	$this->moveStringCursor($positionOffset, $bytes);
707	}
708
709	return [$utf8char, $code, $bytes];
710	}
711
712	/**
713	* Reads next $numberOfChars UTF8 characters from the byte stream.
714	*
715	* @return array{string, int}
716	*/
717	private function readChars(int $charCount): array
718	{
719	$result = '';
720	$totalBytes = 0;
721	$byteOffset = $this->byteStreamPosition;
722
723	for ($i = 0; $i < $charCount; ++$i) {
724	[$char, $code, $bytes] = $this->readChar(false, $byteOffset);
725	$totalBytes += $bytes;
726	$byteOffset += $bytes;
727	$result .= $char;
728	}
729
730	$this->moveStringCursor($charCount, $totalBytes);
731
732	return [$result, $totalBytes];
733	}
734
735	/** Moves internal string cursor position. */
736	private function moveStringCursor(int $positionOffset, int $byteStreamOffset): self
737	{
738	$this->position += $positionOffset;
739	$this->byteStreamPosition += $byteStreamOffset;
740
741	return $this;
742	}
743	}
744