Return null from getTokenName() if token unknown

Author: nikic

ext/tokenizer/tests/PhpToken_methods.phpt

@@ -70,6 +70,10 @@ try {
     echo $e->getMessage(), "\n";
 }
 
+echo "\nName of unknown token:\n";
+$token = new PhpToken(100000, "foo");
+var_dump($token->getTokenName());
+
 ?>
 --EXPECT--
 [ 0] T_OPEN_TAG                 ignorable
@@ -110,3 +114,6 @@ Typed property PhpToken::$id must not be accessed before initialization
 Typed property PhpToken::$text must not be accessed before initialization
 Typed property PhpToken::$id must not be accessed before initialization
 Typed property PhpToken::$text must not be accessed before initialization
+
+Name of unknown token:
+NULL

ext/tokenizer/tokenizer.c

@@ -237,7 +237,12 @@ PHP_METHOD(PhpToken, getTokenName)
 	if (Z_LVAL_P(id_zval) < 256) {
 		RETURN_INTERNED_STR(ZSTR_CHAR(Z_LVAL_P(id_zval)));
 	} else {
-		RETURN_STRING(get_token_type_name(Z_LVAL_P(id_zval)));
+		const char *token_name = get_token_type_name(Z_LVAL_P(id_zval));
+		if (!token_name) {
+			RETURN_NULL();
+		}
+
+		RETURN_STRING(token_name);
 	}
 }
 
@@ -531,6 +536,10 @@ PHP_FUNCTION(token_name)
 		Z_PARAM_LONG(type)
 	ZEND_PARSE_PARAMETERS_END();
 
-	RETVAL_STRING(get_token_type_name(type));
+	const char *token_name = get_token_type_name(type);
+	if (!token_name) {
+		token_name = "UNKNOWN";
+	}
+	RETURN_STRING(token_name);
 }
 /* }}} */

ext/tokenizer/tokenizer.stub.php

@@ -15,5 +15,5 @@ public function is($kind): bool;
 
     public function isIgnorable(): bool;
 
-    public function getTokenName(): string;
+    public function getTokenName(): ?string;
 }

ext/tokenizer/tokenizer_arginfo.h

@@ -28,5 +28,5 @@ ZEND_END_ARG_INFO()
 ZEND_BEGIN_ARG_WITH_RETURN_TYPE_INFO_EX(arginfo_class_PhpToken_isIgnorable, 0, 0, _IS_BOOL, 0)
 ZEND_END_ARG_INFO()
 
-ZEND_BEGIN_ARG_WITH_RETURN_TYPE_INFO_EX(arginfo_class_PhpToken_getTokenName, 0, 0, IS_STRING, 0)
+ZEND_BEGIN_ARG_WITH_RETURN_TYPE_INFO_EX(arginfo_class_PhpToken_getTokenName, 0, 0, IS_STRING, 1)
 ZEND_END_ARG_INFO()

ext/tokenizer/tokenizer_data.c

@@ -306,6 +306,6 @@ char *get_token_type_name(int token_type)
 		case T_BAD_CHARACTER: return "T_BAD_CHARACTER";
 
 	}
-	return "UNKNOWN";
+	return NULL;
 }
 

ext/tokenizer/tokenizer_data_gen.sh

@@ -71,7 +71,7 @@ awk '
 
 echo '
 	}
-	return "UNKNOWN";
+	return NULL;
 }
 ' >> $outfile