Preserve documentation and hook string semantics

lib/class-command.php

@@ -152,7 +152,14 @@ protected function _get_phpdoc_data( $path, $format = 'json' ) {
 		$output = parse_files( $files, $path );
 
 		if ( 'json' == $format ) {
-			return json_encode( $output, JSON_PRETTY_PRINT );
+			$json = json_encode( $output, JSON_PRETTY_PRINT );
+
+			if ( false === $json ) {
+				WP_CLI::error( sprintf( 'Problem encoding the data from %1$s as JSON: %2$s', $path, json_last_error_msg() ) );
+				exit;
+			}
+
+			return $json;
 		}
 
 		return $output;

lib/class-hook-reflector.php

@@ -10,10 +10,16 @@
 class Hook_Reflector extends BaseReflector {
 
 	/**
+	 * Get the hook name as it is spelled in the source.
+	 *
+	 * The name is printed from the source expression instead of read from the
+	 * interpreted string value. Interpreting escape sequences may produce bytes
+	 * that are not valid UTF-8, which cannot be encoded as JSON.
+	 *
 	 * @return string
 	 */
 	public function getName() {
-		$printer = new \PhpParser\PrettyPrinter\Standard();
+		$printer = new Pretty_Printer();
 		return $this->cleanupName( $printer->prettyPrintExpr( $this->node->args[0]->value ) );
 	}
 
@@ -26,8 +32,10 @@ private function cleanupName( $name ) {
 		$matches = array();
 
 		// quotes on both ends of a string
-		if ( preg_match( '/^[\'"]([^\'"]*)[\'"]$/', $name, $matches ) ) {
-			return $matches[1];
+		// The quoted body may contain the other quote character, as in "it's",
+		// or an escaped copy of the quote that delimits it, as in 'it\'s'.
+		if ( preg_match( '/^([\'"])((?:(?!\1)[^\\\\]|\\\\.)*)\1$/s', $name, $matches ) ) {
+			return $matches[2];
 		}
 
 		// two concatenated things, last one of them a variable

lib/class-pretty-printer.php

@@ -5,7 +5,57 @@
 /**
  * Extends default printer for arguments.
  */
-class Pretty_Printer extends \PhpParser\PrettyPrinter\Standard {
+class Pretty_Printer extends \phpDocumentor\Reflection\PrettyPrinter {
+	/**
+	 * Print names as they appeared before PHP-Parser's name resolution.
+	 *
+	 * PHP-Parser represents resolved global names as fully-qualified names. The
+	 * leading namespace separator is useful in an AST, but adding it to exported
+	 * source expressions changes the established JSON output.
+	 *
+	 * Single-segment fully-qualified names are therefore printed without the
+	 * leading backslash. Inside a namespaced file the printed form denotes a
+	 * namespaced symbol rather than the global one, for example `\Foo::BAR` is
+	 * printed as `Foo::BAR`, which in a namespaced file would resolve to
+	 * `Vendor\Foo::BAR`. This is an accepted limitation because the parser
+	 * targets global-namespace WordPress core code.
+	 *
+	 * @param \PhpParser\Node\Name\FullyQualified $node Fully-qualified name.
+	 *
+	 * @return string Printed name.
+	 */
+	protected function pName_FullyQualified( \PhpParser\Node\Name\FullyQualified $node ): string {
+		$name = $node->toString();
+
+		return false === strpos( $name, '\\' ) ? $name : '\\' . $name;
+	}
+
+	/**
+	 * Print heredoc and nowdoc strings with their delimiters.
+	 *
+	 * The parent printer returns PHP-Parser's `rawValue` attribute so that
+	 * escape sequences are not interpreted. For heredoc and nowdoc strings that
+	 * attribute holds the body only, without the `<<<LABEL` delimiters, which is
+	 * no longer a PHP expression. Those are printed by the default printer,
+	 * which does not interpret escape sequences in doc strings either.
+	 *
+	 * @param \PhpParser\Node\Scalar\String_ $node String.
+	 *
+	 * @return string Printed string.
+	 */
+	public function pScalar_String( \PhpParser\Node\Scalar\String_ $node ): string {
+		$kind = $node->getAttribute( 'kind' );
+
+		if (
+			\PhpParser\Node\Scalar\String_::KIND_HEREDOC === $kind ||
+			\PhpParser\Node\Scalar\String_::KIND_NOWDOC === $kind
+		) {
+			return \PhpParser\PrettyPrinter\Standard::pScalar_String( $node );
+		}
+
+		return parent::pScalar_String( $node );
+	}
+
 	/**
 	 * Pretty prints an argument.
 	 *

lib/runner.php

@@ -90,7 +90,7 @@ function parse_files( $files, $root ) {
 				$out['constants'][] = array(
 					'name'  => $constant->getShortName(),
 					'line'  => $constant->getLineNumber(),
-					'value' => $constant->getValue(),
+					'value' => export_expression( $constant->getNode()->value ),
 				);
 			}
 
@@ -102,7 +102,7 @@ function parse_files( $files, $root ) {
 				$func = array(
 					'name'      => $function->getShortName(),
 					'namespace' => $function->getNamespace(),
-					'aliases'   => $function->getNamespaceAliases(),
+					'aliases'   => strip_global_namespace_prefixes( $function->getNamespaceAliases() ),
 					'line'      => $function->getLineNumber(),
 					'end_line'  => $function->getNode()->getAttribute( 'endLine' ),
 					'arguments' => export_arguments( $function->getArguments() ),
@@ -132,8 +132,8 @@ function parse_files( $files, $root ) {
 					'end_line'   => $class->getNode()->getAttribute( 'endLine' ),
 					'final'      => $class->isFinal(),
 					'abstract'   => $class->isAbstract(),
-					'extends'    => $class->getParentClass(),
-					'implements' => $class->getInterfaces(),
+					'extends'    => strip_global_namespace_prefix( $class->getParentClass() ),
+					'implements' => strip_global_namespace_prefixes( $class->getInterfaces() ),
 					'properties' => export_properties( $class->getProperties(), $class_setup_blueprints, $path ),
 					'methods'    => export_methods( $class->getMethods(), $class_setup_blueprints, $path ),
 					'doc'        => $class_doc,
@@ -149,33 +149,62 @@ function parse_files( $files, $root ) {
 		throw $e;
 	}
 
-	/*
-	 * nikic/php-parser in version 3 started adding a namespace prefix
-	 * at the start of global names, but this is different than how the
-	 * documentation was previously generated. this removes those prefixes
-	 * by removing a leading reverse solidus (\) when no other reverse
-	 * solidus appears before the end of a sequence of PHP identifier
-	 * characters.
-	 */
-	array_walk_recursive(
-		$output,
-		static function( &$value ) {
-			if ( is_string( $value ) ) {
-				// "\wp_kses()" -> "wp_kses()"
-				$without_global_namespace = preg_replace(
-					'~(^|\p{Z})\\\\([A-Z_a-z\x80-\xFF][0-9A-Z_a-z\x80-\xFF]*)([:(\p{Z}]|->|$)~',
-					'$1$2$3',
-					$value,
-				);
+	return $output;
+}
 
-				if ( $value !== $without_global_namespace ) {
-					$value = $without_global_namespace;
-				}
-			}
-		}
+/**
+ * Remove a synthetic leading namespace prefix from a global name.
+ *
+ * @param mixed $name Name to normalize.
+ *
+ * @return mixed
+ */
+function strip_global_namespace_prefix( $name ) {
+	if ( ! is_string( $name ) ) {
+		return $name;
+	}
+
+	return preg_replace(
+		'~^\\\\([A-Z_a-z\x80-\xFF][0-9A-Z_a-z\x80-\xFF]*)([:(\p{Z}]|->|$)~',
+		'$1$2',
+		$name
 	);
+}
 
-	return $output;
+/**
+ * Remove synthetic leading namespace prefixes from global names.
+ *
+ * @param array $names Names to normalize.
+ *
+ * @return array
+ */
+function strip_global_namespace_prefixes( array $names ) {
+	foreach ( $names as $key => $name ) {
+		$names[ $key ] = strip_global_namespace_prefix( $name );
+	}
+
+	return $names;
+}
+
+/**
+ * Export an expression without PHP-Parser's synthetic global namespace prefixes.
+ *
+ * @param null|\PhpParser\Node\Expr $expression Expression to export.
+ *
+ * @return null|string
+ */
+function export_expression( $expression ) {
+	if ( null === $expression ) {
+		return null;
+	}
+
+	static $printer = null;
+
+	if ( null === $printer ) {
+		$printer = new Pretty_Printer();
+	}
+
+	return $printer->prettyPrintExpr( $expression );
 }
 
 /**
@@ -530,7 +559,7 @@ function export_docblock( $element, array $inherited_setup_blueprints = array(),
 			'content' => preg_replace( '/[\n\r]+/', ' ', format_description( $tag->getDescription() ) ),
 		);
 		if ( method_exists( $tag, 'getTypes' ) ) {
-			$tag_data['types'] = $tag->getTypes();
+			$tag_data['types'] = strip_global_namespace_prefixes( $tag->getTypes() );
 		}
 		if ( method_exists( $tag, 'getLink' ) ) {
 			$tag_data['link'] = $tag->getLink();
@@ -697,8 +726,8 @@ function export_arguments( array $arguments ) {
 	foreach ( $arguments as $argument ) {
 		$output[] = array(
 			'name'    => $argument->getName(),
-			'default' => $argument->getDefault(),
-			'type'    => $argument->getType(),
+			'default' => export_expression( $argument->getNode()->default ),
+			'type'    => strip_global_namespace_prefix( $argument->getType() ),
 		);
 	}
 
@@ -720,7 +749,7 @@ function export_properties( array $properties, array $inherited_setup_blueprints
 			'name'        => $property->getName(),
 			'line'        => $property->getLineNumber(),
 			'end_line'    => $property->getNode()->getAttribute( 'endLine' ),
-			'default'     => $property->getDefault(),
+			'default'     => export_expression( $property->getNode()->default ),
 //			'final' => $property->isFinal(),
 			'static'      => $property->isStatic(),
 			'visibility'  => $property->getVisibility(),
@@ -746,7 +775,7 @@ function export_methods( array $methods, array $inherited_setup_blueprints = arr
 		$method_data = array(
 			'name'       => $method->getShortName(),
 			'namespace'  => $method->getNamespace(),
-			'aliases'    => $method->getNamespaceAliases(),
+			'aliases'    => strip_global_namespace_prefixes( $method->getNamespaceAliases() ),
 			'line'       => $method->getLineNumber(),
 			'end_line'   => $method->getNode()->getAttribute( 'endLine' ),
 			'final'      => $method->isFinal(),
@@ -1283,7 +1312,7 @@ function export_uses( array $uses ) {
 				case 'methods':
 					$out[ $type ][] = array(
 						'name'     => $name[1],
-						'class'    => $name[0],
+						'class'    => strip_global_namespace_prefix( $name[0] ),
 						'static'   => $element->isStatic(),
 						'line'     => $element->getLineNumber(),
 						'end_line' => $element->getNode()->getAttribute( 'endLine' ),
@@ -1292,6 +1321,7 @@ function export_uses( array $uses ) {
 
 				default:
 				case 'functions':
+					$name = strip_global_namespace_prefix( $name );
 					$out[ $type ][] = array(
 						'name'     => $name,
 						'line'     => $element->getLineNumber(),

tests/phpunit/tests/export/docblocks.inc

@@ -40,6 +40,21 @@ function test_func( $var, $num ) {
 	return true;
 }
 
+/**
+ * Tests special characters in documentation.
+ *
+ * ```php
+ * true === wp_is_valid_utf8( '✏' );
+ * false === wp_is_valid_utf8( "just \xC0 test" );
+ * ```
+ */
+function test_special_characters() {}
+
+/**
+ * \xC0 starts this description.
+ */
+function test_leading_escape_sequence() {}
+
 /**
  * This is a class docblock.
  *

tests/phpunit/tests/export/docblocks.php

@@ -230,6 +230,31 @@ public function test_function_docblocks() {
 		);
 	}
 
+	/**
+	 * Test that special characters in documentation are preserved.
+	 */
+	public function test_special_characters_are_preserved() {
+		$this->assertFunctionHasDocs(
+			'test_special_characters',
+			array(
+				'long_description' => '<pre><code class="language-php">true === wp_is_valid_utf8( \'✏\' );' . "\n"
+					. 'false === wp_is_valid_utf8( "just \\xC0 test" );</code></pre>',
+			)
+		);
+	}
+
+	/**
+	 * Test that a leading escape sequence in documentation is preserved.
+	 */
+	public function test_leading_escape_sequence_is_preserved() {
+		$this->assertFunctionHasDocs(
+			'test_leading_escape_sequence',
+			array(
+				'description' => '\\xC0 starts this description.',
+			)
+		);
+	}
+
 	/**
 	 * Test that class docs are exported.
 	 */

tests/phpunit/tests/export/global-names.inc

@@ -0,0 +1,53 @@
+<?php
+
+use \Global_Alias_Source as Global_Alias;
+
+/**
+ * Documents a global type.
+ *
+ * Calls {@see Global_Doc_Function()} while preserving \xC0 as documentation.
+ *
+ * @param Global_Doc_Type $value Value.
+ */
+function documented_global_type( \Global_Parameter $value, $enabled = true, $nothing = null, $mode = GLOBAL_MODE, $escape = '\xC0', $namespaced = \Vendor\GLOBAL_MODE ) {}
+
+class Global_Child extends \Global_Parent implements \Global_Interface {}
+
+\Global_Class::global_method();
+
+const GLOBAL_CONST = GLOBAL_VALUE;
+
+define( 'GLOBAL_DEFINED_CONST', global_default( GLOBAL_VALUE ) );
+define( 'NAMESPACED_DEFINED_CONST', \Vendor\global_default( \Vendor\GLOBAL_VALUE ) );
+
+class Global_Defaults {
+	public $enabled = false;
+	public $mode = GLOBAL_MODE;
+
+	public function create() {
+		return ( new \Global_Class() )->global_method();
+	}
+
+	public function create_namespaced() {
+		return ( new \Vendor\Global_Class() )->global_method();
+	}
+}
+
+/**
+ * Documents inline references.
+ *
+ * Calls {@see \Global_Doc_Function()} and {@see \Vendor\Thing::m()} in prose.
+ *
+ * Spells `{@see \Global_Inline::method()}` in an inline code span.
+ *
+ * ```php
+ * // Renders {@see \Global_Widget::render()}.
+ * $widget->render();
+ * ```
+ *
+ * @see \Global_See_Target() Compares prefixed references.
+ * @see Global_See_Plain() Compares plain references.
+ * @link https://example.com/reference Explains the behavior.
+ * @param \Global_Prefixed_Type $ignored Unused.
+ */
+function documented_inline_references() {}