From 41ed0b5b5f84ce008b771b279d9e95bd07038438 Mon Sep 17 00:00:00 2001 From: "C. Scott Ananian" Date: Wed, 26 Feb 2025 14:05:40 -0500 Subject: [PATCH] Entity-escape U+0338 where needed to make HTML output NFC-safe Ensure that Unicode NFC normalization can be applied to our HTML output safely. Even though the W3C officially recommends against normalizing HTML https://www.w3.org/International/questions/qa-html-css-normalization#converting this is still easily done inadvertently, especially when using the MediaWiki action API which normalizes parameters and results by default -- see T266140 / I2e78e660ba1867744e34eda7d00ea527ec016b71 for another similar issue. Bug: T387130 Change-Id: I671648603c4635a35585c860b4857f5ea085e47f --- src/Utils/Utils.php | 5 ++++- src/Utils/WTUtils.php | 12 +++++++----- src/Wt2Html/XMLSerializer.php | 12 +++++++++--- tests/parser/badCharacters.txt | Bin 1892 -> 2818 bytes tests/phpunit/Parsoid/Utils/UtilsTest.php | 4 +++- tests/phpunit/Parsoid/Utils/WTUtilsTest.php | 1 + .../Parsoid/Wt2Html/XMLSerializerTest.php | 18 ++++++++++++++++++ 7 files changed, 42 insertions(+), 10 deletions(-) diff --git a/src/Utils/Utils.php b/src/Utils/Utils.php index 76b84fc34..2048b21dd 100644 --- a/src/Utils/Utils.php +++ b/src/Utils/Utils.php @@ -407,7 +407,10 @@ class Utils { */ public static function escapeHtml( string $s ): string { // Only encodes five characters: " ' & < > - return htmlspecialchars( $s, ENT_QUOTES | ENT_HTML5 ); + $s = htmlspecialchars( $s, ENT_QUOTES | ENT_HTML5 ); + // Also encode U+0338 (T387130) + $s = str_replace( "\u{0338}", '̸', $s ); + return $s; } /** diff --git a/src/Utils/WTUtils.php b/src/Utils/WTUtils.php index cf12339a6..e6adfc32b 100644 --- a/src/Utils/WTUtils.php +++ b/src/Utils/WTUtils.php @@ -689,8 +689,10 @@ class WTUtils { * We work around all these problems by using HTML entity encoding inside * the comment body. The characters -, >, and & must be encoded in order * to prevent premature termination of the comment by one of the cases - * above. Encoding other characters is optional; all entities will be - * decoded during wikitext serialization. + * above. We also choose to encode U+0338 to avoid NFC normalization + * issues; see T387130 for more details. Encoding other characters is + * optional, but *all* entities will be decoded during wikitext + * serialization. * * In order to allow *arbitrary* content inside a wikitext comment, * including the forbidden string "-->" we also do some minimal entity @@ -732,10 +734,10 @@ class WTUtils { return Utils::decodeWtEntities( $m[0] ); }, $comment ); - // Now encode '-', '>' and '&' in the "true value" as HTML entities, - // so that they can be safely embedded in an HTML comment. + // Now encode '-', '>', '&', and U+0338 in the "true value" as HTML + // entities, so that they can be safely embedded in an HTML comment. // This part doesn't have to map strings 1-to-1. - return preg_replace_callback( '/[->&]/', static function ( $m ) { + return preg_replace_callback( '/[->&]|\x{338}/u', static function ( $m ) { return Utils::entityEncodeAll( $m[0] ); }, $trueValue ); } diff --git a/src/Wt2Html/XMLSerializer.php b/src/Wt2Html/XMLSerializer.php index ac4ff64d6..b3a5a4b10 100644 --- a/src/Wt2Html/XMLSerializer.php +++ b/src/Wt2Html/XMLSerializer.php @@ -50,7 +50,9 @@ class XMLSerializer { private const ENTITY_ENCODINGS = [ 'single' => [ '<' => '<', '&' => '&', "'" => ''' ], 'double' => [ '<' => '<', '&' => '&', '"' => '"' ], - 'xml' => [ '<' => '<', '&' => '&' ], + # T387130: add U+0338 to the set of encoded entities outside of + # attribute context. + 'xml' => [ '<' => '<', '&' => '&', "\u{0338}" => '̸' ], ]; /** @@ -209,9 +211,13 @@ class XMLSerializer { // we could throw an exception here if node.data would not create // a "well-formed" XML comment. But we use entity encoding when // we create the comment node to ensure that node.data will always - // be okay; see DOMUtils.encodeComment(). + // be okay; see WTUtils::encodeComment(). + // T387130 applies here as well. '@phan-var Comment $node'; // @var Comment $node - $accum( '', $node ); + $data = $node->nodeValue; + Assert::invariant( !str_contains( $data, "-->" ), "bad data" ); + Assert::invariant( !str_contains( $data, "\u{0338}" ), "bad data" ); + $accum( '', $node ); return; default: diff --git a/tests/parser/badCharacters.txt b/tests/parser/badCharacters.txt index 10e5fd3b5f80c9ea2fd5d7bff3c8d9b5acd0fd7d..a312786b9d76c35998aa6aaa33d6100f81d20d5e 100644 GIT binary patch delta 931 zcma)4&1%~~5Iz(_*+VXYo+lgdCAO6)B^bprw4tRvwb1kwf>~Q*BrIu#w5c3Q5rmw3 z$&2)uhY0i)dgz06W+gdFsY_1Uo&9FMAIlZ)1Z{7ZU_PqMMe{gg(>cdRs2r3Og z6@aOU%$3;P1)Z5lr-dj&T^ckC}; z8CK>h`-F2ljL;^(MDdLYsxn$e23JPBLls~ygzrOdZ$+9;;6sYg1K$^I;f#TdxW9w5 zQrC}s<~mJF&?wgEG#W+A=?Ms3h|&O8-XV~a7{6dl$11%{W|(s^GBJ7|$sFcWgM|UT zSa>7KWFG3=iKBG+${xM@yD2G~_JC^Wwc4y$m1HL!Dz{73Fj1|;R6S6=bpOpoM*rrr z-YplI=CId>$3>koa14W_95nX<{V;hTCt{pU>ZHd`Mz_jo0F(cA9&#rqNK;z-%rB`e zQ>m>FPfv%hM>d+Fdo00RWvCavOiJ&qisPA@FW(2$;BZINXQ~I#8@qLK(gfPz>Lp;V zu@kyhe@)pLzoC=2qIS92WYx{beeBW(zriKHJ;7zW9P1ypHqpH7*xh}s^7dxy*B@y= BFs=Xq delta 11 ScmZn?d&0M&k8QFJ*EIkd?F4QB diff --git a/tests/phpunit/Parsoid/Utils/UtilsTest.php b/tests/phpunit/Parsoid/Utils/UtilsTest.php index 2f3782d73..c0ada4528 100644 --- a/tests/phpunit/Parsoid/Utils/UtilsTest.php +++ b/tests/phpunit/Parsoid/Utils/UtilsTest.php @@ -100,8 +100,9 @@ class UtilsTest extends \PHPUnit\Framework\TestCase { public function provideEscapeHtml() { return [ - [ 'only 5 characters escaped', 'only 5 characters escaped' ], + [ 'only 6 characters escaped', 'only 6 characters escaped' ], [ '<>&"\'', '<>&"'' ], + [ "\u{0338} ≮≯", '̸ ≮≯' ], ]; } @@ -119,6 +120,7 @@ class UtilsTest extends \PHPUnit\Framework\TestCase { [ 'Even ASCII', 'Even ASCII' ], [ 'is encoded', 'is encoded' ], [ "&>", '&>' ], + [ "\u{0338}", '̸' ], // Some entities use special forms [ "\u{00A0}", ' ' ], [ "\u{000D}", ' ' ], diff --git a/tests/phpunit/Parsoid/Utils/WTUtilsTest.php b/tests/phpunit/Parsoid/Utils/WTUtilsTest.php index f46a363aa..bc8f2efa9 100644 --- a/tests/phpunit/Parsoid/Utils/WTUtilsTest.php +++ b/tests/phpunit/Parsoid/Utils/WTUtilsTest.php @@ -40,6 +40,7 @@ class WTUtilsTest extends \PHPUnit\Framework\TestCase { return [ [ 'abc', 'abc', 10 ], [ '& - >', '& - >', 12 ], + [ "\u{0338} ̸", '̸ &#x338;', 17 ], [ 'Use > here', 'Use &gt; here', 20 ], [ '-->', '-->', 13 ], [ '--&gt;', '--&gt;', 17 ], diff --git a/tests/phpunit/Parsoid/Wt2Html/XMLSerializerTest.php b/tests/phpunit/Parsoid/Wt2Html/XMLSerializerTest.php index 2f44b5c8b..61ebb3806 100644 --- a/tests/phpunit/Parsoid/Wt2Html/XMLSerializerTest.php +++ b/tests/phpunit/Parsoid/Wt2Html/XMLSerializerTest.php @@ -3,6 +3,7 @@ namespace Test\Parsoid\Wt2Html; use Wikimedia\Parsoid\Utils\DOMCompat; +use Wikimedia\Parsoid\Utils\DOMUtils; use Wikimedia\Parsoid\Wt2Html\XMLSerializer; use Wikimedia\TestingAccessWrapper; @@ -220,6 +221,23 @@ class XMLSerializerTest extends \PHPUnit\Framework\TestCase { $this->assertSame( $html, $ret['html'] ); } + /** + * @covers ::serialize + */ + public function testSerialize_U0338() { + $html = "

\u{0338}

"; + $doc = DOMUtils::parseHTML( $html ); + + $ret = XMLSerializer::serialize( $doc, [ 'smartQuote' => false ] ); + $this->assertIsArray( $ret ); + $this->assertArrayHasKey( 'html', $ret ); + $this->assertSame( << +

̸

+HTML +, $ret['html'] ); + } + /** * @covers ::serialize */ -- 2.43.0