Charset: Add wp_utf8_chunks() to iterate through strings.

This new generator function iterates through valid and invalid spans of bytes in a UTF-8 string. It is a convenience wrapper around the new `_wp_scan_utf8()` which aids a number of operations which should work on strings containing invalid spans of UTF-8, such as: - Identifying where the invalid spans are. - Operating on the valid portions while preserving the invalid spans. - Displaying broken strings. These operations can be useful during validation, sanitization, debugging, and processing uncontrolled inputs which may contain malformed sequences.

Author: dmsnell

src/wp-includes/utf8.php

@@ -133,3 +133,62 @@ function wp_scrub_utf8( $text ) {
 return _wp_scrub_utf8_fallback( $text );
 }
 endif;
+
+/**
+ * Iterate through a string returning lengths of valid or invalid byte spans.
+ *
+ * This is not likely an often-needed function, but it can be used to build
+ * interesting views of a string containing invalid bytes, or to operate on
+ * the valid portions of a UTF-8 string while preserving the existing spans
+ * of invalid bytes.
+ *
+ * This is convenience wrapper around {@see _wp_scan_utf8()}. For non-allocating
+ * and non-yielding functionality consider calling that function directly.
+ *
+ * Example:
+ *
+ * $text = "test\x90wp\E2\x80\xC0test";
+ *
+ * $chunks = iterator_to_array( wp_utf8_chunks( $text ) );
+ * array( 'test', "\x90", 'wp', "\xE2\x80", "\xC0", 'test' ) === $chunks;
+ *
+ * $is_valid = false;
+ * foreach ( wp_utf8_chunks( $text, $is_valid ) as $chunk ) {
+ * if ( $is_valid ) {
+ * echo $chunk;
+ * } else {
+ * $bytes = implode( ' ', array_map( 'bin2hex', str_split( $chunk ) ) );
+ * echo "({$bytes})";
+ * }
+ * }
+ * // test(90)wp(e2 80)(c0)test
+ *
+ * @param string $text Iterate through this string.
+ * @param bool|null $is_valid Optional. If passed, set to whether the currently yielded
+ * chunk is a valid span of UTF-8 bytes.
+ * @return Generator Spans of valid or invalid UTF-8 text; check `$is_valid` to determine
+ * whether the yielded span is valid.
+ */
+function wp_utf8_chunks( string $text, ?bool &$is_valid = null ): Generator {
+$at = 0;
+$was_at = 0;
+$end = strlen( $text );
+$invalid_length = 0;
+
+while ( $at < $end ) {
+_wp_scan_utf8( $text, $at, $invalid_length );
+
+if ( $at > $was_at ) {
+$is_valid = true;
+yield substr( $text, $was_at, $at - $was_at );
+}
+
+if ( $invalid_length > 0 ) {
+$is_valid = false;
+yield substr( $text, $at, $invalid_length );
+}
+
+$at += $invalid_length;
+$was_at = $at;
+}
+}