Improved character matching, updated tests and supporting files (#2)

* Changed regex to match characters and updated/fixed tests
* Updating readme
* Relocating bracketisation
* Updating CONTRIBUTING
* Added test rtf file generator

Author: TomWilford

CONTRIBUTING.md

@@ -3,4 +3,5 @@ Thanks for checking out my project. I'm nowhere near a linguistics expert (or an
 [email me](mailto:hello@jollyblueman.com) or make a PR if you think there's something you could add.
 
 ## Requirements
-* All changes must be compatible with PHP 7.4 and above
+* All changes must be compatible with PHP 7.4 and above
+* Compatible with phpcs PSR12 standard (excluding any pangrams in the test suite)

README.md

@@ -1,6 +1,6 @@
 # PHP UTF8 To RTF
 ## Description
-A simple class for converting UTF8 characters to an RTF safe format in PHP.
+A simple PHP class for converting UTF8 characters to an RTF safe version in a string.
 
 ## Installation
 `composer require tomwilford/php-utf8-to-rtf`
@@ -14,13 +14,13 @@ Instantiate the converter using:
 $converter = new CharacterConverter();
 ```
 
-To find and replace UTF8 characters in a long string use:
+To find and replace UTF8 characters with RTF safe versions in a long string use:
 
 ```php
 $convertedString = $converter->findAndReplace($string);
 ```
 
-Or, for shorter strings, simply convert the whole string:
+Or, for shorter strings and characters, simply convert the whole string:
 
 ```php
 $convertedString = $converter->convertStringToRtf($string);
@@ -32,19 +32,19 @@ You can also convert an array of UTF8 strings:
 $convertedArray = $converter->convertArrayToRtf($arrayOfStrings);
 ```
 
-Finally, if you need to do any additional processing on the UTF8 strings before converting, 
-you can create an array of UTF8 strings using:
+Finally, if you need to do any additional processing on the UTF8 characters before converting, 
+you can extract them as an array using:
 ```php
-$arrayToConvert = $converter->locateWordsInString($string);
+$arrayToConvert = $converter->locateCharactersInString($string);
 ```
 
 ## Testing
-Testing has been done with PHPUnit and a series of pangrams sourced online to try to best capture
+PHPUnit tests have been written using series of pangrams sourced online to try to best capture
 as many characters as possible in different languages being used in a natural way. Please see the 
 [Pangrams trait](tests/Resources/Pangrams.php) for the pangram sources' credits.
 
-Currently, there are four failing tests for `findAndReplace()` whereby some characters are ignored by the 
-regex and not replaced during the conversion.
+There also is a short script that can [generate a test RTF file](tests/GenerateTestFile/generateTestFile.php) that
+can be opened in a word processor to verify the results.
 
 ## Contributing
 Contributions are welcome, please see [CONTRIBUTING](CONTRIBUTING.md) for more information.

src/CharacterConverter.php

@@ -10,64 +10,64 @@
 namespace Wilf\PhpUtf8ToRtf;
 
 /**
- * Offers methods for locating and converting UTF8 strings to an RTF format.
+ * Methods for locating and converting UTF8 strings to an RTF format.
  *
  * @author Tom Wilford <hello@jollyblueman.com>
  */
 class CharacterConverter implements CharacterConverterInterface
 {
     public function findAndReplace(string $string): string
     {
-        $wordsToConvert = $this->locateWordsInString($string);
-        $convertedWords = $this->convertArrayToRtf($wordsToConvert);
+        $stringsToConvert = $this->locateCharactersInString($string);
+        $convertedStrings = $this->convertArrayToRtf($stringsToConvert);
 
-        foreach ($convertedWords as $word => $convertedWord) {
-            $string = str_replace($word, '{' . $convertedWord . '}', $string);
+        foreach ($convertedStrings as $originalString => $convertedString) {
+            $string = str_replace($originalString, $convertedString, $string);
         }
 
         return $string;
     }
 
-    public function locateWordsInString(string $string): array
+    public function locateCharactersInString(string $string): array
     {
-        preg_match_all("/(\p{L}*(?:[^ -~\n]+))/u", $string, $matches);
+        preg_match_all("/[^\x00-\x7F]/u", $string, $matches);
 
-        $words = [];
+        $characters = [];
         foreach ($matches[0] as $match) {
-            foreach (explode(' ', $match) as $item) {
-                $words[] = $item;
+            if (!in_array($match, $characters)) {
+                $characters[] = $match;
             }
         }
 
-        return $words;
+        return $characters ;
     }
 
-    public function convertArrayToRtf(array $words): array
+    public function convertArrayToRtf(array $strings): array
     {
-        $convertedWords = [];
-        foreach ($words as $word) {
-            if (!array_key_exists($word, $convertedWords)) {
-                $convertedWords[$word] = $this->convertStringToRtf($word);
+        $convertedStrings = [];
+        foreach ($strings as $string) {
+            if (!array_key_exists($string, $convertedStrings)) {
+                $convertedStrings[$string] = $this->convertStringToRtf($string);
             }
         }
 
-        return $convertedWords;
+        return $convertedStrings;
     }
 
-    public function convertStringToRtf(string $word): string
+    public function convertStringToRtf(string $string): string
     {
-        $rtf = "";
-        for ($i = 0; $i < mb_strlen($word, "UTF-8"); $i++) {
-            $char = mb_substr($word, $i, 1, "UTF-8");
-            $code = unpack("N", mb_convert_encoding($char, "UCS-4BE", "UTF-8"))[1];
+        $convertedString = "";
+        for ($i = 0; $i < mb_strlen($string, "UTF-8"); $i++) {
+            $character = mb_substr($string, $i, 1, "UTF-8");
+            $convertedCharacter = unpack("N", mb_convert_encoding($character, "UCS-4BE", "UTF-8"))[1];
 
-            if ($code < 0x7f) {
-                $rtf .= $char;
+            if ($convertedCharacter < 0x7f) {
+                $convertedString .= $character;
             } else {
-                $rtf .= sprintf("\\u%d?", $code);
+                $convertedString .= '{' . sprintf("\\u%d?", $convertedCharacter) . '}';
             }
         }
 
-        return $rtf;
+        return $convertedString;
     }
 }

src/CharacterConverterInterface.php

@@ -18,9 +18,9 @@ interface CharacterConverterInterface
 {
     public function findAndReplace(string $string): string;
 
-    public function locateWordsInString(string $string): array;
+    public function locateCharactersInString(string $string): array;
 
-    public function convertArrayToRtf(array $words): array;
+    public function convertArrayToRtf(array $strings): array;
 
-    public function convertStringToRtf(string $word): string;
+    public function convertStringToRtf(string $string): string;
 }

tests/GenerateTestFile/generateTestFile.php

@@ -0,0 +1,56 @@
+<?php
+/*
+ * Copyright (c) 2023. Tom Wilford <hello@jollyblueman.com>
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+require __DIR__ . "/../../vendor/autoload.php";
+
+$file = file_get_contents(__DIR__ . '/../Resources/Pangrams.php');
+$file = str_replace("<?php
+
+namespace Resources;
+
+/**
+ * A list of pangrams for testing, made possible by:
+ *  - Richard Rutter (majority), https://clagnut.com/blog/2380
+ *  - gacontuyenchien1 (Vietnamese), https://10fastfingers.com/text/212157-Vietnamese-pangram
+ */
+trait Pangrams
+{", '', $file);
+$file = str_replace("}", '', $file);
+$file = str_replace("    public string $", "", $file);
+$file = str_replace(" = ", "", $file);
+
+$array = explode('";', $file);
+$strings = [];
+foreach ($array as $item) {
+    $parts = explode('"', $item);
+    if (trim($parts[0])) {
+        $strings[ucfirst(trim($parts[0]))] = $parts[1];
+    }
+}
+
+$converter = new \Wilf\PhpUtf8ToRtf\CharacterConverter();
+foreach ($strings as $language => $string) {
+    $strings[$language] = $converter->findAndReplace($string);
+}
+
+$rtf = "{rtf1
+{\par}";
+
+foreach ($strings as $language => $string) {
+    $rtf .= $language . '{\par}';
+    $rtf .= $string . '{\par}{\par}';
+}
+
+$rtf .= '
+}';
+
+$filePath = __DIR__ . '/test.rtf';
+$fh = fopen($filePath, 'w');
+file_put_contents(__DIR__ . '/test.rtf', $rtf);
+fclose($fh);