Now the tokeniser is using the new textlib->entities_to_utf8()

own function (PHP4 and PHP compatible)

Merged from MOODLE_18_STABLE

diff --git a/lib/tokeniserlib.php b/lib/tokeniserlib.php
index 737875b126b2433d0a93b79da7ef8cdc1e6367f1..4cf3d4c40e8c681f960aa8859382904dc5da371b 100644 (file)
--- a/lib/tokeniserlib.php
+++ b/lib/tokeniserlib.php
@@ -312,7 +312,7 @@ function tokenise_simplify($text, $overlap_cjk, $join_numbers) {
     $textlib = textlib_get_instance();
 
     // Decode entities to UTF-8
-    $text = html_entity_decode($text, ENT_QUOTES, 'UTF-8');
+    $text = $textlib->entities_to_utf8($text, true);
 
     // Lowercase
     $text = $textlib->strtolower($text);