MDL-12234, fixing unicode issues with global search

diff --git a/lang/en_utf8/search.php b/lang/en_utf8/search.php
index 2a08a33dc1b1b52d74fac15ee4db3802696c8ff5..a56404a362592cf50eb9d983c8ef65d6586de1d7 100644 (file)
--- a/lang/en_utf8/search.php
+++ b/lang/en_utf8/search.php
@@ -51,7 +51,7 @@ $string['tofetchtheseresults'] = 'to fetch these results';
 $string['totalsize'] = 'Total Size ';
 $string['type'] = 'Type';
 $string['uncompleteindexingerror'] = 'Indexing was not successfully completed, please restart it.';
-$string['versiontoolow'] = 'Sorry, global search requires PHP 5.0.0 or later (currently using version '.$phpversion.')';
+$string['versiontoolow'] = 'Sorry, global search requires PHP 5.0.0 or later';
 $string['whichmodulestosearch?'] = 'Which modules to search?';
 $string['wordsintitle'] = 'Words in title';
 ?>

diff --git a/search/Zend/Search/Lucene/Analysis/Analyzer.php b/search/Zend/Search/Lucene/Analysis/Analyzer.php
index e57f6a5ed52d243ea448f4665f96893439b517f1..7a29c763d90e54e3bc7c8cade532bdae8623fe9a 100644 (file)
--- a/search/Zend/Search/Lucene/Analysis/Analyzer.php
+++ b/search/Zend/Search/Lucene/Analysis/Analyzer.php
@@ -97,10 +97,9 @@ abstract class Zend_Search_Lucene_Analysis_Analyzer
      * @param string $data
      * @return array
      */
-    public function tokenize($data, $encoding = '')
+    public function tokenize($data, $encoding = 'UTF-8')
     {
         $this->setInput($data, $encoding);
-
         $tokenList = array();
         while (($nextToken = $this->nextToken()) !== null) {
             $tokenList[] = $nextToken;
@@ -161,7 +160,7 @@ abstract class Zend_Search_Lucene_Analysis_Analyzer
     public static function getDefault()
     {
         if (!self::$_defaultImpl instanceof Zend_Search_Lucene_Analysis_Analyzer) {
-            self::$_defaultImpl = new Zend_Search_Lucene_Analysis_Analyzer_Common_Text_CaseInsensitive();
+            self::$_defaultImpl = new Zend_Search_Lucene_Analysis_Analyzer_Common_Utf8();
         }
 
         return self::$_defaultImpl;

diff --git a/search/add.php b/search/add.php
index 6d45795507be48a19c699b1ae89880a4feb59c41..dbe45333ca70bc5342237073c851af095aeaf05f 100644 (file)
--- a/search/add.php
+++ b/search/add.php
@@ -40,7 +40,7 @@ $startindextime = time();
 
 $indexdate = $CFG->search_indexer_run_date;
 
-mtrace('<pre>Starting index update (additions)...');
+mtrace('Starting index update (additions)...');
 mtrace('Index size before: '.$CFG->search_index_size."\n");
 
 //get all modules
@@ -139,6 +139,6 @@ set_config("search_index_size", (int)$CFG->search_index_size + (int)$addition_co
 
 // print some additional info
 mtrace("Added $addition_count documents.");
-mtrace('Index size after: '.$index->count().'</pre>');
+mtrace('Index size after: '.$index->count());
 
 ?>
\ No newline at end of file

diff --git a/search/cron_php5.php b/search/cron_php5.php
index 4aa4b4347ad9224768dbb412bb397683cb8ddd83..e098c7ab71141a3c3fd0306f70ed5e12fd4d097b 100644 (file)
--- a/search/cron_php5.php
+++ b/search/cron_php5.php
@@ -11,15 +11,15 @@ try{
     $maxmemoryamount = ini_get('memory_limit');
     ini_set('memory_limit', '48M');
 
-    mtrace("<pre>Starting cron...\n");
-    mtrace("--DELETE----");
+    mtrace("\n--DELETE----");
     require_once("$CFG->dirroot/search/delete.php");
     mtrace("--UPDATE----");
     require_once("$CFG->dirroot/search/update.php");
     mtrace("--ADD-------");
     require_once("$CFG->dirroot/search/add.php");
     mtrace("------------");
-    mtrace("cron finished.</pre>");
+    //mtrace("cron finished.</pre>");
+    mtrace('done');
 
     // set back normal values for php limits
     ini_set('max_execution_time', $maxtimelimit);

diff --git a/search/delete.php b/search/delete.php
index 3183e7ddb2f627f497532f38b36fcd2730e42262..11effeea0f468a2a845233cbc50ae5022a9c2200 100644 (file)
--- a/search/delete.php
+++ b/search/delete.php
@@ -38,7 +38,7 @@ $dbcontrol = new IndexDBControl();
 $deletion_count = 0;
 $startcleantime = time();
 
-mtrace('<pre>Starting clean-up of removed records...');
+mtrace('Starting clean-up of removed records...');
 mtrace('Index size before: '.$CFG->search_index_size."\n");
 
 if ($mods = get_records_select('modules')) {
@@ -127,6 +127,6 @@ set_config("search_indexer_cleanup_date", $startcleantime);
 set_config("search_index_size", (int)$CFG->search_index_size - (int)$deletion_count);
 
 mtrace("Finished $deletion_count removals.");
-mtrace('Index size after: '.$index->count().'</pre>');
+mtrace('Index size after: '.$index->count());
 
 ?>
\ No newline at end of file

diff --git a/search/indexer.php b/search/indexer.php
index 06021ee38f8af212f4018159217ff639f05d1a9e..74d1b592752330c2d56477b591c0c695183f96ef 100644 (file)
--- a/search/indexer.php
+++ b/search/indexer.php
@@ -64,7 +64,7 @@ require_once("$CFG->dirroot/search/indexlib.php");
 mtrace('<html><head><meta http-equiv="content-type" content="text/html; charset=utf-8" /></head><body>');
 mtrace('<pre>Server Time: '.date('r',time())."\n");
 
-if ($CFG->search_indexer_busy == '1') {
+if (isset($CFG->search_indexer_busy) && $CFG->search_indexer_busy == '1') {
     //means indexing was not finished previously
     mtrace("Warning: Indexing was not successfully completed last time, restarting.\n");
 }

diff --git a/search/indexlib.php b/search/indexlib.php
index 7ed1be77528af5a6e7e6134778ebbf0962adbbe6..d9e4bc8f0bc8b735180df25d8dd68f03c9802076 100644 (file)
--- a/search/indexlib.php
+++ b/search/indexlib.php
@@ -80,7 +80,7 @@ class IndexInfo {
         }
         
         //check if the busy flag is set
-        if ($CFG->search_indexer_busy == '1') {
+        if (isset($CFG->search_indexer_busy) && $CFG->search_indexer_busy == '1') {
             $this->complete = false;
         } 
         else {

diff --git a/search/query.php b/search/query.php
index 97526fd1800980df8f59085b0a48e646d8090d34..b415891835efbeec16f3f068a7828dfeac44d929 100644 (file)
--- a/search/query.php
+++ b/search/query.php
@@ -132,11 +132,14 @@ if (!$site = get_site()) {
     redirect("index.php");
 } 
 
-$strsearch = get_string('search', 'search'); 
+$strsearch = get_string('search', 'search');
 $strquery  = get_string('enteryoursearchquery', 'search');
 
-print_header("$site->shortname: $strsearch: $strquery", "$site->fullname",
-               "<a href=\"index.php\">$strsearch</a> -> $strquery");
+$navlinks[] = array('name' => $strsearch, 'link' => "index.php", 'type' => 'misc');
+$navlinks[] = array('name' => $strquery, 'link' => null, 'type' => 'misc');
+$navigation = build_navigation($navlinks);
+$course = get_site();
+print_header("$strsearch", "$strsearch" , $navigation, "", "", true, "&nbsp;", navmenu($course));
 
 //keep things pretty, even if php5 isn't available
 if (!$check) {
@@ -154,7 +157,9 @@ $vars = get_object_vars($adv);
 
 if (isset($vars)) {
     foreach ($vars as $key => $value) {
-        $adv->$key = stripslashes(htmlentities($value));
+        // htmlentities breaks non-ascii chars
+        $adv->key = stripslashes($value);
+        //$adv->$key = stripslashes(htmlentities($value));
     } 
 }
 ?>
@@ -305,11 +310,15 @@ if ($sq->is_valid()) {
         $scorestr = get_string('score', 'search');
         $authorstr = get_string('author', 'search');
         foreach ($hits as $listing) {
-            if ($CFG->unicodedb) $listing->title = mb_convert_encoding($listing->title, 'auto', 'UTF8');
+            //if ($CFG->unicodedb) {
+            //$listing->title = mb_convert_encoding($listing->title, 'auto', 'UTF8');
+            //}
             $title_post_processing_function = $listing->doctype.'_link_post_processing';
             require_once "{$CFG->dirroot}/search/documents/{$listing->doctype}_document.php";
-            if (function_exists($title_post_processing_function))
+            if (function_exists($title_post_processing_function)) {
                 $listing->title = $title_post_processing_function($listing->title);
+            }
+
             print "<li value='".($listing->number+1)."'><a href='".str_replace('DEFAULT_POPUP_SETTINGS', DEFAULT_POPUP_SETTINGS ,$listing->url)."'>$listing->title</a><br />\n"
                ."<em>".search_shorten_url($listing->url, 70)."</em><br />\n"
                ."{$typestr}: ".$listing->doctype.", {$scorestr}: ".round($listing->score, 3).", {$authorstr}: ".$listing->author."\n"

diff --git a/search/update.php b/search/update.php
index b7ebe4abc5be199ff0727987f0765a0cb10a1244..f641f1b39ff313ef00897bfa5679782163e48fd4 100644 (file)
--- a/search/update.php
+++ b/search/update.php
@@ -39,7 +39,7 @@ $update_count = 0;
 $indexdate = $CFG->search_indexer_update_date;
 $startupdatedate = time();
 
-mtrace("<pre>Starting index update (updates)...\n");
+mtrace("Starting index update (updates)...\n");
 
 if ($mods = get_records_select('modules')) {
     $mods = array_merge($mods, search_get_additional_modules());
@@ -138,6 +138,6 @@ $index->commit();
 //update index date
 set_config("search_indexer_update_date", $startupdatedate);
 
-mtrace("Finished $update_count updates.</pre>");
+mtrace("Finished $update_count updates");
 
 ?>
\ No newline at end of file