From: skodak Date: Mon, 31 Dec 2007 15:37:12 +0000 (+0000) Subject: MDL-12782 magpie now uses download_file_content() - that should solve several remote... X-Git-Url: http://git.mjollnir.org/gw?a=commitdiff_plain;h=6bf55889ddf85dcd967fb004654eb530f197c6f4;p=moodle.git MDL-12782 magpie now uses download_file_content() - that should solve several remote RSS issues including proxy user/password + several download_file_content() improvements + other minor fixes; merged from MOODLE_19_STABLE --- diff --git a/blocks/rss_client/config_instance.html b/blocks/rss_client/config_instance.html index 3066ed61ce..543e575c4f 100644 --- a/blocks/rss_client/config_instance.html +++ b/blocks/rss_client/config_instance.html @@ -182,6 +182,6 @@ print_box_start(); print_box_start(); rss_display_feeds($id, $USER->id, '', $context); rss_print_form($act, $url, $rssid, $preferredtitle, $shared, $id, $context); - // Do NOT print_box_end() here, this is taken care of by blocks/moodleblock.class.php:582 + print_box_end(); } ?> diff --git a/lib/filelib.php b/lib/filelib.php index 2eb61ed596..6aca3e9f89 100644 --- a/lib/filelib.php +++ b/lib/filelib.php @@ -5,37 +5,103 @@ define('BYTESERVING_BOUNDARY', 's1k2o3d4a5k6s7'); //unique string constant /** * Fetches content of file from Internet (using proxy if defined). Uses cURL extension if present. * @param string $url file url + * @param array $headers http headers, null if none + * @param array $postdata array means use POST request with given parameters + * @param bool $fullresponse return headers, responses, etc in a similar way snoopy does + * @param int $timeout connection timeout * @return mixed false if request failed or content of the file as string if ok. */ -function download_file_content($url) { +function download_file_content($url, $headers=null, $postdata=null, $fullresponse=false, $timeout=20) { global $CFG; - if (!extension_loaded('curl')) { + if (!extension_loaded('curl') or ($ch = curl_init($url)) === false) { require_once($CFG->libdir.'/snoopy/Snoopy.class.inc'); $snoopy = new Snoopy(); - $snoopy->proxy_host = $CFG->proxyhost; - $snoopy->proxy_port = $CFG->proxyport; + $snoopy->read_timeout = $timeout; + $snoopy->proxy_host = $CFG->proxyhost; + $snoopy->proxy_port = $CFG->proxyport; if (!empty($CFG->proxyuser) and !empty($CFG->proxypassword)) { // this will probably fail, but let's try it anyway $snoopy->proxy_user = $CFG->proxyuser; $snoopy->proxy_password = $CFG->proxypassword; } - if ($snoopy->fetch($url)) { - if (strpos($snoopy->response_code, '200') === false) { + if (is_array($headers) ) { + $client->rawheaders = $headers; + } + + if (is_array($postdata)) { + $fetch = @$snoopy->fetch($url, $postdata); // use more specific debug code bellow + } else { + $fetch = @$snoopy->fetch($url); // use more specific debug code bellow + } + + if ($fetch) { + if ($fullresponse) { + //fix header line endings + foreach ($snoopy->headers as $key=>$unused) { + $snoopy->headers[$key] = trim($snoopy->headers[$key]); + } + $response = new object(); + $response->status = $snoopy->status; + $response->headers = $snoopy->headers; + $response->response_code = trim($snoopy->response_code); + $response->results = $snoopy->results; + $response->error = $snoopy->error; + return $response; + + } else if ($snoopy->status != 200) { debugging("Snoopy request for \"$url\" failed, http response code: ".$snoopy->response_code, DEBUG_ALL); return false; + } else { return $snoopy->results; } } else { - debugging("Snoopy request for \"$url\" failed with: ".$snoopy->error, DEBUG_ALL); - return false; + if ($fullresponse) { + $response = new object(); + $response->status = $snoopy->status; + $response->headers = array(); + $response->response_code = $snoopy->response_code; + $response->results = ''; + $response->error = $snoopy->error; + return $response; + } else { + debugging("Snoopy request for \"$url\" failed with: ".$snoopy->error, DEBUG_ALL); + return false; + } + } + } + + // set extra headers, trim all newlines for extra security + if (is_array($headers) ) { + $headers2 = array(); + foreach ($headers as $key => $value) { + $value = str_replace("\n", '', $value); + $value = str_replace("\r", '', $value); + $headers2[] = "$key: $value"; } + curl_setopt($ch, CURLOPT_HTTPHEADER, $headers2); + } + + // use POST if requested + if (is_array($postdata)) { + foreach ($postdata as $k=>$v) { + $postdata[$k] = urlencode($k).'='.urlencode($v); + } + $postdata = implode('&', $postdata); + curl_setopt($ch, CURLOPT_POST, true); + curl_setopt($ch, CURLOPT_POSTFIELDS, $postdata); } - $ch = curl_init($url); curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); - curl_setopt($ch, CURLOPT_HEADER, false); + curl_setopt($ch, CURLOPT_HEADER, true); + curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, $timeout); + if (!ini_get('open_basedir') and !ini_get('safe_mode')) { + // TODO: add version check for '7.10.5' + curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true); + curl_setopt($ch, CURLOPT_MAXREDIRS, 5); + } + if (!empty($CFG->proxyhost)) { // SOCKS supported in PHP5 only if (!empty($CFG->proxytype) and ($CFG->proxytype == 'SOCKS5')) { @@ -48,7 +114,6 @@ function download_file_content($url) { } } - // don't CONNECT for non-https connections curl_setopt($ch, CURLOPT_HTTPPROXYTUNNEL, false); if (empty($CFG->proxyport)) { @@ -65,65 +130,62 @@ function download_file_content($url) { } } } + $result = curl_exec($ch); + // detect encoding problems + if ((curl_errno($ch) == 23 or curl_errno($ch) == 61) and defined('CURLOPT_ENCODING')) { + curl_setopt($ch, CURLOPT_ENCODING, 'none'); + $result = curl_exec($ch); + } + if (curl_errno($ch)) { - debugging("CURL request for \"$url\" failed with: ".curl_error($ch), DEBUG_ALL); - $result = false; + $error = curl_error($ch); + $error_no = curl_errno($ch); + curl_close($ch); + + if ($fullresponse) { + $response = new object(); + if ($error_no == 28) { + $response->status = '-100'; // mimic snoopy + } else { + $response->status = '0'; + } + $response->headers = array(); + $response->response_code = $error; + $response->results = ''; + $response->error = $error; + return $response; + } else { + debugging("CURL request for \"$url\" failed with: $error ($error_no)", DEBUG_ALL); + return false; + } } else { - $code = curl_getinfo($ch, CURLINFO_HTTP_CODE); - if ($code != 200) { - $codes = array( - 100=>'Continue', - 101=>'Switching Protocols', - 200=>'OK', - 201=>'Created', - 202=>'Accepted', - 203=>'Non-Authoritative Information', - 204=>'No Content', - 205=>'Reset Content', - 206=>'Partial Content', - 300=>'Multiple Choices', - 301=>'Moved Permanently', - 302=>'Moved Temporarily', - 303=>'See Other', - 304=>'Not Modified', - 305=>'Use Proxy', - 306=>'(Unused)', - 307=>'Temporary Redirect', - 400=>'Bad Request', - 401=>'Unauthorized', - 402=>'Payment Required', - 403=>'Forbidden', - 404=>'Not Found', - 405=>'Method Not Allowed', - 406=>'Not Acceptable', - 407=>'Proxy Authentication Required', - 408=>'Request Timeout', - 409=>'Conflict', - 410=>'Gone', - 411=>'Length Required', - 412=>'Precondition Failed', - 413=>'Request Entity Too Large', - 414=>'Request-URI Too Long', - 415=>'Unsupported Media Type', - 416=>'Requested Range Not Satisfiable', - 417=>'Expectation Failed', - 500=>'Internal Server Error', - 501=>'Not Implemented', - 502=>'Bad Gateway', - 503=>'Service Unavailable', - 504=>'Gateway Timeout', - 505=>'HTTP Version Not Supported'); - - debugging("CURL request for \"$url\" failed, http response code: ".$code.' '.$codes[$code], DEBUG_ALL); - $result = false; + $info = curl_getinfo($ch); + curl_close($ch); + // strip redirect headers and constrct headers array and body + $result = explode("\r\n\r\n", $result, $info['redirect_count'] + 2); + $results = array_pop($result); + $headers = array_pop($result); + $headers = explode("\r\n", trim($headers)); + + $response = new object();; + $response->status = (string)$info['http_code']; + $response->headers = $headers; + $response->response_code = $headers[0]; + $response->results = $results; + $response->error = ''; + + if ($fullresponse) { + return $response; + } else if ($info['http_code'] != 200) { + debugging("CURL request for \"$url\" failed, http response code: $response_code", DEBUG_ALL); + return false; + } else { + return $response->results; } } - - curl_close($ch); - return $result; } /** diff --git a/lib/magpie/rss_fetch.inc b/lib/magpie/rss_fetch.inc index fad5c01a95..1e36854c76 100644 --- a/lib/magpie/rss_fetch.inc +++ b/lib/magpie/rss_fetch.inc @@ -30,13 +30,12 @@ if (!defined('MAGPIE_DIR')) { require_once( MAGPIE_DIR . 'rss_parse.inc' ); require_once( MAGPIE_DIR . 'rss_cache.inc' ); -// for including 3rd party libraries +// moodlefix start // for including 3rd party libraries //define('MAGPIE_EXTLIB', MAGPIE_DIR . 'extlib' . DIR_SEP); //require_once( MAGPIE_EXTLIB . 'Snoopy.class.inc'); -// Modified by Daryl Hawes for Moodle integration - use Snoopy file in moodle/lib/snoopy -require_once( $CFG->libdir .'/snoopy/Snoopy.class.inc'); - +require_once($CFG->libdir.'/filelib.php'); +// modolefix end /* * CONSTANTS - redefine these in your script to change the @@ -200,7 +199,7 @@ function fetch_rss ($url, $postdata=null) { elseif ( $resp->error ) { # compensate for Snoopy's annoying habbit to tacking # on '\n' - $http_error = substr($resp->error, 0, -2); + $http_error = trim($resp->error); // moodle fix $errormsg .= "(HTTP Error: $http_error)"; } else { @@ -277,6 +276,9 @@ function magpie_error ($errormsg="") { Output: an HTTP response object (see Snoopy.class.inc) \*=======================================================================*/ function _fetch_remote_file ($url, $headers = "", $postdata=null ) { + // moodlefix + return download_file_content($url, $headers, $postdata, true, MAGPIE_FETCH_TIME_OUT); +/* // Snoopy is an HTTP client in PHP $client = new Snoopy(); $client->proxy_host = MAGPIE_PROXY_HOST; @@ -293,6 +295,8 @@ function _fetch_remote_file ($url, $headers = "", $postdata=null ) { @$client->fetch($url); } return $client; +*/ + // end of moodlefix } diff --git a/lib/rsslib.php b/lib/rsslib.php index 025783f792..79575542f5 100644 --- a/lib/rsslib.php +++ b/lib/rsslib.php @@ -270,6 +270,7 @@ function rss_geterrorxmlfile() { //XML item if ($return) { + $item = new object(); $item->title = "RSS Error"; $item->link = $CFG->wwwroot; $item->pubdate = time(); @@ -341,9 +342,7 @@ if (!isset($CFG->block_rss_client_timeout) ) { } // Defines for moodle's use of magpierss classes -define('MAGPIE_PROXY_HOST', $CFG->proxyhost); // Could be empty, that's OK -define('MAGPIE_PROXY_PORT', $CFG->proxyport); // Could be empty, that's OK -define('MAGPIE_DIR', $CFG->dirroot.'/lib/magpie/'); +define('MAGPIE_DIR', $CFG->libdir.'/magpie/'); define('MAGPIE_CACHE_DIR', $CFG->dataroot .'/cache/rsscache'); define('MAGPIE_CACHE_ON', true); //might want to expose as an admin config option, but perhaps this is something that should truly just be on unless the code is tweaked define('MAGPIE_CACHE_FRESH_ONLY', false); //should be exposed as an admin config option