From 0b0bfa934578862e848c24fc58a864b69b173bba Mon Sep 17 00:00:00 2001 From: skodak Date: Fri, 8 Aug 2008 10:22:59 +0000 Subject: [PATCH] MDL-15919, MDL-15920 reworked support for archiving --- lib/deprecatedlib.php | 8 +- lib/file/stored_file.php | 24 +- lib/file/zip_archive.php | 11 - lib/filelib.php | 3 +- lib/moodlelib.php | 25 +- lib/packer/file_archive.php | 175 ++++++++++ lib/packer/file_packer.php | 47 +++ lib/packer/zip_archive.php | 304 ++++++++++++++++++ .../file_packer.php => packer/zip_packer.php} | 92 +++--- 9 files changed, 606 insertions(+), 83 deletions(-) delete mode 100644 lib/file/zip_archive.php create mode 100644 lib/packer/file_archive.php create mode 100644 lib/packer/file_packer.php create mode 100644 lib/packer/zip_archive.php rename lib/{file/file_packer.php => packer/zip_packer.php} (80%) diff --git a/lib/deprecatedlib.php b/lib/deprecatedlib.php index cfeac99e1e..c71b5272e1 100644 --- a/lib/deprecatedlib.php +++ b/lib/deprecatedlib.php @@ -374,7 +374,9 @@ function unzip_file($zipfile, $destination = '', $showstatus_ignored = true) { return false; } - $result = get_file_packer()->unzip_files_to_pathname($zipfile, $destpath); + $packer = get_file_packer('application/zip'); + + $result = $packer->extract_to_pathname($zipfile, $destpath); if ($result === false) { return false; @@ -452,9 +454,9 @@ function zip_files ($originalfiles, $destination) { $zipfiles[substr($file, $start)] = $file; } - $packer = get_file_packer(); + $packer = get_file_packer('application/zip'); - return $packer->zip_files_to_pathname($zipfiles, $destfilename); + return $packer->archive_to_pathname($zipfiles, $destfilename); } ///////////////////////////////////////////////////////////// diff --git a/lib/file/stored_file.php b/lib/file/stored_file.php index 02a4799b46..dae35b3a88 100644 --- a/lib/file/stored_file.php +++ b/lib/file/stored_file.php @@ -100,17 +100,18 @@ class stored_file { /** * Unzip file to given file path (real OS filesystem), existing files are overwrited - * @param string $path target directory + * @param object $file_packer + * @param string $pathname target directory * @return mixed list of processed files; false if error */ - public function unzip_files_to_pathname($path) { - $packer = get_file_packer(); - $zipfile = $this->get_content_file_location(); - return $packer->unzip_files_to_pathname($path, $path); + public function extract_to_pathname(file_packer $packer, $pathname) { + $archivefile = $this->get_content_file_location(); + return $packer->extract_to_pathname($archivefile, $pathname); } /** * Unzip file to given file path (real OS filesystem), existing files are overwrited + * @param object $file_packer * @param int $contextid * @param string $filearea * @param int $itemid @@ -118,10 +119,9 @@ class stored_file { * @param int $userid * @return mixed list of processed files; false if error */ - public function unzip_files_to_storage($contextid, $filearea, $itemid, $pathbase, $userid=null) { - $packer = get_file_packer(); - $zipfile = $this->get_content_file_location(); - return $packer->unzip_files_to_storage($zipfile, $contextid, $filearea, $itemid, $pathbase); + public function extract_to_storage(file_packer $packer, $contextid, $filearea, $itemid, $pathbase, $userid=null) { + $archivefile = $this->get_content_file_location(); + return $packer->extract_to_storage($archivefile, $contextid, $filearea, $itemid, $pathbase); } /** @@ -130,15 +130,15 @@ class stored_file { * @param string $archivepath pathname in zip archive * @return bool success */ - public function add_to_ziparchive(zip_archive $ziparch, $archivepath) { + public function archive_file(file_archive $filearch, $archivepath) { if ($this->is_directory()) { - return $ziparch->addEmptyDir($archivepath); + return $filearch->add_directory($archivepath); } else { $path = $this->get_content_file_location(); if (!is_readable($path)) { return false; } - return $ziparch->addFile($path, $archivepath); + return $filearch->add_file_from_pathname($archivepath, $path); } } diff --git a/lib/file/zip_archive.php b/lib/file/zip_archive.php deleted file mode 100644 index d8b78679ec..0000000000 --- a/lib/file/zip_archive.php +++ /dev/null @@ -1,11 +0,0 @@ -libdir/file/file_exceptions.php"); require_once("$CFG->libdir/file/file_storage.php"); require_once("$CFG->libdir/file/file_browser.php"); -require_once("$CFG->libdir/file/file_packer.php"); + +require_once("$CFG->libdir/packer/zip_packer.php"); function get_file_url($path, $options=null, $type='coursefile') { global $CFG; diff --git a/lib/moodlelib.php b/lib/moodlelib.php index 8ca62e5512..4c1d608bbd 100644 --- a/lib/moodlelib.php +++ b/lib/moodlelib.php @@ -4535,22 +4535,33 @@ function get_file_browser() { /** * Returns file packer + * @param string $mimetype * @return object file_storage */ -function get_file_packer() { +function get_file_packer($mimetype='application/zip') { global $CFG; - static $fp = null; + static $fp = array();; - if ($fp) { - return $fp; + if (isset($fp[$mimetype])) { + return $fp[$mimetype]; } - require_once("$CFG->libdir/filelib.php"); + switch ($mimetype) { + case 'application/zip': + $classname = 'zip_packer'; + break; + case 'application/x-tar': +// $classname = 'tar_packer'; +// break; + default: + return false; + } - $fp = new file_packer(); + require_once("$CFG->libdir/packer/$classname.php"); + $fp[$mimetype] = new $classname(); - return $fp; + return $fp[$mimetype]; } /** diff --git a/lib/packer/file_archive.php b/lib/packer/file_archive.php new file mode 100644 index 0000000000..59b7107729 --- /dev/null +++ b/lib/packer/file_archive.php @@ -0,0 +1,175 @@ +encoding === 'utf-8') { + return $localname; + } + $textlib = textlib_get_instance(); + + $converted = $textlib->convert($localname, 'utf-8', $this->encoding); + $original = $textlib->convert($converted, $this->encoding, 'utf-8'); + + if ($original === $localname) { + $result = $converted; + + } else { + // try ascci conversion + $converted2 = $textlib->specialtoascii($localname); + $converted2 = $textlib->convert($converted2, 'utf-8', $this->encoding); + $original2 = $textlib->convert($converted, $this->encoding, 'utf-8'); + + if ($original2 === $localname) { + //this looks much better + $result = $converted2; + } else { + //bad luck - the file name may not be usable at all + $result = $converted; + } + } + + $result = ereg_replace('\.\.+', '', $result); + $result = ltrim($result); // no leadin / + + if ($result === '.') { + $result = ''; + } + + return $result; + } + + /** + * Tries to convert $localname into utf-8 + * please note that it may fail really badly. + * The resulting file name is cleaned. + * + * @param strin $localname in anothe encoding + * @return string in utf-8 + */ + protected function unmangle_pathname($localname) { + if ($this->encoding === 'utf-8') { + return $localname; + } + $textlib = textlib_get_instance(); + + $result = $textlib->convert($localname, $this->encoding, 'utf-8'); + $result = clean_param($result, PARAM_PATH); + $result = ltrim($result); // no leadin / + + return $result; + } + + /** + * Returns current file info + * @return object + */ + //public abstract function current(); + + /** + * Returns the index of current file + * @return int current file index + */ + //public abstract function key(); + + /** + * Moves forward to next file + * @return void + */ + //public abstract function next(); + + /** + * Revinds back to the first file + * @return void + */ + //public abstract function rewind(); + + /** + * Did we reach the end? + * @return boolean + */ + //public abstract function valid(); + +} \ No newline at end of file diff --git a/lib/packer/file_packer.php b/lib/packer/file_packer.php new file mode 100644 index 0000000000..dcc0ddaff9 --- /dev/null +++ b/lib/packer/file_packer.php @@ -0,0 +1,47 @@ +$pathanme or stored file instance + * @param int $contextid + * @param string $filearea + * @param int $itemid + * @param string $filepath + * @param string $filename + * @return mixed false if error stored file instance if ok + */ + public abstract function archive_to_storage($files, $contextid, $filearea, $itemid, $filepath, $filename, $userid=null); + + /** + * Archive files and store the result in os file + * @param array $archivepath=>$pathanme or stored file instance + * @param string $archivefile + * @return bool success + */ + public abstract function archive_to_pathname($files, $archivefile); + + /** + * Extract file to given file path (real OS filesystem), existing files are overwrited + * @param mixed $archivefile full pathname of zip file or stored_file instance + * @param string $pathname target directory + * @return mixed list of processed files; false if error + */ + public abstract function extract_to_pathname($archivefile, $pathname); + + /** + * Extract file to given file path (real OS filesystem), existing files are overwrited + * @param mixed $archivefile full pathname of zip file or stored_file instance + * @param int $contextid + * @param string $filearea + * @param int $itemid + * @param string $filepath + * @return mixed list of processed files; false if error + */ + public abstract function extract_to_storage($archivefile, $contextid, $filearea, $itemid, $pathbase, $userid=null); + +} \ No newline at end of file diff --git a/lib/packer/zip_archive.php b/lib/packer/zip_archive.php new file mode 100644 index 0000000000..4933e1c8e4 --- /dev/null +++ b/lib/packer/zip_archive.php @@ -0,0 +1,304 @@ +libdir/packer/file_archive.php"); + +class zip_archive extends file_archive { + + /** Pathname of archive */ + protected $archivepathname = null; + + /** Used memory tracking */ + protected $usedmem = 0; + + /** Iteration position */ + protected $pos = 0; + + /** TipArchive instance */ + protected $za; + + /** + * Open or create archive (depending on $mode) + * @param string $archivepathname + * @param int $mode OPEN, CREATE or OVERWRITE constant + * @param string $encoding archive local paths encoding + * @return bool success + */ + public function open($archivepathname, $mode=file_archive::CREATE, $encoding='utf-8') { + $this->close(); + + $this->usedmem = 0; + $this->pos = 0; + + $this->za = new ZipArchive(); + + switch($mode) { + case file_archive::OPEN: $flags = 0; break; + case file_archive::OVERWRITE: $flags = ZIPARCHIVE::OVERWRITE; break; + case file_archive::CREATE: + default : $flags = ZIPARCHIVE::CREATE; break; + } + + $result = $this->za->open($archivepathname, $flags); + + if ($result === true) { + $this->encoding = $encoding; + if (file_exists($archivepathname)) { + $this->archivepathname = realpath($archivepathname); + } else { + $this->archivepathname = $archivepathname; + } + return true; + + } else { + $this->za = null; + $this->archivepathname = null; + $this->encooding = 'utf-8'; + // TODO: maybe we should return some error info + return false; + } + } + + /** + * Close archive + * @return bool success + */ + public function close() { + if (!isset($this->za)) { + return false; + } + + $res = $this->za->close(); + $this->za = null; + + return $res; + } + + /** + * Returns file stream for reading of content + * @param int $index of file + * @return stream or false if error + */ + public function get_stream($index) { + if (!isset($this->za)) { + return false; + } + + $name = $this->za->getNameIndex($index); + if ($name === false) { + return false; + } + + return $this->za->getStream($name); + } + + /** + * Returns file information + * @param int $index of file + * @return info object or false if error + */ + public function get_info($index) { + if (!isset($this->za)) { + return false; + } + + if ($index < 0 or $index >=$this->count()) { + return false; + } + + $result = $this->za->statIndex($index); + + if ($result === false) { + return false; + } + + $info = new object(); + $info->index = $index; + $info->original_pathname = $result['name']; + $info->pathname = $this->unmangle_pathname($result['name']); + $info->mtime = (int)$result['mtime']; + + if ($info->pathname[strlen($info->pathname)-1] === '/') { + $info->is_directory = true; + $info->size = 0; + } else { + $info->is_directory = false; + $info->size = (int)$result['size']; + } + + return $info; + } + + /** + * Returns array of info about all files in archive + * @return array of file infos + */ + public function list_files() { + if (!isset($this->za)) { + return false; + } + + $infos = array(); + + for ($i=0; $i<$this->count(); $i++) { + $info = $this->get_info($i); + if ($info === false) { + continue; + } + $infos[$i] = $info; + } + + return $infos; + } + + /** + * Returns number of files in archive + * @return int number of files + */ + public function count() { + if (!isset($this->za)) { + return false; + } + + return $this->za->numFiles; + } + + /** + * Add file into archive + * @param string $localname name of file in archive + * @param string $pathname localtion of file + * @return bool success + */ + public function add_file_from_pathname($localname, $pathname) { + if (!isset($this->za)) { + return false; + } + + if ($this->archivepathname === realpath($pathname)) { + // do not add self into archive + return false; + } + + if (is_null($localname)) { + $localname = clean_param($pathname, PARAM_PATH); + } + $localname = trim($localname, '/'); // no leading slashes in archives + $localname = $this->mangle_pathname($localname); + + if ($localname === '') { + //sorry - conversion failed badly + return false; + } + + if ($this->count() > 0 and $this->count() % 500 === 0) { + // workaround for open file handles problem, ZipArchive uses file locking in order to prevent file modifications before the close() (strange, eh?) + $this->close(); + $res = $this->open($this->archivepathname, file_archive::OPEN, $this->encoding); + if ($res !== true) { + error('Can not open zip file, probably zip extension bug on 64bit os'); //TODO ?? + } + } + + return $this->za->addFile($pathname, $localname); + } + + /** + * Add content of string into archive + * @param string $localname name of file in archive + * @param string $contents + * @return bool success + */ + public function add_file_from_string($localname, $contents) { + if (!isset($this->za)) { + return false; + } + + $localname = trim($localname, '/'); // no leading slashes in archives + $localname = $this->mangle_pathname($localname); + + if ($localname === '') { + //sorry - conversion failed badly + return false; + } + + if ($this->usedmem > 2097151) { + /// this prevents running out of memory when adding many large files using strings + $this->close(); + $res = $this->open($this->archivepathname, file_archive::OPEN, $this->encoding); + if ($res !== true) { + error('Can not open zip file, probably zip extension bug on 64bit os'); //TODO ?? + } + } + $this->usedmem += strlen($contents); + + return $this->za->addFromString($localname, $contents); + + } + + /** + * Add empty directory into archive + * @param string $local + * @return bool success + */ + public function add_directory($localname) { + if (!isset($this->za)) { + return false; + } + $localname = ltrim($localname, '/'). '/'; + $localname = $this->mangle_pathname($localname); + + if ($localname === '/') { + //sorry - conversion failed badly + return false; + } + + return $this->za->addEmptyDir($localname); + } + + /** + * Returns current file info + * @return object + */ + public function current() { + if (!isset($this->za)) { + return false; + } + + return $this->get_info($this->pos); + } + + /** + * Returns the index of current file + * @return int current file index + */ + public function key() { + return $this->pos; + } + + /** + * Moves forward to next file + * @return void + */ + public function next() { + $this->pos++; + } + + /** + * Revinds back to the first file + * @return void + */ + public function rewind() { + $this->pos = 0; + } + + /** + * Did we reach the end? + * @return boolean + */ + public function valid() { + if (!isset($this->za)) { + return false; + } + + return ($this->pos < $this->count()); + } +} \ No newline at end of file diff --git a/lib/file/file_packer.php b/lib/packer/zip_packer.php similarity index 80% rename from lib/file/file_packer.php rename to lib/packer/zip_packer.php index aa8a007c29..880ba2a6c0 100644 --- a/lib/file/file_packer.php +++ b/lib/packer/zip_packer.php @@ -1,9 +1,12 @@ libdir/packer/file_packer.php"); +require_once("$CFG->libdir/packer/zip_archive.php"); + /** * Utility class - handles all zipping and unzipping operations. */ -class file_packer { +class zip_packer extends file_packer { /** * Zip files and store the result in file storage @@ -15,7 +18,7 @@ class file_packer { * @param string $filename * @return mixed false if error stored file instance if ok */ - public function zip_files_to_storage($files, $contextid, $filearea, $itemid, $filepath, $filename, $userid=null) { + public function archive_to_storage($files, $contextid, $filearea, $itemid, $filepath, $filename, $userid=null) { global $CFG; $fs = get_file_storage(); @@ -23,7 +26,7 @@ class file_packer { check_dir_exists($CFG->dataroot.'/temp/zip', true, true); $tmpfile = tempnam($CFG->dataroot.'/temp/zip', 'zipstor'); - if ($result = $this->zip_files_to_pathname($files, $tmpfile)) { + if ($result = $this->archive_to_pathname($files, $tmpfile)) { if ($file = $fs->get_file($contextid, $filearea, $itemid, $filepath, $filename)) { if (!$file->delete()) { @unlink($tmpfile); @@ -37,6 +40,7 @@ class file_packer { $file_record->filepath = $filepath; $file_record->filename = $filename; $file_record->userid = $userid; + $result = $fs->create_file_from_pathname($file_record, $tmpfile); } @unlink($tmpfile); @@ -46,19 +50,18 @@ class file_packer { /** * Zip files and store the result in os file * @param array $archivepath=>$pathanme or stored file instance - * @param string $zipfile + * @param string $archivefile * @return bool success */ - public function zip_files_to_pathname($files, $zipfile) { + public function archive_to_pathname($files, $archivefile) { global $CFG; - require_once("$CFG->libdir/file/zip_archive.php"); if (!is_array($files)) { return false; } $ziparch = new zip_archive(); - if (!$ziparch->open($zipfile, ZIPARCHIVE::OVERWRITE)) { + if (!$ziparch->open($archivefile, file_archive::OVERWRITE)) { return false; } @@ -67,21 +70,21 @@ class file_packer { if (is_null($file)) { // empty directories have null as content - $ziparch->addEmptyDir($archivepath.'/'); + $ziparch->add_directory($archivepath.'/'); } else if (is_string($file)) { - $this->add_os_file_to_zip($ziparch, $archivepath, $file); + $this->archive_pathname($ziparch, $archivepath, $file); } else { - $this->add_stored_file_to_zip($ziparch, $archivepath, $file); + $this->archive_stored($ziparch, $archivepath, $file); } } return $ziparch->close(); } - protected function add_stored_file_to_zip($ziparch, $archivepath, $file) { - $file->add_to_ziparchive($ziparch, $archivepath); + private function archive_stored($ziparch, $archivepath, $file) { + $file->archive_file($ziparch, $archivepath); if (!$file->is_directory()) { return; @@ -98,11 +101,11 @@ class file_packer { if (!$file->is_directory()) { $path = $path.$file->get_filename(); } - $file->add_to_ziparchive($ziparch, $path); + $file->archive_file($ziparch, $path); } } - protected function add_os_file_to_zip( $ziparch, $archivepath, $file) { + private function archive_pathname($ziparch, $archivepath, $file) { if (!file_exists($file)) { return; } @@ -111,13 +114,12 @@ class file_packer { if (!is_readable($file)) { return; } - $ziparch->addFile($file, $archivepath); + $ziparch->add_file_from_pathname($archivepath, $file); return; } if (is_dir($file)) { if ($archivepath !== '') { - $archivepath = $archivepath.'/'; - $ziparch->addEmptyDir($archivepath); + $ziparch->add_directory($archivepath); } $files = new DirectoryIterator($file); foreach ($files as $file) { @@ -125,7 +127,7 @@ class file_packer { continue; } $newpath = $archivepath.$file->getFilename(); - $this->add_os_file_to_zip($ziparch, $newpath, $file->getPathname()); + $this->archive_pathname($ziparch, $newpath, $file->getPathname()); } unset($files); //release file handles return; @@ -134,42 +136,38 @@ class file_packer { /** * Unzip file to given file path (real OS filesystem), existing files are overwrited - * @param mixed $zipfile full pathname of zip file or stored_file instance + * @param mixed $archivefile full pathname of zip file or stored_file instance * @param string $pathname target directory * @return mixed list of processed files; false if error */ - public function unzip_files_to_pathname($zipfile, $pathname) { + public function extract_to_pathname($archivefile, $pathname) { global $CFG; - require_once("$CFG->libdir/file/zip_archive.php"); - if (!is_string($zipfile)) { - return $zipfile->unzip_files_to_pathname($pathname); + if (!is_string($archivefile)) { + return $archivefile->extract_to_pathname($this, $pathname); } $processed = array(); $pathname = rtrim($pathname, '/'); - if (!is_readable($zipfile)) { + if (!is_readable($archivefile)) { return false; } - $ziparch = new zip_archive(); - if (!$ziparch->open($zipfile, ZIPARCHIVE::FL_NOCASE)) { + $ziparch = new zip_archive(); + if (!$ziparch->open($archivefile, file_archive::OPEN)) { return false; } - for ($i=0; $i<$ziparch->numFiles; $i++) { - $index = $ziparch->statIndex($i); - - $size = clean_param($index['size'], PARAM_INT); - $name = clean_param($index['name'], PARAM_PATH); - $name = ltrim($name, '/'); + foreach ($ziparch as $info) { + $size = $info->size; + $name = $info->pathname; if ($name === '' or array_key_exists($name, $processed)) { //probably filename collisions caused by filename cleaning/conversion continue; } - if ($size === 0 and $name[strlen($name)-1] === '/') { + if ($info->is_directory) { $newdir = "$pathname/$name"; // directory if (is_file($newdir) and !unlink($newdir)) { @@ -205,7 +203,7 @@ class file_packer { $processed[$name] = 'Can not write target file'; // TODO: localise continue; } - if (!$fz = $ziparch->getStream($index['name'])) { + if (!$fz = $ziparch->get_stream($info->index)) { $processed[$name] = 'Can not read file from zip archive'; // TODO: localise fclose($fp); continue; @@ -231,18 +229,18 @@ class file_packer { /** * Unzip file to given file path (real OS filesystem), existing files are overwrited - * @param mixed $zipfile full pathname of zip file or stored_file instance + * @param mixed $archivefile full pathname of zip file or stored_file instance * @param int $contextid * @param string $filearea * @param int $itemid * @param string $filepath * @return mixed list of processed files; false if error */ - public function unzip_files_to_storage($zipfile, $contextid, $filearea, $itemid, $pathbase, $userid=null) { + public function extract_to_storage($archivefile, $contextid, $filearea, $itemid, $pathbase, $userid=null) { global $CFG; - if (!is_string($zipfile)) { - return $zipfile->unzip_files_to_pathname($contextid, $filearea, $itemid, $pathbase, $userid); + if (!is_string($archivefile)) { + return $archivefile->extract_to_pathname($this, $contextid, $filearea, $itemid, $pathbase, $userid); } check_dir_exists($CFG->dataroot.'/temp/zip', true, true); @@ -254,24 +252,20 @@ class file_packer { $processed = array(); $ziparch = new zip_archive(); - if (!$ziparch->open($zipfile, ZIPARCHIVE::FL_NOCASE)) { + if (!$ziparch->open($archivefile, file_archive::OPEN)) { return false; } - for ($i=0; $i<$ziparch->numFiles; $i++) { - $index = $ziparch->statIndex($i); - - $size = clean_param($index['size'], PARAM_INT); - $name = clean_param($index['name'], PARAM_PATH); - $name = ltrim($name, '/'); - + foreach ($ziparch as $info) { + $size = $info->size; + $name = $info->pathname; if ($name === '' or array_key_exists($name, $processed)) { //probably filename collisions caused by filename cleaning/conversion continue; } - if ($size === 0 and $name[strlen($name)-1] === '/') { + if ($info->is_directory) { $newfilepath = $pathbase.$name.'/'; $fs->create_directory($contextid, $filearea, $itemid, $newfilepath, $userid); $processed[$name] = true; @@ -287,7 +281,7 @@ class file_packer { if ($size < 2097151) { // small file - if (!$fz = $ziparch->getStream($index['name'])) { + if (!$fz = $ziparch->get_stream($info->index)) { $processed[$name] = 'Can not read file from zip archive'; // TODO: localise continue; } @@ -332,7 +326,7 @@ class file_packer { $processed[$name] = 'Can not write temp file'; // TODO: localise continue; } - if (!$fz = $ziparch->getStream($index['name'])) { + if (!$fz = $ziparch->get_stream($info->index)) { @unlink($tmpfile); $processed[$name] = 'Can not read file from zip archive'; // TODO: localise continue; -- 2.39.5