From ad6a8f6911c13f2dddaba41919dc4fceadd1fc0e Mon Sep 17 00:00:00 2001 From: David Mudrak Date: Mon, 4 Jan 2010 18:14:01 +0000 Subject: [PATCH] Grading evaluation - best subplugin I am not happy with the algorithm at all. We should replace it with some more sophisticated subplugin, using ICC or some similar statistics. --- mod/workshop/aggregate.php | 8 +- mod/workshop/eval/best/lib.php | 268 ++++++++++++++++-- mod/workshop/eval/best/simpletest/testlib.php | 194 +++++++++++++ mod/workshop/form/accumulative/lib.php | 56 +++- mod/workshop/locallib.php | 13 + 5 files changed, 499 insertions(+), 40 deletions(-) create mode 100644 mod/workshop/eval/best/simpletest/testlib.php diff --git a/mod/workshop/aggregate.php b/mod/workshop/aggregate.php index e1e42d0e51..efcab9392f 100644 --- a/mod/workshop/aggregate.php +++ b/mod/workshop/aggregate.php @@ -46,10 +46,10 @@ if ($confirm) { if (!confirm_sesskey()) { throw new moodle_exception('confirmsesskeybad'); } - $workshop->aggregate_submission_grades(); - //$evaluator->update_grading_grades(); - //$workshop->aggregate_grading_grades(); - //$workshop->aggregate_total_grades(); + $workshop->aggregate_submission_grades(); // updates 'grade' in {workshop_submissions} + $evaluator->update_grading_grades(); // updates 'gradinggrade' in {workshop_assessments} + $workshop->aggregate_grading_grades(); // updates 'gradinggrade' in {workshop_aggregations} + $workshop->aggregate_total_grades(); // updates 'totalgrade' in {workshop_aggregations} redirect($workshop->view_url()); } diff --git a/mod/workshop/eval/best/lib.php b/mod/workshop/eval/best/lib.php index b868623414..f2777e2588 100644 --- a/mod/workshop/eval/best/lib.php +++ b/mod/workshop/eval/best/lib.php @@ -48,7 +48,10 @@ class workshop_best_evaluation implements workshop_evaluation { } /** - * TODO + * Calculates the grades for assessment and updates 'gradinggrade' fields in 'workshop_assessments' table + * + * This function relies on the grading strategy subplugin providing get_assessments_recordset() method. + * {@see self::process_assessments()} for the required structure of the recordset. * * @param null|int|array $restrict If null, update all reviewers, otherwise update just grades for the given reviewers(s) * @@ -63,8 +66,11 @@ class workshop_best_evaluation implements workshop_evaluation { support this method of grading evaluation.'); } + // get the information about the assessment dimensions + $diminfo = $grader->eval_best_dimensions_info(); + // fetch a recordset with all assessments to process - $rs = $grader->get_assessments_recordset($restrict); + $rs = $grader->eval_best_get_assessments_recordset($restrict); $batch = array(); // will contain a set of all assessments of a single submission $previous = null; // a previous record in the recordset foreach ($rs as $current) { @@ -76,52 +82,268 @@ class workshop_best_evaluation implements workshop_evaluation { $batch[] = $current; } else { // process all the assessments of a sigle submission - $this->process_assessments($batch); + $this->process_assessments($batch, $diminfo); // start with a new batch to be processed $batch = array($current); $previous = $current; } } // do not forget to process the last batch! - $this->process_assessments($batch); + $this->process_assessments($batch, $diminfo); $rs->close(); } -//////////////////////////////////////////////////////////////////////////////// -// Internal methods // -//////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// + // Internal methods // + //////////////////////////////////////////////////////////////////////////////// /** * Given a list of all assessments of a single submission, updates the grading grades in database * - * @param array $assessments of stdClass Object(->assessmentid ->assessmentweight ->reviewerid ->submissionid - * ->dimensionid ->grade ->dimensionweight) + * @param array $assessments of stdClass (->assessmentid ->assessmentweight ->reviewerid ->gradinggrade ->submissionid ->dimensionid ->grade) + * @param array $diminfo of stdClass (->id ->weight ->max ->min) * @return void */ - protected function process_assessments(array $assessments) { + protected function process_assessments(array $assessments, array $diminfo) { global $DB; - $grades = $this->evaluate_assessments($assessments); + // reindex the passed flat structure to be indexed by assessmentid + $assessments = $this->prepare_data_from_recordset($assessments); + + // normalize the dimension grades to the interval 0 - 100 + $assessments = $this->normalize_grades($assessments, $diminfo); + + // get a hypothetical average assessment + $average = $this->average_assessment($assessments); + + // calculate variance of dimension grades + $variances = $this->weighted_variance($assessments); + foreach ($variances as $dimid => $variance) { + $diminfo[$dimid]->variance = $variance; + } + + // for every assessment, calculate its distance from the average one + $distances = array(); + foreach ($assessments as $asid => $assessment) { + $distances[$asid] = $this->assessments_distance($assessment, $average, $diminfo); + } + + // identify the best assessments - it est those with the shortest distance from the best assessment + $bestids = array_keys($distances, min($distances)); + + // for every assessment, calculate its distance from the nearest best assessment + $distances = array(); + foreach ($bestids as $bestid) { + $best = $assessments[$bestid]; + foreach ($assessments as $asid => $assessment) { + $d = $this->assessments_distance($assessment, $best, $diminfo); + if (!isset($distances[$asid]) or $d < $distances[$asid]) { + $distances[$asid] = $d; + } + } + } + + // calculate the grading grade + foreach ($distances as $asid => $distance) { + $gradinggrade = (100 - $distance); + /** + if ($gradinggrade < 0) { + $gradinggrade = 0; + } + if ($gradinggrade > 100) { + $gradinggrade = 100; + } + */ + $grades[$asid] = grade_floatval($gradinggrade); + } + + // if the new grading grade differs from the one stored in database, update it + // we do not use set_field() here because we want to pass $bulk param foreach ($grades as $assessmentid => $grade) { - $record = new stdClass(); - $record->id = $assessmentid; - $record->gradinggrade = grade_floatval($grade); - $DB->update_record('workshop_assessments', $record, true); + if (grade_floats_different($grade, $assessments[$assessmentid]->gradinggrade)) { + // the value has changed + $record = new stdClass(); + $record->id = $assessmentid; + $record->gradinggrade = grade_floatval($grade); + $DB->update_record('workshop_assessments', $record, true); // bulk operations expected + } } + + // done. easy, heh? ;-) } /** - * Given a list of all assessments of a single submission, calculates the grading grades for them + * Prepares a structure of assessments and given grades * - * @param array $assessments same structure as for {@link self::process_assessments()} - * @return array [(int)assessmentid => (float)gradinggrade] to be saved into {workshop_assessments} + * @param array $assessments batch of recordset items as returned by the grading strategy + * @return array */ - protected function evaluate_assessments(array $assessments) { - $gradinggrades = array(); - foreach ($assessments as $assessment) { - $gradinggrades[$assessment->assessmentid] = grade_floatval(rand(0, 100)); // todo + protected function prepare_data_from_recordset($assessments) { + $data = array(); // to be returned + foreach ($assessments as $a) { + $id = $a->assessmentid; // just an abbrevation + if (!isset($data[$id])) { + $data[$id] = new stdClass(); + $data[$id]->assessmentid = $a->assessmentid; + $data[$id]->weight = $a->assessmentweight; + $data[$id]->reviewerid = $a->reviewerid; + $data[$id]->gradinggrade = $a->gradinggrade; + $data[$id]->submissionid = $a->submissionid; + $data[$id]->dimgrades = array(); + } + $data[$id]->dimgrades[$a->dimensionid] = $a->grade; } - return $gradinggrades; + return $data; } + /** + * Normalizes the dimension grades to the interval 0.00000 - 100.00000 + * + * Note: this heavily relies on PHP5 way of handling references in array of stdClasses. Hopefuly + * it will not change again soon. + * + * @param array $assessments of stdClass as returned by {@see self::prepare_data_from_recordset()} + * @param array $diminfo of stdClass + * @return array of stdClass with the same structure as $assessments + */ + protected function normalize_grades(array $assessments, array $diminfo) { + foreach ($assessments as $asid => $assessment) { + foreach ($assessment->dimgrades as $dimid => $dimgrade) { + $dimmin = $diminfo[$dimid]->min; + $dimmax = $diminfo[$dimid]->max; + $assessment->dimgrades[$dimid] = grade_floatval(($dimgrade - $dimmin) / ($dimmax - $dimmin) * 100); + } + } + return $assessments; + } + + /** + * Given a set of a submission's assessments, returns a hypothetical average assessment + * + * The passed structure must be array of assessments objects with ->weight and ->dimgrades properties. + * + * @param array $assessments as prepared by {@link self::prepare_data_from_recordset()} + * @return null|stdClass + */ + protected function average_assessment(array $assessments) { + $sumdimgrades = array(); + foreach ($assessments as $a) { + foreach ($a->dimgrades as $dimid => $dimgrade) { + if (!isset($sumdimgrades[$dimid])) { + $sumdimgrades[$dimid] = 0; + } + $sumdimgrades[$dimid] += $dimgrade * $a->weight; + } + } + + $sumweights = 0; + foreach ($assessments as $a) { + $sumweights += $a->weight; + } + if ($sumweights == 0) { + // unable to calculate average assessment + return null; + } + + $average = new stdClass(); + $average->dimgrades = array(); + foreach ($sumdimgrades as $dimid => $sumdimgrade) { + $average->dimgrades[$dimid] = grade_floatval($sumdimgrade / $sumweights); + } + return $average; + } + + /** + * Given a set of a submission's assessments, returns standard deviations of all their dimensions + * + * The passed structure must be array of assessments objects with at least ->weight + * and ->dimgrades properties. This implementation uses weighted incremental algorithm as + * suggested in "D. H. D. West (1979). Communications of the ACM, 22, 9, 532-535: + * Updating Mean and Variance Estimates: An Improved Method" + * {@link http://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Weighted_incremental_algorithm} + * + * @param array $assessments as prepared by {@link self::prepare_data_from_recordset()} + * @return null|array indexed by dimension id + */ + protected function weighted_variance(array $assessments) { + $first = reset($assessments); + if (empty($first)) { + return null; + } + $dimids = array_keys($first->dimgrades); + $asids = array_keys($assessments); + $vars = array(); // to be returned + foreach ($dimids as $dimid) { + $n = 0; + $s = 0; + $sumweight = 0; + foreach ($asids as $asid) { + $x = $assessments[$asid]->dimgrades[$dimid]; // value (data point) + $weight = $assessments[$asid]->weight; // the values's weight + if ($weight == 0) { + continue; + } + if ($n == 0) { + $n = 1; + $mean = $x; + $s = 0; + $sumweight = $weight; + } else { + $n++; + $temp = $weight + $sumweight; + $q = $x - $mean; + $r = $q * $weight / $temp; + $s = $s + $sumweight * $q * $r; + $mean = $mean + $r; + $sumweight = $temp; + } + } + if ($sumweight > 0 and $n > 1) { + // for the sample: $vars[$dimid] = ($s * $n) / (($n - 1) * $sumweight); + // for the population: + $vars[$dimid] = $s / $sumweight; + } else { + $vars[$dimid] = null; + } + } + return $vars; + } + + /** + * Measures the distance of the assessment from a referential one + * + * The passed data structures must contain ->dimgrades property. The referential + * assessment is supposed to be close to the average assessment. All dimension grades are supposed to be + * normalized to the interval 0 - 100. + * + * @param stdClass $assessment the assessment being measured + * @param stdClass $referential assessment + * @param array $diminfo of stdClass(->weight ->min ->max ->variance) indexed by dimension id + * @return float|null rounded to 5 valid decimals + */ + protected function assessments_distance(stdClass $assessment, stdClass $referential, array $diminfo) { + $distance = 0; + $n = 0; + foreach (array_keys($assessment->dimgrades) as $dimid) { + $agrade = $assessment->dimgrades[$dimid]; + $rgrade = $referential->dimgrades[$dimid]; + $var = $diminfo[$dimid]->variance; + $weight = $diminfo[$dimid]->weight; + + // variations very close to zero are too sensitive to a small change of data values + if ($var > 0.01 and $agrade != $rgrade) { + $absdelta = abs($agrade - $rgrade); + // todo the following constant is the param. For 1 it is very strict, for 5 it is quite lax + $reldelta = pow($agrade - $rgrade, 2) / (5 * $var); + $distance += $absdelta * $reldelta * $weight; + $n += $weight; + } + } + if ($n > 0) { + // average distance across all dimensions + return grade_floatval($distance / $n); + } else { + return null; + } + } } diff --git a/mod/workshop/eval/best/simpletest/testlib.php b/mod/workshop/eval/best/simpletest/testlib.php new file mode 100644 index 0000000000..82109a02ee --- /dev/null +++ b/mod/workshop/eval/best/simpletest/testlib.php @@ -0,0 +1,194 @@ +. + +/** + * Unit tests for grading evaluation method "best" + * + * @package mod-workshop + * @copyright 2009 David Mudrak + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +defined('MOODLE_INTERNAL') || die(); + +// Include the code to test +require_once($CFG->dirroot . '/mod/workshop/locallib.php'); +require_once($CFG->dirroot . '/mod/workshop/eval/best/lib.php'); +require_once($CFG->libdir . '/gradelib.php'); + +global $DB; +Mock::generate(get_class($DB), 'mockDB'); + +/** + * Test subclass that makes all the protected methods we want to test public. + */ +class testable_workshop_best_evaluation extends workshop_best_evaluation { + + public function normalize_grades(array $assessments, array $diminfo) { + return parent::normalize_grades($assessments, $diminfo); + } + public function average_assessment(array $assessments) { + return parent::average_assessment($assessments); + } + public function weighted_variance(array $assessments) { + return parent::weighted_variance($assessments); + } + +} + +class workshop_best_evaluation_test extends UnitTestCase { + + /** real database */ + protected $realDB; + + /** workshop instance emulation */ + protected $workshop; + + /** instance of the grading evaluator being tested */ + protected $evaluator; + + /** + * Setup testing environment + */ + public function setUp() { + global $DB; + $this->realDB = $DB; + $DB = new mockDB(); + + $cm = new stdClass(); + $course = new stdClass(); + $context = new stdClass(); + $workshop = (object)array('id' => 42, 'evaluation' => 'best'); + $this->workshop = new workshop($workshop, $cm, $course, $context); + $this->evaluator = new testable_workshop_best_evaluation($this->workshop); + } + + public function tearDown() { + global $DB; + $DB = $this->realDB; + + $this->workshop = null; + $this->evaluator = null; + } + + public function test_normalize_grades() { + // fixture set-up + $assessments = array(); + $assessments[1] = (object)array( + 'dimgrades' => array(3 => 1.0000, 4 => 13.42300), + ); + $assessments[3] = (object)array( + 'dimgrades' => array(3 => 2.0000, 4 => 19.1000), + ); + $assessments[7] = (object)array( + 'dimgrades' => array(3 => 3.0000, 4 => 0.00000), + ); + $diminfo = array( + 3 => (object)array('min' => 1, 'max' => 3), + 4 => (object)array('min' => 0, 'max' => 20), + ); + // excersise SUT + $norm = $this->evaluator->normalize_grades($assessments, $diminfo); + // validate + $this->assertIsA($norm, 'array'); + // the following grades from a scale + $this->assertEqual($norm[1]->dimgrades[3], 0); + $this->assertEqual($norm[3]->dimgrades[3], 50); + $this->assertEqual($norm[7]->dimgrades[3], 100); + // the following grades from an interval 0 - 20 + $this->assertEqual($norm[1]->dimgrades[4], grade_floatval(13.423 / 20 * 100)); + $this->assertEqual($norm[3]->dimgrades[4], grade_floatval(19.1 / 20 * 100)); + $this->assertEqual($norm[7]->dimgrades[4], 0); + } + + public function test_average_assessment() { + // fixture set-up + $assessments = array(); + $assessments[11] = (object)array( + 'weight' => 1, + 'dimgrades' => array(3 => 10.0, 4 => 13.4, 5 => 95.0), + 'dimweights' => array(3 => 1, 4 => 1, 5 => 1) + ); + $assessments[13] = (object)array( + 'weight' => 3, + 'dimgrades' => array(3 => 11.0, 4 => 10.1, 5 => 92.0), + 'dimweights' => array(3 => 1, 4 => 1, 5 => 1) + ); + $assessments[17] = (object)array( + 'weight' => 1, + 'dimgrades' => array(3 => 11.0, 4 => 8.1, 5 => 88.0), + 'dimweights' => array(3 => 1, 4 => 1, 5 => 1) + ); + // excersise SUT + $average = $this->evaluator->average_assessment($assessments); + // validate + $this->assertIsA($average->dimgrades, 'array'); + $this->assertEqual(grade_floatval($average->dimgrades[3]), grade_floatval((10.0 + 11.0*3 + 11.0)/5)); + $this->assertEqual(grade_floatval($average->dimgrades[4]), grade_floatval((13.4 + 10.1*3 + 8.1)/5)); + $this->assertEqual(grade_floatval($average->dimgrades[5]), grade_floatval((95.0 + 92.0*3 + 88.0)/5)); + } + + public function test_average_assessment_noweight() { + // fixture set-up + $assessments = array(); + $assessments[11] = (object)array( + 'weight' => 0, + 'dimgrades' => array(3 => 10.0, 4 => 13.4, 5 => 95.0), + 'dimweights' => array(3 => 1, 4 => 1, 5 => 1) + ); + $assessments[17] = (object)array( + 'weight' => 0, + 'dimgrades' => array(3 => 11.0, 4 => 8.1, 5 => 88.0), + 'dimweights' => array(3 => 1, 4 => 1, 5 => 1) + ); + // excersise SUT + $average = $this->evaluator->average_assessment($assessments); + // validate + $this->assertNull($average); + } + + public function test_weighted_variance() { + // fixture set-up + $assessments[11] = (object)array( + 'weight' => 1, + 'dimgrades' => array(3 => 11, 4 => 2), + ); + $assessments[13] = (object)array( + 'weight' => 3, + 'dimgrades' => array(3 => 11, 4 => 4), + ); + $assessments[17] = (object)array( + 'weight' => 2, + 'dimgrades' => array(3 => 11, 4 => 5), + ); + $assessments[20] = (object)array( + 'weight' => 1, + 'dimgrades' => array(3 => 11, 4 => 7), + ); + $assessments[25] = (object)array( + 'weight' => 1, + 'dimgrades' => array(3 => 11, 4 => 9), + ); + // excersise SUT + $variance = $this->evaluator->weighted_variance($assessments); + // validate + // dimension [3] have all the grades equal to 11 + $this->assertEqual($variance[3], 0); + // dimension [4] represents data 2, 4, 4, 4, 5, 5, 7, 9 having stdev=2 (stdev is sqrt of variance) + $this->assertEqual($variance[4], 4); + } +} diff --git a/mod/workshop/form/accumulative/lib.php b/mod/workshop/form/accumulative/lib.php index 5426143924..b0b577e26c 100644 --- a/mod/workshop/form/accumulative/lib.php +++ b/mod/workshop/form/accumulative/lib.php @@ -260,27 +260,25 @@ class workshop_accumulative_strategy implements workshop_strategy { return false; } -//////////////////////////////////////////////////////////////////////////////// -// Methods needed by 'best' evaluation plugin // -//////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// + // Methods required by the 'best' evaluation plugin // + //////////////////////////////////////////////////////////////////////////////// /** - * TODO: short description. + * TODO * * @param resource $restrict * @return TODO */ - public function get_assessments_recordset($restrict) { + public function eval_best_get_assessments_recordset($restrict) { global $DB; - $sql = 'SELECT a.id AS assessmentid, a.weight AS assessmentweight, a.reviewerid, a.gradinggrade, - s.id AS submissionid, - g.dimensionid, g.grade, - d.weight AS dimensionweight + $sql = 'SELECT s.id AS submissionid, + a.id AS assessmentid, a.weight AS assessmentweight, a.reviewerid, a.gradinggrade, + g.dimensionid, g.grade FROM {workshop_submissions} s JOIN {workshop_assessments} a ON (a.submissionid = s.id) JOIN {workshop_grades} g ON (g.assessmentid = a.id AND g.strategy = :strategy) - JOIN {workshopform_accumulative} d ON (d.id = g.dimensionid) WHERE s.example=0 AND s.workshopid=:workshopid'; // to be cont. $params = array('workshopid' => $this->workshop->id, 'strategy' => $this->workshop->strategy); @@ -299,9 +297,41 @@ class workshop_accumulative_strategy implements workshop_strategy { return $DB->get_recordset_sql($sql, $params); } -//////////////////////////////////////////////////////////////////////////////// -// Internal methods // -//////////////////////////////////////////////////////////////////////////////// + /** + * TODO: short description. + * + * @return array [dimid] => stdClass (->id ->max ->min ->weight) + */ + public function eval_best_dimensions_info() { + global $DB; + + $sql = 'SELECT d.id, d.grade, d.weight, s.scale + FROM {workshopform_accumulative} d + LEFT JOIN {scale} s ON (d.grade < 0 AND -d.grade = s.id) + WHERE d.workshopid = :workshopid'; + $params = array('workshopid' => $this->workshop->id); + $dimrecords = $DB->get_records_sql($sql, $params); + $diminfo = array(); + foreach ($dimrecords as $dimid => $dimrecord) { + $diminfo[$dimid] = new stdClass(); + $diminfo[$dimid]->id = $dimid; + $diminfo[$dimid]->weight = $dimrecord->weight; + if ($dimrecord->grade < 0) { + // the dimension uses a scale + $diminfo[$dimid]->min = 1; + $diminfo[$dimid]->max = count(explode(',', $dimrecord->scale)); + } else { + // the dimension uses points + $diminfo[$dimid]->min = 0; + $diminfo[$dimid]->max = grade_floatval($dimrecord->grade); + } + } + return $diminfo; + } + + //////////////////////////////////////////////////////////////////////////////// + // Internal methods // + //////////////////////////////////////////////////////////////////////////////// /** * Loads the fields of the assessment form currently used in this workshop diff --git a/mod/workshop/locallib.php b/mod/workshop/locallib.php index 5ad2f73f19..48511a7baa 100644 --- a/mod/workshop/locallib.php +++ b/mod/workshop/locallib.php @@ -1200,6 +1200,19 @@ class workshop { // todo } + /** + * Calculates the workshop total grades for the given participant(s) + * + * @param null|int|array $restrict If null, update all reviewers, otherwise update just grades for the given reviewer(s) + * @return void + */ + public function aggregate_total_grades($restrict=null) { + global $DB; + + // todo + } + + //////////////////////////////////////////////////////////////////////////////// // Internal methods (implementation details) // //////////////////////////////////////////////////////////////////////////////// -- 2.39.5