if (!confirm_sesskey()) {
throw new moodle_exception('confirmsesskeybad');
}
- $workshop->aggregate_submission_grades();
- //$evaluator->update_grading_grades();
- //$workshop->aggregate_grading_grades();
- //$workshop->aggregate_total_grades();
+ $workshop->aggregate_submission_grades(); // updates 'grade' in {workshop_submissions}
+ $evaluator->update_grading_grades(); // updates 'gradinggrade' in {workshop_assessments}
+ $workshop->aggregate_grading_grades(); // updates 'gradinggrade' in {workshop_aggregations}
+ $workshop->aggregate_total_grades(); // updates 'totalgrade' in {workshop_aggregations}
redirect($workshop->view_url());
}
}
/**
- * TODO
+ * Calculates the grades for assessment and updates 'gradinggrade' fields in 'workshop_assessments' table
+ *
+ * This function relies on the grading strategy subplugin providing get_assessments_recordset() method.
+ * {@see self::process_assessments()} for the required structure of the recordset.
*
* @param null|int|array $restrict If null, update all reviewers, otherwise update just grades for the given reviewers(s)
*
support this method of grading evaluation.');
}
+ // get the information about the assessment dimensions
+ $diminfo = $grader->eval_best_dimensions_info();
+
// fetch a recordset with all assessments to process
- $rs = $grader->get_assessments_recordset($restrict);
+ $rs = $grader->eval_best_get_assessments_recordset($restrict);
$batch = array(); // will contain a set of all assessments of a single submission
$previous = null; // a previous record in the recordset
foreach ($rs as $current) {
$batch[] = $current;
} else {
// process all the assessments of a sigle submission
- $this->process_assessments($batch);
+ $this->process_assessments($batch, $diminfo);
// start with a new batch to be processed
$batch = array($current);
$previous = $current;
}
}
// do not forget to process the last batch!
- $this->process_assessments($batch);
+ $this->process_assessments($batch, $diminfo);
$rs->close();
}
-////////////////////////////////////////////////////////////////////////////////
-// Internal methods //
-////////////////////////////////////////////////////////////////////////////////
+ ////////////////////////////////////////////////////////////////////////////////
+ // Internal methods //
+ ////////////////////////////////////////////////////////////////////////////////
/**
* Given a list of all assessments of a single submission, updates the grading grades in database
*
- * @param array $assessments of stdClass Object(->assessmentid ->assessmentweight ->reviewerid ->submissionid
- * ->dimensionid ->grade ->dimensionweight)
+ * @param array $assessments of stdClass (->assessmentid ->assessmentweight ->reviewerid ->gradinggrade ->submissionid ->dimensionid ->grade)
+ * @param array $diminfo of stdClass (->id ->weight ->max ->min)
* @return void
*/
- protected function process_assessments(array $assessments) {
+ protected function process_assessments(array $assessments, array $diminfo) {
global $DB;
- $grades = $this->evaluate_assessments($assessments);
+ // reindex the passed flat structure to be indexed by assessmentid
+ $assessments = $this->prepare_data_from_recordset($assessments);
+
+ // normalize the dimension grades to the interval 0 - 100
+ $assessments = $this->normalize_grades($assessments, $diminfo);
+
+ // get a hypothetical average assessment
+ $average = $this->average_assessment($assessments);
+
+ // calculate variance of dimension grades
+ $variances = $this->weighted_variance($assessments);
+ foreach ($variances as $dimid => $variance) {
+ $diminfo[$dimid]->variance = $variance;
+ }
+
+ // for every assessment, calculate its distance from the average one
+ $distances = array();
+ foreach ($assessments as $asid => $assessment) {
+ $distances[$asid] = $this->assessments_distance($assessment, $average, $diminfo);
+ }
+
+ // identify the best assessments - it est those with the shortest distance from the best assessment
+ $bestids = array_keys($distances, min($distances));
+
+ // for every assessment, calculate its distance from the nearest best assessment
+ $distances = array();
+ foreach ($bestids as $bestid) {
+ $best = $assessments[$bestid];
+ foreach ($assessments as $asid => $assessment) {
+ $d = $this->assessments_distance($assessment, $best, $diminfo);
+ if (!isset($distances[$asid]) or $d < $distances[$asid]) {
+ $distances[$asid] = $d;
+ }
+ }
+ }
+
+ // calculate the grading grade
+ foreach ($distances as $asid => $distance) {
+ $gradinggrade = (100 - $distance);
+ /**
+ if ($gradinggrade < 0) {
+ $gradinggrade = 0;
+ }
+ if ($gradinggrade > 100) {
+ $gradinggrade = 100;
+ }
+ */
+ $grades[$asid] = grade_floatval($gradinggrade);
+ }
+
+ // if the new grading grade differs from the one stored in database, update it
+ // we do not use set_field() here because we want to pass $bulk param
foreach ($grades as $assessmentid => $grade) {
- $record = new stdClass();
- $record->id = $assessmentid;
- $record->gradinggrade = grade_floatval($grade);
- $DB->update_record('workshop_assessments', $record, true);
+ if (grade_floats_different($grade, $assessments[$assessmentid]->gradinggrade)) {
+ // the value has changed
+ $record = new stdClass();
+ $record->id = $assessmentid;
+ $record->gradinggrade = grade_floatval($grade);
+ $DB->update_record('workshop_assessments', $record, true); // bulk operations expected
+ }
}
+
+ // done. easy, heh? ;-)
}
/**
- * Given a list of all assessments of a single submission, calculates the grading grades for them
+ * Prepares a structure of assessments and given grades
*
- * @param array $assessments same structure as for {@link self::process_assessments()}
- * @return array [(int)assessmentid => (float)gradinggrade] to be saved into {workshop_assessments}
+ * @param array $assessments batch of recordset items as returned by the grading strategy
+ * @return array
*/
- protected function evaluate_assessments(array $assessments) {
- $gradinggrades = array();
- foreach ($assessments as $assessment) {
- $gradinggrades[$assessment->assessmentid] = grade_floatval(rand(0, 100)); // todo
+ protected function prepare_data_from_recordset($assessments) {
+ $data = array(); // to be returned
+ foreach ($assessments as $a) {
+ $id = $a->assessmentid; // just an abbrevation
+ if (!isset($data[$id])) {
+ $data[$id] = new stdClass();
+ $data[$id]->assessmentid = $a->assessmentid;
+ $data[$id]->weight = $a->assessmentweight;
+ $data[$id]->reviewerid = $a->reviewerid;
+ $data[$id]->gradinggrade = $a->gradinggrade;
+ $data[$id]->submissionid = $a->submissionid;
+ $data[$id]->dimgrades = array();
+ }
+ $data[$id]->dimgrades[$a->dimensionid] = $a->grade;
}
- return $gradinggrades;
+ return $data;
}
+ /**
+ * Normalizes the dimension grades to the interval 0.00000 - 100.00000
+ *
+ * Note: this heavily relies on PHP5 way of handling references in array of stdClasses. Hopefuly
+ * it will not change again soon.
+ *
+ * @param array $assessments of stdClass as returned by {@see self::prepare_data_from_recordset()}
+ * @param array $diminfo of stdClass
+ * @return array of stdClass with the same structure as $assessments
+ */
+ protected function normalize_grades(array $assessments, array $diminfo) {
+ foreach ($assessments as $asid => $assessment) {
+ foreach ($assessment->dimgrades as $dimid => $dimgrade) {
+ $dimmin = $diminfo[$dimid]->min;
+ $dimmax = $diminfo[$dimid]->max;
+ $assessment->dimgrades[$dimid] = grade_floatval(($dimgrade - $dimmin) / ($dimmax - $dimmin) * 100);
+ }
+ }
+ return $assessments;
+ }
+
+ /**
+ * Given a set of a submission's assessments, returns a hypothetical average assessment
+ *
+ * The passed structure must be array of assessments objects with ->weight and ->dimgrades properties.
+ *
+ * @param array $assessments as prepared by {@link self::prepare_data_from_recordset()}
+ * @return null|stdClass
+ */
+ protected function average_assessment(array $assessments) {
+ $sumdimgrades = array();
+ foreach ($assessments as $a) {
+ foreach ($a->dimgrades as $dimid => $dimgrade) {
+ if (!isset($sumdimgrades[$dimid])) {
+ $sumdimgrades[$dimid] = 0;
+ }
+ $sumdimgrades[$dimid] += $dimgrade * $a->weight;
+ }
+ }
+
+ $sumweights = 0;
+ foreach ($assessments as $a) {
+ $sumweights += $a->weight;
+ }
+ if ($sumweights == 0) {
+ // unable to calculate average assessment
+ return null;
+ }
+
+ $average = new stdClass();
+ $average->dimgrades = array();
+ foreach ($sumdimgrades as $dimid => $sumdimgrade) {
+ $average->dimgrades[$dimid] = grade_floatval($sumdimgrade / $sumweights);
+ }
+ return $average;
+ }
+
+ /**
+ * Given a set of a submission's assessments, returns standard deviations of all their dimensions
+ *
+ * The passed structure must be array of assessments objects with at least ->weight
+ * and ->dimgrades properties. This implementation uses weighted incremental algorithm as
+ * suggested in "D. H. D. West (1979). Communications of the ACM, 22, 9, 532-535:
+ * Updating Mean and Variance Estimates: An Improved Method"
+ * {@link http://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Weighted_incremental_algorithm}
+ *
+ * @param array $assessments as prepared by {@link self::prepare_data_from_recordset()}
+ * @return null|array indexed by dimension id
+ */
+ protected function weighted_variance(array $assessments) {
+ $first = reset($assessments);
+ if (empty($first)) {
+ return null;
+ }
+ $dimids = array_keys($first->dimgrades);
+ $asids = array_keys($assessments);
+ $vars = array(); // to be returned
+ foreach ($dimids as $dimid) {
+ $n = 0;
+ $s = 0;
+ $sumweight = 0;
+ foreach ($asids as $asid) {
+ $x = $assessments[$asid]->dimgrades[$dimid]; // value (data point)
+ $weight = $assessments[$asid]->weight; // the values's weight
+ if ($weight == 0) {
+ continue;
+ }
+ if ($n == 0) {
+ $n = 1;
+ $mean = $x;
+ $s = 0;
+ $sumweight = $weight;
+ } else {
+ $n++;
+ $temp = $weight + $sumweight;
+ $q = $x - $mean;
+ $r = $q * $weight / $temp;
+ $s = $s + $sumweight * $q * $r;
+ $mean = $mean + $r;
+ $sumweight = $temp;
+ }
+ }
+ if ($sumweight > 0 and $n > 1) {
+ // for the sample: $vars[$dimid] = ($s * $n) / (($n - 1) * $sumweight);
+ // for the population:
+ $vars[$dimid] = $s / $sumweight;
+ } else {
+ $vars[$dimid] = null;
+ }
+ }
+ return $vars;
+ }
+
+ /**
+ * Measures the distance of the assessment from a referential one
+ *
+ * The passed data structures must contain ->dimgrades property. The referential
+ * assessment is supposed to be close to the average assessment. All dimension grades are supposed to be
+ * normalized to the interval 0 - 100.
+ *
+ * @param stdClass $assessment the assessment being measured
+ * @param stdClass $referential assessment
+ * @param array $diminfo of stdClass(->weight ->min ->max ->variance) indexed by dimension id
+ * @return float|null rounded to 5 valid decimals
+ */
+ protected function assessments_distance(stdClass $assessment, stdClass $referential, array $diminfo) {
+ $distance = 0;
+ $n = 0;
+ foreach (array_keys($assessment->dimgrades) as $dimid) {
+ $agrade = $assessment->dimgrades[$dimid];
+ $rgrade = $referential->dimgrades[$dimid];
+ $var = $diminfo[$dimid]->variance;
+ $weight = $diminfo[$dimid]->weight;
+
+ // variations very close to zero are too sensitive to a small change of data values
+ if ($var > 0.01 and $agrade != $rgrade) {
+ $absdelta = abs($agrade - $rgrade);
+ // todo the following constant is the param. For 1 it is very strict, for 5 it is quite lax
+ $reldelta = pow($agrade - $rgrade, 2) / (5 * $var);
+ $distance += $absdelta * $reldelta * $weight;
+ $n += $weight;
+ }
+ }
+ if ($n > 0) {
+ // average distance across all dimensions
+ return grade_floatval($distance / $n);
+ } else {
+ return null;
+ }
+ }
}
--- /dev/null
+<?php
+
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle. If not, see <http://www.gnu.org/licenses/>.
+
+/**
+ * Unit tests for grading evaluation method "best"
+ *
+ * @package mod-workshop
+ * @copyright 2009 David Mudrak <david.mudrak@gmail.com>
+ * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+
+defined('MOODLE_INTERNAL') || die();
+
+// Include the code to test
+require_once($CFG->dirroot . '/mod/workshop/locallib.php');
+require_once($CFG->dirroot . '/mod/workshop/eval/best/lib.php');
+require_once($CFG->libdir . '/gradelib.php');
+
+global $DB;
+Mock::generate(get_class($DB), 'mockDB');
+
+/**
+ * Test subclass that makes all the protected methods we want to test public.
+ */
+class testable_workshop_best_evaluation extends workshop_best_evaluation {
+
+ public function normalize_grades(array $assessments, array $diminfo) {
+ return parent::normalize_grades($assessments, $diminfo);
+ }
+ public function average_assessment(array $assessments) {
+ return parent::average_assessment($assessments);
+ }
+ public function weighted_variance(array $assessments) {
+ return parent::weighted_variance($assessments);
+ }
+
+}
+
+class workshop_best_evaluation_test extends UnitTestCase {
+
+ /** real database */
+ protected $realDB;
+
+ /** workshop instance emulation */
+ protected $workshop;
+
+ /** instance of the grading evaluator being tested */
+ protected $evaluator;
+
+ /**
+ * Setup testing environment
+ */
+ public function setUp() {
+ global $DB;
+ $this->realDB = $DB;
+ $DB = new mockDB();
+
+ $cm = new stdClass();
+ $course = new stdClass();
+ $context = new stdClass();
+ $workshop = (object)array('id' => 42, 'evaluation' => 'best');
+ $this->workshop = new workshop($workshop, $cm, $course, $context);
+ $this->evaluator = new testable_workshop_best_evaluation($this->workshop);
+ }
+
+ public function tearDown() {
+ global $DB;
+ $DB = $this->realDB;
+
+ $this->workshop = null;
+ $this->evaluator = null;
+ }
+
+ public function test_normalize_grades() {
+ // fixture set-up
+ $assessments = array();
+ $assessments[1] = (object)array(
+ 'dimgrades' => array(3 => 1.0000, 4 => 13.42300),
+ );
+ $assessments[3] = (object)array(
+ 'dimgrades' => array(3 => 2.0000, 4 => 19.1000),
+ );
+ $assessments[7] = (object)array(
+ 'dimgrades' => array(3 => 3.0000, 4 => 0.00000),
+ );
+ $diminfo = array(
+ 3 => (object)array('min' => 1, 'max' => 3),
+ 4 => (object)array('min' => 0, 'max' => 20),
+ );
+ // excersise SUT
+ $norm = $this->evaluator->normalize_grades($assessments, $diminfo);
+ // validate
+ $this->assertIsA($norm, 'array');
+ // the following grades from a scale
+ $this->assertEqual($norm[1]->dimgrades[3], 0);
+ $this->assertEqual($norm[3]->dimgrades[3], 50);
+ $this->assertEqual($norm[7]->dimgrades[3], 100);
+ // the following grades from an interval 0 - 20
+ $this->assertEqual($norm[1]->dimgrades[4], grade_floatval(13.423 / 20 * 100));
+ $this->assertEqual($norm[3]->dimgrades[4], grade_floatval(19.1 / 20 * 100));
+ $this->assertEqual($norm[7]->dimgrades[4], 0);
+ }
+
+ public function test_average_assessment() {
+ // fixture set-up
+ $assessments = array();
+ $assessments[11] = (object)array(
+ 'weight' => 1,
+ 'dimgrades' => array(3 => 10.0, 4 => 13.4, 5 => 95.0),
+ 'dimweights' => array(3 => 1, 4 => 1, 5 => 1)
+ );
+ $assessments[13] = (object)array(
+ 'weight' => 3,
+ 'dimgrades' => array(3 => 11.0, 4 => 10.1, 5 => 92.0),
+ 'dimweights' => array(3 => 1, 4 => 1, 5 => 1)
+ );
+ $assessments[17] = (object)array(
+ 'weight' => 1,
+ 'dimgrades' => array(3 => 11.0, 4 => 8.1, 5 => 88.0),
+ 'dimweights' => array(3 => 1, 4 => 1, 5 => 1)
+ );
+ // excersise SUT
+ $average = $this->evaluator->average_assessment($assessments);
+ // validate
+ $this->assertIsA($average->dimgrades, 'array');
+ $this->assertEqual(grade_floatval($average->dimgrades[3]), grade_floatval((10.0 + 11.0*3 + 11.0)/5));
+ $this->assertEqual(grade_floatval($average->dimgrades[4]), grade_floatval((13.4 + 10.1*3 + 8.1)/5));
+ $this->assertEqual(grade_floatval($average->dimgrades[5]), grade_floatval((95.0 + 92.0*3 + 88.0)/5));
+ }
+
+ public function test_average_assessment_noweight() {
+ // fixture set-up
+ $assessments = array();
+ $assessments[11] = (object)array(
+ 'weight' => 0,
+ 'dimgrades' => array(3 => 10.0, 4 => 13.4, 5 => 95.0),
+ 'dimweights' => array(3 => 1, 4 => 1, 5 => 1)
+ );
+ $assessments[17] = (object)array(
+ 'weight' => 0,
+ 'dimgrades' => array(3 => 11.0, 4 => 8.1, 5 => 88.0),
+ 'dimweights' => array(3 => 1, 4 => 1, 5 => 1)
+ );
+ // excersise SUT
+ $average = $this->evaluator->average_assessment($assessments);
+ // validate
+ $this->assertNull($average);
+ }
+
+ public function test_weighted_variance() {
+ // fixture set-up
+ $assessments[11] = (object)array(
+ 'weight' => 1,
+ 'dimgrades' => array(3 => 11, 4 => 2),
+ );
+ $assessments[13] = (object)array(
+ 'weight' => 3,
+ 'dimgrades' => array(3 => 11, 4 => 4),
+ );
+ $assessments[17] = (object)array(
+ 'weight' => 2,
+ 'dimgrades' => array(3 => 11, 4 => 5),
+ );
+ $assessments[20] = (object)array(
+ 'weight' => 1,
+ 'dimgrades' => array(3 => 11, 4 => 7),
+ );
+ $assessments[25] = (object)array(
+ 'weight' => 1,
+ 'dimgrades' => array(3 => 11, 4 => 9),
+ );
+ // excersise SUT
+ $variance = $this->evaluator->weighted_variance($assessments);
+ // validate
+ // dimension [3] have all the grades equal to 11
+ $this->assertEqual($variance[3], 0);
+ // dimension [4] represents data 2, 4, 4, 4, 5, 5, 7, 9 having stdev=2 (stdev is sqrt of variance)
+ $this->assertEqual($variance[4], 4);
+ }
+}
return false;
}
-////////////////////////////////////////////////////////////////////////////////
-// Methods needed by 'best' evaluation plugin //
-////////////////////////////////////////////////////////////////////////////////
+ ////////////////////////////////////////////////////////////////////////////////
+ // Methods required by the 'best' evaluation plugin //
+ ////////////////////////////////////////////////////////////////////////////////
/**
- * TODO: short description.
+ * TODO
*
* @param resource $restrict
* @return TODO
*/
- public function get_assessments_recordset($restrict) {
+ public function eval_best_get_assessments_recordset($restrict) {
global $DB;
- $sql = 'SELECT a.id AS assessmentid, a.weight AS assessmentweight, a.reviewerid, a.gradinggrade,
- s.id AS submissionid,
- g.dimensionid, g.grade,
- d.weight AS dimensionweight
+ $sql = 'SELECT s.id AS submissionid,
+ a.id AS assessmentid, a.weight AS assessmentweight, a.reviewerid, a.gradinggrade,
+ g.dimensionid, g.grade
FROM {workshop_submissions} s
JOIN {workshop_assessments} a ON (a.submissionid = s.id)
JOIN {workshop_grades} g ON (g.assessmentid = a.id AND g.strategy = :strategy)
- JOIN {workshopform_accumulative} d ON (d.id = g.dimensionid)
WHERE s.example=0 AND s.workshopid=:workshopid'; // to be cont.
$params = array('workshopid' => $this->workshop->id, 'strategy' => $this->workshop->strategy);
return $DB->get_recordset_sql($sql, $params);
}
-////////////////////////////////////////////////////////////////////////////////
-// Internal methods //
-////////////////////////////////////////////////////////////////////////////////
+ /**
+ * TODO: short description.
+ *
+ * @return array [dimid] => stdClass (->id ->max ->min ->weight)
+ */
+ public function eval_best_dimensions_info() {
+ global $DB;
+
+ $sql = 'SELECT d.id, d.grade, d.weight, s.scale
+ FROM {workshopform_accumulative} d
+ LEFT JOIN {scale} s ON (d.grade < 0 AND -d.grade = s.id)
+ WHERE d.workshopid = :workshopid';
+ $params = array('workshopid' => $this->workshop->id);
+ $dimrecords = $DB->get_records_sql($sql, $params);
+ $diminfo = array();
+ foreach ($dimrecords as $dimid => $dimrecord) {
+ $diminfo[$dimid] = new stdClass();
+ $diminfo[$dimid]->id = $dimid;
+ $diminfo[$dimid]->weight = $dimrecord->weight;
+ if ($dimrecord->grade < 0) {
+ // the dimension uses a scale
+ $diminfo[$dimid]->min = 1;
+ $diminfo[$dimid]->max = count(explode(',', $dimrecord->scale));
+ } else {
+ // the dimension uses points
+ $diminfo[$dimid]->min = 0;
+ $diminfo[$dimid]->max = grade_floatval($dimrecord->grade);
+ }
+ }
+ return $diminfo;
+ }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ // Internal methods //
+ ////////////////////////////////////////////////////////////////////////////////
/**
* Loads the fields of the assessment form currently used in this workshop
// todo
}
+ /**
+ * Calculates the workshop total grades for the given participant(s)
+ *
+ * @param null|int|array $restrict If null, update all reviewers, otherwise update just grades for the given reviewer(s)
+ * @return void
+ */
+ public function aggregate_total_grades($restrict=null) {
+ global $DB;
+
+ // todo
+ }
+
+
////////////////////////////////////////////////////////////////////////////////
// Internal methods (implementation details) //
////////////////////////////////////////////////////////////////////////////////
projects / moodle.git / commitdiff