Merge branch 'wip-MDL-35678-master' of git://github.com/abgreeve/moodle

grade/import/csv/index.php

@@ -15,15 +15,18 @@
 // You should have received a copy of the GNU General Public License
 // along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
 
-require_once '../../../config.php';
+require_once("../../../config.php");
-require_once $CFG->libdir.'/gradelib.php';
+require_once($CFG->libdir.'/gradelib.php');
-require_once $CFG->dirroot.'/grade/lib.php';
+require_once($CFG->dirroot.'/grade/lib.php');
-require_once '../grade_import_form.php';
+require_once($CFG->dirroot. '/grade/import/grade_import_form.php');
-require_once '../lib.php';
+require_once($CFG->dirroot.'/grade/import/lib.php');
+require_once($CFG->libdir . '/csvlib.class.php');
 
 $id            = required_param('id', PARAM_INT); // course id
 $separator     = optional_param('separator', '', PARAM_ALPHA);
 $verbosescales = optional_param('verbosescales', 1, PARAM_BOOL);
+$iid           = optional_param('iid', null, PARAM_INT);
+$importcode    = optional_param('importcode', '', PARAM_FILE);
 
 $url = new moodle_url('/grade/import/csv/index.php', array('id'=>$id));
 if ($separator !== '') {
@@ -34,8 +37,6 @@ if ($verbosescales !== 1) {
 }
 $PAGE->set_url($url);
 
-define('GRADE_CSV_LINE_LENGTH', 4096);
-
 if (!$course = $DB->get_record('course', array('id'=>$id))) {
     print_error('nocourseid');
 }
@@ -48,33 +49,14 @@ require_capability('gradeimport/csv:view', $context);
 $separatemode = (groups_get_course_groupmode($COURSE) == SEPARATEGROUPS and !has_capability('moodle/site:accessallgroups', $context));
 $currentgroup = groups_get_course_group($course);
 
-// sort out delimiter
-if (isset($CFG->CSV_DELIMITER)) {
-    $csv_delimiter = $CFG->CSV_DELIMITER;
-
-    if (isset($CFG->CSV_ENCODE)) {
-        $csv_encode = '/\&\#' . $CFG->CSV_ENCODE . '/';
-    }
-} else if ($separator == 'tab') {
-    $csv_delimiter = "\t";
-    $csv_encode = "";
-} else {
-    $csv_delimiter = ",";
-    $csv_encode = '/\&\#44/';
-}
-
 print_grade_page_head($course->id, 'import', 'csv', get_string('importcsv', 'grades'));
 
-// set up import form
+// Set up the grade import mapping form.
-$mform = new grade_import_form(null, array('includeseparator'=>!isset($CFG->CSV_DELIMITER), 'verbosescales'=>true));
-
-// set up grade import mapping form
-$header = '';
 $gradeitems = array();
 if ($id) {
     if ($grade_items = grade_item::fetch_all(array('courseid'=>$id))) {
         foreach ($grade_items as $grade_item) {
-            // skip course type and category type
+            // Skip course type and category type.
             if ($grade_item->itemtype == 'course' || $grade_item->itemtype == 'category') {
                 continue;
             }
@@ -90,17 +72,13 @@ if ($id) {
     }
 }
 
-if ($importcode = optional_param('importcode', '', PARAM_FILE)) {
+// Set up the import form.
-    $filename = $CFG->tempdir.'/gradeimport/cvs/'.$USER->id.'/'.$importcode;
+$mform = new grade_import_form(null, array('includeseparator'=>true, 'verbosescales'=>true));
-    $fp = fopen($filename, "r");
-    $headers = fgets($fp, GRADE_CSV_LINE_LENGTH);
-    $header = explode($csv_delimiter, $headers);
-    fclose($fp);
-}
 
-$mform2 = new grade_import_mapping_form(null, array('gradeitems'=>$gradeitems, 'header'=>$header));
+// If the csv file hasn't been imported yet then look for a form submission or
-
+// show the initial submission form.
-// if import form is submitted
+if (!$iid) {
+    // If the import form has been submitted.
     if ($formdata = $mform->get_data()) {
 
         // Large files are likely to take their time and memory. Let PHP know
@@ -109,77 +87,69 @@ if ($formdata = $mform->get_data()) {
         @set_time_limit(0);
         raise_memory_limit(MEMORY_EXTRA);
 
-    // use current (non-conflicting) time stamp
+        // Use current (non-conflicting) time stamp.
         $importcode = get_new_importcode();
-    $filename = make_temp_directory('gradeimport/cvs/'.$USER->id);
-    $filename = $filename.'/'.$importcode;
 
         $text = $mform->get_file_content('userfile');
-    // trim utf-8 bom
+        $iid = csv_import_reader::get_new_iid('grade');
-    /// normalize line endings and do the encoding conversion
+        $csvimport = new csv_import_reader($iid, 'grade');
-    $text = textlib::convert($text, $formdata->encoding);
-    $text = textlib::trim_utf8_bom($text);
-    // Fix mac/dos newlines
-    $text = preg_replace('!\r\n?!',"\n",$text);
-    $fp = fopen($filename, "w");
-    fwrite($fp,$text);
-    fclose($fp);
 
-    if (!$fp = fopen($filename, "r")) {
+        $csvimport->load_csv_content($text, $formdata->encoding, $separator);
-        print_error('cannotopenfile');
-    }
 
         // --- get header (field names) ---
-    $header = explode($csv_delimiter, fgets($fp, GRADE_CSV_LINE_LENGTH));
+        $header = $csvimport->get_columns();
 
-    // print some preview
+        // Print a preview of the data.
-    $numlines = 0; // 0 preview lines displayed
+        $numlines = 0; // 0 lines previewed so far.
 
         echo $OUTPUT->heading(get_string('importpreview', 'grades'));
-    echo '<table>';
-    echo '<tr>';
-    foreach ($header as $h) {
-        $h = clean_param($h, PARAM_RAW);
-        echo '<th>'.$h.'</th>';
-    }
-    echo '</tr>';
-    while (!feof ($fp) && $numlines <= $formdata->previewrows) {
-        $lines = explode($csv_delimiter, fgets($fp, GRADE_CSV_LINE_LENGTH));
-        echo '<tr>';
-        foreach ($lines as $line) {
-            echo '<td>'.$line.'</td>';
-        }
-        $numlines ++;
-        echo '</tr>';
-    }
-    echo '</table>';
-
-    // display the mapping form with header info processed
-    $mform2 = new grade_import_mapping_form(null, array('gradeitems'=>$gradeitems, 'header'=>$header));
-    $mform2->set_data(array('importcode'=>$importcode, 'id'=>$id, 'verbosescales'=>$verbosescales, 'separator'=>$separator));
-    $mform2->display();
-
-//} else if (($formdata = data_submitted()) && !empty($formdata->map)) {
-
-// else if grade import mapping form is submitted
-} else if ($formdata = $mform2->get_data()) {
-
-    $importcode = clean_param($formdata->importcode, PARAM_FILE);
-    $filename = $CFG->tempdir.'/gradeimport/cvs/'.$USER->id.'/'.$importcode;
-
-    if (!file_exists($filename)) {
-        print_error('cannotuploadfile');
-    }
-
-    if ($fp = fopen($filename, "r")) {
-        // --- get header (field names) ---
-        $header = explode($csv_delimiter, clean_param(fgets($fp,GRADE_CSV_LINE_LENGTH), PARAM_RAW));
 
         foreach ($header as $i => $h) {
-            $h = trim($h); $header[$i] = $h; // remove whitespace
+            $h = trim($h); // Remove whitespace.
+            $h = clean_param($h, PARAM_RAW); // Clean the header.
+            $header[$i] = $h;
         }
+
+        $table = new html_table();
+        $table->head = $header;
+        $csvimport->init();
+        $previewdata = array();
+        while ($numlines <= $formdata->previewrows) {
+            $lines = $csvimport->next();
+            if ($lines) {
+                $previewdata[] = $lines;
+            }
+            $numlines ++;
+        }
+        $table->data = $previewdata;
+        echo html_writer::table($table);
     } else {
-        print_error('cannotopenfile');
+        // Display the standard upload file form.
+        groups_print_course_menu($course, 'index.php?id='.$id);
+        echo html_writer::start_tag('div', array('class' => 'clearer'));
+        echo html_writer::end_tag('div');
+
+        $mform->display();
+        echo $OUTPUT->footer();
+        die();
+    }
+}
+
+// Data has already been submitted so we can use the $iid to retrieve it.
+$csvimport = new csv_import_reader($iid, 'grade');
+$header = $csvimport->get_columns();
+
+// we create a form to handle mapping data from the file to the database.
+$mform2 = new grade_import_mapping_form(null, array('gradeitems'=>$gradeitems, 'header'=>$header));
+$mform2->set_data(array('iid' => $iid, 'id' => $id, 'importcode'=>$importcode, 'verbosescales' => $verbosescales));
+
+// Here, if we have data, we process the fields and enter the information into the database.
+if ($formdata = $mform2->get_data()) {
+
+    foreach ($header as $i => $h) {
+        $h = trim($h); // Remove whitespace.
+        $h = clean_param($h, PARAM_RAW); // Clean the header.
+        $header[$i] = $h;
     }
 
     $map = array();
@@ -204,8 +174,6 @@ if ($formdata = $mform->get_data()) {
                 $maperrors[$j] = true;
             } else {
                 // collision
-                fclose($fp);
-                unlink($filename); // needs to be uploaded again, sorry
                 print_error('cannotmapfield', '', '', $j);
             }
         }
@@ -217,19 +185,12 @@ if ($formdata = $mform->get_data()) {
     @set_time_limit(0);
     raise_memory_limit(MEMORY_EXTRA);
 
-    // we only operate if file is readable
+    $csvimport->init();
-    if ($fp = fopen($filename, "r")) {
-
-        // read the first line makes sure this doesn't get read again
-        $header = explode($csv_delimiter, fgets($fp, GRADE_CSV_LINE_LENGTH));
 
     $newgradeitems = array(); // temporary array to keep track of what new headers are processed
     $status = true;
 
-        while (!feof ($fp)) {
+    while ($line = $csvimport->next()) {
-            // add something
-            $line = explode($csv_delimiter, fgets($fp, GRADE_CSV_LINE_LENGTH));
-
         if(count($line) <= 1){
             // there is no data on this line, move on
             continue;
@@ -241,12 +202,9 @@ if ($formdata = $mform->get_data()) {
         $newfeedbacks = array();
         // each line is a student record
         foreach ($line as $key => $value) {
-                //decode encoded commas
+
             $value = clean_param($value, PARAM_RAW);
             $value = trim($value);
-                if (!empty($csv_encode)) {
-                    $value = preg_replace($csv_encode, $csv_delimiter, $value);
-                }
 
             /*
              * the options are
@@ -478,20 +436,8 @@ if ($formdata = $mform->get_data()) {
     if ($status) {
         grade_import_commit($course->id, $importcode);
     }
-        // temporary file can go now
-        fclose($fp);
-        unlink($filename);
 } else {
-        print_error('cannotreadfile');
+    // If data hasn't been submitted then display the data mapping form.
-    }
+    $mform2->display();
-
-} else {
-    groups_print_course_menu($course, 'index.php?id='.$id);
-    echo '<div class="clearer"></div>';
-
-    // display the standard upload file form
-    $mform->display();
-}
-
     echo $OUTPUT->footer();
-
+}

grade/import/grade_import_form.php

@@ -118,11 +118,11 @@ class grade_import_mapping_form extends moodleform {
         $mform->setType('map', PARAM_INT);
         $mform->addElement('hidden', 'id');
         $mform->setType('id', PARAM_INT);
+        $mform->addElement('hidden', 'iid');
+        $mform->setType('iid', PARAM_INT);
         $mform->addElement('hidden', 'importcode');
         $mform->setType('importcode', PARAM_FILE);
         $mform->addElement('hidden', 'verbosescales', 1);
-        $mform->setType('separator', PARAM_ALPHA);
-        $mform->addElement('hidden', 'separator', 'comma');
         $mform->setType('verbosescales', PARAM_INT);
         $mform->addElement('hidden', 'groupid', groups_get_course_group($COURSE));
         $mform->setType('groupid', PARAM_INT);