Revisione procedura di importazione

modules/import/actions.php

@@ -4,8 +4,11 @@ include_once __DIR__.'/../../core.php';
 
 switch (post('op')) {
     case 'import':
-        $first_row = !post('first_row');
+        $include_first_row = post('include_first_row');
-        $selected = post('fields');
+        $selected_fields = post('fields');
+        $page = post('page');
+
+        $limit = 500;
 
         // Pulizia dei campi inutilizzati
         foreach ($selected as $key => $value) {
@@ -16,46 +19,71 @@ switch (post('op')) {
 
         $fields = Import::getFields($id_record);
 
-        $csv = Import::getFile($id_record, $record['id'], [
+        $csv = Import::getCSV($id_record, $record['id']);
-            'headers' => $first_row,
+
-        ]);
+        $offset = isset($page) ? $page * $limit : 0;
+
+        // Ignora la prima riga se composta da header
+        if ($offset == 0 && empty($include_first_row)) {
+            ++$offset;
+        }
+
+        $csv = $csv->setOffset($offset)
+                    ->setLimit($limit);
+
+        // Chiavi per la lettura CSV
+        $keys = [];
+        foreach ($selected_fields as $id => $field_id) {
+            if (is_numeric($field_id)) {
+                $value = $fields[$field_id]['field'];
+            } else {
+                $value = -($id + 1);
+            }
+
+            $keys[] = $value;
+        }
+
+        // Query dei campi selezionati
+        $queries = [];
+        foreach ($fields as $key => $field) {
+            if (!empty($field['query'])) {
+                $queries[$field['field']] = $field['query'];
+            }
+        }
+
+        // Lettura dei record
+        $rows = $csv->fetchAssoc($keys, function ($row) use ($queries, $dbo) {
+            foreach ($row as $key => $value) {
+                if (is_int($key)) {
+                    unset($row[$key]);
+                } elseif (isset($queries[$key])) {
+                    $query = str_replace('|value|', prepare($value), $queries[$key]);
+
+                    $value = $dbo->fetchOne($query)['result'];
+
+                    $row[$key] = $value;
+                }
+            }
+
+            return $row;
+        });
 
         // Gestione automatica dei valori convertiti
-        $csv = Filter::parse($csv);
+        $rows = iterator_to_array($rows);
-
+        $data = Filter::parse($rows);
-        // Interpretazione dei dati
-        $data = [];
-        foreach ($csv as $row) {
-            $data_row = [];
-
-            foreach ($row as $key => $value) {
-                $field = $fields[$selected[$key]];
-
-                if (isset($selected[$key])) {
-                    $name = $field['field'];
-
-                    $query = $field['query'];
-                    if (!empty($query)) {
-                        $query = str_replace('|value|', prepare($value), $query);
-
-                        $value = $dbo->fetchArray($query)[0]['result'];
-                    }
-
-                    $data_row[$name] = $value;
-                }
-            }
-
-            $data[] = $data_row;
-        }
 
         $primary_key = post('primary_key');
 
         // Richiamo delle operazioni specifiche
         include $imports[$id_record]['import'];
 
-        flash()->info(tr('Importazione completata: _COUNT_  righe processate', [
+        $count = count($rows);
-            '_COUNT_' => count($csv),
+        $more = $count == $limit;
-        ]));
+
+        echo json_encode([
+            'more' => $more,
+            'count' => $count,
+        ]);
 
         break;
 }

modules/import/edit.php

@@ -35,16 +35,15 @@ if (empty($id_record)) {
 
     <div class="row">
         <div class="col-md-8">
-            {[ "type": "checkbox", "label": "'.tr('Importa prima riga').'", "name": "first_row", "extra":"", "value": "1"  ]}
+            {[ "type": "checkbox", "label": "'.tr('Importa prima riga').'", "name": "include_first_row", "extra":"", "value": "1"  ]}
         </div>
         <div class="col-md-4">
             {[ "type": "select", "label": "'.tr('Chiave primaria').'", "name": "primary_key", "values": '.json_encode($select2).', "value": "'.$primary_key.'" ]}
         </div>
     </div>';
 
-    $rows = Import::getFile($id_record, $record['id'], [
+    $csv = Import::getCSV($id_record, $record['id']);
-        'limit' => 10,
+    $rows = $csv->setLimit(10)->fetchAll();
-    ]);
 
     $count = count($rows[0]);
 
@@ -67,7 +66,7 @@ if (empty($id_record)) {
         $selected = null;
         foreach ($fields as $key => $value) {
             if (in_array(str_to_lower($rows[0][$column]), $value['names'])) {
-                $first_row = 1;
+                $exclude_first_row = 1;
                 $selected = $key;
                 break;
             }
@@ -111,13 +110,59 @@ if (empty($id_record)) {
 <script>
 $(document).ready(function(){';
 
-    if ($first_row) {
+    if ($exclude_first_row) {
         echo '
-    $("#first_row").prop("checked", false).trigger("change");';
+    $("#include_first_row").prop("checked", false).trigger("change");';
     }
 
     echo '
     $("#save").html("<i class=\"fa fa-flag-checkered\"></i> '.tr('Avvia importazione').'");
+    
+    $("#save").unbind("click");
+    $("#save").on("click", function() {
+        importPage(0);
     });
+});
+
+var count = 0;
+function importPage(page){
+    $("#main_loading").show();
+
+    data = {
+        id_module: "'.$id_module.'",
+        id_plugin: "'.$id_plugin.'",
+        id_record: "'.$id_record.'",
+        page: page,
+    };
+    
+    $("#edit-form").ajaxSubmit({
+        url: globals.rootdir + "/actions.php",
+        data: data,
+        type: "post",
+        success: function(data) {
+            data = JSON.parse(data);
+            
+            count += data.count;
+            
+            if(data.more) {
+                importPage(page + 1);
+            } else {
+                $("#main_loading").fadeOut();
+
+                swal({
+                    title: "'.tr('Importazione completata: _COUNT_  righe processate', [
+                        '_COUNT_' => '" + count + "',
+                    ]).'",
+                    type: "success",
+                });
+            }
+        },
+        error: function(data) {
+            $("#main_loading").fadeOut();
+
+            alert("'.tr('Errore').': " + data);
+        }
+    });
+};
 </script>';
 }

plugins/importFE/edit.php

@@ -31,7 +31,7 @@ echo '
                             launch_modal("'.tr('Righe fattura').'", globals.rootdir + "/actions.php?id_module=" + globals.id_module + "&id_plugin=" + '.$id_plugin.' + "&op=list&filename=" + data.filename);
                         } else {
                             swal({
-                                title: "'.tr('Fattura già importata.').'",
+                                title: "'.tr('Fattura già importata').'.",
                                 type: "info",
                             });
 							

src/Import.php

@@ -104,11 +104,10 @@ class Import
      *
      * @param string|int $module
      * @param int        $file_id
-     * @param array      $options
      *
      * @return array
      */
-    public static function getFile($module, $file_id, $options = [])
+    public static function getCSV($module, $file_id)
     {
         $import = self::get($module);
 
@@ -119,7 +118,7 @@ class Import
             return [];
         }
 
-        $file = DOCROOT.'/files/'.Modules::get('Import')['directory'].'/'.$import['files'][$find]['filename'];
+        $file = Modules::get('Import')->upload_directory.'/'.$import['files'][$find]['filename'];
 
         // Impostazione automatica per i caratteri di fine riga
         if (!ini_get('auto_detect_line_endings')) {
@@ -130,21 +129,6 @@ class Import
         $csv = League\Csv\Reader::createFromPath($file, 'r');
         $csv->setDelimiter(';');
 
-        // Ignora la prima riga
+        return $csv;
-        $offset = 0;
-        if (!empty($options['headers'])) {
-            ++$offset;
-        }
-        $rows = $csv->setOffset($offset);
-
-        // Limite di righe
-        if (!empty($options['limit'])) {
-            $rows = $rows->setLimit($options['limit']);
-        }
-
-        // Lettura
-        $rows = $rows->fetchAll();
-
-        return $rows;
     }
 }