3
\$\begingroup\$

I am implementing a way to import content into Drupal 8 via a CSV. This CSV contains various fields such as node ID, title, body, etc. It has been working great but I am very concerned about how long this takes to run. When there are quite a few rows, it starts taking minutes to complete.

This was a community module I found that was no longer working, and I have spent some time bringing it up to speed with Drupal 8.

ContentImport.php

<?php

namespace Drupal\contentimport\Form;

use Drupal\contentimport\Controller\ContentImportController;
use Drupal\Core\Ajax\AjaxResponse;
use Drupal\Core\Ajax\HtmlCommand;
use Drupal\Core\Form\ConfigFormBase;
use Drupal\Core\Form\FormStateInterface;
use Drupal\file\Entity\File;
use Drupal\node\Entity\Node;
use Drupal\taxonomy\Entity\Term;
use Drupal\taxonomy\Entity\Vocabulary;
use Drupal\user\Entity\User;

/**
* Configure Content Import settings for this site.
*/
class ContentImport extends ConfigFormBase
{

    /**
    * {@inheritdoc}
    */
    public function getFormId()
    {
        return 'contentimport';
    }

    /**
    * @param $contentType
    * @param $fieldNames
    * @param $fieldTypes
    * @param $images
    * @param $data
    * @param $keyIndex
    * @param $nodeArray
    * @param $fieldSettings
    * @return array
    */
    public function buildNodeArray($contentType, $fieldNames, $fieldTypes, $images, $data, $keyIndex, $nodeArray, $fieldSettings)
    {
        $node_id = [];
        for ($f = 0; $f < count($fieldNames); $f++) {
            switch ($fieldTypes[$f]) {
                case 'image':
                break;
                case 'file':
                if (!empty($images[$data[$keyIndex[$fieldNames[$f]]]])) {
                    $nodeArray[$fieldNames[$f]] = [['target_id' => $images[$data[$keyIndex[$fieldNames[$f]]]]->id()]];
                }
                break;

                case 'entity_reference':
                if ($fieldSettings[$f]['target_type'] == 'taxonomy_term') {
                    $reference = explode(",", $data[$keyIndex[$fieldNames[$f]]]);
                    if (is_array($reference) && $reference[0] != '') {
                        $terms = ContentImport::getTermReference($reference[0], $reference[1]);
                        $nodeArray[$fieldNames[$f]] = $terms;
                    }
                } elseif ($fieldSettings[$f]['target_type'] == 'user') {
                    $userArray = explode(', ', $data[$keyIndex[$fieldNames[$f]]]);
                    $users = ContentImport::getUserInfo($userArray);
                    $nodeArray[$fieldNames[$f]] = $users;
                } elseif ($fieldSettings[$f]['target_type'] == 'node') {
                    if (isset($data[$keyIndex[$fieldNames[$f]]]) && !empty($data[$keyIndex[$fieldNames[$f]]])) {
                        $nids = explode(",", $data[$keyIndex[$fieldNames[$f]]]);
                        $fieldName = $fieldNames[$f];
                        $nids = $this->check_node_id($nids, $contentType, $fieldName);
                        $nodeArray[$fieldNames[$f]] = $nids;
                    }
                } else {
                    $nodeArray[$fieldNames[$f]] = '';
                }
                break;
                case 'entity_reference_revisions':
                // TODO Implement this field type
                break;
                case 'text_with_summary':
                // TODO Implement this field type
                break;
                case 'text_long':
                $nodeArray[$fieldNames[$f]] = ['value' => $data[$keyIndex[$fieldNames[$f]]], 'format' => 'full_html'];
                break;
                case 'url':
                if (!empty($data[$keyIndex[$fieldNames[$f]]])) {
                    $nodeArray['path']['alias'] = $data[$keyIndex[$fieldNames[$f]]];
                } else {
                    $nodeArray['path']['alias'] = "/node/" . $node_id["value"];
                }
                break;

                case 'text':
                $nodeArray[$fieldNames[$f]] = ["value" => $data[$keyIndex[$fieldNames[$f]]]];
                break;
                case 'nid':
                $node_id = ["value" => $data[$keyIndex[$fieldNames[$f]]]];
                break;

                case 'datetime':
                if ($data[$keyIndex[$fieldNames[$f]]] != '') {
                    $dateTimeStamp = strtotime($data[$keyIndex[$fieldNames[$f]]]);
                    $newDateString = date('Y-m-d', $dateTimeStamp);
                }
                $nodeArray[$fieldNames[$f]] = ["value" => $newDateString];

                break;

                case 'timestamp':
                $nodeArray[$fieldNames[$f]] = ["value" => $data[$keyIndex[$fieldNames[$f]]]];
                break;

                case 'boolean':
                $nodeArray[$fieldNames[$f]] = ($data[$keyIndex[$fieldNames[$f]]] == 'On' || $data[$keyIndex[$fieldNames[$f]]] == 'Yes') ? 1 : 0;
                break;

                case 'geolocation':
                $geoArray = explode(";", $data[$keyIndex[$fieldNames[$f]]]);
                if (count($geoArray) > 0) {
                    $geoMultiArray = [];
                    for ($g = 0; $g < count($geoArray); $g++) {
                        $latlng = explode(",", $geoArray[$g]);
                        for ($l = 0; $l < count($latlng); $l++) {
                            $latlng[$l] = floatval(preg_replace("/\[^0-9,.]/", "", $latlng[$l]));
                        }
                        array_push($geoMultiArray, [
                            'lat' => $latlng[0],
                            'lng' => $latlng[1],
                        ]);
                    }
                    $nodeArray[$fieldNames[$f]] = $geoMultiArray;
                } else {
                    $latlng = explode(",", $data[$keyIndex[$fieldNames[$f]]]);
                    for ($l = 0; $l < count($latlng); $l++) {
                        $latlng[$l] = floatval(preg_replace("/\[^0-9,.]/", "", $latlng[$l]));
                    }
                    $nodeArray[$fieldNames[$f]] = ['lat' => $latlng[0], 'lng' => $latlng[1]];
                }
                break;

                case 'entity_reference_revisions':
                // TODO Implement this field type
                break;
                default:
                $nodeArray[$fieldNames[$f]] = $data[$keyIndex[$fieldNames[$f]]];
                break;
            }
        }

        $nodeArray['type'] = strtolower($contentType);
        $nodeArray['uid'] = 1;
        $nodeArray['promote'] = 0;
        $nodeArray['langcode'] = 'en';
        $nodeArray['sticky'] = 0;
        return array($node_id, $nodeArray, $data);
    }

    /**
    * {@inheritdoc}
    */
    protected function getEditableConfigNames()
    {
        return [
            'contentimport.settings',
        ];
    }

    /**
    * Content Import Form.
    */
    public function buildForm(array $form, FormStateInterface $form_state)
    {
        $contentTypes = ContentImportController::getAllContentTypes();

        $form['contentimport_contenttype'] = [
            '#type'          => 'select',
            '#title'         => $this->t('Select Content Type'),
            '#options'       => $contentTypes,
            '#default_value' => t('Select'),
            '#required'      => true,
            '#ajax'          => [
                'event'    => 'change',
                'callback' => '::contentImportcallback',
                'wrapper'  => 'content_import_fields_change_wrapper',
                'progress' => [
                    'type'    => 'throbber',
                    'message' => null,
                ],
            ],
        ];

        $form['file_upload'] = [
            '#type'              => 'file',
            '#title'             => $this->t('Import CSV File'),
            '#size'              => 40,
            '#description'       => $this->t('Select the CSV file to be imported.'),
            '#required'          => false,
            '#autoupload'        => true,
            '#upload_validators' => ['file_validate_extensions' => ['csv']],
        ];


        $form['import_ct_markup'] = [
            '#suffix' => '<div id="content_import_fields_change_wrapper"></div>
            <div id="content_submit_change_wrapper"></div>',
        ];

        $form['submit'] = [
            '#type'        => 'submit',
            '#value'       => $this->t('Import'),
            '#button_type' => 'primary',
            '#ajax'          => [
                'event'    => 'click',
                'callback' => '::submitForm',
                'wrapper'  => 'content_submit_change_wrapper',
                'progress' => [
                    'type'    => 'throbber',
                    'message' => 'IMPORTING PLEASE WAIT',
                ],
            ],
        ];
        $form['import_ct_markup2'] = [
            '#suffix' => '<div id="content_submit_change_success_wrapper"></div>',
        ];
        $form['import_ct_markup3'] = [
            '#suffix' => '<div id="content_submit_change_fail_wrapper"></div>',
        ];

        return parent::buildForm($form, $form_state);
    }


    /**
    * @param $nids Titles of the node referenced
    * @param $contentType Dropdown from the form to select content type
    * @param $fieldName
    * @return array  Array of node IDs
    */
    public function check_node_id($nids, $contentType, $fieldName)
    {
        $referencedEntity = '';
        foreach (\Drupal::entityManager()->getFieldDefinitions('node', $contentType) as $field_definition) {
            if (!empty($field_definition->getTargetBundle() && $field_definition->getName() == $fieldName)) {
                $referencedEntity .= reset($field_definition->getSettings()['handler_settings']['target_bundles']);
            }
        }
        $loaded_nids = [];
        if (is_array($nids)) {
            foreach ($nids as $nid) {
                $nid = trim($nid);
                if (Node::load($nid)) {
                    $loaded_nids[] = $nid;
                } else {
                    drupal_set_message($this->t("ERROR: NO EXISTING \"$fieldName\" with value \"$nid\""), 'error');
                }
            }
        } else {
            $nids = trim($nids);
            if (Node::load($nids)) {
                $loaded_nids[] = $nids;
            } else {
                drupal_set_message($this->t("ERROR: NO EXISTING \"$fieldName\" with value \"$nids\""), 'error');
            }
        }
        return $loaded_nids;
    }

    /**
    * Content Import Sample CSV Creation.
    */
    public function contentImportcallback(array &$form, FormStateInterface $form_state)
    {
        global $base_url;
        $ajax_response = new AjaxResponse();
        $contentType = $form_state->getValue('contentimport_contenttype');
        $fields = ContentImport::getFields($contentType);
        $fieldArray = $fields['name'];
        $contentTypeFields = 'node_id,';
        $contentTypeFields .= 'title,';
        $contentTypeFields .= 'path,';
        foreach ($fieldArray as $key => $val) {
            $contentTypeFields .= $val . ',';
        }
        $contentTypeFields = substr($contentTypeFields, 0, -1);
        $sampleFile = $contentType . '.csv';
        $handle = fopen("sites/default/files/" . $sampleFile, "w+") or die("There is no permission to create log file. Please give permission for sites/default/file!");
        fwrite($handle, $contentTypeFields);
        $ajax_response->addCommand(new HtmlCommand('#content_import_fields_change_wrapper', null));
        return $ajax_response;
    }

    /**
    * Content Import Form Submission.
    */
    public function submitForm(array &$form, FormStateInterface $form_state)
    {
        $ajax_response = new AjaxResponse();
        $contentType = $form_state->getValue('contentimport_contenttype');
        ContentImport::updateNode($_FILES, $contentType);
        if ($ajax_response->isSuccessful()) {
            $ajax_response->addCommand(new HtmlCommand('#content_submit_change_success_wrapper', 'The import was successful.'));
            return $ajax_response;
        } else {
            $ajax_response->addCommand(new HtmlCommand('#content_submit_change_fail_wrapper', 'The import was NOT successful.'));
            return $ajax_response;
        }
    }

    /**
    * To get all Content Type Fields.
    */
    public function getFields($contentType)
    {
        $fields = [];
        foreach (\Drupal::entityManager()
        ->getFieldDefinitions('node', $contentType) as $field_definition) {
            if (!empty($field_definition->getTargetBundle())) {
                $fields['name'][] = $field_definition->getName();
                $fields['type'][] = $field_definition->getType();
                $fields['setting'][] = $field_definition->getSettings();
            }
        }
        return $fields;
    }

    /**
    * To get Reference field ids.
    */
    public function getTermReference($voc, $terms)
    {
        $vocName = strtolower($voc);
        $vid = preg_replace('@[^a-z0-9_]+@', '_', $vocName);
        $vocabularies = Vocabulary::loadMultiple();

        /* Create Vocabulary if does not exist */
        if (!isset($vocabularies[$vid])) {
            ContentImport::createVoc($vid, $voc);
        }

        $termArray = array_map('trim', explode(',', $terms));
        $termIds = [];
        foreach ($termArray as $term) {
            $term_id = ContentImport::getTermId($term, $vid);
            if (empty($term_id)) {
                $term_id = ContentImport::createTerm($voc, $term, $vid);
            }
            $termIds[]['target_id'] = $term_id;
        }
        return $termIds;
    }

    /**
    * Create Vocabulary if not available.
    */
    public function createVoc($vid, $voc)
    {
        $vocabulary = Vocabulary::create([
            'vid'          => $vid,
            'machine_name' => $vid,
            'name'         => $voc,
        ]);
        $vocabulary->save();
    }

    /**
    * Create Term if it is not available.
    */
    public function createTerm($voc, $term, $vid)
    {
        Term::create([
            'parent' => [$voc],
            'name'   => $term,
            'vid'    => $vid,
            ])->save();
        $termId = ContentImport::getTermId($term, $vid);
        return $termId;
    }

    /**
    * Get Term id
    */
    public function getTermId($term, $vid)
    {
        $termRes = db_query('SELECT n.tid FROM {taxonomy_term_field_data} n WHERE n.name  = :uid AND n.vid  = :vid', [':uid' => $term, ':vid' => $vid]);
        foreach ($termRes as $val) {
            $term_id = $val->tid;
        }
        return $term_id;
    }


    /**
    * Get user information based on emailIds.
    */
    public static function getUserInfo($userArray)
    {
        $uids = [];
        foreach ($userArray as $usermail) {
            if (filter_var($usermail, FILTER_VALIDATE_EMAIL)) {
                $users = \Drupal::entityTypeManager()->getStorage('user')
                    ->loadByProperties([
                        'mail' => $usermail,
                    ]);
            } else {
                $users = \Drupal::entityTypeManager()->getStorage('user')
                    ->loadByProperties([
                        'name' => $usermail,
                    ]);
            }
            $user = reset($users);
            if ($user) {
                $uids[] = $user->id();
            } else {
                $user = User::create();
                $user->uid = '';
                $user->setUsername($usermail);
                $user->setEmail($usermail);
                $user->set("init", $usermail);
                $user->enforceIsNew();
                $user->activate();
                $user->save();
                $users = \Drupal::entityTypeManager()->getStorage('user')
                    ->loadByProperties(['mail' => $usermail]);
                $uids[] = $user->id();
            }
        }
        return $uids;
    }


    /**
    * To import data as Content type nodes.
    */
    public function updateNode($filedata, $contentType)
    {
        global $base_url;

        $fields = ContentImport::getFields($contentType);
        $fieldNames = $fields['name'];
        $fieldTypes = $fields['type'];
        $fieldSettings = $fields['setting'];

        $files = glob('sites/default/files/' . $contentType . '/images/*.*');
        $images = [];
        foreach ($files as $file_name) {
            $image = File::create(['uri' => 'public://' . $contentType . '/images/' . basename($file_name)]);
            $image->save();
            $images[basename($file_name)] = $image;
        }

        $mimetype = 1;

        if ($mimetype) {
            $location = $filedata['files']['tmp_name']['file_upload'];

            if (($handle = fopen($location, "r")) !== false) {
                $keyIndex = [];
                $index = 0;

                while (($data = fgetcsv($handle)) !== false) {
                    $index++;

                    if ($index < 2) {
                        array_push($fieldNames, 'node_id');
                        array_push($fieldTypes, 'nid');
                        array_push($fieldNames, 'title');
                        array_push($fieldTypes, 'text');
                        array_push($fieldNames, 'path');
                        array_push($fieldTypes, 'url');

                        foreach ($fieldNames as $fieldValues) {
                            $i = 0;
                            foreach ($data as $dataValues) {

                                if ($fieldValues == $dataValues) {
                                    $keyIndex[$fieldValues] = $i;
                                }
                                $i++;
                            }
                        }
                        continue;
                    }

                    $nodeArray = [];
                    list($node_id, $nodeArray, $data) = $this->buildNodeArray($contentType, $fieldNames, $fieldTypes, $images, $data, $keyIndex, $nodeArray, $fieldSettings);
                    if ($nodeArray['title']['value'] != '') {

                        if (empty($node_id["value"])) {
                            $node = Node::create($nodeArray);
                            $node->save();
                        } elseif (is_null(Node::load($node_id["value"]))) {
                            drupal_set_message($this->t("NO NODE USING ID " . $node_id['value']), 'error');
                        } else {
                            foreach ($nodeArray as $key => $value) {
                                $node = Node::load($node_id["value"]);
                                $node->set($key, $value);
                                $node->save();
                            }
                        }
                    }
                }
                fclose($handle);
            }
        }
    }
}

ContentImportController.php

<?php

namespace Drupal\contentimport\Controller;

use Drupal\Core\Controller\ControllerBase;

/**
 * Controller routines for contentimport routes.
 */
class ContentImportController extends ControllerBase {

  /**
   * Get All Content types.
   */
  public static function getAllContentTypes() {
    $contentTypes = \Drupal::service('entity.manager')->getStorage('node_type')->loadMultiple();
    $contentTypesList = [];
    $contentTypesList['none'] = 'Select';
    foreach ($contentTypes as $contentType) {
      $contentTypesList[$contentType->id()] = $contentType->label();
    }
    return $contentTypesList;
  }

}
\$\endgroup\$
1
  • 3
    \$\begingroup\$ To whoever voted to close this question: there is no indication the code is currently broken. It used to, yes, but that's in the past. It wasn't posted while broken or broken while posted. \$\endgroup\$ – Mast Mar 15 '18 at 19:57

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Browse other questions tagged or ask your own question.