3
\$\begingroup\$

I am implementing a way to import content into Drupal 8 via a CSV. This CSV contains various fields such as node ID, title, body, etc. It has been working great but I am very concerned about how long this takes to run. When there are quite a few rows, it starts taking minutes to complete.

This was a community module I found that was no longer working, and I have spent some time bringing it up to speed with Drupal 8.

ContentImport.php

<?php

namespace Drupal\contentimport\Form;

use Drupal\contentimport\Controller\ContentImportController;
use Drupal\Core\Ajax\AjaxResponse;
use Drupal\Core\Ajax\HtmlCommand;
use Drupal\Core\Form\ConfigFormBase;
use Drupal\Core\Form\FormStateInterface;
use Drupal\file\Entity\File;
use Drupal\node\Entity\Node;
use Drupal\taxonomy\Entity\Term;
use Drupal\taxonomy\Entity\Vocabulary;
use Drupal\user\Entity\User;

/**
* Configure Content Import settings for this site.
*/
class ContentImport extends ConfigFormBase
{

    /**
    * {@inheritdoc}
    */
    public function getFormId()
    {
        return 'contentimport';
    }

    /**
    * @param $contentType
    * @param $fieldNames
    * @param $fieldTypes
    * @param $images
    * @param $data
    * @param $keyIndex
    * @param $nodeArray
    * @param $fieldSettings
    * @return array
    */
    public function buildNodeArray($contentType, $fieldNames, $fieldTypes, $images, $data, $keyIndex, $nodeArray, $fieldSettings)
    {
        $node_id = [];
        for ($f = 0; $f < count($fieldNames); $f++) {
            switch ($fieldTypes[$f]) {
                case 'image':
                break;
                case 'file':
                if (!empty($images[$data[$keyIndex[$fieldNames[$f]]]])) {
                    $nodeArray[$fieldNames[$f]] = [['target_id' => $images[$data[$keyIndex[$fieldNames[$f]]]]->id()]];
                }
                break;

                case 'entity_reference':
                if ($fieldSettings[$f]['target_type'] == 'taxonomy_term') {
                    $reference = explode(",", $data[$keyIndex[$fieldNames[$f]]]);
                    if (is_array($reference) && $reference[0] != '') {
                        $terms = ContentImport::getTermReference($reference[0], $reference[1]);
                        $nodeArray[$fieldNames[$f]] = $terms;
                    }
                } elseif ($fieldSettings[$f]['target_type'] == 'user') {
                    $userArray = explode(', ', $data[$keyIndex[$fieldNames[$f]]]);
                    $users = ContentImport::getUserInfo($userArray);
                    $nodeArray[$fieldNames[$f]] = $users;
                } elseif ($fieldSettings[$f]['target_type'] == 'node') {
                    if (isset($data[$keyIndex[$fieldNames[$f]]]) && !empty($data[$keyIndex[$fieldNames[$f]]])) {
                        $nids = explode(",", $data[$keyIndex[$fieldNames[$f]]]);
                        $fieldName = $fieldNames[$f];
                        $nids = $this->check_node_id($nids, $contentType, $fieldName);
                        $nodeArray[$fieldNames[$f]] = $nids;
                    }
                } else {
                    $nodeArray[$fieldNames[$f]] = '';
                }
                break;
                case 'entity_reference_revisions':
                // TODO Implement this field type
                break;
                case 'text_with_summary':
                // TODO Implement this field type
                break;
                case 'text_long':
                $nodeArray[$fieldNames[$f]] = ['value' => $data[$keyIndex[$fieldNames[$f]]], 'format' => 'full_html'];
                break;
                case 'url':
                if (!empty($data[$keyIndex[$fieldNames[$f]]])) {
                    $nodeArray['path']['alias'] = $data[$keyIndex[$fieldNames[$f]]];
                } else {
                    $nodeArray['path']['alias'] = "/node/" . $node_id["value"];
                }
                break;

                case 'text':
                $nodeArray[$fieldNames[$f]] = ["value" => $data[$keyIndex[$fieldNames[$f]]]];
                break;
                case 'nid':
                $node_id = ["value" => $data[$keyIndex[$fieldNames[$f]]]];
                break;

                case 'datetime':
                if ($data[$keyIndex[$fieldNames[$f]]] != '') {
                    $dateTimeStamp = strtotime($data[$keyIndex[$fieldNames[$f]]]);
                    $newDateString = date('Y-m-d', $dateTimeStamp);
                }
                $nodeArray[$fieldNames[$f]] = ["value" => $newDateString];

                break;

                case 'timestamp':
                $nodeArray[$fieldNames[$f]] = ["value" => $data[$keyIndex[$fieldNames[$f]]]];
                break;

                case 'boolean':
                $nodeArray[$fieldNames[$f]] = ($data[$keyIndex[$fieldNames[$f]]] == 'On' || $data[$keyIndex[$fieldNames[$f]]] == 'Yes') ? 1 : 0;
                break;

                case 'geolocation':
                $geoArray = explode(";", $data[$keyIndex[$fieldNames[$f]]]);
                if (count($geoArray) > 0) {
                    $geoMultiArray = [];
                    for ($g = 0; $g < count($geoArray); $g++) {
                        $latlng = explode(",", $geoArray[$g]);
                        for ($l = 0; $l < count($latlng); $l++) {
                            $latlng[$l] = floatval(preg_replace("/\[^0-9,.]/", "", $latlng[$l]));
                        }
                        array_push($geoMultiArray, [
                            'lat' => $latlng[0],
                            'lng' => $latlng[1],
                        ]);
                    }
                    $nodeArray[$fieldNames[$f]] = $geoMultiArray;
                } else {
                    $latlng = explode(",", $data[$keyIndex[$fieldNames[$f]]]);
                    for ($l = 0; $l < count($latlng); $l++) {
                        $latlng[$l] = floatval(preg_replace("/\[^0-9,.]/", "", $latlng[$l]));
                    }
                    $nodeArray[$fieldNames[$f]] = ['lat' => $latlng[0], 'lng' => $latlng[1]];
                }
                break;

                case 'entity_reference_revisions':
                // TODO Implement this field type
                break;
                default:
                $nodeArray[$fieldNames[$f]] = $data[$keyIndex[$fieldNames[$f]]];
                break;
            }
        }

        $nodeArray['type'] = strtolower($contentType);
        $nodeArray['uid'] = 1;
        $nodeArray['promote'] = 0;
        $nodeArray['langcode'] = 'en';
        $nodeArray['sticky'] = 0;
        return array($node_id, $nodeArray, $data);
    }

    /**
    * {@inheritdoc}
    */
    protected function getEditableConfigNames()
    {
        return [
            'contentimport.settings',
        ];
    }

    /**
    * Content Import Form.
    */
    public function buildForm(array $form, FormStateInterface $form_state)
    {
        $contentTypes = ContentImportController::getAllContentTypes();

        $form['contentimport_contenttype'] = [
            '#type'          => 'select',
            '#title'         => $this->t('Select Content Type'),
            '#options'       => $contentTypes,
            '#default_value' => t('Select'),
            '#required'      => true,
            '#ajax'          => [
                'event'    => 'change',
                'callback' => '::contentImportcallback',
                'wrapper'  => 'content_import_fields_change_wrapper',
                'progress' => [
                    'type'    => 'throbber',
                    'message' => null,
                ],
            ],
        ];

        $form['file_upload'] = [
            '#type'              => 'file',
            '#title'             => $this->t('Import CSV File'),
            '#size'              => 40,
            '#description'       => $this->t('Select the CSV file to be imported.'),
            '#required'          => false,
            '#autoupload'        => true,
            '#upload_validators' => ['file_validate_extensions' => ['csv']],
        ];


        $form['import_ct_markup'] = [
            '#suffix' => '<div id="content_import_fields_change_wrapper"></div>
            <div id="content_submit_change_wrapper"></div>',
        ];

        $form['submit'] = [
            '#type'        => 'submit',
            '#value'       => $this->t('Import'),
            '#button_type' => 'primary',
            '#ajax'          => [
                'event'    => 'click',
                'callback' => '::submitForm',
                'wrapper'  => 'content_submit_change_wrapper',
                'progress' => [
                    'type'    => 'throbber',
                    'message' => 'IMPORTING PLEASE WAIT',
                ],
            ],
        ];
        $form['import_ct_markup2'] = [
            '#suffix' => '<div id="content_submit_change_success_wrapper"></div>',
        ];
        $form['import_ct_markup3'] = [
            '#suffix' => '<div id="content_submit_change_fail_wrapper"></div>',
        ];

        return parent::buildForm($form, $form_state);
    }


    /**
    * @param $nids Titles of the node referenced
    * @param $contentType Dropdown from the form to select content type
    * @param $fieldName
    * @return array  Array of node IDs
    */
    public function check_node_id($nids, $contentType, $fieldName)
    {
        $referencedEntity = '';
        foreach (\Drupal::entityManager()->getFieldDefinitions('node', $contentType) as $field_definition) {
            if (!empty($field_definition->getTargetBundle() && $field_definition->getName() == $fieldName)) {
                $referencedEntity .= reset($field_definition->getSettings()['handler_settings']['target_bundles']);
            }
        }
        $loaded_nids = [];
        if (is_array($nids)) {
            foreach ($nids as $nid) {
                $nid = trim($nid);
                if (Node::load($nid)) {
                    $loaded_nids[] = $nid;
                } else {
                    drupal_set_message($this->t("ERROR: NO EXISTING \"$fieldName\" with value \"$nid\""), 'error');
                }
            }
        } else {
            $nids = trim($nids);
            if (Node::load($nids)) {
                $loaded_nids[] = $nids;
            } else {
                drupal_set_message($this->t("ERROR: NO EXISTING \"$fieldName\" with value \"$nids\""), 'error');
            }
        }
        return $loaded_nids;
    }

    /**
    * Content Import Sample CSV Creation.
    */
    public function contentImportcallback(array &$form, FormStateInterface $form_state)
    {
        global $base_url;
        $ajax_response = new AjaxResponse();
        $contentType = $form_state->getValue('contentimport_contenttype');
        $fields = ContentImport::getFields($contentType);
        $fieldArray = $fields['name'];
        $contentTypeFields = 'node_id,';
        $contentTypeFields .= 'title,';
        $contentTypeFields .= 'path,';
        foreach ($fieldArray as $key => $val) {
            $contentTypeFields .= $val . ',';
        }
        $contentTypeFields = substr($contentTypeFields, 0, -1);
        $sampleFile = $contentType . '.csv';
        $handle = fopen("sites/default/files/" . $sampleFile, "w+") or die("There is no permission to create log file. Please give permission for sites/default/file!");
        fwrite($handle, $contentTypeFields);
        $ajax_response->addCommand(new HtmlCommand('#content_import_fields_change_wrapper', null));
        return $ajax_response;
    }

    /**
    * Content Import Form Submission.
    */
    public function submitForm(array &$form, FormStateInterface $form_state)
    {
        $ajax_response = new AjaxResponse();
        $contentType = $form_state->getValue('contentimport_contenttype');
        ContentImport::updateNode($_FILES, $contentType);
        if ($ajax_response->isSuccessful()) {
            $ajax_response->addCommand(new HtmlCommand('#content_submit_change_success_wrapper', 'The import was successful.'));
            return $ajax_response;
        } else {
            $ajax_response->addCommand(new HtmlCommand('#content_submit_change_fail_wrapper', 'The import was NOT successful.'));
            return $ajax_response;
        }
    }

    /**
    * To get all Content Type Fields.
    */
    public function getFields($contentType)
    {
        $fields = [];
        foreach (\Drupal::entityManager()
        ->getFieldDefinitions('node', $contentType) as $field_definition) {
            if (!empty($field_definition->getTargetBundle())) {
                $fields['name'][] = $field_definition->getName();
                $fields['type'][] = $field_definition->getType();
                $fields['setting'][] = $field_definition->getSettings();
            }
        }
        return $fields;
    }

    /**
    * To get Reference field ids.
    */
    public function getTermReference($voc, $terms)
    {
        $vocName = strtolower($voc);
        $vid = preg_replace('@[^a-z0-9_]+@', '_', $vocName);
        $vocabularies = Vocabulary::loadMultiple();

        /* Create Vocabulary if does not exist */
        if (!isset($vocabularies[$vid])) {
            ContentImport::createVoc($vid, $voc);
        }

        $termArray = array_map('trim', explode(',', $terms));
        $termIds = [];
        foreach ($termArray as $term) {
            $term_id = ContentImport::getTermId($term, $vid);
            if (empty($term_id)) {
                $term_id = ContentImport::createTerm($voc, $term, $vid);
            }
            $termIds[]['target_id'] = $term_id;
        }
        return $termIds;
    }

    /**
    * Create Vocabulary if not available.
    */
    public function createVoc($vid, $voc)
    {
        $vocabulary = Vocabulary::create([
            'vid'          => $vid,
            'machine_name' => $vid,
            'name'         => $voc,
        ]);
        $vocabulary->save();
    }

    /**
    * Create Term if it is not available.
    */
    public function createTerm($voc, $term, $vid)
    {
        Term::create([
            'parent' => [$voc],
            'name'   => $term,
            'vid'    => $vid,
            ])->save();
        $termId = ContentImport::getTermId($term, $vid);
        return $termId;
    }

    /**
    * Get Term id
    */
    public function getTermId($term, $vid)
    {
        $termRes = db_query('SELECT n.tid FROM {taxonomy_term_field_data} n WHERE n.name  = :uid AND n.vid  = :vid', [':uid' => $term, ':vid' => $vid]);
        foreach ($termRes as $val) {
            $term_id = $val->tid;
        }
        return $term_id;
    }


    /**
    * Get user information based on emailIds.
    */
    public static function getUserInfo($userArray)
    {
        $uids = [];
        foreach ($userArray as $usermail) {
            if (filter_var($usermail, FILTER_VALIDATE_EMAIL)) {
                $users = \Drupal::entityTypeManager()->getStorage('user')
                    ->loadByProperties([
                        'mail' => $usermail,
                    ]);
            } else {
                $users = \Drupal::entityTypeManager()->getStorage('user')
                    ->loadByProperties([
                        'name' => $usermail,
                    ]);
            }
            $user = reset($users);
            if ($user) {
                $uids[] = $user->id();
            } else {
                $user = User::create();
                $user->uid = '';
                $user->setUsername($usermail);
                $user->setEmail($usermail);
                $user->set("init", $usermail);
                $user->enforceIsNew();
                $user->activate();
                $user->save();
                $users = \Drupal::entityTypeManager()->getStorage('user')
                    ->loadByProperties(['mail' => $usermail]);
                $uids[] = $user->id();
            }
        }
        return $uids;
    }


    /**
    * To import data as Content type nodes.
    */
    public function updateNode($filedata, $contentType)
    {
        global $base_url;

        $fields = ContentImport::getFields($contentType);
        $fieldNames = $fields['name'];
        $fieldTypes = $fields['type'];
        $fieldSettings = $fields['setting'];

        $files = glob('sites/default/files/' . $contentType . '/images/*.*');
        $images = [];
        foreach ($files as $file_name) {
            $image = File::create(['uri' => 'public://' . $contentType . '/images/' . basename($file_name)]);
            $image->save();
            $images[basename($file_name)] = $image;
        }

        $mimetype = 1;

        if ($mimetype) {
            $location = $filedata['files']['tmp_name']['file_upload'];

            if (($handle = fopen($location, "r")) !== false) {
                $keyIndex = [];
                $index = 0;

                while (($data = fgetcsv($handle)) !== false) {
                    $index++;

                    if ($index < 2) {
                        array_push($fieldNames, 'node_id');
                        array_push($fieldTypes, 'nid');
                        array_push($fieldNames, 'title');
                        array_push($fieldTypes, 'text');
                        array_push($fieldNames, 'path');
                        array_push($fieldTypes, 'url');

                        foreach ($fieldNames as $fieldValues) {
                            $i = 0;
                            foreach ($data as $dataValues) {

                                if ($fieldValues == $dataValues) {
                                    $keyIndex[$fieldValues] = $i;
                                }
                                $i++;
                            }
                        }
                        continue;
                    }

                    $nodeArray = [];
                    list($node_id, $nodeArray, $data) = $this->buildNodeArray($contentType, $fieldNames, $fieldTypes, $images, $data, $keyIndex, $nodeArray, $fieldSettings);
                    if ($nodeArray['title']['value'] != '') {

                        if (empty($node_id["value"])) {
                            $node = Node::create($nodeArray);
                            $node->save();
                        } elseif (is_null(Node::load($node_id["value"]))) {
                            drupal_set_message($this->t("NO NODE USING ID " . $node_id['value']), 'error');
                        } else {
                            foreach ($nodeArray as $key => $value) {
                                $node = Node::load($node_id["value"]);
                                $node->set($key, $value);
                                $node->save();
                            }
                        }
                    }
                }
                fclose($handle);
            }
        }
    }
}

ContentImportController.php

<?php

namespace Drupal\contentimport\Controller;

use Drupal\Core\Controller\ControllerBase;

/**
 * Controller routines for contentimport routes.
 */
class ContentImportController extends ControllerBase {

  /**
   * Get All Content types.
   */
  public static function getAllContentTypes() {
    $contentTypes = \Drupal::service('entity.manager')->getStorage('node_type')->loadMultiple();
    $contentTypesList = [];
    $contentTypesList['none'] = 'Select';
    foreach ($contentTypes as $contentType) {
      $contentTypesList[$contentType->id()] = $contentType->label();
    }
    return $contentTypesList;
  }

}
\$\endgroup\$
1
  • 3
    \$\begingroup\$ To whoever voted to close this question: there is no indication the code is currently broken. It used to, yes, but that's in the past. It wasn't posted while broken or broken while posted. \$\endgroup\$
    – Mast
    Commented Mar 15, 2018 at 19:57

0

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service and acknowledge you have read our privacy policy.

Browse other questions tagged or ask your own question.