Crawling thesaurus for a synonym

Question

I have just finished core functionality for Tehsaurux. It crawls three thesauruses and will find a random synonym for you. It works with PHP agnostic and Scrapy for Python.

I think the two main files - Tehsaurux.php and synoSpider.py - are alright. I am concerned that they may not be robust for these reasons:

• It may not work on every server, depending on its OS (I'm running Windows 10)
• It will require either user path configuration or more work on my part to automatically find/set paths
• The user only needs to know PHP, but requires Python. Some hosts may need scripts in certain locations.

I myself am even stumbling to figure out how to integrate it into Laravel, an MVC framework. Do I just dump the entire spider in the models directory, or break it up and spread it out over arbitrary directories to make the MVC framework happy?

The full project is viewable here.

Here's the core PHP file:

Tehsaurux.php

<?php
class Tehsaurux {

    public function __construct()
    {
        $this->tsx = $this->getTsx();
        $this->syn_src_limit = 10;
        $this->mode = 'strict';
    }

    public function find($keyword, $mode = null)
    {
        if ($mode !== null) {
            $this->setMode($mode);
        }

        if (isset($this->tsx[$keyword])) {
            // grab a random synonym
            $syns = $this->tsx[$keyword][$mode];
            $len = count($syns);
            $idx = rand(1, $len) - 1;
            return $syns[$idx];

        } else {
            // keyword not defined in tsx
            $this->makeDefinition($keyword);
            return $this->find($keyword, $mode);
        }
    }

    public function getTsx()
    {
        $path = __DIR__.'\tehsaurux.json';

        if (!file_exists($path)) {
            file_put_contents($path, json_encode(''));
        }

        $contents = file_get_contents($path);

        return json_decode($contents, true);
    }

    public function setMode($mode) {
        if ($mode == 'l' | $mode == 'loose') {
            $this->mode = 'loose';
            return;
        }

        if ($mode == 's' | $mode == 'strict') {
            $this->mode = 'strict';
            return;
        }

        if ($mode == 'f' | $mode == 'favorites') {
            $this->mode = 'favorites';
        }
    }

    private function saveEntry(string $keyword = null, array $entries = null)
    {
        if ($entries !== null) {
            if (array_key_exists('loose', $entries)) {
                $this->tsx[$keyword]['loose'] = $entries['loose'];
            }

            if (array_key_exists('strict', $entries)) {
                $this->tsx[$keyword]['strict'] = $entries['strict'];
            }

            if (array_key_exists('favorites', $entries)) {
                $this->tsx[$keyword]['favorites'] = $entries['favorites'];
            }
        }

        $this->save();
    }

    private function makeDefinition($keyword)
    {
        $this->makeSources($keyword);
        $entry = $this->fetchWords($keyword);
        $this->saveEntry($keyword, $entry);
    }

    private function fetchWords($keyword)
    {
        $loose = [];
        $strict = [];

        $sources = ['thesaurus', 'collins', 'oxford'];

        foreach ($sources as $source) {
            $filename = "\\" . $source . '.txt';
            if (file_exists(__DIR__ . $filename)) {
                $contents = file_get_contents(__DIR__.$filename);
                file_put_contents(__DIR__.$filename, '');
                $synonyms = explode(',', $contents);

                $strict[] = $synonyms;
                foreach ($synonyms as $idx => $syn) {
                    $loose[] = $syn;
                    if ($idx == $this->syn_src_limit) {
                        break;
                    }
                }
            }
        }

        $loose = array_unique($loose);

        $stricter = call_user_func_array('array_intersect', $strict);

        $loose = array_values(array_filter($loose));
        $strict = array_values(array_filter($stricter));

        if (count($strict) < 1) {
            $strict = [$keyword];
        }

        return ['loose' => $loose, 'strict' => $strict];
    }

    private function save()
    {
        try {
            file_put_contents(__DIR__.'\tehsaurux.json', json_encode($this->tsx));
        } catch (Exception $e) {
            echo "Caught an exception trying to save. It says ", $e->getMessage();
        }
    }

    public function makeSources($keyword)
    {
        try {
            $cmd = escapeshellcmd('scrapy crawl tehsaurux -a keyword=' . $keyword);
            system($cmd);
        } catch (Exception $e) {
            echo "Had a hard time getting the words. Here's the error message: " . $e->getMessage();
      }

    }
}

?>

The core Python file:

synoSpider.py

import scrapy, logging, re, traceback

# Utilities
def make_csv(words):
    csv = ''
    for word in words:
        if word[0] == ',':
            word.strip(',')
        if word == ',' or word == ' ':
            continue
        word.strip()
        csv += word + ','
    return csv

def save_words(words, filename, to_csv=True):
    if to_csv:
        words = make_csv(words)

    with open(filename, 'w') as f:
        f.seek(0)
        f.truncate()
        f.write(words)
        f.close

class WordSpider(scrapy.Spider):
    name = "tehsaurux"

    def __init__(self, keyword, **kwargs):
        super(WordSpider, self).__init__(**kwargs)
        self.k = keyword

    def repr(self):
        return repr()

    def start_requests(self):
        urls = {
            'thesaurus': 'http://www.thesaurus.com/browse/%s?s=t' % self.k,
            'oxford': 'https://en.oxforddictionaries.com/thesaurus/%s' % self.k,
            'collins': 'https://www.collinsdictionary.com/dictionary/english-thesaurus/%s' % self.k,
        }

        for site, url in urls.items():
            yield scrapy.Request(url, meta={'site': site}, callback=self.parse)

    def parse(self, response):
        if response:
            parser = getattr(self, response.meta['site'])
            parser(response)

    # site specific parsers
    def thesaurus(self, response):
        filename = 'thesaurus.txt'

        words = response.css('#filters-0 > div.relevancy-block > div > ul> li > a > span.text::text').extract()

        save_words(words, filename)

    def oxford(self, response):
        filename = 'oxford.txt'

        bolds = response.css('#content > div.lex-container > div.main-content > div > div > div > div > section > div > div:nth-child(1) > div > div > div.synList > div > p > strong::text').extract()

        norms = response.css('#content > div.lex-container > div.main-content > div > div > div > div > section > div > div:nth-child(1) > div > div > div.synList > div > p > span:not(.sense-registers)::text').extract()

        edited = []

        for csv in norms:
            edit = re.split(',(\s)+', csv)
            for word in edit:
                if len(word) > 1:
                    edited.append(word)

        words = bolds + edited

        save_words(words, filename)


    def collins(self, response):
        filename = 'collins.txt'
        words = []
        bold = response.css('body > main > div.dictionary > div.res_cell_center > div > div.homograph-entry > div > div.entry.dictionary.thesbase > div > div.hom > div:nth-child(2) > div.synonymBlock > span.sensehead > span.key::text').extract()

        others = response.css('body > main > div.dictionary > div.res_cell_center > div > div.homograph-entry > div > div.entry.dictionary.thesbase > div > div.hom > h2.titleTypeContainer + div.sense > div.synonymBlock > span.firstSyn > span::text').extract()

        others2 = response.css('body > main > div.dictionary > div.res_cell_center > div > div.homograph-entry > div > div.entry.dictionary.thesbase > div > div.hom > h2.titleTypeContainer + div > div.containerBlock > div.blockSyn > div> a > span::text').extract()

        others3 = response.css('body > main > div.dictionary > div.res_cell_center > div > div.homograph-entry > div > div.entry.dictionary.thesbase > div > div.hom > h2.titleTypeContainer + div > div.containerBlock > div.blockSyn > div > span::text').extract()

        words = bold + others + others2 + others3


        save_words(words, filename)

Answer 1

Base Review

It may not work on every server, depending on its OS (I'm running Windows 10)

One thing to make it more portable would be to use the Predefined Constant DIRECTORY_SEPARATOR instead of "\\",

The code may also need to check if the directory is writable - if not, it could throw an exception.

I myself am even stumbling to figure out how to integrate it into Laravel, an MVC framework. Do I just dump the entire spider in the models directory, or break it up and spread it out over arbitrary directories to make the MVC framework happy?

Typically with a Laravel application one starts with defining routes (e.g. in api.php and/or web.php within the routes directory, which also contains console.php for console commands). Routes can be mapped to/connected with an anonymous closure but typically are mapped to a controller method. Many of the methods in the class Tehsaurux would be candidates to put in a controller - e.g. TehsauruxController though some laravel developers add a Helpers directory under app/Http.

Controllers typically have methods for CRUD operations - so the methods like find() and save() would be appropriate there. Other methods could be in a controller, model or a helper depending on how the structure is preferred.

Other comments

Style guides

Consider following Style guides for the sake of readability and maintainability - e.g. PSR-1 and PSR-12 for PHP, and PEP-8 for Python.

I ran the PHP code through a psr-12 checker phan. It had no errors but 22 warnings - mostly undeclared properties referenced, as well as bitwise operators used (see the last section about method saveEntry() below).

In addition to the multiple imports on one line (already mentioned in a comment), pep8online.com reported mostly warnings about line length and extra lines within methods.

Method Tehsaurux::setMode()

Are the OR operators intentionally bitwise Or |? If logical OR operators || were used, then if the first condition evaluated to TRUE then the second condition would not be evaluated. This may not be a big issue for equality comparison but if there was a function call on the right-hand side of the disjunction it wouldn't (need to) be called.

Method Tehsaurux::saveEntry()

Should the default value for the first argument (i.e. $keyword) really be null? Perhaps it never actually gets called without a value but if it did, would you really want the code inside that method to reference the tsx array at index null?

Answer 2

• I don't see $this->syn_src_limit or $path changing anywhere in your code. For this reason, these are good candidates for being declared as constants within the class. Generally, I think the abbreviation should be avoided for clarity.

  const MAX_SYNONYMS = 10;
  const TSX_PATH = __DIR__ . '\tehsaurux.json';
  

• I think it would be handy to allow the $mode to be declared when the class is instantiated. Just add string $mode = 'strict' as the lone parameter in the constructor definition.

  public function __construct(string $mode = 'strict')
  {
      $this->tsx = $this->getTsx();
      $this->mode = $this->setMode;
  }
  

• I don't like the conditional recursion in find() -- it can be written without recursion.

  public function find(string $keyword, ?string $mode = null)
  {
      if ($mode !== null) {
          $this->setMode($mode);
      }
  
      if (!isset($this->tsx[$keyword])) {
          $this->makeDefinition($keyword);
      }
  
      $synonyms = $this->tsx[$keyword][$mode];
      return $synonyms[array_rand($synonyms)];
  }
  

• When you know you've created an empty json string, you can return early instead of bothering to open the just-created json file. The method should unconditionally return an array for consistency.

  public function getTsx(): array
  {
      if (!file_exists(self::TSX_PATH)) {
          file_put_contents(self::TSX_PATH, json_encode([]));
          return [];
      }
      return json_decode(file_get_contents(self::TSX_PATH), true);
  }
  

• I recommend a lookup array to be declared as another class level constant to help set the mode -- again the valid options do not change.

  const MODE_LOOKUP = [
      'l' => 'loose',
      's' => 'strict',
      'f' => 'favorite',
  ];
  
  public function setMode(?string $mode = 'strict'): void
  {
      $this->mode = self::MODE_LOOKUP[$mode]
          ?? (in_array($mode, self::MODE_LOOKUP) ? $mode : 'strict');
  }
  

• I recommend leveraging the lookup array again to make your saveEntry() code D.R.Y.

  private function saveEntry(string $keyword, array $entries)
  {
      if ($entries) {
          foreach (self::MODE_LOOKUP as $mode) {
          if (isset($entries[$mode])) {
              $this->tsx[$keyword][$mode] = $entries[$mode];
          }
      }
      $this->save();
  }
  

• Since $sources remains unchanged, make that a constant too, and implement techniques previously mentioned in my answer.

I don't use Python, so I will not review that part.