4
\$\begingroup\$

I have just finished core functionality for Tehsaurux. It crawls three thesauruses and will find a random synonym for you. It works with PHP agnostic and Scrapy for Python.

I think the two main files - Tehsaurux.php and synoSpider.py - are alright. I am concerned that they may not be robust for these reasons:

  • It may not work on every server, depending on its OS (I'm running Windows 10)
  • It will require either user path configuration or more work on my part to automatically find/set paths
  • The user only needs to know PHP, but requires Python. Some hosts may need scripts in certain locations.

I myself am even stumbling to figure out how to integrate it into Laravel, an MVC framework. Do I just dump the entire spider in the models directory, or break it up and spread it out over arbitrary directories to make the MVC framework happy?

The full project is viewable here.

Here's the core PHP file:

Tehsaurux.php

<?php
class Tehsaurux {

    public function __construct()
    {
        $this->tsx = $this->getTsx();
        $this->syn_src_limit = 10;
        $this->mode = 'strict';
    }

    public function find($keyword, $mode = null)
    {
        if ($mode !== null) {
            $this->setMode($mode);
        }

        if (isset($this->tsx[$keyword])) {
            // grab a random synonym
            $syns = $this->tsx[$keyword][$mode];
            $len = count($syns);
            $idx = rand(1, $len) - 1;
            return $syns[$idx];

        } else {
            // keyword not defined in tsx
            $this->makeDefinition($keyword);
            return $this->find($keyword, $mode);
        }
    }

    public function getTsx()
    {
        $path = __DIR__.'\tehsaurux.json';

        if (!file_exists($path)) {
            file_put_contents($path, json_encode(''));
        }

        $contents = file_get_contents($path);

        return json_decode($contents, true);
    }

    public function setMode($mode) {
        if ($mode == 'l' | $mode == 'loose') {
            $this->mode = 'loose';
            return;
        }

        if ($mode == 's' | $mode == 'strict') {
            $this->mode = 'strict';
            return;
        }

        if ($mode == 'f' | $mode == 'favorites') {
            $this->mode = 'favorites';
        }
    }

    private function saveEntry(string $keyword = null, array $entries = null)
    {
        if ($entries !== null) {
            if (array_key_exists('loose', $entries)) {
                $this->tsx[$keyword]['loose'] = $entries['loose'];
            }

            if (array_key_exists('strict', $entries)) {
                $this->tsx[$keyword]['strict'] = $entries['strict'];
            }

            if (array_key_exists('favorites', $entries)) {
                $this->tsx[$keyword]['favorites'] = $entries['favorites'];
            }
        }

        $this->save();
    }

    private function makeDefinition($keyword)
    {
        $this->makeSources($keyword);
        $entry = $this->fetchWords($keyword);
        $this->saveEntry($keyword, $entry);
    }

    private function fetchWords($keyword)
    {
        $loose = [];
        $strict = [];

        $sources = ['thesaurus', 'collins', 'oxford'];

        foreach ($sources as $source) {
            $filename = "\\" . $source . '.txt';
            if (file_exists(__DIR__ . $filename)) {
                $contents = file_get_contents(__DIR__.$filename);
                file_put_contents(__DIR__.$filename, '');
                $synonyms = explode(',', $contents);

                $strict[] = $synonyms;
                foreach ($synonyms as $idx => $syn) {
                    $loose[] = $syn;
                    if ($idx == $this->syn_src_limit) {
                        break;
                    }
                }
            }
        }

        $loose = array_unique($loose);

        $stricter = call_user_func_array('array_intersect', $strict);

        $loose = array_values(array_filter($loose));
        $strict = array_values(array_filter($stricter));

        if (count($strict) < 1) {
            $strict = [$keyword];
        }

        return ['loose' => $loose, 'strict' => $strict];
    }

    private function save()
    {
        try {
            file_put_contents(__DIR__.'\tehsaurux.json', json_encode($this->tsx));
        } catch (Exception $e) {
            echo "Caught an exception trying to save. It says ", $e->getMessage();
        }
    }

    public function makeSources($keyword)
    {
        try {
            $cmd = escapeshellcmd('scrapy crawl tehsaurux -a keyword=' . $keyword);
            system($cmd);
        } catch (Exception $e) {
            echo "Had a hard time getting the words. Here's the error message: " . $e->getMessage();
      }

    }
}

?>

The core Python file:

synoSpider.py

import scrapy, logging, re, traceback

# Utilities
def make_csv(words):
    csv = ''
    for word in words:
        if word[0] == ',':
            word.strip(',')
        if word == ',' or word == ' ':
            continue
        word.strip()
        csv += word + ','
    return csv

def save_words(words, filename, to_csv=True):
    if to_csv:
        words = make_csv(words)

    with open(filename, 'w') as f:
        f.seek(0)
        f.truncate()
        f.write(words)
        f.close

class WordSpider(scrapy.Spider):
    name = "tehsaurux"

    def __init__(self, keyword, **kwargs):
        super(WordSpider, self).__init__(**kwargs)
        self.k = keyword

    def repr(self):
        return repr()

    def start_requests(self):
        urls = {
            'thesaurus': 'http://www.thesaurus.com/browse/%s?s=t' % self.k,
            'oxford': 'https://en.oxforddictionaries.com/thesaurus/%s' % self.k,
            'collins': 'https://www.collinsdictionary.com/dictionary/english-thesaurus/%s' % self.k,
        }

        for site, url in urls.items():
            yield scrapy.Request(url, meta={'site': site}, callback=self.parse)

    def parse(self, response):
        if response:
            parser = getattr(self, response.meta['site'])
            parser(response)

    # site specific parsers
    def thesaurus(self, response):
        filename = 'thesaurus.txt'

        words = response.css('#filters-0 > div.relevancy-block > div > ul> li > a > span.text::text').extract()

        save_words(words, filename)

    def oxford(self, response):
        filename = 'oxford.txt'

        bolds = response.css('#content > div.lex-container > div.main-content > div > div > div > div > section > div > div:nth-child(1) > div > div > div.synList > div > p > strong::text').extract()

        norms = response.css('#content > div.lex-container > div.main-content > div > div > div > div > section > div > div:nth-child(1) > div > div > div.synList > div > p > span:not(.sense-registers)::text').extract()

        edited = []

        for csv in norms:
            edit = re.split(',(\s)+', csv)
            for word in edit:
                if len(word) > 1:
                    edited.append(word)

        words = bolds + edited

        save_words(words, filename)


    def collins(self, response):
        filename = 'collins.txt'
        words = []
        bold = response.css('body > main > div.dictionary > div.res_cell_center > div > div.homograph-entry > div > div.entry.dictionary.thesbase > div > div.hom > div:nth-child(2) > div.synonymBlock > span.sensehead > span.key::text').extract()

        others = response.css('body > main > div.dictionary > div.res_cell_center > div > div.homograph-entry > div > div.entry.dictionary.thesbase > div > div.hom > h2.titleTypeContainer + div.sense > div.synonymBlock > span.firstSyn > span::text').extract()

        others2 = response.css('body > main > div.dictionary > div.res_cell_center > div > div.homograph-entry > div > div.entry.dictionary.thesbase > div > div.hom > h2.titleTypeContainer + div > div.containerBlock > div.blockSyn > div> a > span::text').extract()

        others3 = response.css('body > main > div.dictionary > div.res_cell_center > div > div.homograph-entry > div > div.entry.dictionary.thesbase > div > div.hom > h2.titleTypeContainer + div > div.containerBlock > div.blockSyn > div > span::text').extract()

        words = bold + others + others2 + others3


        save_words(words, filename)
\$\endgroup\$
  • \$\begingroup\$ I do have one comment on the python side. Instead of importing everything on one line you should import each library on separate lines. Here is the PEP 8 Link that explains the correct way to import. \$\endgroup\$ – Mike - SMT Jun 28 '17 at 19:42
  • \$\begingroup\$ The github project doesn't appear to exist (anymore)... is it still available, or did you get feedback on this code elsewhere? \$\endgroup\$ – Sᴀᴍ Onᴇᴌᴀ Jan 22 '18 at 21:29
  • \$\begingroup\$ I recently cleaned up old, unattractive projects from my repo while recruiters were scoping me out. Sorry mate. \$\endgroup\$ – Naltroc Jan 23 '18 at 1:37
  • \$\begingroup\$ okay- since the GitHub repo isn't available anymore, can you tell us how the python and PHP files are connected/interact? \$\endgroup\$ – Sᴀᴍ Onᴇᴌᴀ Jun 1 '18 at 23:18

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Browse other questions tagged or ask your own question.