3
\$\begingroup\$

I am trying to optimize this python script that is used to process web requests for machine translation. The actual translation executable that is called is quite fast. Also, the perl scripts that are called are fast as well.

The largest performance boost came from removing unnecessary import libraries. I would like to have this code reviewed so I can further optimize the performance. Also, I welcome any advice on a pythonic way of testing performance. My code is littered with timing and print commands that I removed for this post.

#!/usr/bin/env python
# -*- coding: UTF-8 -*-

import time
import sys
import cgi
import subprocess
import string
import xmlrpclib

reload(sys)
sys.setdefaultencoding('utf8')

isTestPerformance = len(sys.argv) == 4

# Parameters
if isTestPerformance:
    source = sys.argv[1]
    target = sys.argv[2]
    sourceText = sys.argv[3]
else:
    # this part is important to tell the browser that output is html text.
    print "Access-Control-Allow-Origin: *"
    print "Content-Type: text/plain;charset=utf-8"
    print

    form = cgi.FieldStorage()
    sourceText = form.getvalue("sourceText").decode('utf8')
    source = form.getvalue("source").lower()
    target = form.getvalue("target").lower()

# Decode the CGI encoded source text
# NOTE: Custom encoding of semicolon (;), (?), (&), (#), etc, is only done here because 
# CGI can not handle them.  Do not used this (decode) if you are not using CGI, 
# or use some other decoding that matches the encoding from the caller of this code
sourceText = sourceText.replace("__QUESTION_MARK__", "?")
sourceText = sourceText.replace("__SEMICOLON__", ";")
sourceText = sourceText.replace("__AMPERSAND__", "&")
sourceText = sourceText.replace("__NUMBER__", "#")
# sourceText = sourceText.replace("__NEWLINE__", "\n")

# Tokenize the Source Text
if source == "zh":
    # Chinese has to do word alignment
    # options are slim: write the text to a file
    # use NLTK Stanford NLP (python>java) to segment chinese phrase
    # then read the file and get the segmented phrase and continue 
    # TODO 
    # solution found (kinda) mini-segmenter
    # https://github.com/alvations/mini-segmenter
    import miniseg.minisegmenter as mini
    src_tok = mini.segmenter(sourceText)
else:
    cmd = "/usr/bin/perl"
    perlscript = "/home/steve/mosesdecoder/scripts/tokenizer/tokenizer.perl"
    option = "-l"
    lang = source
    proc = subprocess.Popen([cmd, perlscript, option, lang], stdin=subprocess.PIPE, stdout=subprocess.PIPE)
    proc.stdin.write(sourceText)
    src_tok = proc.communicate()[0]
    # print src_tok

# Build URL Proxy to call XML-RPC
if source == 'en' and target == 'zh':
    port = '3001'
if source == 'en' and target == 'de':
    port = '3002'
if source == 'en' and target == 'es':
    port = '3003'
if source == 'en' and target == 'fr':
    port = '3004'
if source == 'en' and target == 'it':
    port = '3005'
if source == 'en' and target == 'nl':
    port = '3006'
if source == 'en' and target == 'pl':
    port = '3007'
if source == 'en' and target == 'pt':
    port = '3008'
if source == 'en' and target == 'ro':
    port = '3009'
if source == 'en' and target == 'ru':
    port = '3010'
if source == 'en' and target == 'sl':
    port = '3011'
if source == 'en' and target == 'hr':
    port = '3012'
if source == 'en' and target == 'tr':
    port = '3013'
if source == 'en' and target == 'ar':
    port = '3014'
if source == 'en' and target == 'fa':
    port = '3015'

if source == 'zh' and target == 'en':
    port = '4001'
if source == 'de' and target == 'en':
    port = '4002'
if source == 'es' and target == 'en':
    port = '4003'
if source == 'fr' and target == 'en':
    port = '4004'
if source == 'it' and target == 'en':
    port = '4005'
if source == 'nl' and target == 'en':
    port = '4006'
if source == 'pl' and target == 'en':
    port = '4007'
if source == 'pt' and target == 'en':
    port = '4008'
if source == 'ro' and target == 'en':
    port = '4009'
if source == 'ru' and target == 'en':
    port = '4010'
if source == 'sl' and target == 'en':
    port = '4011'
if source == 'hr' and target == 'en':
    port = '4012'
if source == 'tr' and target == 'en':
    port = '4013'
if source == 'ar' and target == 'en':
    port = '4014'
if source == 'fa' and target == 'en':
    port = '4015'

url = "http://localhost:" + port + "/RPC2"

proxy = xmlrpclib.ServerProxy(url)

# Translate the Source Text
params = {"text": src_tok, "align": "false", "report-all-factors": "false"}
result = proxy.translate(params)

targetText = result['text'].encode('utf-8')

# Post-Process the Output 
# TODO
if target == "zh":
    # Chinese - Get rid of the spaces (word segmentation)
    targetText = targetText.replace(" ", "")

# Post-Processes the translation output (regardless of language)
targetText = targetText.replace("__UNK__,", ",")
targetText = targetText.replace("__UNK__", " ")
#targetText = targetText.replace(" _ _ NEWLINE _ _ ", "\n")
targetText = targetText.replace("  ", " ")

if target in ["en", "fr", "es", "de", "it", "nl", "pl", "pt", "ro", "ru", "sl", "hr", "tr", "ar", "fa"]:
    # Detokenize the target translation 
    cmd = "/usr/bin/perl"
    perlscript = "/home/steve/mosesdecoder/scripts/tokenizer/detokenizer.perl"
    option = "-l"
    # lang = target
    lang = "en" # THIS SHOULD BE target variable
    proc = subprocess.Popen([cmd, perlscript, option, lang], stdin=subprocess.PIPE, stdout=subprocess.PIPE)
    proc.stdin.write(targetText)
    targetText = proc.communicate()[0]

    # Normalize Punctuation
    # French - 
    # Detokenize the target translation
    cmd = "/usr/bin/perl"
    perlscript = "/home/steve/mosesdecoder/scripts/tokenizer/normalize-punctuation.perl"
    option = "-l"
    lang = target
    proc = subprocess.Popen([cmd, perlscript, option, lang], stdin=subprocess.PIPE, stdout=subprocess.PIPE)
    proc.stdin.write(targetText)
    targetText = proc.communicate()[0]
    # True-case the target translation
    # TODO: Do I need to true case the target output?

print targetText
\$\endgroup\$
  • \$\begingroup\$ I'm intrigued by your comment about how "Custom encoding … is only done here because CGI can not handle them." Could you elaborate on that statement? How is it that nobody else has this problem? \$\endgroup\$ – 200_success May 16 '17 at 22:27
  • \$\begingroup\$ @200_success, when a question mark, semicolon, ampersand, or pound (#) is included in the querystring, I am unable to retrieve any parameters using the python CGI library. When I assign the parameters to variables with cgi.FieldStorage().getvalue("sourceText"), for example, all of those variables have a value of 'None'. \$\endgroup\$ – Steve3p0 May 16 '17 at 23:05
  • \$\begingroup\$ How are you passing the text to the CGI? Are you properly applying percent encoding? \$\endgroup\$ – 200_success May 16 '17 at 23:08
  • \$\begingroup\$ @200_success, thanks for the feedback! I am looking up how to properly URL encode in javascript and URL decode in python right now. If you have any suggestions on how to do this, please share. \$\endgroup\$ – Steve3p0 May 16 '17 at 23:26
  • \$\begingroup\$ @200_success, I am having problems encoding and decoding this without using my custom code. I am trying Javascript's encodeURIComponent, but it doesn't appear I can (or know how to) decode that in the python (server-side) script. Any suggestions? \$\endgroup\$ – Steve3p0 May 17 '17 at 0:01
2
\$\begingroup\$

I don't really know what moses or RPC2 are, and so I've not tested your code, or changed it too much. However, there are still some things you can do to make your code much easier to understand and maintain.

You want to use a lot more functions, and I would use a couple of classes. There are two main classes that I'd make Moses and Translater. The former should be your interface with Moses, where Translater should allow you to translate from one language to another with relative ease. Doing this will allow you to re-use the code in an easy way.

This allows you to remove most of your comments, as now you have variable names that tell us the same as your comments. It also means that if you need to explain them in greater detail then you can in a docstring.

I'd also change your massive if block to a dictionary. This will allow you to define the source and target as keys, and then return the ports.

The performance problems that you have will almost definitely be due to executing three other Perl shells, and interfacing with another server. You're unlikely able to fix this, unless you translate the code from Perl to Python / C, or there's a native Python interface, that doesn't have a slow interface.

You don't need a lot of your imports, and so I'd remove them. I'd also recommend against using most of sys, as the docs say they aren't intended for non-internal use, and so you're setting yourself up to have horrific bugs.

Finally, I'd recommend that you use a better web framework, such as Flask or Django. This would allow for a simpler interface, and mean that isTestPerformance, and the odd prints aren't needed. And should allow you to remove the 'CGI Custom encoding' from the file, as these should have solid implementations of percent encoding, or better yet, allow you to post to the server. Which your program doesn't seem to support very well.

In all I changed your code to:

#!/usr/bin/env python
# -*- coding: UTF-8 -*-
import os.path
import subprocess
import sys
import xmlrpclib


PORTS = {
    ('en', 'zh'): '3001',
    ('en', 'de'): '3002',
    ('en', 'es'): '3003',
    ('en', 'fr'): '3004',
    ('en', 'it'): '3005',
    ('en', 'nl'): '3006',
    ('en', 'pl'): '3007',
    ('en', 'pt'): '3008',
    ('en', 'ro'): '3009',
    ('en', 'ru'): '3010',
    ('en', 'sl'): '3011',
    ('en', 'hr'): '3012',
    ('en', 'tr'): '3013',
    ('en', 'ar'): '3014',
    ('en', 'fa'): '3015',

    ('zh', 'en'): '4001',
    ('de', 'en'): '4002',
    ('es', 'en'): '4003',
    ('fr', 'en'): '4004',
    ('it', 'en'): '4005',
    ('nl', 'en'): '4006',
    ('pl', 'en'): '4007',
    ('pt', 'en'): '4008',
    ('ro', 'en'): '4009',
    ('ru', 'en'): '4010',
    ('sl', 'en'): '4011',
    ('hr', 'en'): '4012',
    ('tr', 'en'): '4013',
    ('ar', 'en'): '4014',
    ('fa', 'en'): '4015',
}


class Moses(object):
    def __init__(self, path):
        self.tokenizer_path = path

    def _call(self, path):
        pipe = subprocess.Popen(["perl", path, "-l", lang], stdin=subprocess.PIPE, stdout=subprocess.PIPE)
        return pipe.communicate(text)[0]

    def tokenize(self, lang, text):
        return _call(os.path.join(self.tokenizer_path, "/scripts/tokenizer/tokenizer.perl"))

    def detokenize(self, lang, text):
        return _call(os.path.join(self.tokenizer_path, "/scripts/tokenizer/detokenizer.perl"))

    def normalize_punctuation(self, lang, text):
        return _call(os.path.join(self.tokenizer_path, "/scripts/tokenizer/normalize-punctuation.perl"))


class Translater(object):
    def __init__(self, path):
        self.moses = Moses(path)

    def _tokenize(self, source, target, text):
        if source == "zh":
            # Chinese has to do word alignment
            # options are slim: write the text to a file
            # use NLTK Stanford NLP (python>java) to segment chinese phrase
            # then read the file and get the segmented phrase and continue 
            # TODO 
            # solution found (kinda) mini-segmenter
            # https://github.com/alvations/mini-segmenter
            import miniseg.minisegmenter as mini
            return mini.segmenter(text)
        else:
            return self.moses.tokenize(source, text)

    def _translate(self, source, text):
        port = PORTS[(source, target)]
        proxy = xmlrpclib.ServerProxy("http://localhost:" + port + "/RPC2")
        params = {"text": text, "align": "false", "report-all-factors": "false"}
        result = proxy.translate(params)
        return result['text'].encode('utf-8')

    def _post_process(self, target, text):
        if target == "zh":
            # Chinese - Get rid of the spaces (word segmentation)
            text = text.replace(" ", "")
        # Post-Processes the translation output (regardless of language)
        text = text.replace("__UNK__,", ",")
        text = text.replace("__UNK__", " ")
        #text = text.replace(" _ _ NEWLINE _ _ ", "\n")
        text = text.replace("  ", " ")
        return text

    def _detokenize(self, target, text):
        if target == "zh":
            pass
        else:
            text = self.moses.detokenize("en", text)
            text = self.moses.normalize_punctuation(target, text)
        return text

    def translate(self, source, target, text):
        text = self._tokenize(source, target, text)
        text = self._translate(source, text)
        text = self._post_process(target, text)
        text = self._detokenize(target, text)
        return text


def read_args():
    isTestPerformance = len(sys.argv) == 4
    if isTestPerformance:
        return tuple(sys.argv[1:4])
    else:
        # this part is important to tell the browser that output is html text.
        print "Access-Control-Allow-Origin: *"
        print "Content-Type: text/plain;charset=utf-8"
        print

        form = cgi.FieldStorage()
        return (
            form.getvalue("source").lower(),
            form.getvalue("target").lower(),
            form.getvalue("sourceText").decode('utf8')
        )


def custom_decode(text):
    # Decode the CGI encoded source text
    # NOTE: Custom encoding of semicolon (;), (?), (&), (#), etc, is only done here because 
    # CGI can not handle them.  Do not used this (decode) if you are not using CGI, 
    # or use some other decoding that matches the encoding from the caller of this code
    text = text.replace("__QUESTION_MARK__", "?")
    text = text.replace("__SEMICOLON__", ";")
    text = text.replace("__AMPERSAND__", "&")
    text = text.replace("__NUMBER__", "#")
    #text = text.replace("__NEWLINE__", "\n")
    return text


def main():
    source, target, text = read_args()
    text = custom_decode(text)
    print Translater("/home/steve/mosesdecoder")
            .translate(source, target, text)


main()
\$\endgroup\$

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Not the answer you're looking for? Browse other questions tagged or ask your own question.