Python CGI front-end for web service to perform machine translation

Question

I am trying to optimize this python script that is used to process web requests for machine translation. The actual translation executable that is called is quite fast. Also, the perl scripts that are called are fast as well.

The largest performance boost came from removing unnecessary import libraries. I would like to have this code reviewed so I can further optimize the performance. Also, I welcome any advice on a pythonic way of testing performance. My code is littered with timing and print commands that I removed for this post.

#!/usr/bin/env python
# -*- coding: UTF-8 -*-

import time
import sys
import cgi
import subprocess
import string
import xmlrpclib

reload(sys)
sys.setdefaultencoding('utf8')

isTestPerformance = len(sys.argv) == 4

# Parameters
if isTestPerformance:
    source = sys.argv[1]
    target = sys.argv[2]
    sourceText = sys.argv[3]
else:
    # this part is important to tell the browser that output is html text.
    print "Access-Control-Allow-Origin: *"
    print "Content-Type: text/plain;charset=utf-8"
    print

    form = cgi.FieldStorage()
    sourceText = form.getvalue("sourceText").decode('utf8')
    source = form.getvalue("source").lower()
    target = form.getvalue("target").lower()

# Decode the CGI encoded source text
# NOTE: Custom encoding of semicolon (;), (?), (&), (#), etc, is only done here because 
# CGI can not handle them.  Do not used this (decode) if you are not using CGI, 
# or use some other decoding that matches the encoding from the caller of this code
sourceText = sourceText.replace("__QUESTION_MARK__", "?")
sourceText = sourceText.replace("__SEMICOLON__", ";")
sourceText = sourceText.replace("__AMPERSAND__", "&")
sourceText = sourceText.replace("__NUMBER__", "#")
# sourceText = sourceText.replace("__NEWLINE__", "\n")

# Tokenize the Source Text
if source == "zh":
    # Chinese has to do word alignment
    # options are slim: write the text to a file
    # use NLTK Stanford NLP (python>java) to segment chinese phrase
    # then read the file and get the segmented phrase and continue 
    # TODO 
    # solution found (kinda) mini-segmenter
    # https://github.com/alvations/mini-segmenter
    import miniseg.minisegmenter as mini
    src_tok = mini.segmenter(sourceText)
else:
    cmd = "/usr/bin/perl"
    perlscript = "/home/steve/mosesdecoder/scripts/tokenizer/tokenizer.perl"
    option = "-l"
    lang = source
    proc = subprocess.Popen([cmd, perlscript, option, lang], stdin=subprocess.PIPE, stdout=subprocess.PIPE)
    proc.stdin.write(sourceText)
    src_tok = proc.communicate()[0]
    # print src_tok

# Build URL Proxy to call XML-RPC
if source == 'en' and target == 'zh':
    port = '3001'
if source == 'en' and target == 'de':
    port = '3002'
if source == 'en' and target == 'es':
    port = '3003'
if source == 'en' and target == 'fr':
    port = '3004'
if source == 'en' and target == 'it':
    port = '3005'
if source == 'en' and target == 'nl':
    port = '3006'
if source == 'en' and target == 'pl':
    port = '3007'
if source == 'en' and target == 'pt':
    port = '3008'
if source == 'en' and target == 'ro':
    port = '3009'
if source == 'en' and target == 'ru':
    port = '3010'
if source == 'en' and target == 'sl':
    port = '3011'
if source == 'en' and target == 'hr':
    port = '3012'
if source == 'en' and target == 'tr':
    port = '3013'
if source == 'en' and target == 'ar':
    port = '3014'
if source == 'en' and target == 'fa':
    port = '3015'

if source == 'zh' and target == 'en':
    port = '4001'
if source == 'de' and target == 'en':
    port = '4002'
if source == 'es' and target == 'en':
    port = '4003'
if source == 'fr' and target == 'en':
    port = '4004'
if source == 'it' and target == 'en':
    port = '4005'
if source == 'nl' and target == 'en':
    port = '4006'
if source == 'pl' and target == 'en':
    port = '4007'
if source == 'pt' and target == 'en':
    port = '4008'
if source == 'ro' and target == 'en':
    port = '4009'
if source == 'ru' and target == 'en':
    port = '4010'
if source == 'sl' and target == 'en':
    port = '4011'
if source == 'hr' and target == 'en':
    port = '4012'
if source == 'tr' and target == 'en':
    port = '4013'
if source == 'ar' and target == 'en':
    port = '4014'
if source == 'fa' and target == 'en':
    port = '4015'

url = "http://localhost:" + port + "/RPC2"

proxy = xmlrpclib.ServerProxy(url)

# Translate the Source Text
params = {"text": src_tok, "align": "false", "report-all-factors": "false"}
result = proxy.translate(params)

targetText = result['text'].encode('utf-8')

# Post-Process the Output 
# TODO
if target == "zh":
    # Chinese - Get rid of the spaces (word segmentation)
    targetText = targetText.replace(" ", "")

# Post-Processes the translation output (regardless of language)
targetText = targetText.replace("__UNK__，", "，")
targetText = targetText.replace("__UNK__", " ")
#targetText = targetText.replace(" _ _ NEWLINE _ _ ", "\n")
targetText = targetText.replace("  ", " ")

if target in ["en", "fr", "es", "de", "it", "nl", "pl", "pt", "ro", "ru", "sl", "hr", "tr", "ar", "fa"]:
    # Detokenize the target translation 
    cmd = "/usr/bin/perl"
    perlscript = "/home/steve/mosesdecoder/scripts/tokenizer/detokenizer.perl"
    option = "-l"
    # lang = target
    lang = "en" # THIS SHOULD BE target variable
    proc = subprocess.Popen([cmd, perlscript, option, lang], stdin=subprocess.PIPE, stdout=subprocess.PIPE)
    proc.stdin.write(targetText)
    targetText = proc.communicate()[0]

    # Normalize Punctuation
    # French - 
    # Detokenize the target translation
    cmd = "/usr/bin/perl"
    perlscript = "/home/steve/mosesdecoder/scripts/tokenizer/normalize-punctuation.perl"
    option = "-l"
    lang = target
    proc = subprocess.Popen([cmd, perlscript, option, lang], stdin=subprocess.PIPE, stdout=subprocess.PIPE)
    proc.stdin.write(targetText)
    targetText = proc.communicate()[0]
    # True-case the target translation
    # TODO: Do I need to true case the target output?

print targetText

Answer 1

I don't really know what moses or RPC2 are, and so I've not tested your code, or changed it too much. However, there are still some things you can do to make your code much easier to understand and maintain.

You want to use a lot more functions, and I would use a couple of classes. There are two main classes that I'd make Moses and Translater. The former should be your interface with Moses, where Translater should allow you to translate from one language to another with relative ease. Doing this will allow you to re-use the code in an easy way.

This allows you to remove most of your comments, as now you have variable names that tell us the same as your comments. It also means that if you need to explain them in greater detail then you can in a docstring.

I'd also change your massive if block to a dictionary. This will allow you to define the source and target as keys, and then return the ports.

The performance problems that you have will almost definitely be due to executing three other Perl shells, and interfacing with another server. You're unlikely able to fix this, unless you translate the code from Perl to Python / C, or there's a native Python interface, that doesn't have a slow interface.

You don't need a lot of your imports, and so I'd remove them. I'd also recommend against using most of sys, as the docs say they aren't intended for non-internal use, and so you're setting yourself up to have horrific bugs.

Finally, I'd recommend that you use a better web framework, such as Flask or Django. This would allow for a simpler interface, and mean that isTestPerformance, and the odd prints aren't needed. And should allow you to remove the 'CGI Custom encoding' from the file, as these should have solid implementations of percent encoding, or better yet, allow you to post to the server. Which your program doesn't seem to support very well.

In all I changed your code to:

#!/usr/bin/env python
# -*- coding: UTF-8 -*-
import os.path
import subprocess
import sys
import xmlrpclib


PORTS = {
    ('en', 'zh'): '3001',
    ('en', 'de'): '3002',
    ('en', 'es'): '3003',
    ('en', 'fr'): '3004',
    ('en', 'it'): '3005',
    ('en', 'nl'): '3006',
    ('en', 'pl'): '3007',
    ('en', 'pt'): '3008',
    ('en', 'ro'): '3009',
    ('en', 'ru'): '3010',
    ('en', 'sl'): '3011',
    ('en', 'hr'): '3012',
    ('en', 'tr'): '3013',
    ('en', 'ar'): '3014',
    ('en', 'fa'): '3015',

    ('zh', 'en'): '4001',
    ('de', 'en'): '4002',
    ('es', 'en'): '4003',
    ('fr', 'en'): '4004',
    ('it', 'en'): '4005',
    ('nl', 'en'): '4006',
    ('pl', 'en'): '4007',
    ('pt', 'en'): '4008',
    ('ro', 'en'): '4009',
    ('ru', 'en'): '4010',
    ('sl', 'en'): '4011',
    ('hr', 'en'): '4012',
    ('tr', 'en'): '4013',
    ('ar', 'en'): '4014',
    ('fa', 'en'): '4015',
}


class Moses(object):
    def __init__(self, path):
        self.tokenizer_path = path

    def _call(self, path):
        pipe = subprocess.Popen(["perl", path, "-l", lang], stdin=subprocess.PIPE, stdout=subprocess.PIPE)
        return pipe.communicate(text)[0]

    def tokenize(self, lang, text):
        return _call(os.path.join(self.tokenizer_path, "/scripts/tokenizer/tokenizer.perl"))

    def detokenize(self, lang, text):
        return _call(os.path.join(self.tokenizer_path, "/scripts/tokenizer/detokenizer.perl"))

    def normalize_punctuation(self, lang, text):
        return _call(os.path.join(self.tokenizer_path, "/scripts/tokenizer/normalize-punctuation.perl"))


class Translater(object):
    def __init__(self, path):
        self.moses = Moses(path)

    def _tokenize(self, source, target, text):
        if source == "zh":
            # Chinese has to do word alignment
            # options are slim: write the text to a file
            # use NLTK Stanford NLP (python>java) to segment chinese phrase
            # then read the file and get the segmented phrase and continue 
            # TODO 
            # solution found (kinda) mini-segmenter
            # https://github.com/alvations/mini-segmenter
            import miniseg.minisegmenter as mini
            return mini.segmenter(text)
        else:
            return self.moses.tokenize(source, text)

    def _translate(self, source, text):
        port = PORTS[(source, target)]
        proxy = xmlrpclib.ServerProxy("http://localhost:" + port + "/RPC2")
        params = {"text": text, "align": "false", "report-all-factors": "false"}
        result = proxy.translate(params)
        return result['text'].encode('utf-8')

    def _post_process(self, target, text):
        if target == "zh":
            # Chinese - Get rid of the spaces (word segmentation)
            text = text.replace(" ", "")
        # Post-Processes the translation output (regardless of language)
        text = text.replace("__UNK__，", "，")
        text = text.replace("__UNK__", " ")
        #text = text.replace(" _ _ NEWLINE _ _ ", "\n")
        text = text.replace("  ", " ")
        return text

    def _detokenize(self, target, text):
        if target == "zh":
            pass
        else:
            text = self.moses.detokenize("en", text)
            text = self.moses.normalize_punctuation(target, text)
        return text

    def translate(self, source, target, text):
        text = self._tokenize(source, target, text)
        text = self._translate(source, text)
        text = self._post_process(target, text)
        text = self._detokenize(target, text)
        return text


def read_args():
    isTestPerformance = len(sys.argv) == 4
    if isTestPerformance:
        return tuple(sys.argv[1:4])
    else:
        # this part is important to tell the browser that output is html text.
        print "Access-Control-Allow-Origin: *"
        print "Content-Type: text/plain;charset=utf-8"
        print

        form = cgi.FieldStorage()
        return (
            form.getvalue("source").lower(),
            form.getvalue("target").lower(),
            form.getvalue("sourceText").decode('utf8')
        )


def custom_decode(text):
    # Decode the CGI encoded source text
    # NOTE: Custom encoding of semicolon (;), (?), (&), (#), etc, is only done here because 
    # CGI can not handle them.  Do not used this (decode) if you are not using CGI, 
    # or use some other decoding that matches the encoding from the caller of this code
    text = text.replace("__QUESTION_MARK__", "?")
    text = text.replace("__SEMICOLON__", ";")
    text = text.replace("__AMPERSAND__", "&")
    text = text.replace("__NUMBER__", "#")
    #text = text.replace("__NEWLINE__", "\n")
    return text


def main():
    source, target, text = read_args()
    text = custom_decode(text)
    print Translater("/home/steve/mosesdecoder")
            .translate(source, target, text)


main()