Python library for tio.run interaction

Question

What started as a direct and simple fork of a CodeGolf.SE member's TioJ Java implementation to interact with the tio.run (we'll just call it TIO from herein) website with a bot or other command line script ended up becoming its own project for me to fuss around with to be more Pythonic and adaptive for how I would probably consider using it.

Essentially, this is a Python way to submit code snippets for running on https://tio.run and get back either a response from a valid code execution, or an error if any error happened to occur, either in the code being executed on TIO or some type of error on the computer's side.

It's effectively several separate files, all integrated as a part of a Python library. To that end, I've made all the subfiles containing classes start with _ to indicate they're protected files, and have the package __init__.py import the individual classes from within the subfiles so that we can do from pytio import {CLASSNAME} where we replace the actual name of the class/object type in the import statement.

Any improvements are welcome. Note that this is essentially a near-direct port of the TioJ code, with some Python 2/3 compatibility changes among other things. It's probably got a lot of useless stuff here, but meh, this is the first usable version of the library thus far.

However, keep in mind that both Python 2 and Python 3 need to be supported, so if you give any recommendations, please don't break cross-compatibility, since that's required for Universal-wheel PyPI packages like this one.

Python 2 compatibility is doable because typing exists in PyPI for Python 2, once it was introduced for Python 3.

It's got several parts. Most are within a directory called pytio, while the test_tio.py unit test test-suite is outside that directory.

---

---

pytio/__init__.py:

from ._Tio import Tio
from ._TioFile import TioFile
from ._TioRequest import TioRequest
from ._TioResponse import TioResponse
from ._TioResult import TioResult
from ._TioVariable import TioVariable

__title__ = 'PyTIO'
__author__ = 'Thomas Ward'
__version__ = '0.1.0'
__copyright__ = '2017 Thomas Ward'
__license__ = 'AGPLv3+'
__all__ = ('Tio', 'TioFile', 'TioRequest', 'TioResponse', 'TioResult', 'TioVariable')

pytio/_Tio.py:

import gzip
import io
import json
import platform
from typing import AnyStr, Union

from ._TioRequest import TioRequest
from ._TioResponse import TioResponse

# Version specific import handling.
if platform.python_version() <= '3.0':
    # Python 2: The specific URLLib sections are in urllib2.

    # noinspection PyCompatibility,PyUnresolvedReferences
    from urllib2 import urlopen
    # noinspection PyCompatibility,PyUnresolvedReferences
    from urllib2 import HTTPError, URLError
else:
    # Python 3: The specific URLLib sections are in urllib submodules.

    # noinspection PyCompatibility
    from urllib.request import urlopen
    # noinspection PyCompatibility
    from urllib.error import HTTPError, URLError


class Tio:
    backend = "cgi-bin/run/api/"
    json = "languages.json"

    def __init__(self, url="https://tio.run"):
        # type: (AnyStr) -> None
        self.backend = url + '/' + self.backend
        self.json = url + '/' + self.json

    @staticmethod
    def read_in_chunks(stream_object, chunk_size=1024):
        """Lazy function (generator) to read a file piece by piece.
        Default chunk size: 1k."""
        while True:
            data = stream_object.read(chunk_size)
            if not data:
                break
            yield data

    @staticmethod
    def new_request(*args, **kwargs):
        # type: () -> None
        raise DeprecationWarning("The Tio.new_request() method is to be removed in a later release; please call "
                                 "TioRequest and its constructor directly..")

    def query_languages(self):
        # type: () -> set
        # Used to get a set containing all supported languages on TIO.run.
        try:
            response = urlopen(self.json)
            rawdata = json.loads(response.read().decode('utf-8'))
            return set(rawdata.keys())
        except (HTTPError, URLError):
            return set()
        except Exception:
            return set()

    def send(self, fmt):
        # type: (TioRequest) -> TioResponse
        # Command alias to use send_bytes; this is more or less a TioJ cutover.
        return self.send_bytes(fmt.as_deflated_bytes())

    def send_bytes(self, message):
        # type: (bytes) -> TioResponse
        req = urlopen(self.backend, data=message)
        reqcode = req.getcode()
        if req.code == 200:
            if platform.python_version() >= '3.0':
                content_type = req.info().get_content_type()
            else:
                # URLLib requests/responses in Python 2 don't have info().get_content_type(),
                # so let's get it the old fashioned way.
                content_type = req.info()['content-type']

            # Specially handle GZipped responses from the server, and unzip them.
            if content_type == 'application/octet-stream':
                buf = io.BytesIO(req.read())
                gzip_f = gzip.GzipFile(fileobj=buf)
                fulldata = gzip_f.read()
            else:
                # However, if it's not compressed, just read it directly.
                fulldata = req.read()

            # Return a TioResponse object, containing the returned data from TIO.
            return TioResponse(reqcode, fulldata, None)
        else:
            # If the HTTP request failed, we need to give a TioResponse object with no data.
            return TioResponse(reqcode, None, None)

pytio/_TioFile.py:

from typing import AnyStr


class TioFile:
    _name = str()
    _content = bytes()

    def __init__(self, name, content):
        # type: (AnyStr, bytes) -> None
        self._name = name
        self._content = content

    def get_name(self):
        # type: () -> AnyStr
        return self.name

    def get_content(self):
        # type: () -> bytes
        return self.content

    @property
    def name(self):
        # type: () -> AnyStr
        return self._name

    @property
    def content(self):
        # type: () -> bytes
        return self._content

pytio/_TioRequest.py:

import platform
import zlib
from typing import List, AnyStr, Union
from ._TioFile import TioFile
from ._TioVariable import TioVariable


class TioRequest:

    def __init__(self, lang=None, code=None):
        # type: (AnyStr, Union[AnyStr, bytes]) -> None
        self._files = []
        self._variables = []
        self._bytes = bytes()

        if lang:
            self.set_lang(lang)

        if code:
            self.add_file_bytes('.code.tio', code)

    def add_file(self, file):
        # type: (TioFile) -> None
        if file in self._files:
            self._files.remove(file)
        self._files.append(file)

    def add_file_bytes(self, name, content):
        # type: (AnyStr, bytes) -> None
        self._files.append(TioFile(name, content))

    def add_variable(self, variable):
        # type: (TioVariable) -> None
        self._variables.append(variable)

    def add_variable_string(self, name, value):
        # type: (AnyStr, Union[List[AnyStr], AnyStr]) -> None
        self._variables.append(TioVariable(name, value))

    def set_lang(self, lang):
        # type: (AnyStr) -> None
        self.add_variable_string('lang', lang)

    def set_code(self, code):
        # type: (AnyStr) -> None
        self.add_file_bytes('.code.tio', code)

    def set_input(self, input_data):
        # type: (AnyStr) -> None
        self.add_file_bytes('.input.tio', input_data.encode('utf-8'))

    def set_compiler_flags(self, flags):
        # type: (AnyStr) -> None
        self.add_variable_string('TIO_CFLAGS', flags)

    def set_commandline_flags(self, flags):
        # type: (AnyStr) -> None
        self.add_variable_string('TIO_OPTIONS', flags)

    def set_arguments(self, args):
        # type: (AnyStr) -> None
        self.add_variable_string('args', args)

    def write_variable(self, name, content):
        # type: (AnyStr, AnyStr) -> None
        if content:
            if platform.python_version() >= '3.0':
                self._bytes += bytes("V" + name + '\x00' + str(len(content.split(' '))) + '\x00', 'utf-8')
                self._bytes += bytes(content + '\x00', 'utf-8')
            else:
                # Python 2 is weird - this is effectively a call to just 'str', somehow, so no encoding argument.
                self._bytes += bytes("V" + name + '\x00' + str(len(content.split(' '))) + '\x00')
                self._bytes += bytes(content + '\x00')

    def write_file(self, name, contents):
        # type: (AnyStr, AnyStr) -> None
        # noinspection PyUnresolvedReferences

        if platform.python_version() < '3.0' and isinstance(contents, str):
            # Python 2 has weirdness - if it's Unicode bytes or a string, len() properly detects bytes length, and
            # not # of chars in the specific item.
            length = len(contents)
        elif isinstance(contents, str):
            # However, for Python 3, we need to first encode it in UTF-8 bytes if it is just a plain string...
            length = len(contents.encode('utf-8'))
        elif isinstance(contents, (bytes, bytearray)):
            # ... and in Python 3, we can only call len() directly if we're working with bytes or bytearray directly.
            length = len(contents)
        else:
            # Any other type of value will result in a code failure for now.
            raise ValueError("Can only pass UTF-8 strings or bytes at this time.")

        if platform.python_version() >= '3.0':
            self._bytes += bytes("F" + name + '\x00' + str(length) + '\x00', 'utf-8')
            self._bytes += bytes(contents + '\x00', 'utf-8')
        else:
            # Python 2 is weird - this is effectively a call to just 'str', somehow, so no encoding argument.
            self._bytes += bytes("F" + name + '\x00' + str(length) + '\x00')
            self._bytes += bytes(contents + '\x00')

    def as_bytes(self):
        # type: () -> bytes
        try:
            for var in self._variables:
                if hasattr(var, 'name') and hasattr(var, 'content'):
                    self.write_variable(var.name, var.content)

            for file in self._files:
                if hasattr(file, 'name') and hasattr(file, 'content'):
                    self.write_file(file.name, file.content)

            self._bytes += b'R'

        except IOError:
            raise RuntimeError("IOError generated during bytes conversion.")

        return self._bytes

    def as_deflated_bytes(self):
        # type: () -> bytes
        # This returns a DEFLATE-compressed bytestring, which is what TIO.run's API requires for the request
        # to be proccessed properly.
        return zlib.compress(self.as_bytes(), 9)[2:-4]

pytio/_TioResponse.py:

from typing import Optional, Any, Union, AnyStr


class TioResponse:
    _code = 0
    _data = None
    _result = None
    _error = None

    def __init__(self, code, data=None, error=None):
        # type: (Union[int, AnyStr], Optional[Any], Optional[Any]) -> None
        self._code = code
        self._data = data
        if data is None:
            self._splitdata = [None, error]
        else:
            self._splitdata = self._data.split(self._data[:16])

        if not self._splitdata[1] or self._splitdata[1] == b'':
            self._error = b''.join(self._splitdata[2:])
            self._result = None
        else:
            self._error = None
            self._result = self._splitdata[1]

    @property
    def code(self):
        # type: () -> Union[int, AnyStr]
        return self._code.decode('utf-8')

    @property
    def result(self):
        # type: () -> Optional[AnyStr]
        if self._result:
            return self._result.decode('utf-8')
        else:
            return None

    @property
    def error(self):
        # type: () -> Optional[AnyStr]
        if self._error:
            return self._error.decode('utf-8')
        else:
            return None

    @property
    def raw(self):
        # type: () -> Any
        return self._data

    def get_code(self):
        # type: () -> Union[int, AnyStr]
        return self.code

    def get_result(self):
        # type: () -> Optional[AnyStr]
        return self.result

    def get_error(self):
        # type: () -> Optional[AnyStr]
        return self.error

pytio/_TioResult.py:

from typing import Any, AnyStr, List


class TioResult:
    _pieces = []

    def __init__(self, pieces):
        # type: (List[AnyStr]) -> None
        self._pieces = pieces

        raise NotImplementedError

    @property
    def pieces(self):
        # type: () -> List
        return self._pieces

    def has(self, field):
        # type: (AnyStr) -> bool
        try:
            if field.lower() == "output":
                return len(self._pieces) > 0
            elif field.lower() == "debug":
                return len(self._pieces) > 1
            else:
                return False
        except IndexError:
            return False

    def get(self, field):
        # type: (AnyStr) -> Any
        if self.has('output') and field.lower() == "output":
            return self._pieces[0]
        elif self.has('debug') and field.lower() == "debug":
            return self._pieces[1]

pytio/_TioVariable.py:

from typing import AnyStr, List, Union


class TioVariable:
    _name = str()
    _content = []

    def __init__(self, name, content):
        # type: (AnyStr, Union[List[AnyStr], AnyStr]) -> None
        self._name = name
        self._content = content

    @property
    def name(self):
        # type: () -> AnyStr
        return self._name

    @property
    def content(self):
        # type: () -> List
        return self._content

    def get_name(self):
        # type: () -> AnyStr
        return self.name

    def get_content(self):
        # type: () -> List
        return self.content

... And of course, a unittest test-suite, which can be run in any Python 2.7 or Python 3 shell with:

# Python 2.7:
python -m unittest -v test_tio

# Python 3.x:
python3 -m unittest -v test_tio

test_tio.py (which sits outside the pytio directory, and it's that outer directory you should run the previous unittest execution commands in):

import platform
import unittest
from pytio import Tio, TioRequest


class TestTIOResults(unittest.TestCase):
    tio = Tio()

    def test_valid_python3_request(self):
        request = TioRequest(lang='python3', code="print('Hello, World!')")
        response = self.tio.send(request)
        self.assertIsNone(response.error)
        if platform.python_version() >= '3.0':
            self.assertIsInstance(response.result, str)
        else:
            # noinspection PyUnresolvedReferences
            self.assertIsInstance(response.result, (unicode, str))
        self.assertEqual(response.result.strip('\n'), "Hello, World!")

    def test_invald_python3_request(self):
        request = TioRequest(lang='python3', code="I'm a teapot!")
        response = self.tio.send(request)
        self.assertIsNone(response.result)
        if platform.python_version() >= '3.0':
            self.assertIsInstance(response.error, str)
        else:
            # noinspection PyUnresolvedReferences
            self.assertIsInstance(response.error, (unicode, str))
        self.assertIn('EOL while scanning string literal', response.error)

    def test_valid_apl_request(self):
        request = TioRequest(lang='apl-dyalog', code="⎕←'Hello, World!'")
        response = self.tio.send(request)
        self.assertIsNone(response.error)
        if platform.python_version() >= '3.0':
            self.assertIsInstance(response.result, str)
        else:
            # noinspection PyUnresolvedReferences
            self.assertIsInstance(response.result, (unicode, str))
        self.assertEqual(response.result.strip('\n'), "Hello, World!")

    def test_invalid_apl_request(self):
        request = TioRequest(lang='apl-dyalog', code="I'm a teapot!")
        response = self.tio.send(request)
        self.assertIsNone(response.result)
        if platform.python_version() >= '3.0':
            self.assertIsInstance(response.error, str)
        else:
            # noinspection PyUnresolvedReferences
            self.assertIsInstance(response.error, (unicode, str))
        self.assertIn('error AC0607: unbalanced quotes detected', response.error)


if __name__ == '__main__':
    unittest.main(warnings='ignore')

Answer 1

I'd highly recommend you use collections.namedtuple. Or since you're using typing, typing.NamedTuple. If we change TioFile to use this, then we'll get:

_TioFile = NamedTuple(
    '_TioFile',
    [
        ('name', AnyStr),
        ('content', bytes)
    ]
)


class TioFile(_TioFile):
    def get_name(self):
        # type: () -> AnyStr
        return self.name

    def get_content(self):
        # type: () -> bytes
        return self.content

From this, we know that get_name and get_content are actual not needed, and promote WET. Write it once for the property, once again for the method. And goes against PEP 20:

There should be one-- and preferably only one --obvious way to do it.

And so I'd also change the following classes to use NamedTuple.

• TioVariable can easily be changed to use NamedTuple, I would also remove the get_* parts as highlighted above.
• TioResult requires an EMPTY variable bound to the class. It should also have output and debug set to EMPTY by default. If we were 3.6.1 this would be super simple, as it allows setting default values. However as we have to support Python 2.7, it's simpler to just add a static method new that does this for you.
• TioResponse requires a static method say from_raw, that contains the old __init__ and should convert result and error to be decoded. This is as wrapping __new__ is a little messy, and PyCharm complains a lot.

---

As I don't really want to look at Tio, this leaves TioRequest. Which I'd change:

• You can use self.set_code rather than self.add_file_bytes('.code.tio', code).
• I wouldn't use self._bytes as that means it will duplicate the state of TioRequest if you ever call as_bytes twice. Instead just make it a local variable in as_bytes.
• I would move write_variable and write_file onto TioVariable and TioFile respectively, as as_byte.
• I would add a function bytes_ that in Python 3 is functools.partial(bytes, encoding='utf-8'), and in Python 2 is bytes. This can greatly simplify write_variable and write_file.
• I would move the if content out of write_variable into as_bytes.
• I would change platform.python_version() < '3.0' to be inverted, as then you can group all the len(content)s together.
• I would use str.format, rather than string concatenation in both write_variable and write_file.

And so I'd change your code to:

pytio/_TioObjects.py:

# coding=utf-8

from typing import NamedTuple, AnyStr, Union, List, Optional, Any
import platform
import functools

if platform.python_version() >= '3.0':
    bytes_ = functools.partial(bytes, encoding='utf-8')
else:
    bytes_ = bytes


_TioFile = NamedTuple(
    '_TioFile',
    [
        ('name', AnyStr),
        ('content', bytes)
    ]
)
_TioVariable = NamedTuple(
    '_TioVariable',
    [
        ('name', AnyStr),
        ('content', Union[List[AnyStr], AnyStr])
    ]
)
_TioResult = NamedTuple(
    '_TioResult',
    [
        ('output', Union[AnyStr, object]),
        ('debug', Union[AnyStr, object])
    ]
)
_TioResponse = NamedTuple(
    '_TioResponse',
    [
        ('code', Union[AnyStr, int]),
        ('result', Union[AnyStr, None]),
        ('error', Union[AnyStr, None]),
        ('raw', Any)
    ]
)


class TioFile(_TioFile):
    def as_bytes(self):
        # type: () -> bytes
        content = self.content
        if platform.python_version() >= '3.0' and isinstance(content, str):
            length = len(content.encode('utf-8'))
        elif isinstance(content, (str, bytes, bytearray)):
            length = len(content)
        else:
            raise ValueError("Can only pass UTF-8 strings or bytes at this time.")

        return bytes_(
            'F{name}\x00{length}\x00{content}\x00'
                .format(
                    name=self.name,
                    length=length,
                    content=self.content
                )
            )


class TioVariable(_TioVariable):
    def as_bytes(self):
        # type: () -> bytes
        return bytes_(
            'V{name}\x00{length}\x00{content}\x00'
                .format(
                    name=self.name,
                    length=len(self.content.split(' ')),
                    content=self.content
                )
            )


class TioResult(_TioResult):
    EMPTY = object()

    @staticmethod
    def new(output=EMPTY, debug=EMPTY):
        # type: (Optional[Union[AnyStr, object]], Optional[Union[AnyStr, object]]) -> TioResult
        return TioResult(output, debug)


class TioResponse(_TioResponse):
    @staticmethod
    def from_raw(code, data=None, error=None):
        # type: (Union[int, AnyStr], Optional[Any], Optional[Any]) -> TioResponse
        if data is None:
            splitdata = [None, error]
        else:
            splitdata = data.split(data[:16])

        if not splitdata[1] or splitdata[1] == b'':
            error = b''.join(splitdata[2:])
            result = None
        else:
            error = None
            result = splitdata[1]

        if result is not None:
            result = result.decode('utf-8')

        if error is not None:
            error = error.decode('utf-8')
        return TioResponse(code, result, error, data)

pytio/_TioRequest.py:

# coding=utf-8

import zlib
from typing import List, AnyStr, Union
from ._TioObjects import TioFile, TioVariable


class TioRequest:
    def __init__(self, lang=None, code=None):
        # type: (AnyStr, Union[AnyStr, bytes]) -> None
        self._files = []
        self._variables = []

        if lang:
            self.set_lang(lang)

        if code:
            self.set_code(code)

    def add_file(self, file):
        # type: (TioFile) -> None
        if file in self._files:
            self._files.remove(file)
        self._files.append(file)

    def add_file_bytes(self, name, content):
        # type: (AnyStr, bytes) -> None
        self._files.append(TioFile(name, content))

    def set_code(self, code):
        # type: (AnyStr) -> None
        self.add_file_bytes('.code.tio', code)

    def set_input(self, input_data):
        # type: (AnyStr) -> None
        self.add_file_bytes('.input.tio', input_data.encode('utf-8'))

    def add_variable(self, variable):
        # type: (TioVariable) -> None
        self._variables.append(variable)

    def add_variable_string(self, name, value):
        # type: (AnyStr, Union[List[AnyStr], AnyStr]) -> None
        self._variables.append(TioVariable(name, value))

    def set_lang(self, lang):
        # type: (AnyStr) -> None
        self.add_variable_string('lang', lang)

    def set_compiler_flags(self, flags):
        # type: (AnyStr) -> None
        self.add_variable_string('TIO_CFLAGS', flags)

    def set_commandline_flags(self, flags):
        # type: (AnyStr) -> None
        self.add_variable_string('TIO_OPTIONS', flags)

    def set_arguments(self, args):
        # type: (AnyStr) -> None
        self.add_variable_string('args', args)

    def as_bytes(self):
        # type: () -> bytes
        bytes_ = bytes()
        try:
            for var in self._variables:
                if var.content:
                    bytes_ += var.as_bytes()
            for file in self._files:
                bytes_ += file.as_bytes()
            bytes_ += b'R'
        except IOError:
            raise RuntimeError("IOError generated during bytes conversion.")
        return bytes_

    def as_deflated_bytes(self):
        # type: () -> bytes
        # This returns a DEFLATE-compressed bytestring, which is what TIO.run's API requires for the request
        # to be proccessed properly.
        return zlib.compress(self.as_bytes(), 9)[2:-4]