remote file follow using python

Question

I've created an on-demand class that can be initiated to collected any new data written to file from start_recording call until stop_recording which close the connection and retrieve the collected data so far.

It uses in test cases for obtaining relevant logs during the time where a certain operation was performed in order to verify its success.

I'm currently using this class for tests in remote machine, but would be happy to hear for any idea for improvements, correctness, etc.

import time

import paramiko
import select

from virtual_machine import utils
from multiprocessing import Queue
import multiprocess

class background():
    def __init__(self, config_file):
        self.q = Queue(1000)
        #this methods implemented elsewhere and read a config file into dictionary
        self.config = utils.get_params(config_file)

    @staticmethod
    def linesplit(socket):
        buffer_string = socket.recv(4048).decode('utf-8')
        done = False
        while not done:
            if '\n' in buffer_string:
                (line, buffer_string) = buffer_string.split('\n', 1)
                yield line + "\n"
            else:
                more = socket.recv(4048)
                if not more:
                    done = True
                else:
                    buffer_string = buffer_string + more.decode('utf-8')
        if buffer_string:
            yield buffer_string

    def do_tail(self, log_file):
        data = ""
        client = paramiko.SSHClient()
        client.load_system_host_keys()
        client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
        from os.path import expanduser
        home = expanduser("~")
        client.connect(self.config["ip"],self.config["username"],self.config["password"])

        grep_pattern = "grep_filter"
        remote_command = 'tail -0 -f {} '.format(log_file)
        print(remote_command)
        transport = client.get_transport()
        channel = transport.open_session()
        channel.exec_command(remote_command)

        while 1:
            try:
                rl, _, _ = select.select([channel], [], [], 0.0)
                if len(rl) > 0:
                    print("ready to read")
                    for line in background.linesplit(channel):
                        self.q.put_nowait(line)


            except (KeyboardInterrupt, SystemExit):
                print('got ctrl+c')
                break

        client.close()

        return data

    def start_recording(self, log_file):
        q = Queue(100)
        self.p = multiprocess.Process(target=self.do_tail, args=(log_file,), daemon=True)
        self.p.start()

    def stop_recording(self):
        self.p.terminate()

        data = ""
        while not self.q.empty():
            data += self.q.get()

        return data


#example 
if __name__ == '__main__':
    x = background("config.conf")
    x.start_recording("/tmp/logfile.log")
    # doing some operation 
    data = x.stop_recording()
    print(data)

Answer 1

Your linesplit repeats a little bit of code; you could phrase it as

while True:
    buffer_bytes = socket.recv(4096)
    if len(buffer_bytes) == 0:
        break
    buffer_string = buffer_bytes.decode('utf-8')
    *lines, buffer_string = buffer_string.split('\n')
    yield from lines

'tail -0 -f {} '.format(log_file) could be simply 'tail -0 -f ' + log_file.

background should be Background.

Consider adding some PEP484 type hints.

rl, _, _ = select ...

can be

rl, *_ = select ...

The code between paramiko.SSHClient() and client.close() has no close-guarantee protection. There are various options, including a try/finally, or making your current class (or a factored-out tail-specific class) a context manager.