subprocess.Popen: Klonen von stdout und stderr sowohl an Terminal und Variablen

Question

Ist es möglich, unter Code zu ändern Ausdruck von 'stdout' zu haben und 'stderr':

• auf dem Terminal gedruckt (real Zeit),
• und schließlich gespeichert in outs und Fehler Variablen?

Der Code:

#!/usr/bin/python3 
# -*- coding: utf-8 -*- 

import subprocess 

def run_cmd(command, cwd=None): 
    p = subprocess.Popen(command, cwd=cwd, shell=False, 
         stdout=subprocess.PIPE, 
         stderr=subprocess.PIPE) 
    outs, errs = p.communicate() 
    rc = p.returncode 
    outs = outs.decode('utf-8') 
    errs = errs.decode('utf-8') 

    return (rc, (outs, errs)) 

Dank besonderer Dank geht an @unutbu, für @ jf-sebastian, letzte Funktion:

#!/usr/bin/python3 
# -*- coding: utf-8 -*- 


import sys 
from queue import Queue 
from subprocess import PIPE, Popen 
from threading import Thread 


def read_output(pipe, funcs): 
    for line in iter(pipe.readline, b''): 
     for func in funcs: 
      func(line.decode('utf-8')) 
    pipe.close() 


def write_output(get): 
    for line in iter(get, None): 
     sys.stdout.write(line) 


def run_cmd(command, cwd=None, passthrough=True): 
    outs, errs = None, None 

    proc = Popen(
     command, 
     cwd=cwd, 
     shell=False, 
     close_fds=True, 
     stdout=PIPE, 
     stderr=PIPE, 
     bufsize=1 
     ) 

    if passthrough: 

     outs, errs = [], [] 

     q = Queue() 

     stdout_thread = Thread(
      target=read_output, args=(proc.stdout, [q.put, outs.append]) 
      ) 

     stderr_thread = Thread(
      target=read_output, args=(proc.stderr, [q.put, errs.append]) 
      ) 

     writer_thread = Thread(
      target=write_output, args=(q.get,) 
      ) 

     for t in (stdout_thread, stderr_thread, writer_thread): 
      t.daemon = True 
      t.start() 

     proc.wait() 

     for t in (stdout_thread, stderr_thread): 
      t.join() 

     q.put(None) 

     outs = ' '.join(outs) 
     errs = ' '.join(errs) 

    else: 

     outs, errs = proc.communicate() 
     outs = '' if outs == None else outs.decode('utf-8') 
     errs = '' if errs == None else errs.decode('utf-8') 

    rc = proc.returncode 

    return (rc, (outs, errs)) 

Answer 1

Sie Threads erzeugen konnte die stdout und stderr Rohre lesen , in eine gemeinsame Warteschlange schreiben und an Listen anhängen. Verwenden Sie dann einen dritten Thread, um Elemente aus der Warteschlange zu drucken.

import time 
import Queue 
import sys 
import threading 
import subprocess 
PIPE = subprocess.PIPE 


def read_output(pipe, funcs): 
    for line in iter(pipe.readline, ''): 
     for func in funcs: 
      func(line) 
      # time.sleep(1) 
    pipe.close() 

def write_output(get): 
    for line in iter(get, None): 
     sys.stdout.write(line) 

process = subprocess.Popen(
    ['random_print.py'], stdout=PIPE, stderr=PIPE, close_fds=True, bufsize=1) 
q = Queue.Queue() 
out, err = [], [] 
tout = threading.Thread(
    target=read_output, args=(process.stdout, [q.put, out.append])) 
terr = threading.Thread(
    target=read_output, args=(process.stderr, [q.put, err.append])) 
twrite = threading.Thread(target=write_output, args=(q.get,)) 
for t in (tout, terr, twrite): 
    t.daemon = True 
    t.start() 
process.wait() 
for t in (tout, terr): 
    t.join() 
q.put(None) 
print(out) 
print(err) 

Der Grund für den dritten Faden - anstatt dass die ersten beiden Gewindegänge sowohl Druck direkt zum Terminal - ist es, beide Druckanweisungen, um zu verhindern, gleichzeitig auftretende, die in teilweise verstümmelten Text führen kann.

---

Die obigen Anrufe random_print.py, die zufällig auf stdout und stderr druckt:

import sys 
import time 
import random 

for i in range(50): 
    f = random.choice([sys.stdout,sys.stderr]) 
    f.write(str(i)+'\n') 
    f.flush() 
    time.sleep(0.1) 

---

Diese Lösung leiht Code und Ideen von J. F. Sebastian, here.

---

Hier ist eine alternative Lösung für Unix-ähnliche Systeme, mit select.select:

import collections 
import select 
import fcntl 
import os 
import time 
import Queue 
import sys 
import threading 
import subprocess 
PIPE = subprocess.PIPE 

def make_async(fd): 
    # https://stackoverflow.com/a/7730201/190597 
    '''add the O_NONBLOCK flag to a file descriptor''' 
    fcntl.fcntl(
     fd, fcntl.F_SETFL, fcntl.fcntl(fd, fcntl.F_GETFL) | os.O_NONBLOCK) 

def read_async(fd): 
    # https://stackoverflow.com/a/7730201/190597 
    '''read some data from a file descriptor, ignoring EAGAIN errors''' 
    # time.sleep(1) 
    try: 
     return fd.read() 
    except IOError, e: 
     if e.errno != errno.EAGAIN: 
      raise e 
     else: 
      return '' 

def write_output(fds, outmap): 
    for fd in fds: 
     line = read_async(fd) 
     sys.stdout.write(line) 
     outmap[fd.fileno()].append(line) 

process = subprocess.Popen(
    ['random_print.py'], stdout=PIPE, stderr=PIPE, close_fds=True) 

make_async(process.stdout) 
make_async(process.stderr) 
outmap = collections.defaultdict(list) 
while True: 
    rlist, wlist, xlist = select.select([process.stdout, process.stderr], [], []) 
    write_output(rlist, outmap) 
    if process.poll() is not None: 
     write_output([process.stdout, process.stderr], outmap) 
     break 

fileno = {'stdout': process.stdout.fileno(), 
      'stderr': process.stderr.fileno()} 

print(outmap[fileno['stdout']]) 
print(outmap[fileno['stderr']]) 

Diese Lösung verwendet Code und Ideen von Adam Rosenfield's post, here.

Answer 2

Zur Erfassung und Anzeige zur gleichen Zeit sowohl stdout und stderr von einer Kind-Prozess Zeile für Zeile in einem einzigen Thread, Sie asynchrones I/O verwenden:

#!/usr/bin/env python3 
import asyncio 
import os 
import sys 
from asyncio.subprocess import PIPE 

@asyncio.coroutine 
def read_stream_and_display(stream, display): 
    """Read from stream line by line until EOF, display, and capture the lines. 

    """ 
    output = [] 
    while True: 
     line = yield from stream.readline() 
     if not line: 
      break 
     output.append(line) 
     display(line) # assume it doesn't block 
    return b''.join(output) 

@asyncio.coroutine 
def read_and_display(*cmd): 
    """Capture cmd's stdout, stderr while displaying them as they arrive 
    (line by line). 

    """ 
    # start process 
    process = yield from asyncio.create_subprocess_exec(*cmd, 
      stdout=PIPE, stderr=PIPE) 

    # read child's stdout/stderr concurrently (capture and display) 
    try: 
     stdout, stderr = yield from asyncio.gather(
      read_stream_and_display(process.stdout, sys.stdout.buffer.write), 
      read_stream_and_display(process.stderr, sys.stderr.buffer.write)) 
    except Exception: 
     process.kill() 
     raise 
    finally: 
     # wait for the process to exit 
     rc = yield from process.wait() 
    return rc, stdout, stderr 

# run the event loop 
if os.name == 'nt': 
    loop = asyncio.ProactorEventLoop() # for subprocess' pipes on Windows 
    asyncio.set_event_loop(loop) 
else: 
    loop = asyncio.get_event_loop() 
rc, *output = loop.run_until_complete(read_and_display(*cmd)) 
loop.close()