Explorar el Código

Removed unnecessarily heavy RAM usage from logs in bench/test.py

For long running processes (testing with >1pls) these logs can grow into
multiple gigabytes, humorously we never access more than the last n lines
as requested by --context. Piping the stdout with --stdout does not use
additional RAM.
Christopher Haster hace 3 años
padre
commit
397aa27181
Se han modificado 2 ficheros con 8 adiciones y 8 borrados
  1. 4 4
      scripts/bench.py
  2. 4 4
      scripts/test.py

+ 4 - 4
scripts/bench.py

@@ -799,7 +799,7 @@ def run_stage(name, runner_, ids, stdout_, trace_, output_, **args):
         mpty = os.fdopen(mpty, 'r', 1)
 
         last_id = None
-        last_stdout = []
+        last_stdout = co.deque(maxlen=args.get('context', 5) + 1)
         last_assert = None
         try:
             while True:
@@ -826,7 +826,7 @@ def run_stage(name, runner_, ids, stdout_, trace_, output_, **args):
                     if op == 'running':
                         locals.seen_perms += 1
                         last_id = m.group('id')
-                        last_stdout = []
+                        last_stdout.clear()
                         last_assert = None
                     elif op == 'finished':
                         case = m.group('case')
@@ -862,7 +862,7 @@ def run_stage(name, runner_, ids, stdout_, trace_, output_, **args):
                         if args.get('keep_going'):
                             proc.kill()
         except KeyboardInterrupt:
-            raise BenchFailure(last_id, 1, last_stdout)
+            raise BenchFailure(last_id, 1, list(last_stdout))
         finally:
             children.remove(proc)
             mpty.close()
@@ -872,7 +872,7 @@ def run_stage(name, runner_, ids, stdout_, trace_, output_, **args):
             raise BenchFailure(
                 last_id,
                 proc.returncode,
-                last_stdout,
+                list(last_stdout),
                 last_assert)
 
     def run_job(runner_, ids=[], start=None, step=None):

+ 4 - 4
scripts/test.py

@@ -809,7 +809,7 @@ def run_stage(name, runner_, ids, stdout_, trace_, output_, **args):
         mpty = os.fdopen(mpty, 'r', 1)
 
         last_id = None
-        last_stdout = []
+        last_stdout = co.deque(maxlen=args.get('context', 5) + 1)
         last_assert = None
         try:
             while True:
@@ -836,7 +836,7 @@ def run_stage(name, runner_, ids, stdout_, trace_, output_, **args):
                     if op == 'running':
                         locals.seen_perms += 1
                         last_id = m.group('id')
-                        last_stdout = []
+                        last_stdout.clear()
                         last_assert = None
                     elif op == 'powerloss':
                         last_id = m.group('id')
@@ -867,7 +867,7 @@ def run_stage(name, runner_, ids, stdout_, trace_, output_, **args):
                         if args.get('keep_going'):
                             proc.kill()
         except KeyboardInterrupt:
-            raise TestFailure(last_id, 1, last_stdout)
+            raise TestFailure(last_id, 1, list(last_stdout))
         finally:
             children.remove(proc)
             mpty.close()
@@ -877,7 +877,7 @@ def run_stage(name, runner_, ids, stdout_, trace_, output_, **args):
             raise TestFailure(
                 last_id,
                 proc.returncode,
-                last_stdout,
+                list(last_stdout),
                 last_assert)
 
     def run_job(runner_, ids=[], start=None, step=None):