From cb0a8f2ed9d5895e7b07a3783f00205f33e23203 Mon Sep 17 00:00:00 2001 From: Kevin O'Connor Date: Wed, 16 Sep 2020 22:23:44 -0400 Subject: [PATCH] reactor: Add support for explicit Python garbage collection Add support for performing Python gc work only from the main reactor thread and only when it appears the main thread is idle. Signed-off-by: Kevin O'Connor --- klippy/klippy.py | 3 ++- klippy/reactor.py | 50 ++++++++++++++++++++++++++++++++++------------- 2 files changed, 38 insertions(+), 15 deletions(-) diff --git a/klippy/klippy.py b/klippy/klippy.py index 03095d06..325d6fa3 100644 --- a/klippy/klippy.py +++ b/klippy/klippy.py @@ -306,6 +306,7 @@ def main(): elif not options.debugoutput: logging.warning("No log file specified!" " Severe timing issues may result!") + gc.disable() # Start Printer() class while 1: @@ -313,7 +314,7 @@ def main(): bglogger.clear_rollover_info() bglogger.set_rollover_info('versions', versions) gc.collect() - main_reactor = reactor.Reactor() + main_reactor = reactor.Reactor(gc_checking=True) printer = Printer(main_reactor, bglogger, start_args) res = printer.run() if res in ['exit', 'error_exit']: diff --git a/klippy/reactor.py b/klippy/reactor.py index 0d369cba..0c24b801 100644 --- a/klippy/reactor.py +++ b/klippy/reactor.py @@ -1,9 +1,9 @@ # File descriptor and timer event helper # -# Copyright (C) 2016-2019 Kevin O'Connor +# Copyright (C) 2016-2020 Kevin O'Connor # # This file may be distributed under the terms of the GNU GPLv3 license. -import os, select, math, time, logging, Queue as queue +import os, gc, select, math, time, logging, Queue as queue import greenlet import chelper, util @@ -93,9 +93,10 @@ class ReactorMutex: class SelectReactor: NOW = _NOW NEVER = _NEVER - def __init__(self): + def __init__(self, gc_checking=False): # Main code self._process = False + self._check_gc = gc_checking self.monotonic = chelper.get_ffi()[1].get_monotonic # Timers self._timers = [] @@ -125,8 +126,22 @@ class SelectReactor: timers = list(self._timers) timers.pop(timers.index(timer_handler)) self._timers = timers - def _check_timers(self, eventtime): + def _check_timers(self, eventtime, busy): if eventtime < self._next_timer: + if busy: + return 0. + if self._check_gc: + gi = gc.get_count() + if gi[0] >= 700: + # Reactor looks idle and gc is due - run it + if gi[1] >= 10: + if gi[2] >= 10: + gc.collect(2) + else: + gc.collect(1) + else: + gc.collect(0) + return 0. return min(1., max(.001, self._next_timer - eventtime)) self._next_timer = self.NEVER g_dispatch = self._g_dispatch @@ -140,9 +155,7 @@ class SelectReactor: self._end_greenlet(g_dispatch) return 0. self._next_timer = min(self._next_timer, waketime) - if eventtime >= self._next_timer: - return 0. - return min(1., max(.001, self._next_timer - self.monotonic())) + return 0. # Callbacks and Completions def completion(self): return ReactorCompletion(self) @@ -228,12 +241,15 @@ class SelectReactor: # Main loop def _dispatch_loop(self): self._g_dispatch = g_dispatch = greenlet.getcurrent() + busy = True eventtime = self.monotonic() while self._process: - timeout = self._check_timers(eventtime) + timeout = self._check_timers(eventtime, busy) + busy = False res = select.select(self._fds, [], [], timeout) eventtime = self.monotonic() for fd in res[0]: + busy = True fd.callback(eventtime) if g_dispatch is not self._g_dispatch: self._end_greenlet(g_dispatch) @@ -264,8 +280,8 @@ class SelectReactor: self._pipe_fds = None class PollReactor(SelectReactor): - def __init__(self): - SelectReactor.__init__(self) + def __init__(self, gc_checking=False): + SelectReactor.__init__(self, gc_checking) self._poll = select.poll() self._fds = {} # File descriptors @@ -284,12 +300,15 @@ class PollReactor(SelectReactor): # Main loop def _dispatch_loop(self): self._g_dispatch = g_dispatch = greenlet.getcurrent() + busy = True eventtime = self.monotonic() while self._process: - timeout = self._check_timers(eventtime) + timeout = self._check_timers(eventtime, busy) + busy = False res = self._poll.poll(int(math.ceil(timeout * 1000.))) eventtime = self.monotonic() for fd, event in res: + busy = True self._fds[fd](eventtime) if g_dispatch is not self._g_dispatch: self._end_greenlet(g_dispatch) @@ -298,8 +317,8 @@ class PollReactor(SelectReactor): self._g_dispatch = None class EPollReactor(SelectReactor): - def __init__(self): - SelectReactor.__init__(self) + def __init__(self, gc_checking=False): + SelectReactor.__init__(self, gc_checking) self._epoll = select.epoll() self._fds = {} # File descriptors @@ -318,12 +337,15 @@ class EPollReactor(SelectReactor): # Main loop def _dispatch_loop(self): self._g_dispatch = g_dispatch = greenlet.getcurrent() + busy = True eventtime = self.monotonic() while self._process: - timeout = self._check_timers(eventtime) + timeout = self._check_timers(eventtime, busy) + busy = False res = self._epoll.poll(timeout) eventtime = self.monotonic() for fd, event in res: + busy = True self._fds[fd](eventtime) if g_dispatch is not self._g_dispatch: self._end_greenlet(g_dispatch)