reactor: Add support for explicit Python garbage collection

Add support for performing Python gc work only from the main reactor
thread and only when it appears the main thread is idle.

Signed-off-by: Kevin O'Connor <kevin@koconnor.net>
This commit is contained in:
Kevin O'Connor 2020-09-16 22:23:44 -04:00
parent 73cd8c241c
commit cb0a8f2ed9
2 changed files with 38 additions and 15 deletions

View File

@ -306,6 +306,7 @@ def main():
elif not options.debugoutput: elif not options.debugoutput:
logging.warning("No log file specified!" logging.warning("No log file specified!"
" Severe timing issues may result!") " Severe timing issues may result!")
gc.disable()
# Start Printer() class # Start Printer() class
while 1: while 1:
@ -313,7 +314,7 @@ def main():
bglogger.clear_rollover_info() bglogger.clear_rollover_info()
bglogger.set_rollover_info('versions', versions) bglogger.set_rollover_info('versions', versions)
gc.collect() gc.collect()
main_reactor = reactor.Reactor() main_reactor = reactor.Reactor(gc_checking=True)
printer = Printer(main_reactor, bglogger, start_args) printer = Printer(main_reactor, bglogger, start_args)
res = printer.run() res = printer.run()
if res in ['exit', 'error_exit']: if res in ['exit', 'error_exit']:

View File

@ -1,9 +1,9 @@
# File descriptor and timer event helper # File descriptor and timer event helper
# #
# Copyright (C) 2016-2019 Kevin O'Connor <kevin@koconnor.net> # Copyright (C) 2016-2020 Kevin O'Connor <kevin@koconnor.net>
# #
# This file may be distributed under the terms of the GNU GPLv3 license. # This file may be distributed under the terms of the GNU GPLv3 license.
import os, select, math, time, logging, Queue as queue import os, gc, select, math, time, logging, Queue as queue
import greenlet import greenlet
import chelper, util import chelper, util
@ -93,9 +93,10 @@ class ReactorMutex:
class SelectReactor: class SelectReactor:
NOW = _NOW NOW = _NOW
NEVER = _NEVER NEVER = _NEVER
def __init__(self): def __init__(self, gc_checking=False):
# Main code # Main code
self._process = False self._process = False
self._check_gc = gc_checking
self.monotonic = chelper.get_ffi()[1].get_monotonic self.monotonic = chelper.get_ffi()[1].get_monotonic
# Timers # Timers
self._timers = [] self._timers = []
@ -125,8 +126,22 @@ class SelectReactor:
timers = list(self._timers) timers = list(self._timers)
timers.pop(timers.index(timer_handler)) timers.pop(timers.index(timer_handler))
self._timers = timers self._timers = timers
def _check_timers(self, eventtime): def _check_timers(self, eventtime, busy):
if eventtime < self._next_timer: if eventtime < self._next_timer:
if busy:
return 0.
if self._check_gc:
gi = gc.get_count()
if gi[0] >= 700:
# Reactor looks idle and gc is due - run it
if gi[1] >= 10:
if gi[2] >= 10:
gc.collect(2)
else:
gc.collect(1)
else:
gc.collect(0)
return 0.
return min(1., max(.001, self._next_timer - eventtime)) return min(1., max(.001, self._next_timer - eventtime))
self._next_timer = self.NEVER self._next_timer = self.NEVER
g_dispatch = self._g_dispatch g_dispatch = self._g_dispatch
@ -140,9 +155,7 @@ class SelectReactor:
self._end_greenlet(g_dispatch) self._end_greenlet(g_dispatch)
return 0. return 0.
self._next_timer = min(self._next_timer, waketime) self._next_timer = min(self._next_timer, waketime)
if eventtime >= self._next_timer: return 0.
return 0.
return min(1., max(.001, self._next_timer - self.monotonic()))
# Callbacks and Completions # Callbacks and Completions
def completion(self): def completion(self):
return ReactorCompletion(self) return ReactorCompletion(self)
@ -228,12 +241,15 @@ class SelectReactor:
# Main loop # Main loop
def _dispatch_loop(self): def _dispatch_loop(self):
self._g_dispatch = g_dispatch = greenlet.getcurrent() self._g_dispatch = g_dispatch = greenlet.getcurrent()
busy = True
eventtime = self.monotonic() eventtime = self.monotonic()
while self._process: while self._process:
timeout = self._check_timers(eventtime) timeout = self._check_timers(eventtime, busy)
busy = False
res = select.select(self._fds, [], [], timeout) res = select.select(self._fds, [], [], timeout)
eventtime = self.monotonic() eventtime = self.monotonic()
for fd in res[0]: for fd in res[0]:
busy = True
fd.callback(eventtime) fd.callback(eventtime)
if g_dispatch is not self._g_dispatch: if g_dispatch is not self._g_dispatch:
self._end_greenlet(g_dispatch) self._end_greenlet(g_dispatch)
@ -264,8 +280,8 @@ class SelectReactor:
self._pipe_fds = None self._pipe_fds = None
class PollReactor(SelectReactor): class PollReactor(SelectReactor):
def __init__(self): def __init__(self, gc_checking=False):
SelectReactor.__init__(self) SelectReactor.__init__(self, gc_checking)
self._poll = select.poll() self._poll = select.poll()
self._fds = {} self._fds = {}
# File descriptors # File descriptors
@ -284,12 +300,15 @@ class PollReactor(SelectReactor):
# Main loop # Main loop
def _dispatch_loop(self): def _dispatch_loop(self):
self._g_dispatch = g_dispatch = greenlet.getcurrent() self._g_dispatch = g_dispatch = greenlet.getcurrent()
busy = True
eventtime = self.monotonic() eventtime = self.monotonic()
while self._process: while self._process:
timeout = self._check_timers(eventtime) timeout = self._check_timers(eventtime, busy)
busy = False
res = self._poll.poll(int(math.ceil(timeout * 1000.))) res = self._poll.poll(int(math.ceil(timeout * 1000.)))
eventtime = self.monotonic() eventtime = self.monotonic()
for fd, event in res: for fd, event in res:
busy = True
self._fds[fd](eventtime) self._fds[fd](eventtime)
if g_dispatch is not self._g_dispatch: if g_dispatch is not self._g_dispatch:
self._end_greenlet(g_dispatch) self._end_greenlet(g_dispatch)
@ -298,8 +317,8 @@ class PollReactor(SelectReactor):
self._g_dispatch = None self._g_dispatch = None
class EPollReactor(SelectReactor): class EPollReactor(SelectReactor):
def __init__(self): def __init__(self, gc_checking=False):
SelectReactor.__init__(self) SelectReactor.__init__(self, gc_checking)
self._epoll = select.epoll() self._epoll = select.epoll()
self._fds = {} self._fds = {}
# File descriptors # File descriptors
@ -318,12 +337,15 @@ class EPollReactor(SelectReactor):
# Main loop # Main loop
def _dispatch_loop(self): def _dispatch_loop(self):
self._g_dispatch = g_dispatch = greenlet.getcurrent() self._g_dispatch = g_dispatch = greenlet.getcurrent()
busy = True
eventtime = self.monotonic() eventtime = self.monotonic()
while self._process: while self._process:
timeout = self._check_timers(eventtime) timeout = self._check_timers(eventtime, busy)
busy = False
res = self._epoll.poll(timeout) res = self._epoll.poll(timeout)
eventtime = self.monotonic() eventtime = self.monotonic()
for fd, event in res: for fd, event in res:
busy = True
self._fds[fd](eventtime) self._fds[fd](eventtime)
if g_dispatch is not self._g_dispatch: if g_dispatch is not self._g_dispatch:
self._end_greenlet(g_dispatch) self._end_greenlet(g_dispatch)