3diy
/
moonraker
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

database: refactor initial check for empty keys 

Rather than reading out the entire database, attempt to pop off
empty keys in each database.  This should should speed up
init on platforms with slow disk I/O and prevents reading
large namespaces in memory.

Signed-off-by:  Eric Callahan <arksine.code@gmail.com>
This commit is contained in:
Eric Callahan 2022-01-30 06:17:00 -05:00
parent e754d123dd
commit 4d0a43cb25
1 changed files with 34 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
moonraker/components/database.py
View File

@ -88,16 +88,40 @@ class MoonrakerDatabase:
self.namespaces[key.decode()] = self.lmdb_env.open_db( self.namespaces[key.decode()] = self.lmdb_env.open_db(
key, txn) key, txn)
remaining = cursor.next() remaining = cursor.next()
cursor.close() if "moonraker" not in self.namespaces:
if "moonraker" not in self.namespaces: mrdb = self.lmdb_env.open_db(b"moonraker", txn)
mrdb = self.lmdb_env.open_db(b"moonraker", txn) self.namespaces["moonraker"] = mrdb
self.namespaces["moonraker"] = mrdb txn.put(b'database_version',
txn.put(b'database_version', self._encode_value(DATABASE_VERSION),
self._encode_value(DATABASE_VERSION), db=mrdb)
db=mrdb) # Iterate through all records, checking for invalid keys
# Read out all namespaces to remove any invalid keys on init for ns, db in self.namespaces.items():
for ns in self.namespaces.keys(): with txn.cursor(db=db) as cursor:
self._get_namespace(ns) remaining = cursor.first()
while remaining:
key_buf = cursor.key()
try:
decoded_key = bytes(key_buf).decode()
except Exception:
logging.info("Database Key Decode Error")
decoded_key = ''
if not decoded_key:
hex_key = bytes(key_buf).hex()
try:
invalid_val = self._decode_value(cursor.value())
except Exception:
invalid_val = ""
logging.info(
f"Invalid Key '{hex_key}' found in namespace "
f"'{ns}', dropping value: {repr(invalid_val)}")
try:
remaining = cursor.delete()
except Exception:
logging.exception("Error Deleting LMDB Key")
else:
continue
remaining = cursor.next()
# Protected Namespaces have read-only API access. Write access can # Protected Namespaces have read-only API access. Write access can
# be granted by enabling the debug option. Forbidden namespaces # be granted by enabling the debug option. Forbidden namespaces
# have no API access. This cannot be overridden. # have no API access. This cannot be overridden.