file_manager: update for changes in the database

Signed-off-by:  Eric Callahan <arksine.code@gmail.com>
This commit is contained in:
Eric Callahan 2022-02-01 11:08:10 -05:00
parent b43f4623fc
commit 0dd10ce116
1 changed files with 132 additions and 71 deletions

View File

@ -33,7 +33,6 @@ from typing import (
if TYPE_CHECKING: if TYPE_CHECKING:
from inotify_simple import Event as InotifyEvent from inotify_simple import Event as InotifyEvent
from moonraker import Server
from confighelper import ConfigHelper from confighelper import ConfigHelper
from websockets import WebRequest from websockets import WebRequest
from components import database from components import database
@ -60,7 +59,7 @@ class FileManager:
self.file_paths: Dict[str, str] = {} self.file_paths: Dict[str, str] = {}
db: DBComp = self.server.load_component(config, "database") db: DBComp = self.server.load_component(config, "database")
gc_path: str = db.get_item( gc_path: str = db.get_item(
"moonraker", "file_manager.gcode_path", "") "moonraker", "file_manager.gcode_path", "").result()
self.gcode_metadata = MetadataStorage(config, gc_path, db) self.gcode_metadata = MetadataStorage(config, gc_path, db)
self.inotify_handler = INotifyHandler(config, self, self.inotify_handler = INotifyHandler(config, self,
self.gcode_metadata) self.gcode_metadata)
@ -172,8 +171,7 @@ class FileManager:
self.server.register_static_file_handler(root, path) self.server.register_static_file_handler(root, path)
if root == "gcodes": if root == "gcodes":
db: DBComp = self.server.lookup_component("database") db: DBComp = self.server.lookup_component("database")
moon_db = db.wrap_namespace("moonraker") db.insert_item("moonraker", "file_manager.gcode_path", path)
moon_db["file_manager.gcode_path"] = path
# scan for metadata changes # scan for metadata changes
self.gcode_metadata.update_gcode_path(path) self.gcode_metadata.update_gcode_path(path)
if full_access: if full_access:
@ -1162,10 +1160,10 @@ class INotifyHandler:
new_rel_path = self.file_manager.get_relative_path( new_rel_path = self.file_manager.get_relative_path(
"gcodes", new_path) "gcodes", new_path)
if is_dir: if is_dir:
self.gcode_metadata.move_directory_metadata( await self.gcode_metadata.move_directory_metadata(
prev_rel_path, new_rel_path) prev_rel_path, new_rel_path)
else: else:
return self.gcode_metadata.move_file_metadata( return await self.gcode_metadata.move_file_metadata(
prev_rel_path, new_rel_path) prev_rel_path, new_rel_path)
else: else:
# move from a non-gcodes root to gcodes root needs a rescan # move from a non-gcodes root to gcodes root needs a rescan
@ -1448,29 +1446,53 @@ class MetadataStorage:
self.mddb = db.wrap_namespace( self.mddb = db.wrap_namespace(
METADATA_NAMESPACE, parse_keys=False) METADATA_NAMESPACE, parse_keys=False)
version = db.get_item( version = db.get_item(
"moonraker", "file_manager.metadata_version", 0) "moonraker", "file_manager.metadata_version", 0).result()
if version != METADATA_VERSION: if version != METADATA_VERSION:
# Clear existing metadata when version is bumped # Clear existing metadata when version is bumped
for fname in self.mddb.keys(): self.mddb.clear()
self.remove_file_metadata(fname)
db.insert_item( db.insert_item(
"moonraker", "file_manager.metadata_version", "moonraker", "file_manager.metadata_version",
METADATA_VERSION) METADATA_VERSION)
# Keep a local cache of the metadata. This allows for synchronous
# queries. Metadata is generally under 1KiB per entry, so even at
# 1000 gcode files we are using < 1MiB of additional memory.
# That said, in the future all components that access metadata should
# be refactored to do so asynchronously.
self.metadata: Dict[str, Any] = self.mddb.as_dict()
self.pending_requests: Dict[ self.pending_requests: Dict[
str, Tuple[Dict[str, Any], asyncio.Event]] = {} str, Tuple[Dict[str, Any], asyncio.Event]] = {}
self.busy: bool = False self.busy: bool = False
if self.gc_path:
# Check for removed gcode files while moonraker was shutdown # Check for removed gcode files while moonraker was shutdown
for fname in list(self.mddb.keys()): if self.gc_path:
del_keys: List[str] = []
for fname in list(self.metadata.keys()):
fpath = os.path.join(self.gc_path, fname) fpath = os.path.join(self.gc_path, fname)
if not os.path.isfile(fpath): if not os.path.isfile(fpath):
self.remove_file_metadata(fname) del self.metadata[fname]
logging.info(f"Pruned file: {fname}") del_keys.append(fname)
continue elif "thumbnails" in self.metadata[fname]:
# Check for any stale data entries and remove them
need_sync = False
for thumb in self.metadata[fname]['thumbnails']:
if 'data' in thumb:
del thumb['data']
need_sync = True
if need_sync:
self.mddb[fname] = self.metadata[fname]
# Delete any removed keys from the database
if del_keys:
ret = self.mddb.delete_batch(del_keys).result()
self._remove_thumbs(ret)
pruned = '\n'.join(ret.keys())
if pruned:
logging.info(f"Pruned metadata for the following:\n"
f"{pruned}")
def update_gcode_path(self, path: str) -> None: def update_gcode_path(self, path: str) -> None:
if path == self.gc_path: if path == self.gc_path:
return return
self.metadata.clear()
self.mddb.clear() self.mddb.clear()
self.gc_path = path self.gc_path = path
@ -1478,10 +1500,7 @@ class MetadataStorage:
key: str, key: str,
default: _T = None default: _T = None
) -> Union[_T, Dict[str, Any]]: ) -> Union[_T, Dict[str, Any]]:
return self.mddb.get(key, default) return self.metadata.get(key, default)
def __getitem__(self, key: str) -> Dict[str, Any]:
return self.mddb[key]
def _has_valid_data(self, def _has_valid_data(self,
fname: str, fname: str,
@ -1491,7 +1510,7 @@ class MetadataStorage:
# UFP files always need processing # UFP files always need processing
return False return False
mdata: Dict[str, Any] mdata: Dict[str, Any]
mdata = self.mddb.get(fname, {'size': "", 'modified': 0}) mdata = self.metadata.get(fname, {'size': "", 'modified': 0})
for field in ['size', 'modified']: for field in ['size', 'modified']:
if mdata[field] != path_info.get(field, None): if mdata[field] != path_info.get(field, None):
return False return False
@ -1500,15 +1519,29 @@ class MetadataStorage:
def remove_directory_metadata(self, dir_name: str) -> None: def remove_directory_metadata(self, dir_name: str) -> None:
if dir_name[-1] != "/": if dir_name[-1] != "/":
dir_name += "/" dir_name += "/"
for fname in list(self.mddb.keys()): del_items: Dict[str, Any] = {}
for fname in list(self.metadata.keys()):
if fname.startswith(dir_name): if fname.startswith(dir_name):
self.remove_file_metadata(fname) md = self.metadata.pop(fname, None)
if md:
del_items[fname] = md
if del_items:
# Remove items from persistent storage
self.mddb.delete_batch(list(del_items.keys()))
eventloop = self.server.get_event_loop()
# Remove thumbs in a nother thread
eventloop.run_in_thread(self._remove_thumbs, del_items)
def remove_file_metadata(self, fname: str) -> None: def remove_file_metadata(self, fname: str) -> None:
metadata: Optional[Dict[str, Any]] md: Optional[Dict[str, Any]] = self.metadata.pop(fname, None)
metadata = self.mddb.pop(fname, None) if md is None:
if metadata is None:
return return
self.mddb.pop(fname, None)
eventloop = self.server.get_event_loop()
eventloop.run_in_thread(self._remove_thumbs, {fname: md})
def _remove_thumbs(self, records: Dict[str, Dict[str, Any]]) -> None:
for fname, metadata in records.items():
# Delete associated thumbnails # Delete associated thumbnails
fdir = os.path.dirname(os.path.join(self.gc_path, fname)) fdir = os.path.dirname(os.path.join(self.gc_path, fname))
if "thumbnails" in metadata: if "thumbnails" in metadata:
@ -1525,31 +1558,59 @@ class MetadataStorage:
except Exception: except Exception:
logging.debug(f"Error removing thumb at {thumb_path}") logging.debug(f"Error removing thumb at {thumb_path}")
def move_directory_metadata(self, prev_dir: str, new_dir: str) -> None: async def move_directory_metadata(self,
prev_dir: str,
new_dir: str
) -> None:
if prev_dir[-1] != "/": if prev_dir[-1] != "/":
prev_dir += "/" prev_dir += "/"
for prev_fname in list(self.mddb.keys()): moved: List[Tuple[str, str, Dict[str, Any]]] = []
for prev_fname in list(self.metadata.keys()):
if prev_fname.startswith(prev_dir): if prev_fname.startswith(prev_dir):
new_fname = os.path.join(new_dir, prev_fname[len(prev_dir):]) new_fname = os.path.join(new_dir, prev_fname[len(prev_dir):])
self.move_file_metadata(prev_fname, new_fname, False) md: Optional[Dict[str, Any]]
md = self.metadata.pop(prev_fname, None)
if md is None:
continue
self.metadata[new_fname] = md
moved.append((prev_fname, new_fname, md))
if moved:
source = [m[0] for m in moved]
dest = [m[1] for m in moved]
self.mddb.move_batch(source, dest)
eventloop = self.server.get_event_loop()
await eventloop.run_in_thread(self._move_thumbnails, moved)
def move_file_metadata(self, async def move_file_metadata(self,
prev_fname: str, prev_fname: str,
new_fname: str, new_fname: str,
move_thumbs: bool = True move_thumbs: bool = True
) -> bool: ) -> bool:
metadata: Optional[Dict[str, Any]] metadata: Optional[Dict[str, Any]]
metadata = self.mddb.pop(prev_fname, None) metadata = self.metadata.pop(prev_fname, None)
if metadata is None: if metadata is None:
# If this move overwrites an existing file it is necessary # If this move overwrites an existing file it is necessary
# to rescan which requires that we remove any existing # to rescan which requires that we remove any existing
# metadata. # metadata.
if self.metadata.pop(new_fname, None) is not None:
self.mddb.pop(new_fname, None) self.mddb.pop(new_fname, None)
return False return False
self.mddb[new_fname] = metadata self.metadata[new_fname] = metadata
self.mddb.move_batch([prev_fname], [new_fname])
if move_thumbs:
eventloop = self.server.get_event_loop()
await eventloop.run_in_thread(
self._move_thumbnails,
[(prev_fname, new_fname, metadata)])
return True
def _move_thumbnails(self,
records: List[Tuple[str, str, Dict[str, Any]]]
) -> None:
for (prev_fname, new_fname, metadata) in records:
prev_dir = os.path.dirname(os.path.join(self.gc_path, prev_fname)) prev_dir = os.path.dirname(os.path.join(self.gc_path, prev_fname))
new_dir = os.path.dirname(os.path.join(self.gc_path, new_fname)) new_dir = os.path.dirname(os.path.join(self.gc_path, new_fname))
if "thumbnails" in metadata and move_thumbs: if "thumbnails" in metadata:
thumb: Dict[str, Any] thumb: Dict[str, Any]
for thumb in metadata["thumbnails"]: for thumb in metadata["thumbnails"]:
path: Optional[str] = thumb.get("relative_path", None) path: Optional[str] = thumb.get("relative_path", None)
@ -1565,7 +1626,6 @@ class MetadataStorage:
except Exception: except Exception:
logging.debug(f"Error moving thumb from {thumb_path}" logging.debug(f"Error moving thumb from {thumb_path}"
f" to {new_path}") f" to {new_path}")
return True
def parse_metadata(self, def parse_metadata(self,
fname: str, fname: str,
@ -1609,12 +1669,13 @@ class MetadataStorage:
break break
else: else:
if ufp_path is None: if ufp_path is None:
self.mddb[fname] = { self.metadata[fname] = {
'size': path_info.get('size', 0), 'size': path_info.get('size', 0),
'modified': path_info.get('modified', 0), 'modified': path_info.get('modified', 0),
'print_start_time': None, 'print_start_time': None,
'job_id': None 'job_id': None
} }
self.mddb[fname] = self.metadata[fname]
logging.info( logging.info(
f"Unable to extract medatadata from file: {fname}") f"Unable to extract medatadata from file: {fname}")
self.pending_requests.pop(fname, None) self.pending_requests.pop(fname, None)
@ -1652,8 +1713,8 @@ class MetadataStorage:
# This indicates an error, do not add metadata for this # This indicates an error, do not add metadata for this
raise self.server.error("Unable to extract metadata") raise self.server.error("Unable to extract metadata")
metadata.update({'print_start_time': None, 'job_id': None}) metadata.update({'print_start_time': None, 'job_id': None})
self.mddb[path] = dict(metadata) self.metadata[path] = metadata
metadata['filename'] = path self.mddb[path] = metadata
def load_component(config: ConfigHelper) -> FileManager: def load_component(config: ConfigHelper) -> FileManager:
return FileManager(config) return FileManager(config)