file_manager: update for changes in the database
Signed-off-by: Eric Callahan <arksine.code@gmail.com>
This commit is contained in:
parent
b43f4623fc
commit
0dd10ce116
|
@ -33,7 +33,6 @@ from typing import (
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
from inotify_simple import Event as InotifyEvent
|
from inotify_simple import Event as InotifyEvent
|
||||||
from moonraker import Server
|
|
||||||
from confighelper import ConfigHelper
|
from confighelper import ConfigHelper
|
||||||
from websockets import WebRequest
|
from websockets import WebRequest
|
||||||
from components import database
|
from components import database
|
||||||
|
@ -60,7 +59,7 @@ class FileManager:
|
||||||
self.file_paths: Dict[str, str] = {}
|
self.file_paths: Dict[str, str] = {}
|
||||||
db: DBComp = self.server.load_component(config, "database")
|
db: DBComp = self.server.load_component(config, "database")
|
||||||
gc_path: str = db.get_item(
|
gc_path: str = db.get_item(
|
||||||
"moonraker", "file_manager.gcode_path", "")
|
"moonraker", "file_manager.gcode_path", "").result()
|
||||||
self.gcode_metadata = MetadataStorage(config, gc_path, db)
|
self.gcode_metadata = MetadataStorage(config, gc_path, db)
|
||||||
self.inotify_handler = INotifyHandler(config, self,
|
self.inotify_handler = INotifyHandler(config, self,
|
||||||
self.gcode_metadata)
|
self.gcode_metadata)
|
||||||
|
@ -172,8 +171,7 @@ class FileManager:
|
||||||
self.server.register_static_file_handler(root, path)
|
self.server.register_static_file_handler(root, path)
|
||||||
if root == "gcodes":
|
if root == "gcodes":
|
||||||
db: DBComp = self.server.lookup_component("database")
|
db: DBComp = self.server.lookup_component("database")
|
||||||
moon_db = db.wrap_namespace("moonraker")
|
db.insert_item("moonraker", "file_manager.gcode_path", path)
|
||||||
moon_db["file_manager.gcode_path"] = path
|
|
||||||
# scan for metadata changes
|
# scan for metadata changes
|
||||||
self.gcode_metadata.update_gcode_path(path)
|
self.gcode_metadata.update_gcode_path(path)
|
||||||
if full_access:
|
if full_access:
|
||||||
|
@ -1162,10 +1160,10 @@ class INotifyHandler:
|
||||||
new_rel_path = self.file_manager.get_relative_path(
|
new_rel_path = self.file_manager.get_relative_path(
|
||||||
"gcodes", new_path)
|
"gcodes", new_path)
|
||||||
if is_dir:
|
if is_dir:
|
||||||
self.gcode_metadata.move_directory_metadata(
|
await self.gcode_metadata.move_directory_metadata(
|
||||||
prev_rel_path, new_rel_path)
|
prev_rel_path, new_rel_path)
|
||||||
else:
|
else:
|
||||||
return self.gcode_metadata.move_file_metadata(
|
return await self.gcode_metadata.move_file_metadata(
|
||||||
prev_rel_path, new_rel_path)
|
prev_rel_path, new_rel_path)
|
||||||
else:
|
else:
|
||||||
# move from a non-gcodes root to gcodes root needs a rescan
|
# move from a non-gcodes root to gcodes root needs a rescan
|
||||||
|
@ -1448,29 +1446,53 @@ class MetadataStorage:
|
||||||
self.mddb = db.wrap_namespace(
|
self.mddb = db.wrap_namespace(
|
||||||
METADATA_NAMESPACE, parse_keys=False)
|
METADATA_NAMESPACE, parse_keys=False)
|
||||||
version = db.get_item(
|
version = db.get_item(
|
||||||
"moonraker", "file_manager.metadata_version", 0)
|
"moonraker", "file_manager.metadata_version", 0).result()
|
||||||
if version != METADATA_VERSION:
|
if version != METADATA_VERSION:
|
||||||
# Clear existing metadata when version is bumped
|
# Clear existing metadata when version is bumped
|
||||||
for fname in self.mddb.keys():
|
self.mddb.clear()
|
||||||
self.remove_file_metadata(fname)
|
|
||||||
db.insert_item(
|
db.insert_item(
|
||||||
"moonraker", "file_manager.metadata_version",
|
"moonraker", "file_manager.metadata_version",
|
||||||
METADATA_VERSION)
|
METADATA_VERSION)
|
||||||
|
# Keep a local cache of the metadata. This allows for synchronous
|
||||||
|
# queries. Metadata is generally under 1KiB per entry, so even at
|
||||||
|
# 1000 gcode files we are using < 1MiB of additional memory.
|
||||||
|
# That said, in the future all components that access metadata should
|
||||||
|
# be refactored to do so asynchronously.
|
||||||
|
self.metadata: Dict[str, Any] = self.mddb.as_dict()
|
||||||
self.pending_requests: Dict[
|
self.pending_requests: Dict[
|
||||||
str, Tuple[Dict[str, Any], asyncio.Event]] = {}
|
str, Tuple[Dict[str, Any], asyncio.Event]] = {}
|
||||||
self.busy: bool = False
|
self.busy: bool = False
|
||||||
if self.gc_path:
|
|
||||||
# Check for removed gcode files while moonraker was shutdown
|
# Check for removed gcode files while moonraker was shutdown
|
||||||
for fname in list(self.mddb.keys()):
|
if self.gc_path:
|
||||||
|
del_keys: List[str] = []
|
||||||
|
for fname in list(self.metadata.keys()):
|
||||||
fpath = os.path.join(self.gc_path, fname)
|
fpath = os.path.join(self.gc_path, fname)
|
||||||
if not os.path.isfile(fpath):
|
if not os.path.isfile(fpath):
|
||||||
self.remove_file_metadata(fname)
|
del self.metadata[fname]
|
||||||
logging.info(f"Pruned file: {fname}")
|
del_keys.append(fname)
|
||||||
continue
|
elif "thumbnails" in self.metadata[fname]:
|
||||||
|
# Check for any stale data entries and remove them
|
||||||
|
need_sync = False
|
||||||
|
for thumb in self.metadata[fname]['thumbnails']:
|
||||||
|
if 'data' in thumb:
|
||||||
|
del thumb['data']
|
||||||
|
need_sync = True
|
||||||
|
if need_sync:
|
||||||
|
self.mddb[fname] = self.metadata[fname]
|
||||||
|
# Delete any removed keys from the database
|
||||||
|
if del_keys:
|
||||||
|
ret = self.mddb.delete_batch(del_keys).result()
|
||||||
|
self._remove_thumbs(ret)
|
||||||
|
pruned = '\n'.join(ret.keys())
|
||||||
|
if pruned:
|
||||||
|
logging.info(f"Pruned metadata for the following:\n"
|
||||||
|
f"{pruned}")
|
||||||
|
|
||||||
def update_gcode_path(self, path: str) -> None:
|
def update_gcode_path(self, path: str) -> None:
|
||||||
if path == self.gc_path:
|
if path == self.gc_path:
|
||||||
return
|
return
|
||||||
|
self.metadata.clear()
|
||||||
self.mddb.clear()
|
self.mddb.clear()
|
||||||
self.gc_path = path
|
self.gc_path = path
|
||||||
|
|
||||||
|
@ -1478,10 +1500,7 @@ class MetadataStorage:
|
||||||
key: str,
|
key: str,
|
||||||
default: _T = None
|
default: _T = None
|
||||||
) -> Union[_T, Dict[str, Any]]:
|
) -> Union[_T, Dict[str, Any]]:
|
||||||
return self.mddb.get(key, default)
|
return self.metadata.get(key, default)
|
||||||
|
|
||||||
def __getitem__(self, key: str) -> Dict[str, Any]:
|
|
||||||
return self.mddb[key]
|
|
||||||
|
|
||||||
def _has_valid_data(self,
|
def _has_valid_data(self,
|
||||||
fname: str,
|
fname: str,
|
||||||
|
@ -1491,7 +1510,7 @@ class MetadataStorage:
|
||||||
# UFP files always need processing
|
# UFP files always need processing
|
||||||
return False
|
return False
|
||||||
mdata: Dict[str, Any]
|
mdata: Dict[str, Any]
|
||||||
mdata = self.mddb.get(fname, {'size': "", 'modified': 0})
|
mdata = self.metadata.get(fname, {'size': "", 'modified': 0})
|
||||||
for field in ['size', 'modified']:
|
for field in ['size', 'modified']:
|
||||||
if mdata[field] != path_info.get(field, None):
|
if mdata[field] != path_info.get(field, None):
|
||||||
return False
|
return False
|
||||||
|
@ -1500,15 +1519,29 @@ class MetadataStorage:
|
||||||
def remove_directory_metadata(self, dir_name: str) -> None:
|
def remove_directory_metadata(self, dir_name: str) -> None:
|
||||||
if dir_name[-1] != "/":
|
if dir_name[-1] != "/":
|
||||||
dir_name += "/"
|
dir_name += "/"
|
||||||
for fname in list(self.mddb.keys()):
|
del_items: Dict[str, Any] = {}
|
||||||
|
for fname in list(self.metadata.keys()):
|
||||||
if fname.startswith(dir_name):
|
if fname.startswith(dir_name):
|
||||||
self.remove_file_metadata(fname)
|
md = self.metadata.pop(fname, None)
|
||||||
|
if md:
|
||||||
|
del_items[fname] = md
|
||||||
|
if del_items:
|
||||||
|
# Remove items from persistent storage
|
||||||
|
self.mddb.delete_batch(list(del_items.keys()))
|
||||||
|
eventloop = self.server.get_event_loop()
|
||||||
|
# Remove thumbs in a nother thread
|
||||||
|
eventloop.run_in_thread(self._remove_thumbs, del_items)
|
||||||
|
|
||||||
def remove_file_metadata(self, fname: str) -> None:
|
def remove_file_metadata(self, fname: str) -> None:
|
||||||
metadata: Optional[Dict[str, Any]]
|
md: Optional[Dict[str, Any]] = self.metadata.pop(fname, None)
|
||||||
metadata = self.mddb.pop(fname, None)
|
if md is None:
|
||||||
if metadata is None:
|
|
||||||
return
|
return
|
||||||
|
self.mddb.pop(fname, None)
|
||||||
|
eventloop = self.server.get_event_loop()
|
||||||
|
eventloop.run_in_thread(self._remove_thumbs, {fname: md})
|
||||||
|
|
||||||
|
def _remove_thumbs(self, records: Dict[str, Dict[str, Any]]) -> None:
|
||||||
|
for fname, metadata in records.items():
|
||||||
# Delete associated thumbnails
|
# Delete associated thumbnails
|
||||||
fdir = os.path.dirname(os.path.join(self.gc_path, fname))
|
fdir = os.path.dirname(os.path.join(self.gc_path, fname))
|
||||||
if "thumbnails" in metadata:
|
if "thumbnails" in metadata:
|
||||||
|
@ -1525,31 +1558,59 @@ class MetadataStorage:
|
||||||
except Exception:
|
except Exception:
|
||||||
logging.debug(f"Error removing thumb at {thumb_path}")
|
logging.debug(f"Error removing thumb at {thumb_path}")
|
||||||
|
|
||||||
def move_directory_metadata(self, prev_dir: str, new_dir: str) -> None:
|
async def move_directory_metadata(self,
|
||||||
|
prev_dir: str,
|
||||||
|
new_dir: str
|
||||||
|
) -> None:
|
||||||
if prev_dir[-1] != "/":
|
if prev_dir[-1] != "/":
|
||||||
prev_dir += "/"
|
prev_dir += "/"
|
||||||
for prev_fname in list(self.mddb.keys()):
|
moved: List[Tuple[str, str, Dict[str, Any]]] = []
|
||||||
|
for prev_fname in list(self.metadata.keys()):
|
||||||
if prev_fname.startswith(prev_dir):
|
if prev_fname.startswith(prev_dir):
|
||||||
new_fname = os.path.join(new_dir, prev_fname[len(prev_dir):])
|
new_fname = os.path.join(new_dir, prev_fname[len(prev_dir):])
|
||||||
self.move_file_metadata(prev_fname, new_fname, False)
|
md: Optional[Dict[str, Any]]
|
||||||
|
md = self.metadata.pop(prev_fname, None)
|
||||||
|
if md is None:
|
||||||
|
continue
|
||||||
|
self.metadata[new_fname] = md
|
||||||
|
moved.append((prev_fname, new_fname, md))
|
||||||
|
if moved:
|
||||||
|
source = [m[0] for m in moved]
|
||||||
|
dest = [m[1] for m in moved]
|
||||||
|
self.mddb.move_batch(source, dest)
|
||||||
|
eventloop = self.server.get_event_loop()
|
||||||
|
await eventloop.run_in_thread(self._move_thumbnails, moved)
|
||||||
|
|
||||||
def move_file_metadata(self,
|
async def move_file_metadata(self,
|
||||||
prev_fname: str,
|
prev_fname: str,
|
||||||
new_fname: str,
|
new_fname: str,
|
||||||
move_thumbs: bool = True
|
move_thumbs: bool = True
|
||||||
) -> bool:
|
) -> bool:
|
||||||
metadata: Optional[Dict[str, Any]]
|
metadata: Optional[Dict[str, Any]]
|
||||||
metadata = self.mddb.pop(prev_fname, None)
|
metadata = self.metadata.pop(prev_fname, None)
|
||||||
if metadata is None:
|
if metadata is None:
|
||||||
# If this move overwrites an existing file it is necessary
|
# If this move overwrites an existing file it is necessary
|
||||||
# to rescan which requires that we remove any existing
|
# to rescan which requires that we remove any existing
|
||||||
# metadata.
|
# metadata.
|
||||||
|
if self.metadata.pop(new_fname, None) is not None:
|
||||||
self.mddb.pop(new_fname, None)
|
self.mddb.pop(new_fname, None)
|
||||||
return False
|
return False
|
||||||
self.mddb[new_fname] = metadata
|
self.metadata[new_fname] = metadata
|
||||||
|
self.mddb.move_batch([prev_fname], [new_fname])
|
||||||
|
if move_thumbs:
|
||||||
|
eventloop = self.server.get_event_loop()
|
||||||
|
await eventloop.run_in_thread(
|
||||||
|
self._move_thumbnails,
|
||||||
|
[(prev_fname, new_fname, metadata)])
|
||||||
|
return True
|
||||||
|
|
||||||
|
def _move_thumbnails(self,
|
||||||
|
records: List[Tuple[str, str, Dict[str, Any]]]
|
||||||
|
) -> None:
|
||||||
|
for (prev_fname, new_fname, metadata) in records:
|
||||||
prev_dir = os.path.dirname(os.path.join(self.gc_path, prev_fname))
|
prev_dir = os.path.dirname(os.path.join(self.gc_path, prev_fname))
|
||||||
new_dir = os.path.dirname(os.path.join(self.gc_path, new_fname))
|
new_dir = os.path.dirname(os.path.join(self.gc_path, new_fname))
|
||||||
if "thumbnails" in metadata and move_thumbs:
|
if "thumbnails" in metadata:
|
||||||
thumb: Dict[str, Any]
|
thumb: Dict[str, Any]
|
||||||
for thumb in metadata["thumbnails"]:
|
for thumb in metadata["thumbnails"]:
|
||||||
path: Optional[str] = thumb.get("relative_path", None)
|
path: Optional[str] = thumb.get("relative_path", None)
|
||||||
|
@ -1565,7 +1626,6 @@ class MetadataStorage:
|
||||||
except Exception:
|
except Exception:
|
||||||
logging.debug(f"Error moving thumb from {thumb_path}"
|
logging.debug(f"Error moving thumb from {thumb_path}"
|
||||||
f" to {new_path}")
|
f" to {new_path}")
|
||||||
return True
|
|
||||||
|
|
||||||
def parse_metadata(self,
|
def parse_metadata(self,
|
||||||
fname: str,
|
fname: str,
|
||||||
|
@ -1609,12 +1669,13 @@ class MetadataStorage:
|
||||||
break
|
break
|
||||||
else:
|
else:
|
||||||
if ufp_path is None:
|
if ufp_path is None:
|
||||||
self.mddb[fname] = {
|
self.metadata[fname] = {
|
||||||
'size': path_info.get('size', 0),
|
'size': path_info.get('size', 0),
|
||||||
'modified': path_info.get('modified', 0),
|
'modified': path_info.get('modified', 0),
|
||||||
'print_start_time': None,
|
'print_start_time': None,
|
||||||
'job_id': None
|
'job_id': None
|
||||||
}
|
}
|
||||||
|
self.mddb[fname] = self.metadata[fname]
|
||||||
logging.info(
|
logging.info(
|
||||||
f"Unable to extract medatadata from file: {fname}")
|
f"Unable to extract medatadata from file: {fname}")
|
||||||
self.pending_requests.pop(fname, None)
|
self.pending_requests.pop(fname, None)
|
||||||
|
@ -1652,8 +1713,8 @@ class MetadataStorage:
|
||||||
# This indicates an error, do not add metadata for this
|
# This indicates an error, do not add metadata for this
|
||||||
raise self.server.error("Unable to extract metadata")
|
raise self.server.error("Unable to extract metadata")
|
||||||
metadata.update({'print_start_time': None, 'job_id': None})
|
metadata.update({'print_start_time': None, 'job_id': None})
|
||||||
self.mddb[path] = dict(metadata)
|
self.metadata[path] = metadata
|
||||||
metadata['filename'] = path
|
self.mddb[path] = metadata
|
||||||
|
|
||||||
def load_component(config: ConfigHelper) -> FileManager:
|
def load_component(config: ConfigHelper) -> FileManager:
|
||||||
return FileManager(config)
|
return FileManager(config)
|
||||||
|
|
Loading…
Reference in New Issue