3diy
/
moonraker
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

file_manager: add support for streaming file uploads 

Signed-off-by:  Eric Callahan <arksine.code@gmail.com>
This commit is contained in:
Arksine 2021-03-04 20:32:18 -05:00
parent f1edaa1f61
commit 4c914d7b4d
1 changed files with 110 additions and 107 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

217
moonraker/plugins/file_manager.py
View File

@ -6,10 +6,11 @@
import os import os
import sys import sys
import shutil import shutil
import io
import zipfile import zipfile
import logging import logging
import json import json
import tempfile
from concurrent.futures import ThreadPoolExecutor
from tornado.ioloop import IOLoop, PeriodicCallback from tornado.ioloop import IOLoop, PeriodicCallback
from tornado.locks import Event from tornado.locks import Event
@ -324,31 +325,76 @@ class FileManager:
path_info = {'modified': modified, 'size': size} path_info = {'modified': modified, 'size': size}
return path_info return path_info
async def process_file_upload(self, request): def gen_temp_upload_path(self):
ioloop = IOLoop.current()
return os.path.join(
tempfile.gettempdir(),
f"moonraker.upload-{int(ioloop.time())}.mru")
async def finalize_upload(self, form_args):
# lookup root file path # lookup root file path
root = self._get_argument(request, 'root', "gcodes") try:
if root == "gcodes": upload_info = self._parse_upload_args(form_args)
result = await self._do_gcode_upload(request) root = upload_info['root']
elif root in FULL_ACCESS_ROOTS: if root == "gcodes":
result = self._do_standard_upload(request, root) result = await self._finish_gcode_upload(upload_info)
else: elif root in FULL_ACCESS_ROOTS:
raise self.server.error(f"Invalid root request: {root}") result = await self._finish_standard_upload(upload_info)
else:
raise self.server.error(f"Invalid root request: {root}")
except Exception:
try:
os.remove(form_args['tmp_file_path'])
except Exception:
pass
raise
return result return result
async def _do_gcode_upload(self, request): def _parse_upload_args(self, upload_args):
start_print = print_ongoing = False if 'filename' not in upload_args:
root_path = self.file_paths.get("gcodes", "") raise self.server.error(
if not root_path: "No file name specifed in upload form")
raise self.server.error("Gcodes root not available") # check relative path
start_print = self._get_argument(request, 'print', "false") == "true" root = upload_args.get('root', "gcodes").lower()
upload = self._get_upload_info(request, root_path) if root not in self.file_paths:
fparts = os.path.splitext(upload['full_path']) raise self.server.error(f"Root {root} not available")
is_ufp = fparts[-1].lower() == ".ufp" root_path = self.file_paths[root]
dir_path = upload_args.get('path', "")
if os.path.isfile(root_path):
filename = os.path.basename(root_path)
dest_path = root_path
dir_path = ""
else:
filename = upload_args['filename'].strip().lstrip("/")
if dir_path:
filename = os.path.join(dir_path, filename)
dest_path = os.path.abspath(os.path.join(root_path, filename))
# Validate the path. Don't allow uploads to a parent of the root
if not dest_path.startswith(root_path):
raise self.server.error(
f"Cannot write to path: {dest_path}")
start_print = upload_args.get('print', "false") == "true"
f_ext = os.path.splitext(dest_path)[-1].lower()
unzip_ufp = f_ext == ".ufp" and root == "gcodes"
if unzip_ufp:
filename = os.path.splitext(filename)[0] + ".gcode"
dest_path = os.path.splitext(dest_path)[0] + ".gcode"
return {
'root': root,
'filename': filename,
'dir_path': dir_path,
'dest_path': dest_path,
'tmp_file_path': upload_args['tmp_file_path'],
'start_print': start_print,
'unzip_ufp': unzip_ufp
}
async def _finish_gcode_upload(self, upload_info):
print_ongoing = False
start_print = upload_info['start_print']
# Verify that the operation can be done if attempting to upload a gcode # Verify that the operation can be done if attempting to upload a gcode
try: try:
check_path = upload['full_path'] check_path = upload_info['dest_path']
if is_ufp:
check_path = fparts[0] + ".gcode"
print_ongoing = await self._handle_operation_check( print_ongoing = await self._handle_operation_check(
check_path) check_path)
except self.server.error as e: except self.server.error as e:
@ -361,112 +407,73 @@ class FileManager:
start_print = False start_print = False
# Don't start if another print is currently in progress # Don't start if another print is currently in progress
start_print = start_print and not print_ongoing start_print = start_print and not print_ongoing
self._write_file(upload, is_ufp) ioloop = IOLoop.current()
with ThreadPoolExecutor(max_workers=1) as tpe:
await ioloop.run_in_executor(
tpe, self._process_uploaded_file, upload_info)
# Fetch Metadata # Fetch Metadata
finfo = self._get_path_info(upload['full_path']) finfo = self._get_path_info(upload_info['dest_path'])
evt = self.gcode_metadata.parse_metadata( evt = self.gcode_metadata.parse_metadata(
upload['filename'], finfo['size'], finfo['modified']) upload_info['filename'], finfo['size'], finfo['modified'])
await evt.wait() await evt.wait()
if start_print: if start_print:
# Make a Klippy Request to "Start Print" # Make a Klippy Request to "Start Print"
klippy_apis = self.server.lookup_plugin('klippy_apis') klippy_apis = self.server.lookup_plugin('klippy_apis')
try: try:
await klippy_apis.start_print(upload['filename']) await klippy_apis.start_print(upload_info['filename'])
except self.server.error: except self.server.error:
# Attempt to start print failed # Attempt to start print failed
start_print = False start_print = False
self.notify_filelist_changed( self.notify_filelist_changed(
'upload_file', upload['filename'], "gcodes") 'upload_file', upload_info['filename'], "gcodes")
return {'result': upload['filename'], 'print_started': start_print}
def _do_standard_upload(self, request, root):
path = self.file_paths.get(root, None)
if path is None:
raise self.server.error(f"Unknown root path: {root}")
upload = self._get_upload_info(request, path)
self._write_file(upload)
self.notify_filelist_changed('upload_file', upload['filename'], root)
return {'result': upload['filename']}
def _get_argument(self, request, name, default=None):
args = request.arguments.get(name, None)
if args is not None:
return args[0].decode().strip()
return default
def _get_upload_info(self, request, root_path):
# check relative path
dir_path = self._get_argument(request, 'path', "")
# fetch the upload from the request
if len(request.files) != 1:
raise self.server.error(
"Bad Request, can only process a single file upload")
f_list = list(request.files.values())[0]
if len(f_list) != 1:
raise self.server.error(
"Bad Request, can only process a single file upload")
upload = f_list[0]
if os.path.isfile(root_path):
filename = os.path.basename(root_path)
full_path = root_path
dir_path = ""
else:
filename = upload['filename'].strip().lstrip("/")
if dir_path:
filename = os.path.join(dir_path, filename)
full_path = os.path.abspath(os.path.join(root_path, filename))
# Validate the path. Don't allow uploads to a parent of the root
if not full_path.startswith(root_path):
raise self.server.error(
f"Cannot write to path: {full_path}")
return { return {
'filename': filename, 'result': upload_info['filename'],
'body': upload['body'], 'print_started': start_print
'dir_path': dir_path, }
'full_path': full_path}
def _write_file(self, upload, unzip_ufp=False): async def _finish_standard_upload(self, upload_info):
ioloop = IOLoop.current()
with ThreadPoolExecutor(max_workers=1) as tpe:
await ioloop.run_in_executor(
tpe, self._process_uploaded_file, upload_info)
self.notify_filelist_changed(
'upload_file', upload_info['filename'], upload_info['root'])
return {'result': upload_info['filename']}
def _process_uploaded_file(self, upload_info):
try: try:
if upload['dir_path']: if upload_info['dir_path']:
os.makedirs(os.path.dirname( os.makedirs(os.path.dirname(
upload['full_path']), exist_ok=True) upload_info['dest_path']), exist_ok=True)
if unzip_ufp: if upload_info['unzip_ufp']:
self._unzip_ufp(upload) self._unzip_ufp(upload_info['tmp_file_path'],
upload_info['dest_path'])
else: else:
with open(upload['full_path'], 'wb') as fh: shutil.move(upload_info['tmp_file_path'],
fh.write(upload['body']) upload_info['dest_path'])
except Exception: except Exception:
raise self.server.error("Unable to save file", 500) raise self.server.error("Unable to save file", 500)
# UFP Extraction Implementation inspired by by GitHub user @cdkeito # UFP Extraction Implementation inspired by GitHub user @cdkeito
def _unzip_ufp(self, upload): def _unzip_ufp(self, ufp_path, dest_path):
base_name = os.path.splitext(
os.path.basename(upload['filename']))[0]
working_dir = os.path.dirname(upload['full_path'])
thumb_dir = os.path.join(working_dir, "thumbs")
ufp_bytes = io.BytesIO(upload['body'])
gc_bytes = img_bytes = None gc_bytes = img_bytes = None
with zipfile.ZipFile(ufp_bytes) as zf: with zipfile.ZipFile(ufp_path) as zf:
gc_bytes = zf.read("/3D/model.gcode") gc_bytes = zf.read("/3D/model.gcode")
try: try:
img_bytes = zf.read("/Metadata/thumbnail.png") img_bytes = zf.read("/Metadata/thumbnail.png")
except Exception: except Exception:
img_bytes = None img_bytes = None
if gc_bytes is not None: if gc_bytes is not None:
gc_name = base_name + ".gcode" with open(dest_path, "wb") as gc_file:
gc_path = os.path.join(working_dir, gc_name)
with open(gc_path, "wb") as gc_file:
gc_file.write(gc_bytes) gc_file.write(gc_bytes)
# update upload file name to extracted gcode file
upload['full_path'] = gc_path
upload['filename'] = os.path.join(
os.path.dirname(upload['filename']), gc_name)
else: else:
raise self.server.error( raise self.server.error(
f"UFP file {upload['filename']} does not " f"UFP file {dest_path} does not "
"contain a gcode file") "contain a gcode file")
if img_bytes is not None: if img_bytes is not None:
thumb_name = base_name + ".png" thumb_name = os.path.splitext(
os.path.basename(dest_path))[0] + ".png"
thumb_dir = os.path.join(os.path.dirname(dest_path), "thumbs")
thumb_path = os.path.join(thumb_dir, thumb_name) thumb_path = os.path.join(thumb_dir, thumb_name)
try: try:
if not os.path.exists(thumb_dir): if not os.path.exists(thumb_dir):
@ -475,19 +482,15 @@ class FileManager:
thumb_file.write(img_bytes) thumb_file.write(img_bytes)
except Exception: except Exception:
logging.exception("Unable to write Image") logging.exception("Unable to write Image")
try:
os.remove(ufp_path)
except Exception:
logging.exception(f"Error removing ufp file: {ufp_path}")
def _process_ufp_from_refresh(self, ufp_path): def _process_ufp_from_refresh(self, ufp_path):
filename = os.path.split(ufp_path)[-1] dest_path = os.path.splitext(ufp_path)[0] + ".gcode"
ul = { self._unzip_ufp(ufp_path, dest_path)
'filename': filename, return dest_path
'full_path': ufp_path
}
with open(ufp_path, 'rb') as ufp:
body = ufp.read()
ul['body'] = body
self._unzip_ufp(ul)
os.remove(ufp_path)
return ul['full_path']
def get_file_list(self, root, list_format=False, notify=False): def get_file_list(self, root, list_format=False, notify=False):
# Use os.walk find files in sd path and subdirs # Use os.walk find files in sd path and subdirs