From b4af85f405a1a0bd90814f6f65bb6dabf1e272ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gina=20H=C3=A4u=C3=9Fge?= Date: Tue, 5 Aug 2014 11:26:13 +0200 Subject: [PATCH] Finalizing upload streaming support Major refactoring of octoprint.server.util (divided into smaller submodules), extended Tornado to allow for request-specific max content lengths, introduced settings parameters to configure maximum upload size, maximum request body size and file suffixes See #455 --- src/octoprint/server/__init__.py | 52 +- src/octoprint/server/api/__init__.py | 14 +- src/octoprint/server/api/connection.py | 6 +- src/octoprint/server/api/files.py | 27 +- src/octoprint/server/api/job.py | 6 +- src/octoprint/server/api/log.py | 9 +- src/octoprint/server/api/printer.py | 8 +- src/octoprint/server/api/settings.py | 6 +- src/octoprint/server/api/timelapse.py | 9 +- src/octoprint/server/api/users.py | 6 +- src/octoprint/server/util.py | 1450 ------------------------ src/octoprint/server/util/__init__.py | 100 ++ src/octoprint/server/util/flask.py | 154 +++ src/octoprint/server/util/sockjs.py | 124 ++ src/octoprint/server/util/tornado.py | 853 ++++++++++++++ src/octoprint/server/util/watchdog.py | 87 ++ src/octoprint/settings.py | 14 +- src/octoprint/util/__init__.py | 4 + 18 files changed, 1422 insertions(+), 1507 deletions(-) delete mode 100644 src/octoprint/server/util.py create mode 100644 src/octoprint/server/util/__init__.py create mode 100644 src/octoprint/server/util/flask.py create mode 100644 src/octoprint/server/util/sockjs.py create mode 100644 src/octoprint/server/util/tornado.py create mode 100644 src/octoprint/server/util/watchdog.py diff --git a/src/octoprint/server/__init__.py b/src/octoprint/server/__init__.py index 8f29dd53..96890a40 100644 --- a/src/octoprint/server/__init__.py +++ b/src/octoprint/server/__init__.py @@ -1,13 +1,13 @@ # coding=utf-8 -import uuid +from __future__ import absolute_import __author__ = "Gina Häußge " __license__ = 'GNU Affero General Public License http://www.gnu.org/licenses/agpl.html' +__copyright__ = "Copyright (C) 2014 The OctoPrint Project - Released under terms of the AGPLv3 License" -import flask -import tornado.wsgi +import uuid from sockjs.tornado import SockJSRouter -from flask import Flask, render_template, send_from_directory, make_response +from flask import Flask, render_template, send_from_directory from flask.ext.login import LoginManager from flask.ext.principal import Principal, Permission, RoleNeed, identity_loaded, UserNeed from watchdog.observers import Observer @@ -33,18 +33,16 @@ admin_permission = Permission(RoleNeed("admin")) user_permission = Permission(RoleNeed("user")) # only import the octoprint stuff down here, as it might depend on things defined above to be initialized already -from octoprint.server.util import LargeResponseHandler, ReverseProxied, restricted_access, PrinterStateConnection, admin_validator, \ - UrlForwardHandler, user_validator, GcodeWatchdogHandler, UploadCleanupWatchdogHandler, \ - access_validation_factory, WsgiInputContainer, StreamingFallbackHandler, PrintableFilesUploadHandler, \ - UploadStorageFallbackHandler from octoprint.printer import Printer, getConnectionOptions from octoprint.settings import settings import octoprint.gcodefiles as gcodefiles -import octoprint.util as util import octoprint.users as users import octoprint.events as events import octoprint.timelapse import octoprint._version +import octoprint.util + +from . import util UI_API_KEY = ''.join('%02X' % ord(z) for z in uuid.uuid4().bytes) @@ -121,10 +119,8 @@ class Server(): global loginManager global debug - from tornado.wsgi import WSGIContainer - from tornado.httpserver import HTTPServer from tornado.ioloop import IOLoop - from tornado.web import Application, FallbackHandler + from tornado.web import Application debug = self._debug @@ -152,12 +148,12 @@ class Server(): if settings().getBoolean(["accessControl", "enabled"]): userManagerName = settings().get(["accessControl", "userManager"]) try: - clazz = util.getClass(userManagerName) + clazz = octoprint.util.getClass(userManagerName) userManager = clazz() except AttributeError, e: logger.exception("Could not instantiate user manager %s, will run with accessControl disabled!" % userManagerName) - app.wsgi_app = ReverseProxied(app.wsgi_app) + app.wsgi_app = util.ReverseProxied(app.wsgi_app) app.secret_key = "k3PuVYgtxNm8DXKKTw2nWmFQQun9qceV" loginManager = LoginManager() @@ -182,14 +178,18 @@ class Server(): self._router = SockJSRouter(self._createSocketConnection, "/sockjs") + upload_suffixes = dict(name=settings().get(["server", "uploads", "nameSuffix"]), path=settings().get(["server", "uploads", "pathSuffix"])) self._tornado_app = Application(self._router.urls + [ - (r"/downloads/timelapse/([^/]*\.mpg)", LargeResponseHandler, dict(path=settings().getBaseFolder("timelapse"), as_attachment=True)), - (r"/downloads/files/local/([^/]*\.(gco|gcode))", LargeResponseHandler, dict(path=settings().getBaseFolder("uploads"), as_attachment=True)), - (r"/downloads/logs/([^/]*)", LargeResponseHandler, dict(path=settings().getBaseFolder("logs"), as_attachment=True, access_validation=access_validation_factory(app, loginManager, admin_validator))), - (r"/downloads/camera/current", UrlForwardHandler, dict(url=settings().get(["webcam", "snapshot"]), as_attachment=True, access_validation=access_validation_factory(app, loginManager, user_validator))), - (r".*", UploadStorageFallbackHandler, dict(fallback=WsgiInputContainer(app.wsgi_app), file_prefix="octoprint-file-upload-", file_suffix=".tmp")) + (r"/downloads/timelapse/([^/]*\.mpg)", util.tornado.LargeResponseHandler, dict(path=settings().getBaseFolder("timelapse"), as_attachment=True)), + (r"/downloads/files/local/([^/]*\.(gco|gcode))", util.tornado.LargeResponseHandler, dict(path=settings().getBaseFolder("uploads"), as_attachment=True)), + (r"/downloads/logs/([^/]*)", util.tornado.LargeResponseHandler, dict(path=settings().getBaseFolder("logs"), as_attachment=True, access_validation=util.tornado.access_validation_factory(app, loginManager, util.flask.admin_validator))), + (r"/downloads/camera/current", util.tornado.UrlForwardHandler, dict(url=settings().get(["webcam", "snapshot"]), as_attachment=True, access_validation=util.tornado.access_validation_factory(app, loginManager, util.flask.user_validator))), + (r".*", util.tornado.UploadStorageFallbackHandler, dict(fallback=util.tornado.WsgiInputContainer(app.wsgi_app), file_prefix="octoprint-file-upload-", file_suffix=".tmp", suffixes=upload_suffixes)) ]) - self._server = HTTPServer(self._tornado_app, max_body_size=1*1024*1024*1024) + max_body_sizes = [ + ("POST", r"/api/files/([^/]*)", settings().getInt(["server", "uploads", "maxSize"])) + ] + self._server = util.tornado.CustomHTTPServer(self._tornado_app, max_body_sizes=max_body_sizes, default_max_body_size=10*1024) self._server.listen(self._port, address=self._host) eventManager.fire(events.Events.STARTUP) @@ -201,8 +201,8 @@ class Server(): # start up watchdogs observer = Observer() - observer.schedule(GcodeWatchdogHandler(gcodeManager, printer), settings().getBaseFolder("watched")) - observer.schedule(UploadCleanupWatchdogHandler(gcodeManager), settings().getBaseFolder("uploads")) + observer.schedule(util.watchdog.GcodeWatchdogHandler(gcodeManager, printer), settings().getBaseFolder("watched")) + observer.schedule(util.watchdog.UploadCleanupWatchdogHandler(gcodeManager), settings().getBaseFolder("uploads")) observer.start() try: @@ -218,7 +218,7 @@ class Server(): def _createSocketConnection(self, session): global printer, gcodeManager, userManager, eventManager - return PrinterStateConnection(printer, gcodeManager, userManager, eventManager, session) + return util.sockjs.PrinterStateConnection(printer, gcodeManager, userManager, eventManager, session) def _checkForRoot(self): if "geteuid" in dir(os) and os.geteuid() == 0: @@ -283,7 +283,7 @@ class Server(): with open(logConf, "r") as f: configFromFile = yaml.safe_load(f) - config = util.dict_merge(defaultConfig, configFromFile) + config = octoprint.util.dict_merge(defaultConfig, configFromFile) logging.config.dictConfig(config) if settings().getBoolean(["serial", "log"]): @@ -292,5 +292,5 @@ class Server(): logging.getLogger("SERIAL").debug("Enabling serial logging") if __name__ == "__main__": - octoprint = Server() - octoprint.run() + server = Server() + server.run() diff --git a/src/octoprint/server/api/__init__.py b/src/octoprint/server/api/__init__.py index 2af44726..5e021ac1 100644 --- a/src/octoprint/server/api/__init__.py +++ b/src/octoprint/server/api/__init__.py @@ -1,8 +1,9 @@ # coding=utf-8 -from octoprint.server.util import getApiKey, getUserForApiKey +from __future__ import absolute_import __author__ = "Gina Häußge " __license__ = 'GNU Affero General Public License http://www.gnu.org/licenses/agpl.html' +__copyright__ = "Copyright (C) 2014 The OctoPrint Project - Released under terms of the AGPLv3 License" import logging import netaddr @@ -15,8 +16,11 @@ from flask.ext.principal import Identity, identity_changed, AnonymousIdentity import octoprint.util as util import octoprint.users import octoprint.server -from octoprint.server import restricted_access, admin_permission, NO_CONTENT, UI_API_KEY +from octoprint.server import admin_permission, NO_CONTENT, UI_API_KEY from octoprint.settings import settings as s, valid_boolean_trues +from octoprint.server.util import get_api_key, get_user_for_apikey +from octoprint.server.util.flask import restricted_access + #~~ init api blueprint, including sub modules @@ -67,7 +71,7 @@ def beforeApiRequests(): if request.method == 'OPTIONS' and s().getBoolean(["api", "allowCrossOrigin"]): return optionsAllowOrigin(request) - apikey = getApiKey(request) + apikey = get_api_key(request) if apikey is None: # no api key => 401 return make_response("No API key provided", 401) @@ -84,7 +88,7 @@ def beforeApiRequests(): # global api key => continue regular request processing return - user = getUserForApiKey(apikey) + user = get_user_for_apikey(apikey) if user is not None: # user specific api key => continue regular request processing return @@ -143,7 +147,7 @@ def apiPrinterState(): def apiVersion(): return jsonify({ "server": octoprint.server.VERSION, - "api": octoprint.server.api.VERSION + "api": VERSION }) #~~ system control diff --git a/src/octoprint/server/api/connection.py b/src/octoprint/server/api/connection.py index 683c4e9d..f947e860 100644 --- a/src/octoprint/server/api/connection.py +++ b/src/octoprint/server/api/connection.py @@ -1,13 +1,17 @@ # coding=utf-8 +from __future__ import absolute_import + __author__ = "Gina Häußge " __license__ = 'GNU Affero General Public License http://www.gnu.org/licenses/agpl.html' +__copyright__ = "Copyright (C) 2014 The OctoPrint Project - Released under terms of the AGPLv3 License" from flask import request, jsonify, make_response from octoprint.settings import settings from octoprint.printer import getConnectionOptions -from octoprint.server import printer, restricted_access, NO_CONTENT +from octoprint.server import printer, NO_CONTENT from octoprint.server.api import api +from octoprint.server.util.flask import restricted_access import octoprint.util as util diff --git a/src/octoprint/server/api/files.py b/src/octoprint/server/api/files.py index eb71c14c..8f0dd062 100644 --- a/src/octoprint/server/api/files.py +++ b/src/octoprint/server/api/files.py @@ -1,17 +1,20 @@ # coding=utf-8 -from octoprint.events import Events +from __future__ import absolute_import __author__ = "Gina Häußge " __license__ = 'GNU Affero General Public License http://www.gnu.org/licenses/agpl.html' +__copyright__ = "Copyright (C) 2014 The OctoPrint Project - Released under terms of the AGPLv3 License" -from flask import request, jsonify, make_response, url_for, redirect +from flask import request, jsonify, make_response, url_for import octoprint.gcodefiles as gcodefiles import octoprint.util as util from octoprint.filemanager.destinations import FileDestinations from octoprint.settings import settings, valid_boolean_trues -from octoprint.server import printer, gcodeManager, eventManager, restricted_access, NO_CONTENT +from octoprint.server import printer, gcodeManager, eventManager, NO_CONTENT +from octoprint.server.util.flask import restricted_access from octoprint.server.api import api +from octoprint.events import Events #~~ GCODE file handling @@ -89,13 +92,16 @@ def uploadGcodeFile(target): if not target in [FileDestinations.LOCAL, FileDestinations.SDCARD]: return make_response("Unknown target: %s" % target, 404) - if "file.name" in request.values and "file.path" in request.values: + input_name = "file" + input_upload_name = input_name + "." + settings().get(["server", "uploads", "nameSuffix"]) + input_upload_path = input_name + "." + settings().get(["server", "uploads", "pathSuffix"]) + if input_upload_name in request.values and input_upload_path in request.values: import shutil - upload = Object() - upload.filename = request.values["file.name"] - upload.save = lambda new_path: shutil.move(request.values["file.path"], new_path) - elif "file" in request.files: - upload = request.files["file"] + upload = util.Object() + upload.filename = request.values[input_upload_name] + upload.save = lambda new_path: shutil.move(request.values[input_upload_path], new_path) + elif input_name in request.files: + upload = request.files[input_name] else: return make_response("No file included", 400) @@ -280,6 +286,3 @@ def deleteGcodeFile(filename, target): return NO_CONTENT - -class Object(object): - pass diff --git a/src/octoprint/server/api/job.py b/src/octoprint/server/api/job.py index 3ee86a1a..84eacd8a 100644 --- a/src/octoprint/server/api/job.py +++ b/src/octoprint/server/api/job.py @@ -1,10 +1,14 @@ # coding=utf-8 +from __future__ import absolute_import + __author__ = "Gina Häußge " __license__ = 'GNU Affero General Public License http://www.gnu.org/licenses/agpl.html' +__copyright__ = "Copyright (C) 2014 The OctoPrint Project - Released under terms of the AGPLv3 License" from flask import request, make_response, jsonify -from octoprint.server import printer, restricted_access, NO_CONTENT +from octoprint.server import printer, NO_CONTENT +from octoprint.server.util.flask import restricted_access from octoprint.server.api import api import octoprint.util as util diff --git a/src/octoprint/server/api/log.py b/src/octoprint/server/api/log.py index 44159d27..b1131b32 100644 --- a/src/octoprint/server/api/log.py +++ b/src/octoprint/server/api/log.py @@ -1,6 +1,9 @@ # coding=utf-8 +from __future__ import absolute_import + __author__ = "Marc Hannappel Salandora" __license__ = 'GNU Affero General Public License http://www.gnu.org/licenses/agpl.html' +__copyright__ = "Copyright (C) 2014 The OctoPrint Project - Released under terms of the AGPLv3 License" import os @@ -9,8 +12,8 @@ from werkzeug.utils import secure_filename from octoprint.settings import settings -from octoprint.server import restricted_access, NO_CONTENT, admin_permission -from octoprint.server.util import redirectToTornado +from octoprint.server import NO_CONTENT, admin_permission +from octoprint.server.util.flask import redirect_to_tornado, restricted_access from octoprint.server.api import api from octoprint.util import getFreeBytes @@ -27,7 +30,7 @@ def getLogFiles(): @restricted_access @admin_permission.require(403) def downloadLog(filename): - return redirectToTornado(request, url_for("index") + "downloads/logs/" + filename) + return redirect_to_tornado(request, url_for("index") + "downloads/logs/" + filename) @api.route("/logs/", methods=["DELETE"]) diff --git a/src/octoprint/server/api/printer.py b/src/octoprint/server/api/printer.py index 39189870..55ae6d3b 100644 --- a/src/octoprint/server/api/printer.py +++ b/src/octoprint/server/api/printer.py @@ -1,13 +1,17 @@ # coding=utf-8 +from __future__ import absolute_import + __author__ = "Gina Häußge " __license__ = 'GNU Affero General Public License http://www.gnu.org/licenses/agpl.html' +__copyright__ = "Copyright (C) 2014 The OctoPrint Project - Released under terms of the AGPLv3 License" -from flask import request, jsonify, make_response +from flask import request, jsonify, make_response, Response import re from octoprint.settings import settings, valid_boolean_trues -from octoprint.server import printer, restricted_access, NO_CONTENT +from octoprint.server import printer, NO_CONTENT from octoprint.server.api import api +from octoprint.server.util.flask import restricted_access import octoprint.util as util #~~ Printer diff --git a/src/octoprint/server/api/settings.py b/src/octoprint/server/api/settings.py index b2306d9e..8c7a5c56 100644 --- a/src/octoprint/server/api/settings.py +++ b/src/octoprint/server/api/settings.py @@ -1,6 +1,9 @@ # coding=utf-8 +from __future__ import absolute_import + __author__ = "Gina Häußge " __license__ = 'GNU Affero General Public License http://www.gnu.org/licenses/agpl.html' +__copyright__ = "Copyright (C) 2014 The OctoPrint Project - Released under terms of the AGPLv3 License" import logging @@ -9,8 +12,9 @@ from flask import request, jsonify from octoprint.settings import settings from octoprint.printer import getConnectionOptions -from octoprint.server import restricted_access, admin_permission +from octoprint.server import admin_permission from octoprint.server.api import api +from octoprint.server.util.flask import restricted_access #~~ settings diff --git a/src/octoprint/server/api/timelapse.py b/src/octoprint/server/api/timelapse.py index 91898e18..11f27e5b 100644 --- a/src/octoprint/server/api/timelapse.py +++ b/src/octoprint/server/api/timelapse.py @@ -1,6 +1,9 @@ # coding=utf-8 +from __future__ import absolute_import + __author__ = "Gina Häußge " __license__ = 'GNU Affero General Public License http://www.gnu.org/licenses/agpl.html' +__copyright__ = "Copyright (C) 2014 The OctoPrint Project - Released under terms of the AGPLv3 License" import os @@ -11,8 +14,8 @@ import octoprint.timelapse import octoprint.util as util from octoprint.settings import settings, valid_boolean_trues -from octoprint.server import restricted_access, admin_permission -from octoprint.server.util import redirectToTornado +from octoprint.server import admin_permission +from octoprint.server.util.flask import redirect_to_tornado, restricted_access from octoprint.server.api import api @@ -46,7 +49,7 @@ def getTimelapseData(): @api.route("/timelapse/", methods=["GET"]) def downloadTimelapse(filename): - return redirectToTornado(request, url_for("index") + "downloads/timelapse/" + filename) + return redirect_to_tornado(request, url_for("index") + "downloads/timelapse/" + filename) @api.route("/timelapse/", methods=["DELETE"]) diff --git a/src/octoprint/server/api/users.py b/src/octoprint/server/api/users.py index 9228acc9..2b6f4022 100644 --- a/src/octoprint/server/api/users.py +++ b/src/octoprint/server/api/users.py @@ -1,14 +1,18 @@ # coding=utf-8 +from __future__ import absolute_import + __author__ = "Gina Häußge " __license__ = 'GNU Affero General Public License http://www.gnu.org/licenses/agpl.html' +__copyright__ = "Copyright (C) 2014 The OctoPrint Project - Released under terms of the AGPLv3 License" from flask import request, jsonify, abort, make_response from flask.ext.login import current_user import octoprint.users as users -from octoprint.server import restricted_access, SUCCESS, admin_permission, userManager +from octoprint.server import SUCCESS, admin_permission, userManager from octoprint.server.api import api +from octoprint.server.util.flask import restricted_access #~~ user settings diff --git a/src/octoprint/server/util.py b/src/octoprint/server/util.py deleted file mode 100644 index f4e6f4ab..00000000 --- a/src/octoprint/server/util.py +++ /dev/null @@ -1,1450 +0,0 @@ -# coding=utf-8 -from tempfile import TemporaryFile -from tornado.httputil import HTTPHeaders -from octoprint.filemanager.destinations import FileDestinations - -__author__ = "Gina Häußge " -__license__ = 'GNU Affero General Public License http://www.gnu.org/licenses/agpl.html' - -from flask.ext.principal import identity_changed, Identity -from tornado.web import StaticFileHandler, HTTPError, RequestHandler, asynchronous, stream_request_body, FallbackHandler -import tornado.escape, tornado.httputil -from tornado.httpclient import AsyncHTTPClient, HTTPRequest -from flask import url_for, make_response, request, current_app -from flask.ext.login import login_required, login_user, current_user -from werkzeug.utils import redirect -from sockjs.tornado import SockJSConnection - -import datetime -import stat -import mimetypes -import email -import time -import os -import threading -import logging -from functools import wraps -from watchdog.events import PatternMatchingEventHandler - -from octoprint.settings import settings, valid_boolean_trues -import octoprint.timelapse -import octoprint.server -from octoprint.users import ApiUser -from octoprint.events import Events -from octoprint import gcodefiles -import octoprint.util as util - -def restricted_access(func, apiEnabled=True): - """ - If you decorate a view with this, it will ensure that first setup has been - done for OctoPrint's Access Control plus that any conditions of the - login_required decorator are met. It also allows to login using the masterkey or any - of the user's apikeys if API access is enabled globally and for the decorated view. - - If OctoPrint's Access Control has not been setup yet (indicated by the "firstRun" - flag from the settings being set to True and the userManager not indicating - that it's user database has been customized from default), the decorator - will cause a HTTP 403 status code to be returned by the decorated resource. - - If an API key is provided and it matches a known key, the user will be logged in and - the view will be called directly. If the provided key doesn't match any known key, - a HTTP 403 status code will be returned by the decorated resource. - - Otherwise the result of calling login_required will be returned. - """ - @wraps(func) - def decorated_view(*args, **kwargs): - # if OctoPrint hasn't been set up yet, abort - if settings().getBoolean(["server", "firstRun"]) and (octoprint.server.userManager is None or not octoprint.server.userManager.hasBeenCustomized()): - return make_response("OctoPrint isn't setup yet", 403) - - # if API is globally enabled, enabled for this request and an api key is provided that is not the current UI API key, try to use that - apikey = getApiKey(request) - if settings().get(["api", "enabled"]) and apiEnabled and apikey is not None and apikey != octoprint.server.UI_API_KEY: - if apikey == settings().get(["api", "key"]): - # master key was used - user = ApiUser() - else: - # user key might have been used - user = octoprint.server.userManager.findUser(apikey=apikey) - - if user is None: - return make_response("Invalid API key", 401) - if login_user(user, remember=False): - identity_changed.send(current_app._get_current_object(), identity=Identity(user.get_id())) - return func(*args, **kwargs) - - # call regular login_required decorator - return login_required(func)(*args, **kwargs) - return decorated_view - - -def api_access(func): - @wraps(func) - def decorated_view(*args, **kwargs): - if not settings().get(["api", "enabled"]): - make_response("API disabled", 401) - apikey = getApiKey(request) - if apikey is None: - make_response("No API key provided", 401) - if apikey != settings().get(["api", "key"]): - make_response("Invalid API key", 403) - return func(*args, **kwargs) - return decorated_view - - -def getUserForApiKey(apikey): - if settings().get(["api", "enabled"]) and apikey is not None: - if apikey == settings().get(["api", "key"]): - # master key was used - return ApiUser() - else: - # user key might have been used - return octoprint.server.userManager.findUser(apikey=apikey) - else: - return None - - -def getApiKey(request): - # Check Flask GET/POST arguments - if hasattr(request, "values") and "apikey" in request.values: - return request.values["apikey"] - - # Check Tornado GET/POST arguments - if hasattr(request, "arguments") and "apikey" in request.arguments \ - and len(request.arguments["apikey"]) > 0 and len(request.arguments["apikey"].strip()) > 0: - return request.arguments["apikey"] - - # Check Tornado and Flask headers - if "X-Api-Key" in request.headers.keys(): - return request.headers.get("X-Api-Key") - - return None - - -#~~ Printer state - - -class PrinterStateConnection(SockJSConnection): - EVENTS = [Events.UPDATED_FILES, Events.METADATA_ANALYSIS_FINISHED, Events.MOVIE_RENDERING, Events.MOVIE_DONE, - Events.MOVIE_FAILED, Events.SLICING_STARTED, Events.SLICING_DONE, Events.SLICING_FAILED, - Events.TRANSFER_STARTED, Events.TRANSFER_DONE] - - def __init__(self, printer, gcodeManager, userManager, eventManager, session): - SockJSConnection.__init__(self, session) - - self._logger = logging.getLogger(__name__) - - self._temperatureBacklog = [] - self._temperatureBacklogMutex = threading.Lock() - self._logBacklog = [] - self._logBacklogMutex = threading.Lock() - self._messageBacklog = [] - self._messageBacklogMutex = threading.Lock() - - self._printer = printer - self._gcodeManager = gcodeManager - self._userManager = userManager - self._eventManager = eventManager - - def _getRemoteAddress(self, info): - forwardedFor = info.headers.get("X-Forwarded-For") - if forwardedFor is not None: - return forwardedFor.split(",")[0] - return info.ip - - def on_open(self, info): - remoteAddress = self._getRemoteAddress(info) - self._logger.info("New connection from client: %s" % remoteAddress) - - # connected => update the API key, might be necessary if the client was left open while the server restarted - self._emit("connected", {"apikey": octoprint.server.UI_API_KEY, "version": octoprint.server.VERSION}) - - self._printer.registerCallback(self) - self._gcodeManager.registerCallback(self) - octoprint.timelapse.registerCallback(self) - - self._eventManager.fire(Events.CLIENT_OPENED, {"remoteAddress": remoteAddress}) - for event in PrinterStateConnection.EVENTS: - self._eventManager.subscribe(event, self._onEvent) - - octoprint.timelapse.notifyCallbacks(octoprint.timelapse.current) - - def on_close(self): - self._logger.info("Client connection closed") - self._printer.unregisterCallback(self) - self._gcodeManager.unregisterCallback(self) - octoprint.timelapse.unregisterCallback(self) - - self._eventManager.fire(Events.CLIENT_CLOSED) - for event in PrinterStateConnection.EVENTS: - self._eventManager.unsubscribe(event, self._onEvent) - - def on_message(self, message): - pass - - def sendCurrentData(self, data): - # add current temperature, log and message backlogs to sent data - with self._temperatureBacklogMutex: - temperatures = self._temperatureBacklog - self._temperatureBacklog = [] - - with self._logBacklogMutex: - logs = self._logBacklog - self._logBacklog = [] - - with self._messageBacklogMutex: - messages = self._messageBacklog - self._messageBacklog = [] - - data.update({ - "temps": temperatures, - "logs": logs, - "messages": messages - }) - self._emit("current", data) - - def sendHistoryData(self, data): - self._emit("history", data) - - def sendEvent(self, type, payload=None): - self._emit("event", {"type": type, "payload": payload}) - - def sendFeedbackCommandOutput(self, name, output): - self._emit("feedbackCommandOutput", {"name": name, "output": output}) - - def sendTimelapseConfig(self, timelapseConfig): - self._emit("timelapse", timelapseConfig) - - def addLog(self, data): - with self._logBacklogMutex: - self._logBacklog.append(data) - - def addMessage(self, data): - with self._messageBacklogMutex: - self._messageBacklog.append(data) - - def addTemperature(self, data): - with self._temperatureBacklogMutex: - self._temperatureBacklog.append(data) - - def _onEvent(self, event, payload): - self.sendEvent(event, payload) - - def _emit(self, type, payload): - self.send({type: payload}) - - -@stream_request_body -class UploadStorageFallbackHandler(RequestHandler): - """ - A `RequestHandler` similar to `tornado.web.FallbackHandler` which fetches any files contained in the request bodies - of content type `multipart`, stores them in temporary files and supplies the `fallback` with the file's `name`, - `content_type`, `path` and `size` instead via a rewritten body. - - Basically similar to what the nginx upload module does. - - Basic request body example: - - ------WebKitFormBoundarypYiSUx63abAmhT5C - Content-Disposition: form-data; name="file"; filename="test.gcode" - Content-Type: application/octet-stream - - ... - ------WebKitFormBoundarypYiSUx63abAmhT5C - Content-Disposition: form-data; name="apikey" - - my_funny_apikey - ------WebKitFormBoundarypYiSUx63abAmhT5C - Content-Disposition: form-data; name="select" - - true - ------WebKitFormBoundarypYiSUx63abAmhT5C-- - - That would get turned into: - - ------WebKitFormBoundarypYiSUx63abAmhT5C - Content-Disposition: form-data; name="apikey" - - my_funny_apikey - ------WebKitFormBoundarypYiSUx63abAmhT5C - Content-Disposition: form-data; name="select" - - true - ------WebKitFormBoundarypYiSUx63abAmhT5C - Content-Disposition: form-data; name="file.path" - - /tmp/tmpzupkro - ------WebKitFormBoundarypYiSUx63abAmhT5C - Content-Disposition: form-data; name="file.name" - - test.gcode - ------WebKitFormBoundarypYiSUx63abAmhT5C - Content-Disposition: form-data; name="file.content_type" - - application/octet-stream - ------WebKitFormBoundarypYiSUx63abAmhT5C - Content-Disposition: form-data; name="file.size" - - 349182 - ------WebKitFormBoundarypYiSUx63abAmhT5C-- - - The underlying application can then access the contained files via their respective paths and just move them - where necessary. - """ - - # the request methods that may contain a request body - BODY_METHODS = ("POST", "PATCH", "PUT") - - def initialize(self, fallback, file_prefix="tmp", file_suffix="", path=None, suffixes=None): - if not suffixes: - suffixes = dict() - - self._fallback = fallback - self._file_prefix = file_prefix - self._file_suffix = file_suffix - self._path = path - - self._suffixes = dict((key, key) for key in ("name", "path", "content_type", "size")) - for suffix_type, suffix in suffixes.iteritems(): - if suffix_type in self._suffixes and suffix is not None: - self._suffixes[suffix_type] = suffix - - # Parts, files and values will be stored here - self._parts = dict() - self._files = [] - - # Part currently being processed - self._current_part = None - - # content type of request body - self._content_type = None - - # bytes left to read according to content_length of request body - self._bytes_left = 0 - - # buffer needed for identifying form data parts - self._buffer = b"" - - # buffer for new body - self._new_body = b"" - - # logger - self._logger = logging.getLogger(__name__) - - def prepare(self): - """ - Prepares the processing of the request. If it's a request that may contain a request body (as defined in - `UploadStorageFallbackHandler.BODY_METHODS) prepares the multipart parsing if content type fits. If it's a - body-less request, just calls the `fallback` with an empty body and finishes the request. - """ - if self.request.method in UploadStorageFallbackHandler.BODY_METHODS: - self._bytes_left = self.request.headers.get("Content-Length", 0) - self._content_type = self.request.headers.get("Content-Type", None) - - # request might contain a body - if self.is_multipart(): - if not self._bytes_left: - # we don't support requests without a content-length - raise HTTPError(400, reason="No Content-Length supplied") - - # extract the multipart boundary - fields = self._content_type.split(";") - for field in fields: - k, sep, v = field.strip().partition("=") - if k == "boundary" and v: - if v.startswith(b'"') and v.endswith(b'"'): - self._multipart_boundary = tornado.escape.utf8(v[1:-1]) - else: - self._multipart_boundary = tornado.escape.utf8(v) - break - else: - self._multipart_boundary = None - else: - self._fallback(self.request, b"") - self._finished = True - - def data_received(self, chunk): - """ - Called by Tornado on receiving a chunk of the request body. If request is a multipart request, takes care of - processing the multipart data structure via `self._process_multipart_data`. If not, just adds the chunk to - internal in-memory buffer. - - :param chunk: chunk of data received from Tornado - """ - - data = self._buffer + chunk - if self.is_multipart(): - self._process_multipart_data(data) - else: - self._buffer = data - - def is_multipart(self): - """Checks whether this request is a `multipart` request""" - return self._content_type is not None and self._content_type.startswith("multipart") - - def _process_multipart_data(self, data): - """ - Processes the given data, parsing it for multipart definitions and calling the appropriate methods. - - :param data: the data to process as a string - """ - - # check for boundary - delimiter = b"--%s" % self._multipart_boundary - delimiter_loc = data.find(delimiter) - delimiter_len = len(delimiter) - end_of_header = None - if delimiter_loc != -1: - # found the delimiter in the currently available data - data, self._buffer = data[0:delimiter_loc], data[delimiter_loc:] - end_of_header = self._buffer.find("\r\n\r\n") - else: - # make sure any boundary (with single or double ==) contained at the end of chunk does not get - # truncated by this processing round => save it to the buffer for next round - endlen = len(self._multipart_boundary) + 4 - data, self._buffer = data[0:-endlen], data[-endlen:] - - # stream data to part handler - if data: - if self._current_part: - self._on_data(self._current_part, data) - - if end_of_header >= 0: - self._on_header(self._buffer[delimiter_len+2:end_of_header]) - self._buffer = self._buffer[end_of_header + 4:] - - if delimiter_loc != -1 and self._buffer[delimiter_len:delimiter_len+2] == "--": - # we saw the last boundary and are at the end of our request - if self._current_part: - self._on_close(self._current_part) - self._current_part = None - self._buffer = b"" - self._on_finish() - - def _on_header(self, header): - """ - Called for a new multipart header, takes care of parsing the header and calling `self._on_part` with the - relevant data, setting the current part in the process. - - :param header: header to parse - """ - - # close any open parts - if self._current_part: - self._on_close(self._current_part) - self._current_part = None - - header_check = header.find(self._multipart_boundary) - if header_check != -1: - self._logger.warn("Header still contained multipart boundary, stripping it...") - header = header[header_check:] - - # convert to dict - header = HTTPHeaders.parse(header.decode("utf-8")) - disp_header = header.get("Content-Disposition", "") - disposition, disp_params = tornado.httputil._parse_header(disp_header) - - if disposition != "form-data": - self._logger.warn("Got a multipart header without form-data content disposition, ignoring that one") - return - if not disp_params.get("name"): - self._logger.warn("Got a multipart header without name, ignoring that one") - return - - self._current_part = self._on_part(disp_params["name"], header.get("Content-Type", None), filename=disp_params["filename"] if "filename" in disp_params else None) - - def _on_part(self, name, content_type, filename=None): - """ - Called for new parts in the multipart stream. If `filename` is given creates new `file` part (which leads - to storage of the data as temporary file on disk), if not creates a new `data` part (which stores - incoming data in memory). - - Structure of `file` parts: - - * `name`: name of the part - * `filename`: filename associated with the part - * `path`: path to the temporary file storing the file's data - * `content_type`: content type of the part - * `file`: file handle for the temporary file (mode "wb", not deleted on close!) - - Structure of `data` parts: - - * `name`: name of the part - * `content_type`: content type of the part - * `data`: bytes of the part (initialized to "") - - :param name: name of the part - :param content_type: content type of the part - :param filename: filename associated with the part. - :return: dict describing the new part - """ - if filename is not None: - # this is a file - import tempfile - handle = tempfile.NamedTemporaryFile(mode="wb", prefix=self._file_prefix, suffix=self._file_suffix, dir=self._path, delete=False) - return dict(name=tornado.escape.utf8(name), - filename=tornado.escape.utf8(filename), - path=tornado.escape.utf8(handle.name), - content_type=tornado.escape.utf8(content_type), - file=handle) - - else: - return dict(name=tornado.escape.utf8(name), content_type=content_type, data=b"") - - def _on_data(self, part, data): - """ - Called when new bytes are received for the given `part`, takes care of writing them to their storage. - - :param part: part for which data was received - :param data: data chunk which was received - """ - if "file" in part: - part["file"].write(data) - else: - part["data"] += data - - def _on_close(self, part): - """ - Called when a part gets closed, takes care of storing the finished part in the internal parts storage and for - `file` parts closing the temporary file and storing the part in the internal files storage. - - :param part: part which was closed - """ - name = part["name"] - self._parts[name] = part - if "file" in part: - self._files.append(part["path"]) - part["file"].close() - del part["file"] - - def _on_finish(self): - """ - Called when the request body has been read completely. Takes care of creating the replacement body out of the - logged parts, turning `file` parts into new - :return: - """ - - self._new_body = b"" - for name, part in self._parts.iteritems(): - if "filename" in part: - # add form fields for filename, path, size and content_type for all files contained in the request - fields = dict((self._suffixes[key], value) for (key, value) in dict(name=part["filename"], path=part["path"], size=str(os.stat(part["path"]).st_size), content_type=part["content_type"]).iteritems()) - for n, p in fields.iteritems(): - key = name + "." + n - self._new_body += b"--%s\r\n" % self._multipart_boundary - self._new_body += b"Content-Disposition: form-data; name=\"%s\"\r\n" % key - self._new_body += b"\r\n" - self._new_body += p + b"\r\n" - elif "data" in part: - self._new_body += b"--%s\r\n" % self._multipart_boundary - value = part["data"] - self._new_body += b"Content-Disposition: form-data; name=\"%s\"\r\n" % name - if "content_type" in part and part["content_type"] is not None: - self._new_body += b"Content-Type: %s\r\n" % part["content_type"] - self._new_body += b"\r\n" - self._new_body += value - self._new_body += b"--%s--\r\n" % self._multipart_boundary - - def _handle_method(self, *args, **kwargs): - """ - Handler for any request method, takes care of defining the new request body if necessary and forwarding - the current request and changed body to the `fallback`. - """ - - # determine which body to supply - body = b"" - if self.is_multipart(): - # make sure we really processed all data in the buffer - while len(self._buffer): - self._process_multipart_data(self._buffer) - - # use rewritten body - body = self._new_body - - elif self.request.method in UploadStorageFallbackHandler.BODY_METHODS: - # directly use data from buffer - body = self._buffer - - # rewrite content length - self.request.headers["Content-Length"] = len(body) - - try: - # call the configured fallback with request and body to use - self._fallback(self.request, body) - finally: - # make sure the temporary files are removed again - for f in self._files: - util.silentRemove(f) - - # make all http methods trigger _handle_method - get = _handle_method - post = _handle_method - put = _handle_method - patch = _handle_method - delete = _handle_method - head = _handle_method - options = _handle_method - - -@stream_request_body -class StreamingFallbackHandler(FallbackHandler): - """A `RequestHandler` that wraps another HTTP server callback. - - The fallback is a callable object that accepts an - `~.httputil.HTTPServerRequest`, such as an `Application` or - `octoprint.server.util.StreamedWSGIContainer`. This is most useful to use both - Tornado ``RequestHandlers`` and WSGI in the same server. Typical - usage:: - - wsgi_app = octoprint.server.util.StreamedWSGIContainer( - django.core.handlers.wsgi.WSGIHandler()) - application = tornado.web.Application([ - (r"/foo", FooHandler), - (r".*", StreamingFallbackHandler, dict(fallback=wsgi_app), - ]) - """ - - NO_BODY_METHODS = ("GET", "HEAD", "OPTIONS") - - def initialize(self, fallback, max_size=50*1024*1024*1024, file_prefix="tmp"): - self._fallback = fallback - self._max_size = max_size - self._file_prefix = file_prefix - - def prepare(self): - if self.request.method not in StreamingFallbackHandler.NO_BODY_METHODS: - import tempfile - self._length_left = int(self.request.headers.get("Content-Length", 0)) - self._tempfile = tempfile.TemporaryFile(prefix=self._file_prefix) - else: - import io - self._fallback(self.request, io.BytesIO()) - self._finished = True - - def data_received(self, chunk): - if self.request.method not in StreamingFallbackHandler.NO_BODY_METHODS: - self._tempfile.write(chunk) - self._length_left -= len(chunk) - if self._length_left <= 0: - self.finished_body() - - def finished_body(self): - self._tempfile.seek(0) - - def get(self, *args, **kwargs): - return self._handle_method() - - def post(self, *args, **kwargs): - return self._handle_method() - - def put(self, *args, **kwargs): - return self._handle_method() - - def delete(self, *args, **kwargs): - return self._handle_method() - - def head(self, *args, **kwargs): - return self._handle_method() - - def options(self, *args, **kwargs): - return self._handle_method() - - def _handle_method(self): - try: - self._fallback(self.request, self._tempfile) - finally: - self._tempfile.close() - - -class WsgiInputContainer(object): - """ - A WSGI container for use with Tornado that allows supplying the request body to be used for `wsgi.input` in the - generated WSGI environment upon call. - - A `RequestHandler` can thus provide the WSGI application with a stream for the request body, or a modified body. - - Example usage: - - wsgi_app = octoprint.server.util.WsgiInputContainer(octoprint_app) - application = tornado.web.Application([ - (r".*", UploadStorageFallbackHandler, dict(fallback=wsgi_app), - ]) - - The implementation logic is basically the same as `tornado.wsgi.WSGIContainer` but the `__call__` and `environ` - - """ - - def __init__(self, wsgi_application): - self.wsgi_application = wsgi_application - - def __call__(self, request, body=None): - """ - Wraps the call against the WSGI app, deriving the WSGI environment from the supplied Tornado `HTTPServerRequest`. - - :param request: the `tornado.httpserver.HTTPServerRequest` to derive the WSGI environment from - :param body: an optional body to use as `wsgi.input` instead of `request.body`, can be a string or a stream - """ - - data = {} - response = [] - - def start_response(status, response_headers, exc_info=None): - data["status"] = status - data["headers"] = response_headers - return response.append - app_response = self.wsgi_application( - WsgiInputContainer.environ(request, body), start_response) - try: - response.extend(app_response) - body = b"".join(response) - finally: - if hasattr(app_response, "close"): - app_response.close() - if not data: - raise Exception("WSGI app did not call start_response") - - status_code = int(data["status"].split()[0]) - headers = data["headers"] - header_set = set(k.lower() for (k, v) in headers) - body = tornado.escape.utf8(body) - if status_code != 304: - if "content-length" not in header_set: - headers.append(("Content-Length", str(len(body)))) - if "content-type" not in header_set: - headers.append(("Content-Type", "text/html; charset=UTF-8")) - if "server" not in header_set: - headers.append(("Server", "TornadoServer/%s" % tornado.version)) - - parts = [tornado.escape.utf8("HTTP/1.1 " + data["status"] + "\r\n")] - for key, value in headers: - parts.append(tornado.escape.utf8(key) + b": " + tornado.escape.utf8(value) + b"\r\n") - parts.append(b"\r\n") - parts.append(body) - request.write(b"".join(parts)) - request.finish() - self._log(status_code, request) - - @staticmethod - def environ(request, body=None): - """ - Converts a `tornado.httputil.HTTPServerRequest` to a WSGI environment. - - An optional `body` to be used for populating `wsgi.input` can be supplied (either a string or a stream). If not - supplied, `request.body` will be wrapped into a `io.BytesIO` stream and used instead. - - :param request: the `tornado.httpserver.HTTPServerRequest` to derive the WSGI environment from - :param body: an optional body to use as `wsgi.input` instead of `request.body`, can be a string or a stream - """ - from tornado.wsgi import to_wsgi_str - import sys - import io - - # determine the request_body to supply as wsgi.input - if body is not None: - if isinstance(body, (bytes, str)): - request_body = io.BytesIO(tornado.escape.utf8(body)) - else: - request_body = body - else: - request_body = io.BytesIO(tornado.escape.utf8(request.body)) - - hostport = request.host.split(":") - if len(hostport) == 2: - host = hostport[0] - port = int(hostport[1]) - else: - host = request.host - port = 443 if request.protocol == "https" else 80 - environ = { - "REQUEST_METHOD": request.method, - "SCRIPT_NAME": "", - "PATH_INFO": to_wsgi_str(tornado.escape.url_unescape( - request.path, encoding=None, plus=False)), - "QUERY_STRING": request.query, - "REMOTE_ADDR": request.remote_ip, - "SERVER_NAME": host, - "SERVER_PORT": str(port), - "SERVER_PROTOCOL": request.version, - "wsgi.version": (1, 0), - "wsgi.url_scheme": request.protocol, - "wsgi.input": request_body, - "wsgi.errors": sys.stderr, - "wsgi.multithread": False, - "wsgi.multiprocess": True, - "wsgi.run_once": False, - } - if "Content-Type" in request.headers: - environ["CONTENT_TYPE"] = request.headers.pop("Content-Type") - if "Content-Length" in request.headers: - environ["CONTENT_LENGTH"] = request.headers.pop("Content-Length") - for key, value in request.headers.items(): - environ["HTTP_" + key.replace("-", "_").upper()] = value - return environ - - def _log(self, status_code, request): - access_log = logging.getLogger("tornado.access") - - if status_code < 400: - log_method = access_log.info - elif status_code < 500: - log_method = access_log.warning - else: - log_method = access_log.error - request_time = 1000.0 * request.request_time() - summary = request.method + " " + request.uri + " (" + \ - request.remote_ip + ")" - log_method("%d %s %.2fms", status_code, summary, request_time) - - -def access_validation_factory(app, login_manager, validator): - """ - Creates an access validation wrapper using the supplied validator. - - :param validator: the access validator to use inside the validation wrapper - :return: an access validation wrapper taking a request as parameter and performing the request validation - """ - def f(request): - """ - Creates a custom wsgi and Flask request context in order to be able to process user information - stored in the current session. - - :param request: The Tornado request for which to create the environment and context - """ - import flask - import tornado.wsgi - - wsgi_environ = tornado.wsgi.WSGIContainer.environ(request) - with app.request_context(wsgi_environ): - app.session_interface.open_session(app, flask.request) - login_manager.reload_user() - validator(flask.request) - return f - - -@stream_request_body -class PrintableFilesUploadHandler(RequestHandler): - - def initialize(self, path, postfix=".tmp", files_only=False, access_validation=None): - self._path = path - self._postfix = postfix - self._files_only = files_only - self._access_validation = access_validation - - # Parts information will be stored here - self.parts = dict() - - # files will be stored here - self.files = dict() - - # values will be stored here - self.values = dict() - - # Part currently being processed - self._current_part = None - # buffer needed for identifying form data parts - self._buffer = b"" - - # we only support multipart/form-data content, so check for that - bytes_left = self.request.headers.get("Content-Length", 0) - content_type = self.request.headers.get("Content-Type", "") - if not bytes_left or not content_type.startswith("multipart"): - raise HTTPError(405) - - # extract the multipart boundary - fields = content_type.split(";") - for field in fields: - k, sep, v = field.strip().partition("=") - if k == "boundary" and v: - if v.startswith(b'"') and v.endswith(b'"'): - self.boundary = tornado.escape.utf8(v[1:-1]) - else: - self.boundary = tornado.escape.utf8(v) - break - else: - raise HTTPError(400) - - def data_received(self, chunk): - data = self._buffer + chunk - self.process_data(data) - - def process_data(self, data): - # check for boundary - delimiter = b"--%s" % self.boundary - delimiter_loc = data.find(delimiter) - delimiter_len = len(delimiter) - end_of_header = None - if delimiter_loc != -1: - # found the delimiter in the currently available data - data, self._buffer = data[0:delimiter_loc], data[delimiter_loc:] - end_of_header = self._buffer.find("\r\n\r\n") - else: - # make sure any boundary (with single or double ==) contained at the end of chunk does not get - # truncated by this processing round => save it to the buffer for next round - endlen = len(self.boundary) + 4 - data, self._buffer = data[0:-endlen], data[-endlen:] - - # stream data to part handler - if data: - if self._current_part: - self.ondata(self._current_part, data) - - if end_of_header >= 0: - self._header(self._buffer[delimiter_len+2:end_of_header]) - self._buffer = self._buffer[end_of_header + 4:] - - if delimiter_loc != -1 and self._buffer[delimiter_len:delimiter_len+2] == "--": - # we saw the last boundary and are at the end of our request - if self._current_part: - self.onclose(self._current_part) - self._current_part = None - self._buffer = b"" - self.onfinish() - - def _header(self, header): - # close any open parts - if self._current_part: - self.onclose(self._current_part) - self._current_part = None - - header_check = header.find(self.boundary) - if header_check != -1: - # TODO log warning - header = header[header_check:] - - # convert to dict - header = HTTPHeaders.parse(header.decode("utf-8")) - disp_header = header.get("Content-Disposition", "") - disposition, disp_params = tornado.httputil._parse_header(disp_header) - - if disposition != "form-data": - # TODO log warning - return - if not disp_params.get("name"): - # TODO log warning - return - - if self._files_only and "filename" not in disp_params: - # TODO log warning - return - else: - self._current_part = self.onpart(disp_params["name"], header.get("Content-Type", None), filename=disp_params["filename"] if "filename" in disp_params else None) - - def onpart(self, name, content_type, filename=None): - from octoprint.server import gcodeManager - - if content_type is None: - # we got a key-value-pair - return dict(name=name, value=b"") - elif filename is not None: - # this is a file - upload = Object() - upload.filename = filename - - sane_filename = gcodeManager.getFutureFilename(upload) - if sane_filename is None: - return dict() - - local_path = os.path.join(self._path, sane_filename + self._postfix) - handle = open(local_path, "wb") - return dict(name=name, filename=filename, sane_filename=sane_filename, content_type=content_type, local_path=local_path, file=handle) - else: - return dict() - - def ondata(self, part, data): - if "value" in part: - part["value"] += data - elif "file" in part: - part["file"].write(data) - - def onclose(self, part): - escaped_name = tornado.escape.utf8(part["name"]) - - self.parts[escaped_name] = part - if "file" in part: - part["file"].close() - del part["file"] - self.files[escaped_name] = part - elif "value" in part: - escaped_value = tornado.escape.utf8(part["value"]) - self.values[escaped_name] = escaped_value - self.request.body_arguments.setdefault(escaped_name, []).append(escaped_value) - - def onfinish(self): - # we now do something horrible and replace the body by a version stripped of all files. Yes, I feel bad for this - import io - new_body = b"" - for name, value in self.values.iteritems(): - new_body += b"--%s\r\n" % self.boundary - new_body += b"Content-Disposition: form-data; name=\"%s\"\r\n\r\n" % name - new_body += value - new_body += b"--%s--\r\n" % self.boundary - self.request.body = new_body - self.request.headers["Content-Length"] = len(new_body) - - def post(self, *args, **kwargs): - while len(self._buffer): - self.process_data(self._buffer) - - from octoprint.server import gcodeManager, printer, eventManager - - if self._access_validation is not None: - self._access_validation(self.request) - - target = self.request.path.split("/")[-1] - - if not target in [FileDestinations.LOCAL, FileDestinations.SDCARD]: - self.set_status(404, reason="Unknown target: %s" % target) - return - - if not "file" in self.files: - self.set_status(400, reason="No file included") - return - - if target == FileDestinations.SDCARD and not settings().getBoolean(["feature", "sdSupport"]): - self.set_status(404, reason="SD card support is disabled") - return - - import octoprint.util - - upload = Object() - upload.filename = self.files["file"]["filename"] - upload.sane_filename = self.files["file"]["sane_filename"] - upload.save = lambda new_name: octoprint.util.safeRename(self.files["file"]["local_path"], new_name) - - sd = target == FileDestinations.SDCARD - selectAfterUpload = "select" in self.values and self.values["select"] in valid_boolean_trues - printAfterSelect = "print" in self.values and self.values["print"] in valid_boolean_trues - - if sd: - # validate that all preconditions for SD upload are met before attempting it - if not (printer.isOperational() and not (printer.isPrinting() or printer.isPaused())): - self.set_status(409, reason="Can not upload to SD card, printer is either not operational or already busy") - return - if not printer.isSdReady(): - self.set_status(409, "Can not upload to SD card, not yet initialized") - return - - # determine current job - currentFilename = None - currentOrigin = None - currentJob = printer.getCurrentJob() - if currentJob is not None and "file" in currentJob.keys(): - currentJobFile = currentJob["file"] - if "name" in currentJobFile.keys() and "origin" in currentJobFile.keys(): - currentFilename = currentJobFile["name"] - currentOrigin = currentJobFile["origin"] - - # determine future filename of file to be uploaded, abort if it can't be uploaded - futureFilename = upload.sane_filename - if futureFilename is None or (not settings().getBoolean(["cura", "enabled"]) and not gcodefiles.isGcodeFileName(futureFilename)): - self.set_status(415, reason="Can not upload file %s, wrong format?" % upload.filename) - return - - # prohibit overwriting currently selected file while it's being printed - if futureFilename == currentFilename and target == currentOrigin and printer.isPrinting() or printer.isPaused(): - self.set_status(409, reason="Trying to overwrite file that is currently being printed: %s" % currentFilename) - return - - def fileProcessingFinished(filename, absFilename, destination): - """ - Callback for when the file processing (upload, optional slicing, addition to analysis queue) has - finished. - - Depending on the file's destination triggers either streaming to SD card or directly calls selectAndOrPrint. - """ - if destination == FileDestinations.SDCARD: - return filename, printer.addSdFile(filename, absFilename, selectAndOrPrint) - else: - selectAndOrPrint(filename, absFilename, destination) - return filename - - def selectAndOrPrint(filename, absFilename, destination): - """ - Callback for when the file is ready to be selected and optionally printed. For SD file uploads this is only - the case after they have finished streaming to the printer, which is why this callback is also used - for the corresponding call to addSdFile. - - Selects the just uploaded file if either selectAfterUpload or printAfterSelect are True, or if the - exact file is already selected, such reloading it. - """ - if selectAfterUpload or printAfterSelect or (currentFilename == filename and currentOrigin == destination): - printer.selectFile(absFilename, destination == FileDestinations.SDCARD, printAfterSelect) - - filename, done = gcodeManager.addFile(upload, target, fileProcessingFinished) - if filename is None: - return make_response("Could not upload the file %s" % upload.filename, 500) - - sdFilename = None - if isinstance(filename, tuple): - filename, sdFilename = filename - - eventManager.fire(Events.UPLOAD, {"file": filename, "target": target}) - - """ - files = {} - location = url_for(".readGcodeFile", target=FileDestinations.LOCAL, filename=filename, _external=True) - files.update({ - FileDestinations.LOCAL: { - "name": filename, - "origin": FileDestinations.LOCAL, - "refs": { - "resource": location, - "download": url_for("index", _external=True) + "downloads/files/" + FileDestinations.LOCAL + "/" + filename - } - } - }) - - if sd and sdFilename: - location = url_for(".readGcodeFile", target=FileDestinations.SDCARD, filename=sdFilename, _external=True) - files.update({ - FileDestinations.SDCARD: { - "name": sdFilename, - "origin": FileDestinations.SDCARD, - "refs": { - "resource": location - } - } - }) - """ - - import tornado.escape - - self.set_status(201) - #self.set_header("Location", location) - self.finish(tornado.escape.json_encode(dict(files={}, done=done))) - - -#~~ customized large response handler - - -class LargeResponseHandler(StaticFileHandler): - - CHUNK_SIZE = 16 * 1024 - - def initialize(self, path, default_filename=None, as_attachment=False, access_validation=None): - StaticFileHandler.initialize(self, path, default_filename) - self._as_attachment = as_attachment - self._access_validation = access_validation - - def get(self, path, include_body=True): - if self._access_validation is not None: - self._access_validation(self.request) - - path = self.parse_url_path(path) - abspath = os.path.abspath(os.path.join(self.root, path)) - # os.path.abspath strips a trailing / - # it needs to be temporarily added back for requests to root/ - if not (abspath + os.path.sep).startswith(self.root): - raise HTTPError(403, "%s is not in root static directory", path) - if os.path.isdir(abspath) and self.default_filename is not None: - # need to look at the request.path here for when path is empty - # but there is some prefix to the path that was already - # trimmed by the routing - if not self.request.path.endswith("/"): - self.redirect(self.request.path + "/") - return - abspath = os.path.join(abspath, self.default_filename) - if not os.path.exists(abspath): - raise HTTPError(404) - if not os.path.isfile(abspath): - raise HTTPError(403, "%s is not a file", path) - - stat_result = os.stat(abspath) - modified = datetime.datetime.fromtimestamp(stat_result[stat.ST_MTIME]) - - self.set_header("Last-Modified", modified) - - mime_type, encoding = mimetypes.guess_type(abspath) - if mime_type: - self.set_header("Content-Type", mime_type) - - cache_time = self.get_cache_time(path, modified, mime_type) - - if cache_time > 0: - self.set_header("Expires", datetime.datetime.utcnow() + - datetime.timedelta(seconds=cache_time)) - self.set_header("Cache-Control", "max-age=" + str(cache_time)) - - self.set_extra_headers(path) - - # Check the If-Modified-Since, and don't send the result if the - # content has not been modified - ims_value = self.request.headers.get("If-Modified-Since") - if ims_value is not None: - date_tuple = email.utils.parsedate(ims_value) - if_since = datetime.datetime.fromtimestamp(time.mktime(date_tuple)) - if if_since >= modified: - self.set_status(304) - return - - if not include_body: - assert self.request.method == "HEAD" - self.set_header("Content-Length", stat_result[stat.ST_SIZE]) - else: - with open(abspath, "rb") as file: - while True: - data = file.read(LargeResponseHandler.CHUNK_SIZE) - if not data: - break - self.write(data) - self.flush() - - def set_extra_headers(self, path): - if self._as_attachment: - self.set_header("Content-Disposition", "attachment") - - -##~~ URL Forward Handler for forwarding requests to a preconfigured static URL - - -class UrlForwardHandler(RequestHandler): - - def initialize(self, url=None, as_attachment=False, basename=None, access_validation=None): - RequestHandler.initialize(self) - self._url = url - self._as_attachment = as_attachment - self._basename = basename - self._access_validation = access_validation - - @asynchronous - def get(self, *args, **kwargs): - if self._access_validation is not None: - self._access_validation(self.request) - - if self._url is None: - raise HTTPError(404) - - client = AsyncHTTPClient() - r = HTTPRequest(url=self._url, method=self.request.method, body=self.request.body, headers=self.request.headers, follow_redirects=False, allow_nonstandard_methods=True) - - try: - return client.fetch(r, self.handle_response) - except HTTPError as e: - if hasattr(e, "response") and e.response: - self.handle_response(e.response) - else: - raise HTTPError(500) - - def handle_response(self, response): - if response.error and not isinstance(response.error, HTTPError): - raise HTTPError(500) - - filename = None - - self.set_status(response.code) - for name in ("Date", "Cache-Control", "Server", "Content-Type", "Location"): - value = response.headers.get(name) - if value: - self.set_header(name, value) - - if name == "Content-Type": - filename = self.get_filename(value) - - if self._as_attachment: - if filename is not None: - self.set_header("Content-Disposition", "attachment; filename=%s" % filename) - else: - self.set_header("Content-Disposition", "attachment") - - if response.body: - self.write(response.body) - self.finish() - - def get_filename(self, content_type): - if not self._basename: - return None - - typeValue = map(str.strip, content_type.split(";")) - if len(typeValue) == 0: - return None - - extension = mimetypes.guess_extension(typeValue[0]) - if not extension: - return None - - return "%s%s" % (self._basename, extension) - - -#~~ admin access validator for use with tornado - - -def admin_validator(request): - """ - Validates that the given request is made by an admin user, identified either by API key or existing Flask - session. - - Must be executed in an existing Flask request context! - - :param request: The Flask request object - """ - - apikey = getApiKey(request) - if settings().get(["api", "enabled"]) and apikey is not None: - user = getUserForApiKey(apikey) - else: - user = current_user - - if user is None or not user.is_authenticated() or not user.is_admin(): - raise HTTPError(403) - - -#~~ user access validator for use with tornado - - -def user_validator(request): - """ - Validates that the given request is made by an authenticated user, identified either by API key or existing Flask - session. - - Must be executed in an existing Flask request context! - - :param request: The Flask request object - """ - - apikey = getApiKey(request) - if settings().get(["api", "enabled"]) and apikey is not None: - user = getUserForApiKey(apikey) - else: - user = current_user - - if user is None or not user.is_authenticated(): - raise HTTPError(403) - - -#~~ reverse proxy compatible wsgi middleware - - -class ReverseProxied(object): - """ - Wrap the application in this middleware and configure the - front-end server to add these headers, to let you quietly bind - this to a URL other than / and to an HTTP scheme that is - different than what is used locally. - - In nginx: - location /myprefix { - proxy_pass http://192.168.0.1:5001; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Scheme $scheme; - proxy_set_header X-Script-Name /myprefix; - } - - Alternatively define prefix and scheme via config.yaml: - server: - baseUrl: /myprefix - scheme: http - - :param app: the WSGI application - """ - - def __init__(self, app): - self.app = app - - def __call__(self, environ, start_response): - script_name = environ.get('HTTP_X_SCRIPT_NAME', '') - if not script_name: - script_name = settings().get(["server", "baseUrl"]) - - if script_name: - environ['SCRIPT_NAME'] = script_name - path_info = environ['PATH_INFO'] - if path_info.startswith(script_name): - environ['PATH_INFO'] = path_info[len(script_name):] - - scheme = environ.get('HTTP_X_SCHEME', '') - if not scheme: - scheme = settings().get(["server", "scheme"]) - - if scheme: - environ['wsgi.url_scheme'] = scheme - return self.app(environ, start_response) - - -def redirectToTornado(request, target, code=302): - requestUrl = request.url - appBaseUrl = requestUrl[:requestUrl.find(url_for("index") + "api")] - - redirectUrl = appBaseUrl + target - if "?" in requestUrl: - fragment = requestUrl[requestUrl.rfind("?"):] - redirectUrl += fragment - return redirect(redirectUrl, code=code) - - -class UploadCleanupWatchdogHandler(PatternMatchingEventHandler): - """ - Takes care of automatically deleting metadata entries for files that get deleted from the uploads folder - """ - - patterns = map(lambda x: "*.%s" % x, gcodefiles.GCODE_EXTENSIONS) - - def __init__(self, gcode_manager): - PatternMatchingEventHandler.__init__(self) - self._gcode_manager = gcode_manager - - def on_deleted(self, event): - filename = self._gcode_manager._getBasicFilename(event.src_path) - if not filename: - return - - self._gcode_manager.removeFileFromMetadata(filename) - - -class GcodeWatchdogHandler(PatternMatchingEventHandler): - """ - Takes care of automatically "uploading" files that get added to the watched folder. - """ - - patterns = map(lambda x: "*.%s" % x, gcodefiles.SUPPORTED_EXTENSIONS) - - def __init__(self, gcodeManager, printer): - PatternMatchingEventHandler.__init__(self) - - self._logger = logging.getLogger(__name__) - - self._gcodeManager = gcodeManager - self._printer = printer - - def _upload(self, path): - class WatchdogFileWrapper(object): - - def __init__(self, path): - self._path = path - self.filename = os.path.basename(self._path) - - def save(self, target): - util.safeRename(self._path, target) - - fileWrapper = WatchdogFileWrapper(path) - - # determine current job - currentFilename = None - currentOrigin = None - currentJob = self._printer.getCurrentJob() - if currentJob is not None and "file" in currentJob.keys(): - currentJobFile = currentJob["file"] - if "name" in currentJobFile.keys() and "origin" in currentJobFile.keys(): - currentFilename = currentJobFile["name"] - currentOrigin = currentJobFile["origin"] - - # determine future filename of file to be uploaded, abort if it can't be uploaded - futureFilename = self._gcodeManager.getFutureFilename(fileWrapper) - if futureFilename is None or (not settings().getBoolean(["cura", "enabled"]) and not gcodefiles.isGcodeFileName(futureFilename)): - self._logger.warn("Could not add %s: Invalid file" % fileWrapper.filename) - return - - # prohibit overwriting currently selected file while it's being printed - if futureFilename == currentFilename and not currentOrigin == FileDestinations.SDCARD and self._printer.isPrinting() or self._printer.isPaused(): - self._logger.warn("Could not add %s: Trying to overwrite file that is currently being printed" % fileWrapper.filename) - return - - self._gcodeManager.addFile(fileWrapper, FileDestinations.LOCAL) - - def on_created(self, event): - self._upload(event.src_path) - - -class Object(object): - pass \ No newline at end of file diff --git a/src/octoprint/server/util/__init__.py b/src/octoprint/server/util/__init__.py new file mode 100644 index 00000000..66e67301 --- /dev/null +++ b/src/octoprint/server/util/__init__.py @@ -0,0 +1,100 @@ +# coding=utf-8 +from __future__ import absolute_import + +__author__ = "Gina Häußge " +__license__ = 'GNU Affero General Public License http://www.gnu.org/licenses/agpl.html' +__copyright__ = "Copyright (C) 2014 The OctoPrint Project - Released under terms of the AGPLv3 License" + +from octoprint.settings import settings +import octoprint.timelapse +import octoprint.server +from octoprint.users import ApiUser + +from . import flask +from . import sockjs +from . import tornado +from . import watchdog + + +def get_user_for_apikey(apikey): + if settings().get(["api", "enabled"]) and apikey is not None: + if apikey == settings().get(["api", "key"]): + # master key was used + return ApiUser() + else: + # user key might have been used + return octoprint.server.userManager.findUser(apikey=apikey) + else: + return None + + +def get_api_key(request): + # Check Flask GET/POST arguments + if hasattr(request, "values") and "apikey" in request.values: + return request.values["apikey"] + + # Check Tornado GET/POST arguments + if hasattr(request, "arguments") and "apikey" in request.arguments \ + and len(request.arguments["apikey"]) > 0 and len(request.arguments["apikey"].strip()) > 0: + return request.arguments["apikey"] + + # Check Tornado and Flask headers + if "X-Api-Key" in request.headers.keys(): + return request.headers.get("X-Api-Key") + + return None + + +#~~ reverse proxy compatible WSGI middleware + + +class ReverseProxied(object): + """ + Wrap the application in this middleware and configure the + front-end server to add these headers, to let you quietly bind + this to a URL other than / and to an HTTP scheme that is + different than what is used locally. + + In nginx: + location /myprefix { + proxy_pass http://192.168.0.1:5001; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Scheme $scheme; + proxy_set_header X-Script-Name /myprefix; + } + + Alternatively define prefix and scheme via config.yaml: + server: + baseUrl: /myprefix + scheme: http + + :param app: the WSGI application + :param header_script_name: the HTTP header in the wsgi environment from which to determine the prefix + :param header_scheme: the HTTP header in the wsgi environment from which to determine the scheme + """ + + def __init__(self, app, header_script_name="HTTP_X_SCRIPT_NAME", header_scheme="HTTP_X_SCHEME"): + self.app = app + self._header_script_name = header_script_name + self._header_scheme = header_scheme + + def __call__(self, environ, start_response): + script_name = environ.get(self._header_script_name, '') + if not script_name: + script_name = settings().get(["server", "baseUrl"]) + + if script_name: + environ['SCRIPT_NAME'] = script_name + path_info = environ['PATH_INFO'] + if path_info.startswith(script_name): + environ['PATH_INFO'] = path_info[len(script_name):] + + scheme = environ.get(self._header_scheme, '') + if not scheme: + scheme = settings().get(["server", "scheme"]) + + if scheme: + environ['wsgi.url_scheme'] = scheme + return self.app(environ, start_response) + diff --git a/src/octoprint/server/util/flask.py b/src/octoprint/server/util/flask.py new file mode 100644 index 00000000..414f5be5 --- /dev/null +++ b/src/octoprint/server/util/flask.py @@ -0,0 +1,154 @@ +# coding=utf-8 +from __future__ import absolute_import + +__author__ = "Gina Häußge " +__license__ = 'GNU Affero General Public License http://www.gnu.org/licenses/agpl.html' +__copyright__ = "Copyright (C) 2014 The OctoPrint Project - Released under terms of the AGPLv3 License" + +import tornado.web +import flask +import flask.ext.login +import flask.ext.principal +import functools + +from octoprint.settings import settings +import octoprint.server +import octoprint.users + + +#~~ access validators for use with tornado + + +def admin_validator(request): + """ + Validates that the given request is made by an admin user, identified either by API key or existing Flask + session. + + Must be executed in an existing Flask request context! + + :param request: The Flask request object + """ + + user = _get_flask_user_from_request(request) + if user is None or not user.is_authenticated() or not user.is_admin(): + raise tornado.web.HTTPError(403) + + +def user_validator(request): + """ + Validates that the given request is made by an authenticated user, identified either by API key or existing Flask + session. + + Must be executed in an existing Flask request context! + + :param request: The Flask request object + """ + + user = _get_flask_user_from_request(request) + if user is None or not user.is_authenticated(): + raise tornado.web.HTTPError(403) + + +def _get_flask_user_from_request(request): + """ + Retrieves the current flask user from the request context. Uses API key if available, otherwise the current + user session if available. + + :param request: flask request from which to retrieve the current user + :return: the user or None if no user could be determined + """ + import octoprint.server.util + import flask.ext.login + from octoprint.settings import settings + + apikey = octoprint.server.util.get_api_key(request) + if settings().get(["api", "enabled"]) and apikey is not None: + user = octoprint.server.util.get_user_for_apikey(apikey) + else: + user = flask.ext.login.current_user + + return user + + +def redirect_to_tornado(request, target, code=302): + """ + Redirects from flask to tornado, flask request context must exist. + + :param request: + :param target: + :param code: + :return: + """ + + import flask + + requestUrl = request.url + appBaseUrl = requestUrl[:requestUrl.find(flask.url_for("index") + "api")] + + redirectUrl = appBaseUrl + target + if "?" in requestUrl: + fragment = requestUrl[requestUrl.rfind("?"):] + redirectUrl += fragment + return flask.redirect(redirectUrl, code=code) + + +def restricted_access(func, api_enabled=True): + """ + If you decorate a view with this, it will ensure that first setup has been + done for OctoPrint's Access Control plus that any conditions of the + login_required decorator are met. It also allows to login using the masterkey or any + of the user's apikeys if API access is enabled globally and for the decorated view. + + If OctoPrint's Access Control has not been setup yet (indicated by the "firstRun" + flag from the settings being set to True and the userManager not indicating + that it's user database has been customized from default), the decorator + will cause a HTTP 403 status code to be returned by the decorated resource. + + If an API key is provided and it matches a known key, the user will be logged in and + the view will be called directly. If the provided key doesn't match any known key, + a HTTP 403 status code will be returned by the decorated resource. + + Otherwise the result of calling login_required will be returned. + """ + @functools.wraps(func) + def decorated_view(*args, **kwargs): + # if OctoPrint hasn't been set up yet, abort + if settings().getBoolean(["server", "firstRun"]) and (octoprint.server.userManager is None or not octoprint.server.userManager.hasBeenCustomized()): + return flask.make_response("OctoPrint isn't setup yet", 403) + + # if API is globally enabled, enabled for this request and an api key is provided that is not the current UI API key, try to use that + apikey = octoprint.server.util.get_api_key(flask.request) + if settings().get(["api", "enabled"]) and api_enabled and apikey is not None and apikey != octoprint.server.UI_API_KEY: + if apikey == settings().get(["api", "key"]): + # master key was used + user = octoprint.users.ApiUser() + else: + # user key might have been used + user = octoprint.server.userManager.findUser(apikey=apikey) + + if user is None: + return flask.make_response("Invalid API key", 401) + if flask.ext.login.login_user(user, remember=False): + flask.ext.principal.identity_changed.send(flask.current_app._get_current_object(), identity=flask.ext.principal.Identity(user.get_id())) + return func(*args, **kwargs) + + # call regular login_required decorator + return flask.ext.login.login_required(func)(*args, **kwargs) + return decorated_view + + +def api_access(func): + @functools.wraps(func) + def decorated_view(*args, **kwargs): + if not settings().get(["api", "enabled"]): + flask.make_response("API disabled", 401) + apikey = octoprint.server.util.get_api_key(flask.request) + if apikey is None: + flask.make_response("No API key provided", 401) + if apikey != settings().get(["api", "key"]): + flask.make_response("Invalid API key", 403) + return func(*args, **kwargs) + return decorated_view + + + diff --git a/src/octoprint/server/util/sockjs.py b/src/octoprint/server/util/sockjs.py new file mode 100644 index 00000000..bbd96863 --- /dev/null +++ b/src/octoprint/server/util/sockjs.py @@ -0,0 +1,124 @@ +# coding=utf-8 +from __future__ import absolute_import + +__author__ = "Gina Häußge " +__license__ = 'GNU Affero General Public License http://www.gnu.org/licenses/agpl.html' +__copyright__ = "Copyright (C) 2014 The OctoPrint Project - Released under terms of the AGPLv3 License" + +import logging +import threading +import sockjs.tornado + +import octoprint.timelapse +import octoprint.server +from octoprint.events import Events + + +class PrinterStateConnection(sockjs.tornado.SockJSConnection): + EVENTS = [Events.UPDATED_FILES, Events.METADATA_ANALYSIS_FINISHED, Events.MOVIE_RENDERING, Events.MOVIE_DONE, + Events.MOVIE_FAILED, Events.SLICING_STARTED, Events.SLICING_DONE, Events.SLICING_FAILED, + Events.TRANSFER_STARTED, Events.TRANSFER_DONE] + + def __init__(self, printer, gcodeManager, userManager, eventManager, session): + sockjs.tornado.SockJSConnection.__init__(self, session) + + self._logger = logging.getLogger(__name__) + + self._temperatureBacklog = [] + self._temperatureBacklogMutex = threading.Lock() + self._logBacklog = [] + self._logBacklogMutex = threading.Lock() + self._messageBacklog = [] + self._messageBacklogMutex = threading.Lock() + + self._printer = printer + self._gcodeManager = gcodeManager + self._userManager = userManager + self._eventManager = eventManager + + def _getRemoteAddress(self, info): + forwardedFor = info.headers.get("X-Forwarded-For") + if forwardedFor is not None: + return forwardedFor.split(",")[0] + return info.ip + + def on_open(self, info): + remoteAddress = self._getRemoteAddress(info) + self._logger.info("New connection from client: %s" % remoteAddress) + + # connected => update the API key, might be necessary if the client was left open while the server restarted + self._emit("connected", {"apikey": octoprint.server.UI_API_KEY, "version": octoprint.server.VERSION}) + + self._printer.registerCallback(self) + self._gcodeManager.registerCallback(self) + octoprint.timelapse.registerCallback(self) + + self._eventManager.fire(Events.CLIENT_OPENED, {"remoteAddress": remoteAddress}) + for event in PrinterStateConnection.EVENTS: + self._eventManager.subscribe(event, self._onEvent) + + octoprint.timelapse.notifyCallbacks(octoprint.timelapse.current) + + def on_close(self): + self._logger.info("Client connection closed") + self._printer.unregisterCallback(self) + self._gcodeManager.unregisterCallback(self) + octoprint.timelapse.unregisterCallback(self) + + self._eventManager.fire(Events.CLIENT_CLOSED) + for event in PrinterStateConnection.EVENTS: + self._eventManager.unsubscribe(event, self._onEvent) + + def on_message(self, message): + pass + + def sendCurrentData(self, data): + # add current temperature, log and message backlogs to sent data + with self._temperatureBacklogMutex: + temperatures = self._temperatureBacklog + self._temperatureBacklog = [] + + with self._logBacklogMutex: + logs = self._logBacklog + self._logBacklog = [] + + with self._messageBacklogMutex: + messages = self._messageBacklog + self._messageBacklog = [] + + data.update({ + "temps": temperatures, + "logs": logs, + "messages": messages + }) + self._emit("current", data) + + def sendHistoryData(self, data): + self._emit("history", data) + + def sendEvent(self, type, payload=None): + self._emit("event", {"type": type, "payload": payload}) + + def sendFeedbackCommandOutput(self, name, output): + self._emit("feedbackCommandOutput", {"name": name, "output": output}) + + def sendTimelapseConfig(self, timelapseConfig): + self._emit("timelapse", timelapseConfig) + + def addLog(self, data): + with self._logBacklogMutex: + self._logBacklog.append(data) + + def addMessage(self, data): + with self._messageBacklogMutex: + self._messageBacklog.append(data) + + def addTemperature(self, data): + with self._temperatureBacklogMutex: + self._temperatureBacklog.append(data) + + def _onEvent(self, event, payload): + self.sendEvent(event, payload) + + def _emit(self, type, payload): + self.send({type: payload}) diff --git a/src/octoprint/server/util/tornado.py b/src/octoprint/server/util/tornado.py new file mode 100644 index 00000000..38a5252e --- /dev/null +++ b/src/octoprint/server/util/tornado.py @@ -0,0 +1,853 @@ +# coding=utf-8 +from __future__ import absolute_import + +__author__ = "Gina Häußge " +__license__ = 'GNU Affero General Public License http://www.gnu.org/licenses/agpl.html' +__copyright__ = "Copyright (C) 2014 The OctoPrint Project - Released under terms of the AGPLv3 License" + +import logging +import os +import datetime +import stat +import mimetypes +import email +import time + +import tornado +import tornado.web +import tornado.gen +import tornado.escape +import tornado.httputil +import tornado.httpserver +import tornado.httpclient +import tornado.http1connection +import tornado.iostream +import tornado.tcpserver + +import octoprint.util + + +#~~ WSGI middleware + + +@tornado.web.stream_request_body +class UploadStorageFallbackHandler(tornado.web.RequestHandler): + """ + A `RequestHandler` similar to `tornado.web.FallbackHandler` which fetches any files contained in the request bodies + of content type `multipart`, stores them in temporary files and supplies the `fallback` with the file's `name`, + `content_type`, `path` and `size` instead via a rewritten body. + + Basically similar to what the nginx upload module does. + + Basic request body example: + + ------WebKitFormBoundarypYiSUx63abAmhT5C + Content-Disposition: form-data; name="file"; filename="test.gcode" + Content-Type: application/octet-stream + + ... + ------WebKitFormBoundarypYiSUx63abAmhT5C + Content-Disposition: form-data; name="apikey" + + my_funny_apikey + ------WebKitFormBoundarypYiSUx63abAmhT5C + Content-Disposition: form-data; name="select" + + true + ------WebKitFormBoundarypYiSUx63abAmhT5C-- + + That would get turned into: + + ------WebKitFormBoundarypYiSUx63abAmhT5C + Content-Disposition: form-data; name="apikey" + + my_funny_apikey + ------WebKitFormBoundarypYiSUx63abAmhT5C + Content-Disposition: form-data; name="select" + + true + ------WebKitFormBoundarypYiSUx63abAmhT5C + Content-Disposition: form-data; name="file.path" + + /tmp/tmpzupkro + ------WebKitFormBoundarypYiSUx63abAmhT5C + Content-Disposition: form-data; name="file.name" + + test.gcode + ------WebKitFormBoundarypYiSUx63abAmhT5C + Content-Disposition: form-data; name="file.content_type" + + application/octet-stream + ------WebKitFormBoundarypYiSUx63abAmhT5C + Content-Disposition: form-data; name="file.size" + + 349182 + ------WebKitFormBoundarypYiSUx63abAmhT5C-- + + The underlying application can then access the contained files via their respective paths and just move them + where necessary. + """ + + # the request methods that may contain a request body + BODY_METHODS = ("POST", "PATCH", "PUT") + + def initialize(self, fallback, file_prefix="tmp", file_suffix="", path=None, suffixes=None): + if not suffixes: + suffixes = dict() + + self._fallback = fallback + self._file_prefix = file_prefix + self._file_suffix = file_suffix + self._path = path + + self._suffixes = dict((key, key) for key in ("name", "path", "content_type", "size")) + for suffix_type, suffix in suffixes.iteritems(): + if suffix_type in self._suffixes and suffix is not None: + self._suffixes[suffix_type] = suffix + + # Parts, files and values will be stored here + self._parts = dict() + self._files = [] + + # Part currently being processed + self._current_part = None + + # content type of request body + self._content_type = None + + # bytes left to read according to content_length of request body + self._bytes_left = 0 + + # buffer needed for identifying form data parts + self._buffer = b"" + + # buffer for new body + self._new_body = b"" + + # logger + self._logger = logging.getLogger(__name__) + + def prepare(self): + """ + Prepares the processing of the request. If it's a request that may contain a request body (as defined in + `UploadStorageFallbackHandler.BODY_METHODS) prepares the multipart parsing if content type fits. If it's a + body-less request, just calls the `fallback` with an empty body and finishes the request. + """ + if self.request.method in UploadStorageFallbackHandler.BODY_METHODS: + self._bytes_left = self.request.headers.get("Content-Length", 0) + self._content_type = self.request.headers.get("Content-Type", None) + + # request might contain a body + if self.is_multipart(): + if not self._bytes_left: + # we don't support requests without a content-length + raise tornado.web.HTTPError(400, reason="No Content-Length supplied") + + # extract the multipart boundary + fields = self._content_type.split(";") + for field in fields: + k, sep, v = field.strip().partition("=") + if k == "boundary" and v: + if v.startswith(b'"') and v.endswith(b'"'): + self._multipart_boundary = tornado.escape.utf8(v[1:-1]) + else: + self._multipart_boundary = tornado.escape.utf8(v) + break + else: + self._multipart_boundary = None + else: + self._fallback(self.request, b"") + self._finished = True + + def data_received(self, chunk): + """ + Called by Tornado on receiving a chunk of the request body. If request is a multipart request, takes care of + processing the multipart data structure via `self._process_multipart_data`. If not, just adds the chunk to + internal in-memory buffer. + + :param chunk: chunk of data received from Tornado + """ + + data = self._buffer + chunk + if self.is_multipart(): + self._process_multipart_data(data) + else: + self._buffer = data + + def is_multipart(self): + """Checks whether this request is a `multipart` request""" + return self._content_type is not None and self._content_type.startswith("multipart") + + def _process_multipart_data(self, data): + """ + Processes the given data, parsing it for multipart definitions and calling the appropriate methods. + + :param data: the data to process as a string + """ + + # check for boundary + delimiter = b"--%s" % self._multipart_boundary + delimiter_loc = data.find(delimiter) + delimiter_len = len(delimiter) + end_of_header = None + if delimiter_loc != -1: + # found the delimiter in the currently available data + data, self._buffer = data[0:delimiter_loc], data[delimiter_loc:] + end_of_header = self._buffer.find("\r\n\r\n") + else: + # make sure any boundary (with single or double ==) contained at the end of chunk does not get + # truncated by this processing round => save it to the buffer for next round + endlen = len(self._multipart_boundary) + 4 + data, self._buffer = data[0:-endlen], data[-endlen:] + + # stream data to part handler + if data: + if self._current_part: + self._on_data(self._current_part, data) + + if end_of_header >= 0: + self._on_header(self._buffer[delimiter_len+2:end_of_header]) + self._buffer = self._buffer[end_of_header + 4:] + + if delimiter_loc != -1 and self._buffer[delimiter_len:delimiter_len+2] == "--": + # we saw the last boundary and are at the end of our request + if self._current_part: + self._on_close(self._current_part) + self._current_part = None + self._buffer = b"" + self._on_finish() + + def _on_header(self, header): + """ + Called for a new multipart header, takes care of parsing the header and calling `self._on_part` with the + relevant data, setting the current part in the process. + + :param header: header to parse + """ + + # close any open parts + if self._current_part: + self._on_close(self._current_part) + self._current_part = None + + header_check = header.find(self._multipart_boundary) + if header_check != -1: + self._logger.warn("Header still contained multipart boundary, stripping it...") + header = header[header_check:] + + # convert to dict + header = tornado.httputil.HTTPHeaders.parse(header.decode("utf-8")) + disp_header = header.get("Content-Disposition", "") + disposition, disp_params = tornado.httputil._parse_header(disp_header) + + if disposition != "form-data": + self._logger.warn("Got a multipart header without form-data content disposition, ignoring that one") + return + if not disp_params.get("name"): + self._logger.warn("Got a multipart header without name, ignoring that one") + return + + self._current_part = self._on_part(disp_params["name"], header.get("Content-Type", None), filename=disp_params["filename"] if "filename" in disp_params else None) + + def _on_part(self, name, content_type, filename=None): + """ + Called for new parts in the multipart stream. If `filename` is given creates new `file` part (which leads + to storage of the data as temporary file on disk), if not creates a new `data` part (which stores + incoming data in memory). + + Structure of `file` parts: + + * `name`: name of the part + * `filename`: filename associated with the part + * `path`: path to the temporary file storing the file's data + * `content_type`: content type of the part + * `file`: file handle for the temporary file (mode "wb", not deleted on close!) + + Structure of `data` parts: + + * `name`: name of the part + * `content_type`: content type of the part + * `data`: bytes of the part (initialized to "") + + :param name: name of the part + :param content_type: content type of the part + :param filename: filename associated with the part. + :return: dict describing the new part + """ + if filename is not None: + # this is a file + import tempfile + handle = tempfile.NamedTemporaryFile(mode="wb", prefix=self._file_prefix, suffix=self._file_suffix, dir=self._path, delete=False) + return dict(name=tornado.escape.utf8(name), + filename=tornado.escape.utf8(filename), + path=tornado.escape.utf8(handle.name), + content_type=tornado.escape.utf8(content_type), + file=handle) + + else: + return dict(name=tornado.escape.utf8(name), content_type=content_type, data=b"") + + def _on_data(self, part, data): + """ + Called when new bytes are received for the given `part`, takes care of writing them to their storage. + + :param part: part for which data was received + :param data: data chunk which was received + """ + if "file" in part: + part["file"].write(data) + else: + part["data"] += data + + def _on_close(self, part): + """ + Called when a part gets closed, takes care of storing the finished part in the internal parts storage and for + `file` parts closing the temporary file and storing the part in the internal files storage. + + :param part: part which was closed + """ + name = part["name"] + self._parts[name] = part + if "file" in part: + self._files.append(part["path"]) + part["file"].close() + del part["file"] + + def _on_finish(self): + """ + Called when the request body has been read completely. Takes care of creating the replacement body out of the + logged parts, turning `file` parts into new + :return: + """ + + self._new_body = b"" + for name, part in self._parts.iteritems(): + if "filename" in part: + # add form fields for filename, path, size and content_type for all files contained in the request + fields = dict((self._suffixes[key], value) for (key, value) in dict(name=part["filename"], path=part["path"], size=str(os.stat(part["path"]).st_size), content_type=part["content_type"]).iteritems()) + for n, p in fields.iteritems(): + key = name + "." + n + self._new_body += b"--%s\r\n" % self._multipart_boundary + self._new_body += b"Content-Disposition: form-data; name=\"%s\"\r\n" % key + self._new_body += b"\r\n" + self._new_body += p + b"\r\n" + elif "data" in part: + self._new_body += b"--%s\r\n" % self._multipart_boundary + value = part["data"] + self._new_body += b"Content-Disposition: form-data; name=\"%s\"\r\n" % name + if "content_type" in part and part["content_type"] is not None: + self._new_body += b"Content-Type: %s\r\n" % part["content_type"] + self._new_body += b"\r\n" + self._new_body += value + self._new_body += b"--%s--\r\n" % self._multipart_boundary + + def _handle_method(self, *args, **kwargs): + """ + Handler for any request method, takes care of defining the new request body if necessary and forwarding + the current request and changed body to the `fallback`. + """ + + # determine which body to supply + body = b"" + if self.is_multipart(): + # make sure we really processed all data in the buffer + while len(self._buffer): + self._process_multipart_data(self._buffer) + + # use rewritten body + body = self._new_body + + elif self.request.method in UploadStorageFallbackHandler.BODY_METHODS: + # directly use data from buffer + body = self._buffer + + # rewrite content length + self.request.headers["Content-Length"] = len(body) + + try: + # call the configured fallback with request and body to use + self._fallback(self.request, body) + finally: + # make sure the temporary files are removed again + for f in self._files: + octoprint.util.silentRemove(f) + + # make all http methods trigger _handle_method + get = _handle_method + post = _handle_method + put = _handle_method + patch = _handle_method + delete = _handle_method + head = _handle_method + options = _handle_method + + +class WsgiInputContainer(object): + """ + A WSGI container for use with Tornado that allows supplying the request body to be used for `wsgi.input` in the + generated WSGI environment upon call. + + A `RequestHandler` can thus provide the WSGI application with a stream for the request body, or a modified body. + + Example usage: + + wsgi_app = octoprint.server.util.WsgiInputContainer(octoprint_app) + application = tornado.web.Application([ + (r".*", UploadStorageFallbackHandler, dict(fallback=wsgi_app), + ]) + + The implementation logic is basically the same as `tornado.wsgi.WSGIContainer` but the `__call__` and `environ` + methods have been adjusted to for an optionally supplied `body` argument which is then used for `wsgi.input`. + """ + + def __init__(self, wsgi_application): + self.wsgi_application = wsgi_application + + def __call__(self, request, body=None): + """ + Wraps the call against the WSGI app, deriving the WSGI environment from the supplied Tornado `HTTPServerRequest`. + + :param request: the `tornado.httpserver.HTTPServerRequest` to derive the WSGI environment from + :param body: an optional body to use as `wsgi.input` instead of `request.body`, can be a string or a stream + """ + + data = {} + response = [] + + def start_response(status, response_headers, exc_info=None): + data["status"] = status + data["headers"] = response_headers + return response.append + app_response = self.wsgi_application( + WsgiInputContainer.environ(request, body), start_response) + try: + response.extend(app_response) + body = b"".join(response) + finally: + if hasattr(app_response, "close"): + app_response.close() + if not data: + raise Exception("WSGI app did not call start_response") + + status_code = int(data["status"].split()[0]) + headers = data["headers"] + header_set = set(k.lower() for (k, v) in headers) + body = tornado.escape.utf8(body) + if status_code != 304: + if "content-length" not in header_set: + headers.append(("Content-Length", str(len(body)))) + if "content-type" not in header_set: + headers.append(("Content-Type", "text/html; charset=UTF-8")) + if "server" not in header_set: + headers.append(("Server", "TornadoServer/%s" % tornado.version)) + + parts = [tornado.escape.utf8("HTTP/1.1 " + data["status"] + "\r\n")] + for key, value in headers: + parts.append(tornado.escape.utf8(key) + b": " + tornado.escape.utf8(value) + b"\r\n") + parts.append(b"\r\n") + parts.append(body) + request.write(b"".join(parts)) + request.finish() + self._log(status_code, request) + + @staticmethod + def environ(request, body=None): + """ + Converts a `tornado.httputil.HTTPServerRequest` to a WSGI environment. + + An optional `body` to be used for populating `wsgi.input` can be supplied (either a string or a stream). If not + supplied, `request.body` will be wrapped into a `io.BytesIO` stream and used instead. + + :param request: the `tornado.httpserver.HTTPServerRequest` to derive the WSGI environment from + :param body: an optional body to use as `wsgi.input` instead of `request.body`, can be a string or a stream + """ + from tornado.wsgi import to_wsgi_str + import sys + import io + + # determine the request_body to supply as wsgi.input + if body is not None: + if isinstance(body, (bytes, str)): + request_body = io.BytesIO(tornado.escape.utf8(body)) + else: + request_body = body + else: + request_body = io.BytesIO(tornado.escape.utf8(request.body)) + + hostport = request.host.split(":") + if len(hostport) == 2: + host = hostport[0] + port = int(hostport[1]) + else: + host = request.host + port = 443 if request.protocol == "https" else 80 + environ = { + "REQUEST_METHOD": request.method, + "SCRIPT_NAME": "", + "PATH_INFO": to_wsgi_str(tornado.escape.url_unescape( + request.path, encoding=None, plus=False)), + "QUERY_STRING": request.query, + "REMOTE_ADDR": request.remote_ip, + "SERVER_NAME": host, + "SERVER_PORT": str(port), + "SERVER_PROTOCOL": request.version, + "wsgi.version": (1, 0), + "wsgi.url_scheme": request.protocol, + "wsgi.input": request_body, + "wsgi.errors": sys.stderr, + "wsgi.multithread": False, + "wsgi.multiprocess": True, + "wsgi.run_once": False, + } + if "Content-Type" in request.headers: + environ["CONTENT_TYPE"] = request.headers.pop("Content-Type") + if "Content-Length" in request.headers: + environ["CONTENT_LENGTH"] = request.headers.pop("Content-Length") + for key, value in request.headers.items(): + environ["HTTP_" + key.replace("-", "_").upper()] = value + return environ + + def _log(self, status_code, request): + access_log = logging.getLogger("tornado.access") + + if status_code < 400: + log_method = access_log.info + elif status_code < 500: + log_method = access_log.warning + else: + log_method = access_log.error + request_time = 1000.0 * request.request_time() + summary = request.method + " " + request.uri + " (" + \ + request.remote_ip + ")" + log_method("%d %s %.2fms", status_code, summary, request_time) + + +#~~ customized HTTP1Connection implementation + + +class CustomHTTPServer(tornado.httpserver.HTTPServer): + """ + Custom implementation of `tornado.httpserver.HTTPServer` that allows defining max body sizes depending on path and + method. + + The implementation is mostly taken from `tornado.httpserver.HTTPServer`, the only difference is the creation + of a `CustomHTTP1ConnectionParameters` instance instead of `tornado.http1connection.HTTP1ConnectionParameters` + which is supplied with the two new constructor arguments `max_body_sizes` and `max_default_body_size` and the + creation of a `CustomHTTP1ServerConnection` instead of a `tornado.http1connection.HTTP1ServerConnection` upon + connection by a client. + + `max_body_sizes` is expected to be an iterable containing tuples of the form (method, path regex, maximum body size), + with method and path regex having to match in order for maximum body size to take affect. + + `default_max_body_size` is the default maximum body size to apply if no specific one from `max_body_sizes` matches. + """ + + def __init__(self, request_callback, no_keep_alive=False, io_loop=None, + xheaders=False, ssl_options=None, protocol=None, + decompress_request=False, + chunk_size=None, max_header_size=None, + idle_connection_timeout=None, body_timeout=None, + max_body_sizes=None, default_max_body_size=None, max_buffer_size=None): + self.request_callback = request_callback + self.no_keep_alive = no_keep_alive + self.xheaders = xheaders + self.protocol = protocol + self.conn_params = CustomHTTP1ConnectionParameters( + decompress=decompress_request, + chunk_size=chunk_size, + max_header_size=max_header_size, + header_timeout=idle_connection_timeout or 3600, + max_body_sizes=max_body_sizes, + default_max_body_size=default_max_body_size, + body_timeout=body_timeout) + tornado.tcpserver.TCPServer.__init__(self, io_loop=io_loop, ssl_options=ssl_options, + max_buffer_size=max_buffer_size, + read_chunk_size=chunk_size) + self._connections = set() + + + def handle_stream(self, stream, address): + context = tornado.httpserver._HTTPRequestContext(stream, address, + self.protocol) + conn = CustomHTTP1ServerConnection( + stream, self.conn_params, context) + self._connections.add(conn) + conn.start_serving(self) + + +class CustomHTTP1ServerConnection(tornado.http1connection.HTTP1ServerConnection): + """ + A custom implementation of `tornado.http1connection.HTTP1ServerConnection` which utilizes a `CustomHTTP1Connection` + instead of a `tornado.http1connection.HTTP1Connection` in `_server_request_loop`. The implementation logic is + otherwise the same as `tornado.http1connection.HTTP1ServerConnection`. + """ + + @tornado.gen.coroutine + def _server_request_loop(self, delegate): + try: + while True: + conn = CustomHTTP1Connection(self.stream, False, + self.params, self.context) + request_delegate = delegate.start_request(self, conn) + try: + ret = yield conn.read_response(request_delegate) + except (tornado.iostream.StreamClosedError, + tornado.iostream.UnsatisfiableReadError): + return + except tornado.http1connection._QuietException: + # This exception was already logged. + conn.close() + return + except Exception: + tornado.http1connection.gen_log.error("Uncaught exception", exc_info=True) + conn.close() + return + if not ret: + return + yield tornado.gen.moment + finally: + delegate.on_close(self) + + +class CustomHTTP1Connection(tornado.http1connection.HTTP1Connection): + """ + A custom implementation of `tornado.http1connection.HTTP1Connection` which upon checking the `Content-Length` of + the request against the configured maximum utilizes `max_body_sizes` and `default_max_body_size` as a fallback. + """ + + def __init__(self, stream, is_client, params=None, context=None): + tornado.http1connection.HTTP1Connection.__init__(self, stream, is_client, params=params, context=context) + + import re + self._max_body_sizes = map(lambda x: (x[0], re.compile(x[1]), x[2]), self.params.max_body_sizes or dict()) + self._default_max_body_size = self.params.default_max_body_size or self.stream.max_buffer_size + + def _read_body(self, headers, delegate): + """ + Basically the same as `tornado.http1connection.HTTP1Connection._read_body`, but determines the maximum + content length individually for the request (utilizing `._get_max_content_length`). + + If the individual max content length is 0 or smaller no content length is checked. If the content length of the + current request exceeds the individual max content length, the request processing is aborted and an + `HTTPInputError` is raised. + """ + content_length = headers.get("Content-Length") + if content_length: + content_length = int(content_length) + max_content_length = self._get_max_content_length(self._request_start_line.method, self._request_start_line.path) + if 0 <= max_content_length < content_length: + raise tornado.httputil.HTTPInputError("Content-Length too long") + return self._read_fixed_body(content_length, delegate) + if headers.get("Transfer-Encoding") == "chunked": + return self._read_chunked_body(delegate) + if self.is_client: + return self._read_body_until_close(delegate) + return None + + def _get_max_content_length(self, method, path): + """ + Gets the max content length for the given method and path. Checks whether method and path match against any + of the specific maximum content lengths supplied in `max_body_sizes` and returns that as the maximum content + length if available, otherwise returns `default_max_body_size`. + + :param method: method of the request to match against + :param path: path od the request to match against + :return: determine maximum content length to apply to this request, max return 0 for unlimited allowed content + length + """ + + for m, p, s in self._max_body_sizes: + if method == m and p.match(path): + return s + return self._default_max_body_size + + +class CustomHTTP1ConnectionParameters(tornado.http1connection.HTTP1ConnectionParameters): + """ + An implementation of `tornado.http1connection.HTTP1ConnectionParameters` that adds to new parameters + `max_body_sizes` and `default_max_body_size`. + + For a description of these please see the documentation of `CustomHTTPServer` above. + """ + + def __init__(self, *args, **kwargs): + tornado.http1connection.HTTP1ConnectionParameters.__init__(self, args, kwargs) + self.max_body_sizes = kwargs["max_body_sizes"] if "max_body_sizes" in kwargs else dict() + self.default_max_body_size = kwargs["default_max_body_size"] if "default_max_body_size" in kwargs else dict() + +#~~ customized large response handler + + +class LargeResponseHandler(tornado.web.StaticFileHandler): + + CHUNK_SIZE = 16 * 1024 + + def initialize(self, path, default_filename=None, as_attachment=False, access_validation=None): + tornado.web.StaticFileHandler.initialize(self, path, default_filename) + self._as_attachment = as_attachment + self._access_validation = access_validation + + def get(self, path, include_body=True): + if self._access_validation is not None: + self._access_validation(self.request) + + path = self.parse_url_path(path) + abspath = os.path.abspath(os.path.join(self.root, path)) + # os.path.abspath strips a trailing / + # it needs to be temporarily added back for requests to root/ + if not (abspath + os.path.sep).startswith(self.root): + raise tornado.web.HTTPError(403, "%s is not in root static directory", path) + if os.path.isdir(abspath) and self.default_filename is not None: + # need to look at the request.path here for when path is empty + # but there is some prefix to the path that was already + # trimmed by the routing + if not self.request.path.endswith("/"): + self.redirect(self.request.path + "/") + return + abspath = os.path.join(abspath, self.default_filename) + if not os.path.exists(abspath): + raise tornado.web.HTTPError(404) + if not os.path.isfile(abspath): + raise tornado.web.HTTPError(403, "%s is not a file", path) + + stat_result = os.stat(abspath) + modified = datetime.datetime.fromtimestamp(stat_result[stat.ST_MTIME]) + + self.set_header("Last-Modified", modified) + + mime_type, encoding = mimetypes.guess_type(abspath) + if mime_type: + self.set_header("Content-Type", mime_type) + + cache_time = self.get_cache_time(path, modified, mime_type) + + if cache_time > 0: + self.set_header("Expires", datetime.datetime.utcnow() + + datetime.timedelta(seconds=cache_time)) + self.set_header("Cache-Control", "max-age=" + str(cache_time)) + + self.set_extra_headers(path) + + # Check the If-Modified-Since, and don't send the result if the + # content has not been modified + ims_value = self.request.headers.get("If-Modified-Since") + if ims_value is not None: + date_tuple = email.utils.parsedate(ims_value) + if_since = datetime.datetime.fromtimestamp(time.mktime(date_tuple)) + if if_since >= modified: + self.set_status(304) + return + + if not include_body: + assert self.request.method == "HEAD" + self.set_header("Content-Length", stat_result[stat.ST_SIZE]) + else: + with open(abspath, "rb") as file: + while True: + data = file.read(LargeResponseHandler.CHUNK_SIZE) + if not data: + break + self.write(data) + self.flush() + + def set_extra_headers(self, path): + if self._as_attachment: + self.set_header("Content-Disposition", "attachment") + + +##~~ URL Forward Handler for forwarding requests to a preconfigured static URL + + +class UrlForwardHandler(tornado.web.RequestHandler): + + def initialize(self, url=None, as_attachment=False, basename=None, access_validation=None): + tornado.web.RequestHandler.initialize(self) + self._url = url + self._as_attachment = as_attachment + self._basename = basename + self._access_validation = access_validation + + @tornado.web.asynchronous + def get(self, *args, **kwargs): + if self._access_validation is not None: + self._access_validation(self.request) + + if self._url is None: + raise tornado.web.HTTPError(404) + + client = tornado.httpclient.AsyncHTTPClient() + r = tornado.httpclient.HTTPRequest(url=self._url, method=self.request.method, body=self.request.body, headers=self.request.headers, follow_redirects=False, allow_nonstandard_methods=True) + + try: + return client.fetch(r, self.handle_response) + except tornado.web.HTTPError as e: + if hasattr(e, "response") and e.response: + self.handle_response(e.response) + else: + raise tornado.web.HTTPError(500) + + def handle_response(self, response): + if response.error and not isinstance(response.error, tornado.web.HTTPError): + raise tornado.web.HTTPError(500) + + filename = None + + self.set_status(response.code) + for name in ("Date", "Cache-Control", "Server", "Content-Type", "Location"): + value = response.headers.get(name) + if value: + self.set_header(name, value) + + if name == "Content-Type": + filename = self.get_filename(value) + + if self._as_attachment: + if filename is not None: + self.set_header("Content-Disposition", "attachment; filename=%s" % filename) + else: + self.set_header("Content-Disposition", "attachment") + + if response.body: + self.write(response.body) + self.finish() + + def get_filename(self, content_type): + if not self._basename: + return None + + typeValue = map(str.strip, content_type.split(";")) + if len(typeValue) == 0: + return None + + extension = mimetypes.guess_extension(typeValue[0]) + if not extension: + return None + + return "%s%s" % (self._basename, extension) + + +#~~ Factory method for creating Flask access validation wrappers from the Tornado request context + + +def access_validation_factory(app, login_manager, validator): + """ + Creates an access validation wrapper using the supplied validator. + + :param validator: the access validator to use inside the validation wrapper + :return: an access validation wrapper taking a request as parameter and performing the request validation + """ + def f(request): + """ + Creates a custom wsgi and Flask request context in order to be able to process user information + stored in the current session. + + :param request: The Tornado request for which to create the environment and context + """ + import flask + + wsgi_environ = WsgiInputContainer.environ(request) + with app.request_context(wsgi_environ): + app.session_interface.open_session(app, flask.request) + login_manager.reload_user() + validator(flask.request) + return f diff --git a/src/octoprint/server/util/watchdog.py b/src/octoprint/server/util/watchdog.py new file mode 100644 index 00000000..e6ced528 --- /dev/null +++ b/src/octoprint/server/util/watchdog.py @@ -0,0 +1,87 @@ +# coding=utf-8 +from __future__ import absolute_import + +__author__ = "Gina Häußge " +__license__ = 'GNU Affero General Public License http://www.gnu.org/licenses/agpl.html' +__copyright__ = "Copyright (C) 2014 The OctoPrint Project - Released under terms of the AGPLv3 License" + +import logging +import os +import watchdog.events + +import octoprint.gcodefiles +import octoprint.util +from octoprint.settings import settings + + +class UploadCleanupWatchdogHandler(watchdog.events.PatternMatchingEventHandler): + """ + Takes care of automatically deleting metadata entries for files that get deleted from the uploads folder + """ + + patterns = map(lambda x: "*.%s" % x, octoprint.gcodefiles.GCODE_EXTENSIONS) + + def __init__(self, gcode_manager): + watchdog.events.PatternMatchingEventHandler.__init__(self) + self._gcode_manager = gcode_manager + + def on_deleted(self, event): + filename = self._gcode_manager._getBasicFilename(event.src_path) + if not filename: + return + + self._gcode_manager.removeFileFromMetadata(filename) + + +class GcodeWatchdogHandler(watchdog.events.PatternMatchingEventHandler): + """ + Takes care of automatically "uploading" files that get added to the watched folder. + """ + + patterns = map(lambda x: "*.%s" % x, octoprint.gcodefiles.SUPPORTED_EXTENSIONS) + + def __init__(self, gcodeManager, printer): + watchdog.events.PatternMatchingEventHandler.__init__(self) + + self._logger = logging.getLogger(__name__) + + self._gcodeManager = gcodeManager + self._printer = printer + + def _upload(self, path): + class WatchdogFileWrapper(object): + + def __init__(self, path): + self._path = path + self.filename = os.path.basename(self._path) + + def save(self, target): + octoprint.util.safeRename(self._path, target) + + fileWrapper = WatchdogFileWrapper(path) + + # determine current job + currentFilename = None + currentOrigin = None + currentJob = self._printer.getCurrentJob() + if currentJob is not None and "file" in currentJob.keys(): + currentJobFile = currentJob["file"] + if "name" in currentJobFile.keys() and "origin" in currentJobFile.keys(): + currentFilename = currentJobFile["name"] + currentOrigin = currentJobFile["origin"] + + # determine future filename of file to be uploaded, abort if it can't be uploaded + futureFilename = self._gcodeManager.getFutureFilename(fileWrapper) + if futureFilename is None or (not settings().getBoolean(["cura", "enabled"]) and not octoprint.gcodefiles.isGcodeFileName(futureFilename)): + self._logger.warn("Could not add %s: Invalid file" % fileWrapper.filename) + return + + # prohibit overwriting currently selected file while it's being printed + if futureFilename == currentFilename and not currentOrigin == octoprint.gcodefiles.FileDestinations.SDCARD and self._printer.isPrinting() or self._printer.isPaused(): + self._logger.warn("Could not add %s: Trying to overwrite file that is currently being printed" % fileWrapper.filename) + return + + self._gcodeManager.addFile(fileWrapper, octoprint.gcodefiles.FileDestinations.LOCAL) + + def on_created(self, event): + self._upload(event.src_path) diff --git a/src/octoprint/settings.py b/src/octoprint/settings.py index d599791e..a31b0780 100644 --- a/src/octoprint/settings.py +++ b/src/octoprint/settings.py @@ -42,7 +42,13 @@ default_settings = { "port": 5000, "firstRun": True, "baseUrl": "", - "scheme": "" + "scheme": "", + "uploads": { + "maxSize": 1 * 1024 * 1024 * 1024, # 1GB + "nameSuffix": ".name", + "pathSuffix": ".path" + }, + "maxSize": 100 * 1024, # 100 KB }, "webcam": { "stream": None, @@ -375,7 +381,11 @@ class Settings(object): return None if isinstance(value, bool): return value - return value.lower() in valid_boolean_trues + if isinstance(value, (int, float)): + return value != 0 + if isinstance(value, (str, unicode)): + return value.lower() in valid_boolean_trues + return value is not None def getBaseFolder(self, type): if type not in default_settings["folder"].keys(): diff --git a/src/octoprint/util/__init__.py b/src/octoprint/util/__init__.py index 5df97e7f..d889ddce 100644 --- a/src/octoprint/util/__init__.py +++ b/src/octoprint/util/__init__.py @@ -248,3 +248,7 @@ def dict_merge(a, b): else: result[k] = deepcopy(v) return result + + +class Object(object): + pass