Merge branch 'fix/utf8BomResilience' into devel

Conflicts:
	src/octoprint/static/js/app/dataupdater.js
	src/octoprint/util/comm.py

src/octoprint/util/__init__.py

@@ -510,6 +510,29 @@ def atomic_write(filename, mode="w+b", prefix="tmp", suffix=""):
 	shutil.move(temp_config.name, filename)
 
 
+def bom_aware_open(filename, encoding="ascii", mode="r", **kwargs):
+	import codecs
+
+	codec = codecs.lookup(encoding)
+	encoding = codec.name
+
+	if kwargs is None:
+		kwargs = dict()
+
+	potential_bom_attribute = "BOM_" + codec.name.replace("utf-", "utf").upper()
+	if "r" in mode and hasattr(codecs, potential_bom_attribute):
+		# these encodings might have a BOM, so let's see if there is one
+		bom = getattr(codecs, potential_bom_attribute)
+
+		with open(filename, "rb") as f:
+			header = f.read(4)
+
+		if header.startswith(bom):
+			encoding += "-sig"
+
+	return codecs.open(filename, encoding=encoding, **kwargs)
+
+
 class RepeatedTimer(threading.Thread):
 	"""
 	This class represents an action that should be run repeatedly in an interval. It is similar to python's

src/octoprint/util/comm.py

@@ -24,7 +24,7 @@ from octoprint.settings import settings, default_settings
 from octoprint.events import eventManager, Events
 from octoprint.filemanager import valid_file_type
 from octoprint.filemanager.destinations import FileDestinations
-from octoprint.util import get_exception_string, sanitize_ascii, filter_non_ascii, CountedEvent, RepeatedTimer
+from octoprint.util import get_exception_string, sanitize_ascii, filter_non_ascii, CountedEvent, RepeatedTimer, to_unicode, bom_aware_open
 
 try:
 	import _winreg
@@ -548,7 +548,7 @@ class MachineCom(object):
 		self._clear_to_send.set()
 
 	def sendCommand(self, cmd, cmd_type=None, processed=False, force=False):
-		cmd = cmd.encode('ascii', 'replace')
+		cmd = to_unicode(cmd, errors="replace")
 		if not processed:
 			cmd = process_gcode_line(cmd)
 			if not cmd:
@@ -1631,10 +1631,11 @@ class MachineCom(object):
 						command_allowing_checksum = gcode is not None or self._sendChecksumWithUnknownCommands
 						checksum_enabled = self._alwaysSendChecksum or (self.isPrinting() and not self._neverSendChecksum)
 
+						command_to_send = command.encode("ascii", errors="replace")
 						if command_requiring_checksum or (command_allowing_checksum and checksum_enabled):
-							self._doIncrementAndSendWithChecksum(command)
+							self._doIncrementAndSendWithChecksum(command_to_send)
 						else:
-							self._doSendWithoutChecksum(command)
+							self._doSendWithoutChecksum(command_to_send)
 
 					# trigger "sent" phase and use up one "ok"
 					self._process_command_phase("sent", command, command_type, gcode=gcode)
@@ -2039,7 +2040,7 @@ class PrintingGcodeFileInformation(PrintingFileInformation):
 		Opens the file for reading and determines the file size.
 		"""
 		PrintingFileInformation.start(self)
-		self._handle = open(self._filename, "r")
+		self._handle = bom_aware_open(self._filename, encoding="utf-8", errors="replace")
 
 	def close(self):
 		"""
@@ -2069,7 +2070,7 @@ class PrintingGcodeFileInformation(PrintingFileInformation):
 				if self._handle is None:
 					# file got closed just now
 					return None
-				line = self._handle.readline()
+				line = to_unicode(self._handle.readline())
 				if not line:
 					self.close()
 				processed = process_gcode_line(line, offsets=offsets, current_tool=current_tool)

src/octoprint/util/gcodeInterpreter.py

@@ -35,7 +35,9 @@ class gcode(object):
 		if os.path.isfile(filename):
 			self.filename = filename
 			self._fileSize = os.stat(filename).st_size
-			with open(filename, "r") as f:
+
+			import codecs
+			with codecs.open(filename, encoding="utf-8", errors="replace") as f:
 				self._load(f, printer_profile, throttle=throttle)
 
 	def abort(self):