configfile: Add "include" support (#1359)

Allows configuration files to include other configuration files using
[include filename.cfg] syntax.  Klippy loads include files in the
position of the include header; subsequent definitions override
included values.  Supports wildcards (e.g. [include macros/*.cfg).
Allows included files to include other files but blocks recursion.

Signed-off-by: Greg Lauckhart <greg@lauckhart.com>

klippy/configfile.py

@@ -3,7 +3,7 @@
 # Copyright (C) 2016-2018  Kevin O'Connor <kevin@koconnor.net>
 #
 # This file may be distributed under the terms of the GNU GPLv3 license.
-import os, re, time, logging, ConfigParser, StringIO
+import os, glob, re, time, logging, ConfigParser, StringIO
 
 error = ConfigParser.Error
 
@@ -149,33 +149,75 @@ class PrinterConfig:
                 is_dup_field = True
                 lines[lineno] = '#' + lines[lineno]
         return "\n".join(lines)
-    def _build_config_wrapper(self, data):
+    def _parse_config_buffer(self, buffer, filename, fileconfig):
-        # Strip trailing comments from config
+        if not buffer:
+            return
+        data = '\n'.join(buffer)
+        del buffer[:]
+        sbuffer = StringIO.StringIO(data)
+        fileconfig.readfp(sbuffer, filename)
+    def _resolve_include(self, source_filename, include_spec, fileconfig,
+                         visited):
+        dirname = os.path.dirname(source_filename)
+        include_spec = include_spec.strip()
+        include_glob = os.path.join(dirname, include_spec)
+        include_filenames = glob.glob(include_glob)
+        if not include_filenames and not glob.has_magic(include_glob):
+            # Empty set is OK if wildcard but not for direct file reference
+            raise error("Include file '%s' does not exist", include_glob)
+        include_filenames.sort()
+        for include_filename in include_filenames:
+            include_data = self._read_config_file(include_filename)
+            self._parse_config(include_data, include_filename, fileconfig,
+                               visited)
+        return include_filenames
+    def _parse_config(self, data, filename, fileconfig, visited):
+        path = os.path.abspath(filename)
+        if path in visited:
+            raise error("Recursive include of config file '%s'" % (filename))
+        visited.add(path)
         lines = data.split('\n')
-        for i, line in enumerate(lines):
+        # Buffer lines between includes and parse as a unit so that overrides
+        # in includes apply linearly as they do within a single file
+        buffer = []
+        for line in lines:
+            # Strip trailing comment
             pos = line.find('#')
             if pos >= 0:
-                lines[i] = line[:pos]
+                line = line[:pos]
-        data = '\n'.join(lines)
+            # Process include or buffer line
-        # Read and process config file
+            mo = ConfigParser.RawConfigParser.SECTCRE.match(line)
-        sfile = StringIO.StringIO(data)
+            header = mo and mo.group('header')
+            if header and header.startswith('include '):
+                self._parse_config_buffer(buffer, filename, fileconfig)
+                include_spec = header[8:].strip()
+                self._resolve_include(filename, include_spec, fileconfig,
+                                      visited)
+            else:
+                buffer.append(line)
+        self._parse_config_buffer(buffer, filename, fileconfig)
+        visited.remove(path)
+    def _build_config_wrapper(self, data, filename):
         fileconfig = ConfigParser.RawConfigParser()
-        fileconfig.readfp(sfile)
+        self._parse_config(data, filename, fileconfig, set())
         return ConfigWrapper(self.printer, fileconfig, {}, 'printer')
     def _build_config_string(self, config):
         sfile = StringIO.StringIO()
         config.fileconfig.write(sfile)
         return sfile.getvalue().strip()
     def read_config(self, filename):
-        return self._build_config_wrapper(self._read_config_file(filename))
+        return self._build_config_wrapper(self._read_config_file(filename),
+                                          filename)
     def read_main_config(self):
         filename = self.printer.get_start_args()['config_file']
         data = self._read_config_file(filename)
         regular_data, autosave_data = self._find_autosave_data(data)
-        regular_config = self._build_config_wrapper(regular_data)
+        regular_config = self._build_config_wrapper(regular_data, filename)
         autosave_data = self._strip_duplicates(autosave_data, regular_config)
-        self.autosave = self._build_config_wrapper(autosave_data)
+        self.autosave = self._build_config_wrapper(autosave_data, filename)
-        return self._build_config_wrapper(regular_data + autosave_data)
+        self._config = self._build_config_wrapper(regular_data + autosave_data,
+                                                  filename)
+        return self._config
     def check_unused_options(self, config):
         fileconfig = config.fileconfig
         objects = dict(self.printer.lookup_objects())
@@ -210,6 +252,14 @@ class PrinterConfig:
         logging.info("save_config: set [%s] %s = %s", section, option, svalue)
     def remove_section(self, section):
         self.autosave.fileconfig.remove_section(section)
+    def _disallow_include_conflicts(self, regular_data, cfgname, gcode):
+        config = self._build_config_wrapper(regular_data, cfgname)
+        for section in self.autosave.fileconfig.sections():
+            for option in self.autosave.fileconfig.options(section):
+                if config.fileconfig.has_option(section, option):
+                    msg = "SAVE_CONFIG section '%s' option '%s' conflicts " \
+                          "with included value" % (section, option)
+                    raise gcode.error(msg)
     cmd_SAVE_CONFIG_help = "Overwrite config file and restart"
     def cmd_SAVE_CONFIG(self, params):
         if not self.autosave.fileconfig.sections():
@@ -227,12 +277,13 @@ class PrinterConfig:
         try:
             data = self._read_config_file(cfgname)
             regular_data, old_autosave_data = self._find_autosave_data(data)
-            config = self._build_config_wrapper(regular_data)
+            config = self._build_config_wrapper(regular_data, cfgname)
         except error as e:
             msg = "Unable to parse existing config on SAVE_CONFIG"
             logging.exception(msg)
             raise gcode.error(msg)
         regular_data = self._strip_duplicates(regular_data, self.autosave)
+        self._disallow_include_conflicts(regular_data, cfgname, gcode)
         data = regular_data.rstrip() + autosave_data
         # Determine filenames
         datestr = time.strftime("-%Y%m%d_%H%M%S")