generate-config: display line numbers in errors

test/genconf/apache.conf.template

@@ -199,5 +199,5 @@ key_to_writer = {
 
 }$# @@__template_header@@
 
-${write_cfg(key_to_writer, config)
+${write_cfg(key_to_writer, config, key_to_node)
 }$

test/genconf/genconf.py

@@ -52,32 +52,38 @@ def get_key_path(o):
     else:
         return get_key_path(c2)
 
-def ast_node_to_dict(node, dest, lookup, parent_key):
+def ast_node_to_dict(node, dest, lookup, parent_key, key_to_node):
+    key_to_node[parent_key] = node
+
     if Node is None:
         return None
     elif isinstance(node, Dict):
         dest = OrderedDict()
         c = node.getChildren()
         for n in range(0, len(c), 2):
             key = c[n].getChildren()[0]
+            key_to_node["%s.%s" % (parent_key, key)] = c[n]
             dest[key] = ast_node_to_dict(c[n + 1], dest, lookup,
-                    '%s.%s' % (parent_key, key))
+                    '%s.%s' % (parent_key, key), key_to_node)
             lookup[parent_key] = dest
     elif isinstance(node, List):
         dest = []
         for (index, child) in enumerate(node.getChildren()):
+            key_to_node["%s.%s" % (parent_key, str(index))] = child
             cn = ast_node_to_dict(child, None, lookup, parent_key +
-                                  '.' + repr(index))
+                                  '.' + repr(index), key_to_node)
             dest.append(cn)
             lookup[parent_key] = dest
     elif isinstance(node, UnarySub):
         if parent_key in lookup:
-            raise Error("%s: already defined" % parent_key)
+            raise Error("line %d: %s: already defined" %
+                        (node.lineno,parent_key))
         lookup[parent_key] = '-' + repr(node.getChildren()[0].getChildren()[0])
         return '-' + repr(node.getChildren()[0].getChildren()[0])
     elif isinstance(node, Const):
         if parent_key in lookup:
-            raise Error("%s: already defined" % parent_key)
+            raise Error("line %d: %s: already defined" % (node.lineno,
+                                                          parent_key))
         lookup[parent_key] = node.getChildren()[0]
         return node.getChildren()[0]
     elif isinstance(node, Node):
@@ -86,7 +92,8 @@ def ast_node_to_dict(node, dest, lookup, parent_key):
             val = lookup[key_path]
             lookup[parent_key] = val
         else:
-            raise Error("Lookup failed for [%s]" % repr(node))
+            raise Error("line %d: Lookup failed for [%s]" % (node.lineno,
+                                                             repr(node)))
         return val
     return dest
 
@@ -129,8 +136,6 @@ def pre_process_text(cfg, conditions, placeholders):
     re_placeholders = r'@@([^\s]*)@@'
     cfg = re.sub(re_placeholders, lambda x: \
                  fill_placeholders(placeholders, x), cfg)
-    re_empty_line = r'^\s*$'
-    cfg = re.sub(re_empty_line, '', cfg, flags=re.MULTILINE)
     return cfg
 
 def pre_process_ifdefs(cfg,conditions):
@@ -142,23 +147,37 @@ def pre_process_ifdefs(cfg,conditions):
         if line.startswith("#ifdef"):
             condition = line[len("#ifdef"):].strip()
             ifstack.append(condition in conditions)
-        if line.startswith("#ifndef "):
+            ret.append("## filtered out")
+        elif line.startswith("#ifndef"):
             condition = line[len("#ifndef"):].strip()
             ifstack.append(not condition in conditions)
+            ret.append("## filtered out")
         elif line.startswith("#endif"):
             if len(ifstack) == 1:
                 raise Error("unmatched #endif found in input")
             ifstack.pop()
+            ret.append("## filtered out")
         else:
             if not False in ifstack:
                 ret.append(line)
+            else:
+                ret.append("## filtered out")
 
     if not len(ifstack) == 1:
         raise Error("#ifdef not matched with an #endif")
 
     return "\n".join(ret)
 
-def copy_locations_to_virtual_hosts(config):
+def copy_prefix(key_to_node, prefix, new_prefix):
+    copy_list = {}
+    for key in key_to_node:
+        if key[:len(prefix)] == prefix:
+            copy_list[new_prefix + key[len(prefix):]] = key_to_node[key]
+
+    for key in copy_list:
+        key_to_node[key] = copy_list[key]
+
+def copy_locations_to_virtual_hosts(config, key_to_node):
     # we clone locations that are defined at the root level
     # to all defined servers. that way, we do not have
     # to rely on inheritance being available in the targeted
@@ -169,10 +188,18 @@ def copy_locations_to_virtual_hosts(config):
     move = ["locations"]
     servers = config["servers"]
     for m in move:
-        for server in servers:
+        for server_index, server in enumerate(servers):
             if not m in server:
                server[m] = []
-            server[m].extend(copy.deepcopy(config[m]))
+
+            offset = len(server[m])
+            for (location_index, location) in enumerate(config[m]):
+                new_prefix = ".servers.%s.%s.%s" % (server_index, m,
+                                                    location_index + offset)
+                server[m].append(copy.deepcopy(location))
+                copy_prefix(key_to_node,
+                            ".locations.%s" % location_index, new_prefix)
+
         del config[m]
 
 def move_configuration_to_locations(config):
@@ -223,13 +250,28 @@ def execute_template(pyconf_path, conditions,
 
     with open(pyconf_path) as config_file:
         config_text = config_file.read()
-
+    original_len = len(config_text.split("\n"))
     config_text = pre_process_ifdefs(config_text, conditions)
+    new_len = len(config_text.split("\n"))
+
+    # Make sure the number of lines doesn't get changed by the preprocessor.
+    # We want to be able to report correct line numbers for templates that
+    # choke on a .pyconf file
+    if (new_len != original_len):
+        raise Error("preprocessor step 1: linecount changed from %s to %s:\n"
+                    % (original_len, new_len))
+
     config_text = pre_process_text(config_text, conditions,
                                    placeholders)
+    new_len = len(config_text.split("\n"))
+
+    if (new_len != original_len): 
+        raise Error("preprocessor step 2: linecount changed from %s to %s:\n"
+                    % (original_len, new_len))
 
     ast = parse_python_struct(config_text)
-    config = ast_node_to_dict(ast, None, {}, '')
+    key_to_node = {}
+    config = ast_node_to_dict(ast, None, {}, '', key_to_node)
     template_file = ""
 
     with open(template_path) as template_file:
@@ -239,7 +281,7 @@ def execute_template(pyconf_path, conditions,
                                      placeholders)
     template = Templite(template_text)
 
-    copy_locations_to_virtual_hosts(config)
+    copy_locations_to_virtual_hosts(config, key_to_node)
     move_configuration_to_locations(config)
-    text = template.render(config=config)
+    text = template.render(config=config, key_to_node=key_to_node)
     return text

test/genconf/nginx.conf.template

@@ -223,5 +223,5 @@ events {
 http {
     include       @@NGX_CONF_DIRECTORY@@/mime.types;
     default_type  application/octet-stream;
-${write_cfg(key_to_writer, config, 1)}$
+${write_cfg(key_to_writer, config, key_to_node, 1)}$
 } #http

test/genconf/pagespeed.debug.pyconf

@@ -408,15 +408,15 @@ locations: [
 #ifdef PER_VHOST_STATS
    {
        path: "/mod_pagespeed_global_statistics",
-       #ifdef apache
+#ifdef apache
        literal: {
            value: """
            Order allow,deny
            Allow from localhost
            Allow from 127.0.0.1
            SetHandler mod_pagespeed_global_statistics\n""",
        }
-       #endif
+#endif
    },
 #endif
 #ifdef STRESS

test/genconf/util.py

@@ -14,49 +14,77 @@
 #
 # Author: [email protected] (Otto van der Schaaf)
 
+import sys
+import traceback
 
 class Error(Exception):
     pass
 
-def write_cfg(key_to_writer, config, level=0):
+def pyconf_fatal(lineno, e = None):
+    message = "Exception while processing line [%s] from pyconf" % lineno
+    if e: message = "%s\n\n%s\n" % (message,traceback.format_exc())
+    sys.exit(message)
+
+def exec_template_call(func, lineno, item, level):
+    try:
+        return func(item, level)
+    except Exception as e:
+        pyconf_fatal(lineno, e)
+        raise
+
+def write_cfg(key_to_writer, config, key_to_node, level=0, parent_key = ""):
     global global_writer
 
     for key in config:
+        key_path = "%s.%s" % (parent_key, key)
         next_level = level
+        lineno = "unknown"
+        handled = False
+
+        if key_path in key_to_node:
+            lineno = key_to_node[key_path].lineno
+
         if key + '_open' in key_to_writer:
             w = key_to_writer[key + '_open']
             if w != write_void:
                 next_level = next_level + 1
-            handled = w(config[key], level)
+
+            handled = exec_template_call(w, lineno, config[key], level)
+
             if not handled:
                 if key + '_open_item' in key_to_writer \
                     and isinstance(config[key], list):
                     for (index, item) in enumerate(config[key]):
                         w = key_to_writer[key + '_open_item']
-                        handled = w(item, next_level)
+                        child_key_path = "%s.%s" % (key_path, str(index))
+                        handled = exec_template_call(
+                            w, lineno, item, next_level)
+
                         if not handled:
-                            write_cfg(key_to_writer, item, level + 1)
+                            write_cfg(key_to_writer, item, key_to_node,
+                                      level + 1, child_key_path)
                         if key + '_close_item' in key_to_writer:
                             w = key_to_writer[key + '_close_item']
-                            w(item, next_level)
+                            exec_template_call(w, lineno, item, next_level)
                 else:
-                    write_cfg(key_to_writer, config[key], level)
+                    write_cfg(key_to_writer, config[key], key_to_node, level
+                              ,key_path)
 
             if key + '_close' in key_to_writer:
                 w = key_to_writer[key + '_close']
-                w(config[key], level)
+                exec_template_call(w, lineno, config[key], level)
         else:
             if not isinstance(config[key], str) \
                 and not isinstance(config[key], int):
-                raise Error("no tranform handler for [%s]" % key)
+                raise Error("no tranform handler for [%s] at line [%s]"
+                            % (key_path, lineno))
 
 def indent(txt, level):
     lines = txt.split("\n")
     r = []
     for line in lines:
         if not line: continue
         r.append("%s%s\n" % (' ' * (level * 4), line))
-
     return ''.join(r)
 
 def emit_indent(txt,level):