From c0135d5a5638f6186edd327869a9c8e8cf809f7f Mon Sep 17 00:00:00 2001 From: Phil Ruffwind Date: Sun, 27 Mar 2016 04:59:50 -0400 Subject: [PATCH 1/4] makegen.hs [WIP] --- bin/makegen | 23 +++++++--- lib/makegen/Makegen.hs | 77 ++++++++++++++++++++++++++++++++++ lib/makegen/Makegen/Prelude.hs | 3 ++ lib/makegen/pp | 11 +++++ make.hs | 1 + 5 files changed, 109 insertions(+), 6 deletions(-) create mode 100644 lib/makegen/Makegen.hs create mode 100644 lib/makegen/Makegen/Prelude.hs create mode 100755 lib/makegen/pp create mode 100644 make.hs diff --git a/bin/makegen b/bin/makegen index e9aaeee..188a021 100755 --- a/bin/makegen +++ b/bin/makegen @@ -1,12 +1,23 @@ #!/bin/sh -set -eu if [ $# -lt 1 ]; then if [ -f make.py ]; then set make.py - else + elif [ -f Makefile.py ]; then set Makefile.py + else + set make.hs fi -fi -libdir=`dirname "$0"`/../lib/`basename "$0"` -PYTHONPATH=$libdir${PYTHONPATH+:}${PYTHONPATH-} \ - exec ${PYTHON-python} "$@" +fi && +libdir=`dirname "$0"`/../lib/`basename "$0"` && +case $1 in + *.py) + PYTHONPATH=$libdir${PYTHONPATH+:}${PYTHONPATH-} \ + exec ${PYTHON-python} "$@";; + *.hs) + name=`basename "$1"` && + dir=`dirname "$1"` && + exe=${dir}/.#makegen_${name}.tmp && + ghc -F -pgmF "$libdir/pp" -optF "$1" \ + -o "$exe" "-i$libdir" "$@" && + exec "$exe";; +esac diff --git a/lib/makegen/Makegen.hs b/lib/makegen/Makegen.hs new file mode 100644 index 0000000..9c2d328 --- /dev/null +++ b/lib/makegen/Makegen.hs @@ -0,0 +1,77 @@ +{-# LANGUAGE NoMonomorphismRestriction #-} +{-# LANGUAGE FlexibleInstances #-} +module Makegen + ( module Makegen + , mempty + , (<>) + ) where +import Control.Arrow +import Control.Monad +import Data.Dynamic +import Data.Maybe +import Data.Monoid +import Control.Monad.Trans.Writer.Strict +import Data.Map.Strict (Map) +import Data.Sequence (Seq) +import Data.Set (Set) +import qualified Data.Map.Strict as Map + +data Makefile = + Makefile + { _rules :: Map String ([String], [String]) + , _macros :: Map String String + } + +unionEqMap :: (Ord k, Eq a) => + Map k a + -> Map k a + -> Either [(k, a, a)] (Map k a) +unionEqMap m1 m2 + | null conflicts = Right (m1 <> m2) + | otherwise = Left conflicts + where + conflicts = do + (k, v1) <- Map.toAscList (Map.intersection m1 m2) + let v2 = m2 Map.! k + guard (v1 /= v2) + pure (k, v1, v2) + +data Error + = ERuleConflict (String, ([String], [String]), ([String], [String])) + | EMacroConflict (String, String, String) + deriving (Eq, Ord, Read, Show) + +instance Monoid (Either [Error] Makefile) where + mempty = Right (Makefile mempty mempty) + mappend mx my = do + Makefile x1 x2 <- mx + Makefile y1 y2 <- my + z1 <- left (ERuleConflict <$>) (unionEqMap x1 y1) + z2 <- left (EMacroConflict <$>) (unionEqMap x2 y2) + pure (Makefile z1 z2) + +newtype VarMap + = VarMap (Map TypeRep Dynamic) + deriving Show + +type Make a = Writer (Makefile, VarMap) a + +-- | Laws are @isMempty mempty ≡ True@ and if @a ≡ t b@ and @Foldable t@ then +-- @isMempty ≡ null@. +class Monoid a => MemptyComparable a where + isMempty :: a -> Bool + +field :: (Functor f, MemptyComparable a, Typeable a) => + (a -> f a) -> VarMap -> f VarMap +field f m = (`set` m) <$> f (get m) + where + set x (VarMap m) = + VarMap $ + if isMempty x + then Map.delete typeRep m + else Map.insert typeRep (toDyn x) m + where typeRep = typeOf x + get (VarMap m) = x + where x = fromMaybe mempty (fromDynamic =<< Map.lookup (typeOf x) m) + +data Mk m a = Mk m a diff --git a/lib/makegen/Makegen/Prelude.hs b/lib/makegen/Makegen/Prelude.hs new file mode 100644 index 0000000..d2adb6e --- /dev/null +++ b/lib/makegen/Makegen/Prelude.hs @@ -0,0 +1,3 @@ +module Makegen.Prelude (module Prelude, module Makegen) where +import Prelude +import Makegen diff --git a/lib/makegen/pp b/lib/makegen/pp new file mode 100755 index 0000000..102c377 --- /dev/null +++ b/lib/makegen/pp @@ -0,0 +1,11 @@ +#!/bin/sh +exec >"$3" +if [ "$2" = "$4" ]; then + cat < Date: Sat, 23 Apr 2016 05:36:33 -0400 Subject: [PATCH 2/4] More WIP --- lib/makegen/Makegen.hs | 68 +++++++++++++++++++++++------------------- 1 file changed, 38 insertions(+), 30 deletions(-) diff --git a/lib/makegen/Makegen.hs b/lib/makegen/Makegen.hs index 9c2d328..4f76e2f 100644 --- a/lib/makegen/Makegen.hs +++ b/lib/makegen/Makegen.hs @@ -16,19 +16,27 @@ import Data.Sequence (Seq) import Data.Set (Set) import qualified Data.Map.Strict as Map -data Makefile = - Makefile - { _rules :: Map String ([String], [String]) - , _macros :: Map String String - } +-- | Equivalent to @Control.Lens.At.at@ for 'Map'. +at_Map :: (Functor f, Ord k) => + k -> (Maybe a -> f (Maybe a)) -> Map k a -> f (Map k a) +at_Map k f m = set <$> f (Map.lookup k m) + where + set Nothing = Map.delete k m + set (Just x) = Map.insert k x m -unionEqMap :: (Ord k, Eq a) => - Map k a - -> Map k a - -> Either [(k, a, a)] (Map k a) -unionEqMap m1 m2 +newtype VarMap + = VarMap (Map TypeRep Dynamic) + deriving Show + +-- | Union two 'Map's. If the 'Map's contain conflicting elements, the list +-- of conflicting elements is returned via 'Left'. +unionEq_Map :: (Ord k, Eq a) => + Map k a + -> Map k a + -> Either [(k, a, a)] (Map k a) +unionEq_Map m1 m2 | null conflicts = Right (m1 <> m2) - | otherwise = Left conflicts + | otherwise = Left conflicts where conflicts = do (k, v1) <- Map.toAscList (Map.intersection m1 m2) @@ -36,12 +44,29 @@ unionEqMap m1 m2 guard (v1 /= v2) pure (k, v1, v2) -data Error +field :: (Functor f, Eq a, Monoid a, Typeable a) => + (a -> f a) -> VarMap -> f VarMap +field f (VarMap m) = + case undefined of + dummy_a -> + let set x | x == mempty = Nothing + | otherwise = Just (toDyn (x `asTypeOf` dummy_a)) + get y = fromMaybe mempty (fromDynamic =<< y) + upd y = set <$> f (get y) + in VarMap <$> at_Map (typeOf dummy_a) upd m + +data Makefile = + Makefile + { _rules :: Map String ([String], [String]) + , _macros :: Map String String + } + +data MakefileError = ERuleConflict (String, ([String], [String]), ([String], [String])) | EMacroConflict (String, String, String) deriving (Eq, Ord, Read, Show) -instance Monoid (Either [Error] Makefile) where +instance Monoid (Either [MakefileError] Makefile) where mempty = Right (Makefile mempty mempty) mappend mx my = do Makefile x1 x2 <- mx @@ -50,10 +75,6 @@ instance Monoid (Either [Error] Makefile) where z2 <- left (EMacroConflict <$>) (unionEqMap x2 y2) pure (Makefile z1 z2) -newtype VarMap - = VarMap (Map TypeRep Dynamic) - deriving Show - type Make a = Writer (Makefile, VarMap) a -- | Laws are @isMempty mempty ≡ True@ and if @a ≡ t b@ and @Foldable t@ then @@ -61,17 +82,4 @@ type Make a = Writer (Makefile, VarMap) a class Monoid a => MemptyComparable a where isMempty :: a -> Bool -field :: (Functor f, MemptyComparable a, Typeable a) => - (a -> f a) -> VarMap -> f VarMap -field f m = (`set` m) <$> f (get m) - where - set x (VarMap m) = - VarMap $ - if isMempty x - then Map.delete typeRep m - else Map.insert typeRep (toDyn x) m - where typeRep = typeOf x - get (VarMap m) = x - where x = fromMaybe mempty (fromDynamic =<< Map.lookup (typeOf x) m) - data Mk m a = Mk m a From d6c5e16a4df9b7f87289df4e18e8512f58bc77e3 Mon Sep 17 00:00:00 2001 From: Phil Ruffwind Date: Sun, 16 Apr 2017 04:53:36 -0400 Subject: [PATCH 3/4] Experimental makegen2 --- lib/makegen/makegen2.py | 441 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 441 insertions(+) create mode 100644 lib/makegen/makegen2.py diff --git a/lib/makegen/makegen2.py b/lib/makegen/makegen2.py new file mode 100644 index 0000000..2eeadd0 --- /dev/null +++ b/lib/makegen/makegen2.py @@ -0,0 +1,441 @@ +import functools, itertools, logging, os, re, subprocess, tempfile, traceback + +logger = logging.getLogger(__name__) + +class MakeError(Exception): + pass + +class Symbol(object): + def __init__(self, name): + self.name = name + + def __repr__(self): + return self.name + +class UnionDict(object): + def __init__(self, d=()): + self.inner = dict(d) + + def __repr__(self): + return "UnionDict({!r})".format(self.inner) + + def __ior__(self, other): + for k, v in other.items(): + try: + self_v = self.inner[k] + except KeyError: + pass + else: + if self_v != v: + raise ValueError("conflicting values for {!r}: {!r} != {!r}" + .format(k, self_v, v)) + continue + self.inner[k] = v + return self + +class Identity(object): + def __init__(self, value): + self.value = value + + def __repr__(self): + return "Identity({!r})".format(self.value) + + def __hash__(self): + __hash__ = getattr(self.value, "__hash__", None) + if __hash__ is not None: + return __hash__() + return hash(id(self.value)) + + def __eq__(self, other): + __hash__ = getattr(self.value, "__hash__", None) + if __hash__ is not None: + return self.value == other + return id(self.value) == id(other.value) + +class Make(object): + + @staticmethod + def ensure(make): + if isinstance(make, str): + return Make(make) + elif isinstance(make, Make): + return make + else: + raise TypeError("an object of {!r} cannot be converted to Make" + .format(type(make))) + + def __init__(self, name, raw_deps=(), cmds=(), *attrs): + '''The 'raw_deps' argument can be either [deps] or (cache) -> deps. + In the latter case the '.deps' field will remain unavailable until + '.populate_deps()' is called. + + 'attrs' can contain either (attr_key, value), (name) -> (attr_key, + value), or (attr_key, None). The last one prevents attributes of + dependencies from propagating upward.''' + if name is None and cmds: + raise TypeError("if name is None, cmds must be None too: {!r}" + .format(cmds)) + if not (isinstance(cmds, tuple) or isinstance(cmds, list)): + raise TypeError("cmds must either list or tuple: {!r}" + .format(cmds)) + for cmd in cmds: + if not isinstance(cmd, str): + raise TypeError("cmds must be a list of str: {!r}" + .format(cmds)) + if not callable(raw_deps): + raw_deps = [Make.ensure(dep) for dep in raw_deps] + self.name = name + self.raw_deps = raw_deps + self.cmds = cmds + self.attrs = {} + self.update_attrs(*attrs) + self._stack = traceback.extract_stack()[:-1] + + def __repr__(self): + deps = "" if callable(self.raw_deps) else "" + tup = (self.name, Symbol(deps), self.cmds) + tuple( + ((Symbol(k.__name__) if callable(k) else k), v) + for k, v in self.attrs.items()) + return "Make{!r}".format(tup) + + def with_attrs(self, *attrs): + copy = Make(self.name, self.raw_deps, self.cmds) + copy.attrs = dict(self.attrs) + copy.update_attrs(*attrs) + return copy + + def update_attrs(self, *attrs): + attrs = list(attrs) + for i, attr in enumerate(attrs): + if not attr: + continue + if callable(attr): + if self.name is None: + raise TypeError("function attributes are not supported " + "if name is None") + attrs[i] = attr(self.name) + self.attrs.update(attrs) + + @property + def macros(self): + try: + return self.attrs[MACROS] + except KeyError: + pass + macros = {} + self.attrs[MACROS] = macros + return macros + + @property + def is_trivial(self): + return not self.deps and not self.cmds and ( + not self.attrs or (len(self.attrs) == 1 and + AUXILIARY in self.attrs)) + + @property + def deps(self): + raw_deps = self.raw_deps + if callable(raw_deps): + raise ValueError("dependencies haven't been populated yet") + return raw_deps + + def populate_deps(self, cache): + raw_deps = self.raw_deps + if callable(raw_deps): + deps = raw_deps(cache) + self.raw_deps = [Make.ensure(dep) for dep in deps] + + def append(self, dep): + self.deps.append(dep) + return dep + + def traverse(self): + '''Traverse first through self and then through all its unique + transitive dependencies in an unspecified order. Send a true-ish + value to skip the dependencies of a particular node.''' + seen = set() + candidates = [self] + while candidates: + candidate = candidates.pop() + candidate_id = id(candidate) + if candidate_id in seen: + continue + seen.add(candidate_id) + skip_deps = yield candidate + if not skip_deps: + candidates.extend(reversed(candidate.deps)) + candidates.extend(candidate.attrs.get(AUXILIARY, ())) + + def merged_attr(self, attr_key, accum): + '''Attributes are expected to form a semilattice with respect to the + |= operator.''' + traversal = self.traverse() + for m in traversal: + while True: + try: + attr = m.attrs[attr_key] + except KeyError: + break + if attr is None: + try: + m = traversal.send(True) + except StopIteration: + return + continue + accum |= attr + break + return accum + + def default_target(self): + t = self.name + if t is not None: + return t + for dep in self.deps: + t = dep.default_target() + if t is not None: + return t + return None + + def gather_suffixes(self): + suffixes = sorted(self.merged_attr(SUFFIXES, set())) + return Make(".SUFFIXES", suffixes) if suffixes else MakeSet() + + def gather_phony(self): + phonys = sorted(self.merged_attr(PHONY, set())) + return Make(".PHONY", phonys) if phonys else MakeSet() + + def gather_clean(self, exclusions): + clean_cmds = self.merged_attr(CLEAN_CMDS, set()) + cleans = self.merged_attr(CLEAN, set()) + cleans.difference_update(exclusions) + if cleans: + clean_cmds.add("rm -fr " + " ".join(cleans)) + return (Make("clean", (), sorted(clean_cmds)) + if clean_cmds else MakeSet()) + + def render_rule(self): + if self.name is None: + raise TypeError("cannot render a rule whose name is None: {!r}" + .format(self)) + return "".join(( + self.name, + ":", + "".join(" " + dep.name for dep in self.deps), + "\n", + "".join("\t{}\n".format(cmd) for cmd in self.cmds), + )) + + def render(self, f): + # make sure modifications are localized + self = self.with_attrs() + self.raw_deps = list(self.deps) + + cache = {} + for make in self.traverse(): + make.populate_deps(cache) + + macros = self.merged_attr(MACROS, UnionDict()).inner + suffixes = self.gather_suffixes() + phony = self.gather_phony() + clean = self.gather_clean(dep.name for dep in phony.deps) + extras = (suffixes, phony, clean) + + rules = {} + default_target = self.default_target() + for make in itertools.chain(itertools.islice(self.traverse(), 1, None), + extras): + # skip trivial rules to avoid unnecessary bloat + conflicts + if make.is_trivial: + continue + name = make.name + rule = (make.render_rule(), make) + old_rule = rules.get(name) + if old_rule is not None and old_rule[0] != rule[0]: + old_stack = "".join(traceback.format_list(old_rule[1]._stack)) + stack = "".join(traceback.format_list(rule[1]._stack)) + raise MakeError("conflicting rules:\n {!r}\n {!r}\n\n" + "Origin of first rule:\n{}\n" + "Origin of second rule:\n{}" + .format(old_rule[1], rule[1], old_stack, stack)) + rules[name] = rule + first_rule = (() if default_target is None + else (rules.pop(default_target),)) + + for k, v in macros.items(): + f.write("{}={}\n".format(k, v)) + if macros: + f.write("\n") + + for rule, _ in itertools.chain(first_rule, sorted(rules.values())): + f.write(rule + "\n") + +def make(name, deps=(), cmds=(), *attrs, clean=True, mkdir=True): + m = Make(name, deps, cmds, *attrs) + if CLEAN not in m.attrs and not m.attrs.get(PHONY): + if clean: + m.update_attrs(CLEAN) + if mkdir: + m.cmds = ("mkdir -p $(@D)",) + tuple(m.cmds) + return m + +def MACROS(macros=()): + assert not isinstance(macros, str) + return MACROS, macros + +def SUFFIXES(suffixes): + return SUFFIXES, set(suffixes) + +def AUXILIARY(rules): + if isinstance(rules, str): + raise TypeError("expected an iterable of of Make") + return AUXILIARY, set(rules) + +def PHONY(name): + return PHONY, set((name,)) + +def CLEAN(name): + return CLEAN, set((name,)) + +def CLEAN_CMDS(cmds): + assert not isinstance(cmds, str) + return CLEAN_CMDS, set(cmds) + +def memoize(cache, key): + def inner(f): + @functools.wraps(f) + def wrapper(*args, **kwargs): + try: + return cache[key] + except (IndexError, KeyError): + pass + result = f(*args, **kwargs) + cache[key] = result + return result + return wrapper + return inner + +def make_rule_header_lex(string): + '''This is a simplified lexer that is not faithful to the actual makefile + syntax (colons and tabs are ignored) but it is sufficient for parsing the + output of cpp -M.''' + import re + token = "" + escaping = False + for m in re.finditer(r"([^ \n\\]*)([ \n\\]?)", string): + s, c = m.groups() + token += s + if escaping: + escaping = False + if not s: + if c == "\n": + token = token[:-1] + if token: + yield token[:-1] + token = "" + else: + token += c + continue + if c == "\\": + token += "\\" + escaping = True + else: + if token: + yield token + token = "" + if c == "\n": + break + if token: + yield token + +def make_rule_header_parse(string): + import re + # ignore the target (btw, this regex is surprisingly robust) + m = re.match("(?s).*?: (.*)", string) + if not m: + raise ValueError("could not parse dependency tool output:\n\n" + string) + return make_rule_header_lex(m.group(1)) + +def run_dep_tool(dep_tool, path, cache=None): + '''dep_tool(source_path: str, output_path: str) -> [str]''' + @memoize(cache, (run_dep_tool, dep_tool, path)) + def get(): + with tempfile.NamedTemporaryFile(mode="r") as f: + with open(os.devnull, "wb") as fnull: + p = subprocess.Popen(dep_tool(path, f.name), + stdin=fnull, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + universal_newlines=True) + err, _ = p.communicate() + out = f.read() + if p.returncode or not out: + raise MakeError("failed to run dependency tool for {0}:\n\n{1}" + .format(repr(path), err)) + # warn about missing dependencies because these dependencies + # themselves may also contain dependencies that we don't know; to fix + # this, it is recommended to run `make` to generate the missing files, + # and then run `makegen` again. + for dep_path in make_rule_header_parse(out): + if not os.path.exists(dep_path): + logger.warning("missing dependency: {0}".format(dep_path)) + yield dep_path + return get() + +def c_dep_tool(in_fn, out_fn): + return ("cc", "-MM", "-MF", out_fn, in_fn) + +C_DEPS_FUNC = functools.partial(run_dep_tool, c_dep_tool) + +class InferenceRule(object): + def __init__(self, src_ext, tar_ext, cmds): + self.src_ext = src_ext + self.tar_ext = tar_ext + self.cmds = cmds + + def __repr__(self): + return "InferenceRule{!r}".format( + (self.src_ext, self.tar_ext, self.cmds)) + + def make(self, mkdir=True): + suffixes = [self.src_ext] + if self.tar_ext: + suffixes.append(self.tar_ext) + return Make(self.src_ext + self.tar_ext, + [], + (("mkdir -p $(@D)",) if mkdir else ()) + tuple(self.cmds), + SUFFIXES(suffixes)) + +C_INFERENCE_RULE = InferenceRule(".c", ".o", + ("$(CC) $(CPPFLAGS) $(CFLAGS) -c -o $@ $<",)) + +def make_c_obj(path, via=C_INFERENCE_RULE): + ''''via' can either be an InferenceRule or (name, cmds)''' + if isinstance(path, Make): + return path + if not path.endswith(".c"): + raise ValueError("expected source file (*.c), got: {!r}" + .format(path)) + if isinstance(via, InferenceRule): + stem, _ = os.path.splitext(path) + name = stem + via.tar_ext + cmds = () + attrs = (AUXILIARY((via.make(),)),) + else: + name, cmds = via + attrs = () + return make(name, functools.partial(C_DEPS_FUNC, path), cmds, *attrs, + mkdir=False) + +def make_bin(name, + deps, + ld="$(CC) $(CFLAGS)", + libs="$(LIBS)", + make_obj=make_c_obj): + if isinstance(deps, str): + raise TypeError("'deps' argument must be a list, not str") + deps = tuple(tuple(dep if isinstance(dep, Make) else make_obj(dep) + for dep in deps)) + dep_names = " ".join(dep.name for dep in deps) + return make(name, + deps, + ("{ld} -o $@ {dep_names} {libs}\n".format(**locals()),)) From 7000672661b660adb641a5def4af7dedb30541ba Mon Sep 17 00:00:00 2001 From: Phil Ruffwind Date: Sun, 16 Apr 2017 16:21:37 -0400 Subject: [PATCH 4/4] Clean up makegen2 --- lib/makegen/makegen2.py | 388 ++++++++++++++++++++++++++-------------- 1 file changed, 253 insertions(+), 135 deletions(-) diff --git a/lib/makegen/makegen2.py b/lib/makegen/makegen2.py index 2eeadd0..7af0b17 100644 --- a/lib/makegen/makegen2.py +++ b/lib/makegen/makegen2.py @@ -2,8 +2,8 @@ logger = logging.getLogger(__name__) -class MakeError(Exception): - pass +def _format_stack(stack): + return "".join(traceback.format_list(stack)) class Symbol(object): def __init__(self, name): @@ -12,9 +12,33 @@ def __init__(self, name): def __repr__(self): return self.name +class Meta(object): + '''An object provides equality over the first value, ignoring the second.''' + def __init__(self, value, meta): + self.value = value + self.meta = meta + + def __repr__(self): + return "Meta{!r}".format((self.value, self.meta)) + + def __eq__(self, other): + return self.value == other.value + +class UnionConflictError(ValueError): + def __init__(self, key, value1, value2, *args): + self.key = key + self.value1 = value1 + self.value2 = value2 + message = ("conflicting values for {!r}: {!r} != {!r}" + .format(key, value1, value2)) + super(UnionConflictError, self).__init__(message, *args) + class UnionDict(object): - def __init__(self, d=()): - self.inner = dict(d) + '''A wrapper over a dictionary where |= is overloaded to perform merges. + The merge fails if there are keys with differing values.''' + + def __init__(self, inner=()): + self.inner = dict(inner) def __repr__(self): return "UnionDict({!r})".format(self.inner) @@ -27,32 +51,44 @@ def __ior__(self, other): pass else: if self_v != v: - raise ValueError("conflicting values for {!r}: {!r} != {!r}" - .format(k, self_v, v)) + raise UnionConflictError(k, self_v, v) continue self.inner[k] = v return self -class Identity(object): - def __init__(self, value): - self.value = value - - def __repr__(self): - return "Identity({!r})".format(self.value) - - def __hash__(self): - __hash__ = getattr(self.value, "__hash__", None) - if __hash__ is not None: - return __hash__() - return hash(id(self.value)) - - def __eq__(self, other): - __hash__ = getattr(self.value, "__hash__", None) - if __hash__ is not None: - return self.value == other - return id(self.value) == id(other.value) - class Make(object): + '''A `Make` object represents either: + + - a rule: a single rule along with a collection of its dependencies; or + - a ruleset: an anonymous collection of rules (dependencies). + + A rule contains a name, dependencies, commands, and attributes. + + - A name is a string. It is the target of the makefile rule. + - Dependencies are an ordered sequence of `Make` objects. A dependency + is included in the prerequisites of the makefile rule if and only if + it is a rule, not a ruleset. + - Commands are an ordered sequences of strings. In the makefile, each + string is translated to a single line preceded by a tab. + - Attributes are lists of pairs of the form (key, value). The key is a + unique identifier for this particular attribute type. The value is an + arbitrary object that forms a semilattice using the `|=` operator. + + Ruleset contain only dependencies and attributes, just like rules. Their + names are always `None` and their commands must be an empty sequence. + + A useful trick is to anonymize a rule by wrapping it: `Make(None, rule)`. + This allows them to be attached as a dependency without being included + as an explicit prerequisite. This is done automatically by + `InferenceRule.make()`, for example. + + A `Make` object may be created with a lazily-populated sequence of + dependencies. This means passing a function rather than a list to the + `deps` argument of the constructor. When `populate_deps(cache)` is + called, the function will be called with the `cache` argument, where + `cache` should be a `dict`-like object. Until `populate_deps` is called, + the `.deps` attribute will raise an exception. + ''' @staticmethod def ensure(make): @@ -64,47 +100,71 @@ def ensure(make): raise TypeError("an object of {!r} cannot be converted to Make" .format(type(make))) - def __init__(self, name, raw_deps=(), cmds=(), *attrs): - '''The 'raw_deps' argument can be either [deps] or (cache) -> deps. - In the latter case the '.deps' field will remain unavailable until - '.populate_deps()' is called. - - 'attrs' can contain either (attr_key, value), (name) -> (attr_key, - value), or (attr_key, None). The last one prevents attributes of - dependencies from propagating upward.''' + @staticmethod + def _check_args(name, cmds, attrs): if name is None and cmds: raise TypeError("if name is None, cmds must be None too: {!r}" .format(cmds)) if not (isinstance(cmds, tuple) or isinstance(cmds, list)): - raise TypeError("cmds must either list or tuple: {!r}" - .format(cmds)) + raise TypeError("cmds must either list or tuple: {!r}".format(cmds)) for cmd in cmds: if not isinstance(cmd, str): - raise TypeError("cmds must be a list of str: {!r}" - .format(cmds)) - if not callable(raw_deps): - raw_deps = [Make.ensure(dep) for dep in raw_deps] + raise TypeError("cmds must be a list of str: {!r}".format(cmds)) + try: + valid_attrs = all(callable(attr) or + (isinstance(attr, tuple) and + len(attr) == 2) + for attr in attrs) + except TypeError: + valid_attrs = False + if not valid_attrs: + raise TypeError("attrs must be a list of callables and/or " + "pairs: {!r}".format(attrs)) + + def __init__(self, name=None, deps=(), cmds=(), attrs=(), _stack=None): + '''The 'deps' argument can be either `deps` or a function `(cache) -> + deps`. In the latter case the '.deps' field will remain unavailable + until '.populate_deps()' is called. + + The string dependencies are automatically promoted to `Make` objects + using `Make.ensure`. + + 'attrs' can contain be either `(key, value)` pairs, or functions of + the form `(name) -> (attr_key, value)`.''' + self._check_args(name, cmds, attrs) + if not callable(deps): + deps = [Make.ensure(dep) for dep in deps] self.name = name - self.raw_deps = raw_deps + self.raw_deps = deps self.cmds = cmds self.attrs = {} self.update_attrs(*attrs) - self._stack = traceback.extract_stack()[:-1] + self._stack = _stack or traceback.extract_stack()[:-1] def __repr__(self): - deps = "" if callable(self.raw_deps) else "" + deps = ("" if callable(self.raw_deps) else + "[{}]".format(", ".join("" + .format(dep.name, hex(id(dep))) + for dep in self.deps))) tup = (self.name, Symbol(deps), self.cmds) + tuple( ((Symbol(k.__name__) if callable(k) else k), v) for k, v in self.attrs.items()) return "Make{!r}".format(tup) + def __ior__(self, other): + self.deps.append(Make(deps=(other,))) + return self + def with_attrs(self, *attrs): + '''Make a copy with the given attributes added.''' copy = Make(self.name, self.raw_deps, self.cmds) copy.attrs = dict(self.attrs) + copy._stack = self._stack copy.update_attrs(*attrs) return copy def update_attrs(self, *attrs): + '''Add the given attributes.''' attrs = list(attrs) for i, attr in enumerate(attrs): if not attr: @@ -116,43 +176,48 @@ def update_attrs(self, *attrs): attrs[i] = attr(self.name) self.attrs.update(attrs) - @property - def macros(self): - try: - return self.attrs[MACROS] - except KeyError: - pass - macros = {} - self.attrs[MACROS] = macros - return macros - @property def is_trivial(self): - return not self.deps and not self.cmds and ( - not self.attrs or (len(self.attrs) == 1 and - AUXILIARY in self.attrs)) + ''' + Return whether the `Make` object needs to be rendered. + + Raises an error if the list of dependencies has not yet been populated. + ''' + return self.name is None or not (self.deps or self.cmds or self.attrs) @property def deps(self): + '''Obtain the dependencies of the rule. This will raise an error if + the dependencies have not yet been populated.''' raw_deps = self.raw_deps if callable(raw_deps): raise ValueError("dependencies haven't been populated yet") return raw_deps def populate_deps(self, cache): + '''Populate the dependencies of the current node. + + This has no effect if the dependencies are already populated.''' raw_deps = self.raw_deps if callable(raw_deps): deps = raw_deps(cache) self.raw_deps = [Make.ensure(dep) for dep in deps] def append(self, dep): + ''' + Append `dep` to the list of dependencies and return `dep`. + + Raises an error if the list of dependencies has not yet been populated. + ''' self.deps.append(dep) return dep - def traverse(self): + def walk(self): '''Traverse first through self and then through all its unique - transitive dependencies in an unspecified order. Send a true-ish - value to skip the dependencies of a particular node.''' + transitive dependencies in depth-first order. Send a true-ish + value to skip the dependencies of a particular node. + + Requires a tree with fully populated dependencies.''' seen = set() candidates = [self] while candidates: @@ -164,52 +229,69 @@ def traverse(self): skip_deps = yield candidate if not skip_deps: candidates.extend(reversed(candidate.deps)) - candidates.extend(candidate.attrs.get(AUXILIARY, ())) def merged_attr(self, attr_key, accum): '''Attributes are expected to form a semilattice with respect to the - |= operator.''' - traversal = self.traverse() - for m in traversal: - while True: - try: - attr = m.attrs[attr_key] - except KeyError: - break - if attr is None: - try: - m = traversal.send(True) - except StopIteration: - return - continue - accum |= attr - break + |= operator. + + Requires a tree with fully populated dependencies.''' + for m in self.walk(): + try: + attr = m.attrs[attr_key] + except KeyError: + continue + accum |= attr return accum def default_target(self): - t = self.name - if t is not None: - return t - for dep in self.deps: - t = dep.default_target() + ''' + Find the name of the first rule within the transitive closure. + Returns `None` if there are no rules. + + Requires a tree with fully populated dependencies.''' + for m in self.walk(): + t = self.name if t is not None: return t return None + def gather_macros(self): + '''Requires a tree with fully populated dependencies.''' + try: + macros = self.merged_attr(MACROS, UnionDict()).inner + err = None + except UnionConflictError as e: + err = e + if err: + raise ValueError("conflicting attributes for {!r}:\n " + "{!r}\n {!r}\n\n" + "The former attribute comes from:\n{}\n" + "The latter attribute comes from:\n{}" + .format(err.key, + err.value1.value, + err.value2.value, + _format_stack(err.value1.meta), + _format_stack(err.value2.meta)) + .rstrip()) + return dict((k, v.value) for k, v in macros.items()) + def gather_suffixes(self): + '''Requires a tree with fully populated dependencies.''' suffixes = sorted(self.merged_attr(SUFFIXES, set())) return Make(".SUFFIXES", suffixes) if suffixes else MakeSet() def gather_phony(self): + '''Requires a tree with fully populated dependencies.''' phonys = sorted(self.merged_attr(PHONY, set())) return Make(".PHONY", phonys) if phonys else MakeSet() def gather_clean(self, exclusions): + '''Requires a tree with fully populated dependencies.''' clean_cmds = self.merged_attr(CLEAN_CMDS, set()) cleans = self.merged_attr(CLEAN, set()) cleans.difference_update(exclusions) if cleans: - clean_cmds.add("rm -fr " + " ".join(cleans)) + clean_cmds.add("rm -fr " + " ".join(sorted(cleans))) return (Make("clean", (), sorted(clean_cmds)) if clean_cmds else MakeSet()) @@ -220,57 +302,97 @@ def render_rule(self): return "".join(( self.name, ":", - "".join(" " + dep.name for dep in self.deps), + "".join(" " + dep.name + for dep in self.deps + if dep.name is not None), "\n", "".join("\t{}\n".format(cmd) for cmd in self.cmds), )) - def render(self, f): + def render(self, f, out_name="Makefile"): # make sure modifications are localized self = self.with_attrs() self.raw_deps = list(self.deps) cache = {} - for make in self.traverse(): + for make in self.walk(): make.populate_deps(cache) - macros = self.merged_attr(MACROS, UnionDict()).inner + default_target = self.default_target() + macros = self.gather_macros() suffixes = self.gather_suffixes() phony = self.gather_phony() - clean = self.gather_clean(dep.name for dep in phony.deps) + phony_names = set(dep.name for dep in phony.deps) + assert None not in phony_names + clean = self.gather_clean(phony_names) extras = (suffixes, phony, clean) - rules = {} - default_target = self.default_target() - for make in itertools.chain(itertools.islice(self.traverse(), 1, None), - extras): - # skip trivial rules to avoid unnecessary bloat + conflicts + seen = {} + # categorize rules so we can re-order them later + first_rule = [] + phony_rules = [] + normal_rules = [] + self_rule = [] + inference_rules = [] + special_rules = [] + special_phony_rule = [] + for make in itertools.chain(self.walk(), extras): + # skip trivial rules to avoid unnecessary bloat and conflicts if make.is_trivial: continue + name = make.name - rule = (make.render_rule(), make) - old_rule = rules.get(name) - if old_rule is not None and old_rule[0] != rule[0]: - old_stack = "".join(traceback.format_list(old_rule[1]._stack)) - stack = "".join(traceback.format_list(rule[1]._stack)) - raise MakeError("conflicting rules:\n {!r}\n {!r}\n\n" - "Origin of first rule:\n{}\n" - "Origin of second rule:\n{}" - .format(old_rule[1], rule[1], old_stack, stack)) - rules[name] = rule - first_rule = (() if default_target is None - else (rules.pop(default_target),)) + rendered = make.render_rule() + old_rule = seen.get(name) + + # check for conflicting rules + if old_rule is not None and old_rule[0] != rendered: + old_make = old_rule[1] + raise ValueError("conflicting rules:\n {!r}\n {!r}\n\n" + "The former rule comes from:\n{}\n" + "The latter rule comes from:\n{}" + .format(old_make, make, + _format_stack(old_make._stack), + _format_stack(make._stack)) + .rstrip()) + + seen[name] = (rendered, make) + if name == default_target: + first_rule.append(rendered) + elif name == out_name: + self_rule.append(rendered) + elif re.match("\.[^.]+(\.[^.]+)", name) and not make.deps: + inference_rules.append(rendered) + elif name == ".PHONY": + special_phony_rule.append(rendered) + elif re.match("\.[A-Z]", name) or name == out_name: + special_rules.append(rendered) + elif name in phony_names: + phony_rules.append(rendered) + else: + normal_rules.append(rendered) + phony_rules.sort() + inference_rules.sort() + special_rules.sort() + normal_rules.sort() + rules = itertools.chain(first_rule, + phony_rules, + normal_rules, + self_rule, + inference_rules, + special_rules, + special_phony_rule) for k, v in macros.items(): f.write("{}={}\n".format(k, v)) - if macros: - f.write("\n") - - for rule, _ in itertools.chain(first_rule, sorted(rules.values())): - f.write(rule + "\n") - -def make(name, deps=(), cmds=(), *attrs, clean=True, mkdir=True): - m = Make(name, deps, cmds, *attrs) + for i, rule in enumerate(rules): + if i != 0 or macros: + f.write("\n") + f.write(rule) + +def make(name, deps=(), cmds=(), attrs=(), clean=True, mkdir=True, _stack=None): + stack = _stack or traceback.extract_stack()[:-1] + m = Make(name, deps, cmds, attrs, _stack=stack) if CLEAN not in m.attrs and not m.attrs.get(PHONY): if clean: m.update_attrs(CLEAN) @@ -279,17 +401,16 @@ def make(name, deps=(), cmds=(), *attrs, clean=True, mkdir=True): return m def MACROS(macros=()): - assert not isinstance(macros, str) - return MACROS, macros + try: + macros = dict(macros) + except TypeError as e: + raise TypeError("'macros' should be a dict-like object") + stack = traceback.extract_stack()[:-1] + return MACROS, dict((k, Meta(v, stack)) for k, v in macros.items()) def SUFFIXES(suffixes): return SUFFIXES, set(suffixes) -def AUXILIARY(rules): - if isinstance(rules, str): - raise TypeError("expected an iterable of of Make") - return AUXILIARY, set(rules) - def PHONY(name): return PHONY, set((name,)) @@ -369,8 +490,8 @@ def get(): err, _ = p.communicate() out = f.read() if p.returncode or not out: - raise MakeError("failed to run dependency tool for {0}:\n\n{1}" - .format(repr(path), err)) + raise RuntimeError("failed to run dependency tool for {0}:\n\n{1}" + .format(repr(path), err)) # warn about missing dependencies because these dependencies # themselves may also contain dependencies that we don't know; to fix # this, it is recommended to run `make` to generate the missing files, @@ -400,10 +521,10 @@ def make(self, mkdir=True): suffixes = [self.src_ext] if self.tar_ext: suffixes.append(self.tar_ext) - return Make(self.src_ext + self.tar_ext, - [], - (("mkdir -p $(@D)",) if mkdir else ()) + tuple(self.cmds), - SUFFIXES(suffixes)) + cmds = (("mkdir -p $(@D)",) if mkdir else ()) + tuple(self.cmds) + return Make(deps=(Make(self.src_ext + self.tar_ext, + cmds=cmds, + attrs=(SUFFIXES(suffixes),)),)) C_INFERENCE_RULE = InferenceRule(".c", ".o", ("$(CC) $(CPPFLAGS) $(CFLAGS) -c -o $@ $<",)) @@ -412,19 +533,17 @@ def make_c_obj(path, via=C_INFERENCE_RULE): ''''via' can either be an InferenceRule or (name, cmds)''' if isinstance(path, Make): return path - if not path.endswith(".c"): - raise ValueError("expected source file (*.c), got: {!r}" - .format(path)) + base_deps_func = functools.partial(C_DEPS_FUNC, path) if isinstance(via, InferenceRule): stem, _ = os.path.splitext(path) name = stem + via.tar_ext cmds = () - attrs = (AUXILIARY((via.make(),)),) + deps_func = lambda cache: itertools.chain(base_deps_func(cache), + (via.make(),)) else: name, cmds = via - attrs = () - return make(name, functools.partial(C_DEPS_FUNC, path), cmds, *attrs, - mkdir=False) + deps_func = base_deps_func + return make(name, deps_func, cmds, mkdir=False) def make_bin(name, deps, @@ -435,7 +554,6 @@ def make_bin(name, raise TypeError("'deps' argument must be a list, not str") deps = tuple(tuple(dep if isinstance(dep, Make) else make_obj(dep) for dep in deps)) - dep_names = " ".join(dep.name for dep in deps) - return make(name, - deps, - ("{ld} -o $@ {dep_names} {libs}\n".format(**locals()),)) + dep_names = " ".join(dep.name for dep in deps if dep.name is not None) + cmds = ("{ld} -o $@ {dep_names} {libs}".format(**locals()),) + return make(name, deps, cmds)