Praise our setuptools god

byrd/config.py

@@ -0,0 +1,178 @@
+from collections import OrderedDict
+
+from .utils import (ByrdException, ObjectDict, spell,
+                    gen_candidates)
+
+
+try:
+    import yaml
+except ImportError:
+    pass
+
+def yaml_load(stream):
+    class OrderedLoader(yaml.Loader):
+        pass
+
+    def construct_mapping(loader, node):
+        loader.flatten_mapping(node)
+        return OrderedDict(loader.construct_pairs(node))
+    OrderedLoader.add_constructor(
+        yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
+        construct_mapping)
+    return yaml.load(stream, OrderedLoader)
+
+
+class Node:
+
+    @staticmethod
+    def fail(path, kind):
+        msg = 'Error while parsing config: expecting "%s" while parsing "%s"'
+        raise ByrdException(msg % (kind, '->'.join(path)))
+
+    @classmethod
+    def parse(cls, cfg, path=tuple()):
+        children = getattr(cls, '_children', None)
+        type_name = children and type(children).__name__ \
+                    or ' or '.join((c.__name__ for c in cls._type))
+        res = None
+        if type_name == 'dict':
+            if not isinstance(cfg, dict):
+                cls.fail(path, type_name)
+            res = ObjectDict()
+
+            if '*' in children:
+                assert len(children) == 1, "Don't mix '*' and other keys"
+                child_class = children['*']
+                for name, value in cfg.items():
+                    res[name] = child_class.parse(value, path + (name,))
+            else:
+                # Enforce known pre-defined
+                for key in cfg:
+                    if key not in children:
+                        path = ' -> '.join(path)
+                        if path:
+                            msg = 'Attribute "%s" not understood in %s' % (
+                                key, path)
+                        else:
+                            msg = 'Top-level attribute "%s" not understood' % (
+                                key)
+                        candidates = gen_candidates(children.keys())
+                        matches = spell(candidates, key)
+                        if matches:
+                            msg += ', try: %s' % ' or '.join(matches)
+                        raise ByrdException(msg)
+
+                for name, child_class in children.items():
+                    if name not in cfg:
+                        continue
+                    res[name] = child_class.parse(cfg[name], path + (name,))
+
+        elif type_name == 'list':
+            if not isinstance(cfg, list):
+                cls.fail(path, type_name)
+            child_class = children[0]
+            res = [child_class.parse(c, path+ ('[%s]' % pos,))
+                   for pos, c in enumerate(cfg)]
+
+        else:
+            if not isinstance(cfg, cls._type):
+                cls.fail(path, type_name)
+            res = cfg
+
+        return cls.setup(res, path)
+
+    @classmethod
+    def setup(cls, values, path):
+        if isinstance(values, ObjectDict):
+            values._path = '->'.join(path)
+        return values
+
+
+class Atom(Node):
+    _type = (str, bool)
+
+class AtomList(Node):
+    _children = [Atom]
+
+class Hosts(Node):
+    _children = [Atom]
+
+class Auth(Node):
+    _children = {'*': Atom}
+
+class EnvNode(Node):
+    _children = {'*': Atom}
+
+class HostGroup(Node):
+    _children = {
+        'hosts': Hosts,
+    }
+
+class Network(Node):
+    _children = {
+        '*': HostGroup,
+    }
+
+class Multi(Node):
+    _children = {
+        'task': Atom,
+        'export': Atom,
+        'network': Atom,
+    }
+
+class MultiList(Node):
+    _children = [Multi]
+
+class Task(Node):
+    _children = {
+        'desc': Atom,
+        'local': Atom,
+        'python': Atom,
+        'once': Atom,
+        'run': Atom,
+        'sudo': Atom,
+        'send': Atom,
+        'to': Atom,
+        'assert': Atom,
+        'env': EnvNode,
+        'multi': MultiList,
+        'fmt': Atom,
+    }
+
+    @classmethod
+    def setup(cls, values, path):
+        values['name'] = path and path[-1] or ''
+        if 'desc' not in values:
+            values['desc'] = values.get('name', '')
+        super().setup(values, path)
+        return values
+
+# Multi can also accept any task attribute:
+Multi._children.update(Task._children)
+
+
+class TaskGroup(Node):
+    _children = {
+        '*': Task,
+    }
+
+class LoadNode(Node):
+    _children = {
+        'file': Atom,
+        'pkg': Atom,
+        'as': Atom,
+    }
+
+class LoadList(Node):
+    _children = [LoadNode]
+
+class ConfigRoot(Node):
+    _children = {
+        'networks': Network,
+        'tasks': TaskGroup,
+        'auth': Auth,
+        'env': EnvNode,
+        'load': LoadList,
+    }
+
+

byrd.py → byrd/main.py

@@ -1,368 +1,33 @@
-from contextlib import contextmanager
 from getpass import getpass
 from hashlib import md5
 from itertools import chain
-from collections import ChainMap, OrderedDict, defaultdict
 from itertools import islice
 from string import Formatter
 import argparse
 import io
-import logging
 import os
 import posixpath
 import subprocess
 import sys
 import threading
 
+from .config import Task, yaml_load, ConfigRoot
+from .utils import (ByrdException, LocalException, ObjectDict, RemoteException,
+                   DummyClient, Env, spellcheck, spell, enable_logging_color,
+                   logger)
+
 try:
     # This file is imported by setup.py at install time
     import keyring
     import paramiko
-    import yaml
 except ImportError:
     pass
 
-__version__ = '0.0'
-
-
-log_fmt = '%(levelname)s:%(asctime).19s: %(message)s'
-logger = logging.getLogger('byrd')
-logger.setLevel(logging.INFO)
-log_handler = logging.StreamHandler()
-log_handler.setLevel(logging.INFO)
-log_handler.setFormatter(logging.Formatter(log_fmt))
-logger.addHandler(log_handler)
-
+__version__ = '0.0.2'
 basedir, _ = os.path.split(__file__)
 PKG_DIR = os.path.join(basedir, 'pkg')
 TAB = '\n    '
 
-class ByrdException(Exception):
-    pass
-
-class FmtException(ByrdException):
-    pass
-
-class ExecutionException(ByrdException):
-    pass
-
-class RemoteException(ExecutionException):
-    pass
-
-class LocalException(ExecutionException):
-    pass
-
-def enable_logging_color():
-    try:
-        import colorama
-    except ImportError:
-        return
-
-    colorama.init()
-    MAGENTA = colorama.Fore.MAGENTA
-    RED = colorama.Fore.RED
-    RESET = colorama.Style.RESET_ALL
-
-    # We define custom handler ..
-    class Handler(logging.StreamHandler):
-        def format(self, record):
-            if record.levelname == 'INFO':
-                record.msg = MAGENTA + record.msg + RESET
-            elif record.levelname in ('WARNING', 'ERROR', 'CRITICAL'):
-                record.msg = RED + record.msg + RESET
-            return super(Handler, self).format(record)
-
-    #  .. and plug it
-    logger.removeHandler(log_handler)
-    handler = Handler()
-    handler.setFormatter(logging.Formatter(log_fmt))
-    logger.addHandler(handler)
-    logger.propagate = 0
-
-
-def yaml_load(stream):
-    class OrderedLoader(yaml.Loader):
-        pass
-
-    def construct_mapping(loader, node):
-        loader.flatten_mapping(node)
-        return OrderedDict(loader.construct_pairs(node))
-    OrderedLoader.add_constructor(
-        yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
-        construct_mapping)
-    return yaml.load(stream, OrderedLoader)
-
-
-def edits(word):
-    yield word
-    splits = ((word[:i], word[i:]) for i in range(len(word) + 1))
-    for left, right in splits:
-        if right:
-            yield left + right[1:]
-
-
-def gen_candidates(wordlist):
-    candidates = defaultdict(set)
-    for word in wordlist:
-        for ed1 in edits(word):
-            for ed2 in edits(ed1):
-                candidates[ed2].add(word)
-    return candidates
-
-
-def spell(candidates,  word):
-    matches = set(chain.from_iterable(
-        candidates[ed] for ed in edits(word) if ed in candidates
-    ))
-    return matches
-
-
-def spellcheck(objdict, word):
-    if word in objdict:
-        return
-
-    candidates = objdict.get('_candidates')
-    if not candidates:
-        candidates = gen_candidates(list(objdict))
-        objdict._candidates = candidates
-
-    msg = '"%s" not found in %s' % (word, objdict._path)
-    matches = spell(candidates, word)
-    if matches:
-        msg += ', try: %s' % ' or '.join(matches)
-    raise ByrdException(msg)
-
-
-class ObjectDict(dict):
-    """
-    Simple objet sub-class that allows to transform a dict into an
-    object, like: `ObjectDict({'ham': 'spam'}).ham == 'spam'`
-    """
-    _meta = {}
-
-    def copy(self):
-        res = ObjectDict(super().copy())
-        ObjectDict._meta[id(res)] = ObjectDict._meta.get(id(self), {}).copy()
-        return res
-
-    def __getattr__(self, key):
-        if key.startswith('_'):
-            return ObjectDict._meta[id(self), key]
-
-        if key in self:
-            return self[key]
-        else:
-            return None
-
-    def __setattr__(self, key, value):
-        if key.startswith('_'):
-            ObjectDict._meta[id(self), key] = value
-        else:
-            self[key] = value
-
-class Node:
-
-    @staticmethod
-    def fail(path, kind):
-        msg = 'Error while parsing config: expecting "%s" while parsing "%s"'
-        raise ByrdException(msg % (kind, '->'.join(path)))
-
-    @classmethod
-    def parse(cls, cfg, path=tuple()):
-        children = getattr(cls, '_children', None)
-        type_name = children and type(children).__name__ \
-                    or ' or '.join((c.__name__ for c in cls._type))
-        res = None
-        if type_name == 'dict':
-            if not isinstance(cfg, dict):
-                cls.fail(path, type_name)
-            res = ObjectDict()
-
-            if '*' in children:
-                assert len(children) == 1, "Don't mix '*' and other keys"
-                child_class = children['*']
-                for name, value in cfg.items():
-                    res[name] = child_class.parse(value, path + (name,))
-            else:
-                # Enforce known pre-defined
-                for key in cfg:
-                    if key not in children:
-                        path = ' -> '.join(path)
-                        if path:
-                            msg = 'Attribute "%s" not understood in %s' % (
-                                key, path)
-                        else:
-                            msg = 'Top-level attribute "%s" not understood' % (
-                                key)
-                        candidates = gen_candidates(children.keys())
-                        matches = spell(candidates, key)
-                        if matches:
-                            msg += ', try: %s' % ' or '.join(matches)
-                        raise ByrdException(msg)
-
-                for name, child_class in children.items():
-                    if name not in cfg:
-                        continue
-                    res[name] = child_class.parse(cfg[name], path + (name,))
-
-        elif type_name == 'list':
-            if not isinstance(cfg, list):
-                cls.fail(path, type_name)
-            child_class = children[0]
-            res = [child_class.parse(c, path+ ('[%s]' % pos,))
-                   for pos, c in enumerate(cfg)]
-
-        else:
-            if not isinstance(cfg, cls._type):
-                cls.fail(path, type_name)
-            res = cfg
-
-        return cls.setup(res, path)
-
-    @classmethod
-    def setup(cls, values, path):
-        if isinstance(values, ObjectDict):
-            values._path = '->'.join(path)
-        return values
-
-
-class Atom(Node):
-    _type = (str, bool)
-
-class AtomList(Node):
-    _children = [Atom]
-
-class Hosts(Node):
-    _children = [Atom]
-
-class Auth(Node):
-    _children = {'*': Atom}
-
-class EnvNode(Node):
-    _children = {'*': Atom}
-
-class HostGroup(Node):
-    _children = {
-        'hosts': Hosts,
-    }
-
-class Network(Node):
-    _children = {
-        '*': HostGroup,
-    }
-
-class Multi(Node):
-    _children = {
-        'task': Atom,
-        'export': Atom,
-        'network': Atom,
-    }
-
-class MultiList(Node):
-    _children = [Multi]
-
-class Task(Node):
-    _children = {
-        'desc': Atom,
-        'local': Atom,
-        'python': Atom,
-        'once': Atom,
-        'run': Atom,
-        'sudo': Atom,
-        'send': Atom,
-        'to': Atom,
-        'assert': Atom,
-        'env': EnvNode,
-        'multi': MultiList,
-        'fmt': Atom,
-    }
-
-    @classmethod
-    def setup(cls, values, path):
-        values['name'] = path and path[-1] or ''
-        if 'desc' not in values:
-            values['desc'] = values.get('name', '')
-        super().setup(values, path)
-        return values
-
-# Multi can also accept any task attribute:
-Multi._children.update(Task._children)
-
-
-class TaskGroup(Node):
-    _children = {
-        '*': Task,
-    }
-
-class LoadNode(Node):
-    _children = {
-        'file': Atom,
-        'pkg': Atom,
-        'as': Atom,
-    }
-
-class LoadList(Node):
-    _children = [LoadNode]
-
-class ConfigRoot(Node):
-    _children = {
-        'networks': Network,
-        'tasks': TaskGroup,
-        'auth': Auth,
-        'env': EnvNode,
-        'load': LoadList,
-    }
-
-
-class Env(ChainMap):
-
-    def __init__(self, *dicts):
-        self.fmt_kind = 'new'
-        return super().__init__(*filter(lambda x: x is not None, dicts))
-
-    def fmt_env(self, child_env, kind=None):
-        new_env = {}
-        for key, val in child_env.items():
-            # env wrap-around!
-            new_val = self.fmt(val, kind=kind)
-            if new_val == val:
-                continue
-            new_env[key] = new_val
-        return Env(new_env, child_env)
-
-    def fmt_string(self, string, kind=None):
-        fmt_kind = kind or self.fmt_kind
-        try:
-            if fmt_kind == 'old':
-                return string % self
-            else:
-                return string.format(**self)
-        except KeyError as exc:
-            msg = 'Unable to format "%s" (missing: "%s")'% (string, exc.args[0])
-            candidates = gen_candidates(self.keys())
-            key = exc.args[0]
-            matches = spell(candidates, key)
-            if matches:
-                msg += ', try: %s' % ' or '.join(matches)
-            raise FmtException(msg )
-        except IndexError as exc:
-            msg = 'Unable to format "%s", positional argument not supported'
-            raise FmtException(msg)
-
-    def fmt(self, what, kind=None):
-        if isinstance(what, str):
-            return self.fmt_string(what, kind=kind)
-        return self.fmt_env(what, kind=kind)
-
-
-class DummyClient:
-    '''
-    Dummy Paramiko client, mainly usefull for testing & dry runs
-    '''
-
-    @contextmanager
-    def open_sftp(self):
-        yield None
 
 
 def get_secret(service, resource, resource_id=None):
@@ -571,7 +236,6 @@ def run_remote(task, host, env, cli):
 
     elif task.send:
         local_path = env.fmt(task.send)
-        remote_path = env.fmt(task.to)
         if not os.path.exists(local_path):
             raise ByrdException('Path "%s" not found'  % local_path)
         else:
@@ -619,7 +283,8 @@ def send_file(sftp, local_path, remote_path, env, dry_run=False, fmt=None):
             sftp.put(local_path, remote_path)
         return
     # Format file content and save it on remote
-    logger.info(f'[fmt] {local_path} -> {remote_path}')
+    local_relpath = os.path.relpath(local_path)
+    logger.info(f'[fmt] {local_relpath} -> {remote_path}')
     content = env.fmt(open(local_path).read(), kind=fmt)
     lines = islice(content.splitlines(), 30)
     logger.debug('File head:' + TAB.join(lines))
@@ -731,7 +396,7 @@ def load_cfg(path, prefix=None):
     load_sections = ('networks', 'tasks', 'auth', 'env')
 
     if os.path.isfile(path):
-        logger.debug('Load config %s' % path)
+        logger.debug('Load config %s' % os.path.relpath(path))
         cfg = yaml_load(open(path))
         cfg = ConfigRoot.parse(cfg)
     else:
@@ -743,18 +408,10 @@ def load_cfg(path, prefix=None):
 
     # Create backrefs between tasks to the local config
     if cfg.get('tasks'):
-        items = cfg['tasks'].items()
-        for k, v in items:
-            v._cfg = ObjectDict(cfg.copy())
-
-    if prefix:
-        key_fn = lambda x: '/'.join(prefix + [x])
-        # Apply prefix
-        for section in load_sections:
-            if not section in cfg:
-                continue
-            items = cfg[section].items()
-            cfg[section] = {key_fn(k): v for k, v in items}
+        cfg_cp = cfg.copy()
+        for k, v in cfg['tasks'].items():
+            v._cfg = cfg_cp
+
 
     # Recursive load
     if cfg.load:
@@ -772,11 +429,13 @@ def load_cfg(path, prefix=None):
             else:
                 child_prefix, _ = os.path.splitext(rel_path)
 
-            child_cfg = load_cfg(child_path, child_prefix.split('/'))
+            child_cfg = load_cfg(child_path, child_prefix)
+            key_fn = lambda x: '/'.join([child_prefix, x])
             for section in load_sections:
-                if not section in cfg:
+                if not section in child_cfg:
                     continue
-                cfg[section].update(child_cfg.get(section, {}))
+                items = {key_fn(k): v for k, v in child_cfg[section].items()}
+                cfg[section].update(items)
     return cfg
 
 
@@ -899,7 +558,7 @@ def main():
             enable_logging_color()
         cli.verbose = max(0, 1 + cli.verbose - cli.quiet)
         level = ['WARNING', 'INFO', 'DEBUG'][min(cli.verbose, 2)]
-        log_handler.setLevel(level)
+        print(level)
         logger.setLevel(level)
 
         if cli.info:

byrd/pkg/az.yaml

@@ -0,0 +1,63 @@
+tasks:
+  # Creation, deletion and mgmt of VM
+  create-vm:
+     desc: Create a new azure VM
+     local: >
+       az vm create -n "{vm_name}" -g "{ressource_group}"
+       --image UbuntuLTS  --admin-username {vm_admin} --ssh-key-value {ssh_pubkey}
+       --data-disk-sizes-gb {vm_disk_size} --public-ip-address "" --subnet "{subnet}"
+     once: true
+  delete-vm-only:
+     desc: Delete azure VM
+     local: az vm delete -n "{vm_name}"  -g "{ressource_group}" -y
+     once: true
+  show-disk:
+     desc: Read disk id
+     local: >-
+       az vm show -g "{ressource_group}" --query "{query}"
+       -n "{vm_name}"
+     once: true
+  delete-disk:
+     desc: Delete azure disk
+     local: az disk delete -n "{disk_name}" -g "{ressource_group}" -y
+     once: true
+  delete-vm:
+     desc: Delete both VM and attached disk
+     multi:
+      - task: az-show-disk
+        export: os_disk
+        env:
+          query: "storageProfile.osDisk.name"
+      - task: az-show-disk
+        export: data_disk
+        env:
+          query: "storageProfile.dataDisks[0].name"
+      - task: az-delete-vm-only
+      - task: az-delete-disk
+        env:
+          disk_name: "{data_disk}"
+      - task: az-delete-disk
+        env:
+          disk_name: "{os_disk}"
+  show-ip:
+     desc: Query azure vm by name for ip
+     local: >
+       az vm list-ip-addresses
+       --query "[?virtualMachine.name=='{vm_name}']
+       .virtualMachine.network.privateIpAddresses[0]"
+     once: true
+  vm-info:
+     desc: Query azure vm by name for info
+     local: az vm list --query "[?name=='{vm_name}']"
+     once: true
+  vm-search:
+    desc: Search VM by name
+    local: az vm list --query "[?contains(name, '{vm_name}')].name"
+    once: true
+  fix-hosts:
+    desc: "See: https://github.com/Microsoft/WSL/issues/491"
+    run: |
+      if grep -q $(hostname) /etc/hosts
+      then true
+      else sudo sed -i "s/127.0.0.1 localhost/127.0.0.1 localhost $(hostname)/g" /etc/hosts
+      fi

byrd/pkg/git.yaml

@@ -0,0 +1,10 @@
+tasks:
+  clone:
+    desc: Clone git repo
+    run: |
+      if test ! -d {path}
+        then  git clone {repo_uri} {path}
+      fi
+  pull:
+    desc: Update codebase
+    run: cd src/prove && git pull

byrd/pkg/misc.yaml

@@ -0,0 +1,5 @@
+tasks:
+  random-string:
+    python: |
+      import random, string
+      print(''.join(random.sample(string.hexdigits, 8)))

byrd/pkg/os.yaml

@@ -0,0 +1,133 @@
+load:
+  - pkg: misc.yaml
+
+tasks:
+  # Generic tasks
+  bash:
+    desc: Interactive prompt
+    run: "bash"
+  create-dir:
+    desc: Add a new directory
+    run: "mkdir -p {path}"
+  symlink:
+    desc: Create symlink
+    run: |
+      if test ! -e {to}
+      then ln -s {from} {to}
+      fi
+  mount:
+    run: mount | grep ' {path} ' &> /dev/null || mount {path}
+  move:
+    desc: Move a file or directory (if destination does not exists)
+    run: |
+      if test ! -e {to}
+      then mv {from} {to}
+      fi
+  copy: # XXX move is lazy but copy is not !?
+    desc: Copy a file or directory
+    run: |
+      if test -f {from}
+      then cp {from} {to}
+      else cp -Tr {from} {to}
+      fi
+  remove:
+    desc: Remove (-r) path
+    run: "rm -r {path}"
+  chown:
+    desc: Chown a file or directory
+    run: "chown -R {user}. {path}"
+  chmod:
+    desc: Chmod a file or directory
+    run: "chmod -R {mode} {path}"
+  apt-install:
+    desc: Run apt install
+    run: apt update && apt install -y {packages}
+    sudo: true
+  systemctl:
+    desc: Call systemctl
+    run: systemctl {action} {service}
+    sudo: true
+  send:
+    desc: Send a file or a directory
+    send: "{send}"
+    to: "{to}"
+  send-tpl:
+    desc: Format a template and send it (can be file or directory)
+    send: "{send}"
+    to: "{to}"
+    fmt: "{fmt}"
+  sudo-send:
+    desc: Combine send & sudo-move
+    multi:
+      - task: misc/random-string
+        export: tmppath
+      - task: send
+        env:
+          to: "/tmp/{tmppath}"
+      - task: copy
+        sudo: true
+        env:
+          from: "/tmp/{tmppath}"
+      - task: remove
+        env:
+          path: "/tmp/{tmppath}"
+  parted:
+    desc: Create primary partition && mkfs
+    sudo: true
+    run: |
+      if test ! -e {device}1
+      then
+      parted {device} -a optimal --script mklabel gpt mkpart primary 0% 100%
+      sleep 1
+      mkfs -t ext4 {device}1
+      fi
+  append-line:
+    desc: Append line to file
+    run: >-
+      grep {guard} {file} &> /dev/null
+      || echo {line} >> {file}
+  add-user:
+    desc: Create a new user
+    run: id {user} || sudo adduser {user} --disabled-login --gecos ""
+  add-group:
+    desc: Add group to user
+    run: usermod -a -G {group} {user}
+  wget:
+    desc: Download a file
+    run: test -f {file} || wget -O {file} {url}
+  unzip:
+    desc: Unzip a zip file
+    run: test -d {dir} || unzip {file} -d {dir}
+  patch:
+    desc: Patch a file with a specific local diff file
+    run: |
+      patch --ignore-whitespace --reject-file=/dev/null -uN {file} << EOF
+      {diff_string}
+      EOF
+  send-rmcr:
+    desc: Send a file but remove its carriage returns before.
+    multi:
+      - task: send
+      - run: "tr -d '\r' < {to} > {to}-tmp"
+      - task: move
+        env:
+          from: "{to}-tmp"
+  sudo-send-rmcr:
+    desc: Combine send & sudo-move, and remove carriage returns
+    multi:
+      - task: send
+        env:
+          to: "/tmp/{tmppath}"
+      - task: rmcr
+        env:
+          from: "/tmp/{tmppath}"
+        sudo: true
+      - task: remove
+        env:
+          path: "/tmp/{tmppath}"
+  rmcr:
+    desc: remove carriage returns and move file
+    run: "tr -d '\r' < {from} | tee {to} > /dev/null"
+  unlink:
+    desc: remove a symlink
+    run: "test -L {path} && unlink {path}"

byrd/pkg/pg.yaml

@@ -0,0 +1,24 @@
+tasks:
+  createuser:
+    desc: Add postgres user
+    sudo: postgres
+    run: |
+      if ! psql -c "SELECT usename FROM pg_user" | grep {pg_user} &> /dev/null
+      then createuser {pg_user} -d
+      fi
+  createdb:
+    desc: Create a new db for default user
+    multi:
+      - task: createdb-with-owner
+    env:
+      db_owner: "{ssh_user}"
+  alter-passwd:
+    desc: Alter user password
+    sudo: postgres
+    run: psql -c "ALTER USER {pg_user} WITH PASSWORD '{pg_password}'"
+  createdb-with-owner:
+    desc: Create a new db with a specific owner
+    run: |
+      if ! psql -l | grep {db_name} &> /dev/null
+      then createdb -O {db_owner} {db_name}
+      fi

byrd/pkg/py.yaml

@@ -0,0 +1,7 @@
+tasks:
+  create-venv:
+    desc: Create python venv
+    run: |
+      if ! -e {venv_path}
+      then python3 -m venv {venv_path}
+      fi

byrd/utils.py

@@ -0,0 +1,176 @@
+from itertools import chain
+from collections import defaultdict, ChainMap
+from contextlib import contextmanager
+import logging
+
+log_fmt = '%(levelname)s:%(asctime).19s: %(message)s'
+logger = logging.getLogger('byrd')
+logger.setLevel(logging.INFO)
+log_handler = logging.StreamHandler()
+log_handler.setLevel(logging.INFO)
+log_handler.setFormatter(logging.Formatter(log_fmt))
+logger.addHandler(log_handler)
+
+
+class ByrdException(Exception):
+    pass
+
+class FmtException(ByrdException):
+    pass
+
+class ExecutionException(ByrdException):
+    pass
+
+class RemoteException(ExecutionException):
+    pass
+
+class LocalException(ExecutionException):
+    pass
+
+def enable_logging_color():
+    try:
+        import colorama
+    except ImportError:
+        return
+
+    colorama.init()
+    MAGENTA = colorama.Fore.MAGENTA
+    RED = colorama.Fore.RED
+    RESET = colorama.Style.RESET_ALL
+
+    # We define custom handler ..
+    class Handler(logging.StreamHandler):
+        def format(self, record):
+            if record.levelname == 'INFO':
+                record.msg = MAGENTA + record.msg + RESET
+            elif record.levelname in ('WARNING', 'ERROR', 'CRITICAL'):
+                record.msg = RED + record.msg + RESET
+            return super(Handler, self).format(record)
+
+    #  .. and plug it
+    logger.removeHandler(log_handler)
+    handler = Handler()
+    handler.setFormatter(logging.Formatter(log_fmt))
+    logger.addHandler(handler)
+    logger.propagate = 0
+
+
+
+def edits(word):
+    yield word
+    splits = ((word[:i], word[i:]) for i in range(len(word) + 1))
+    for left, right in splits:
+        if right:
+            yield left + right[1:]
+
+
+def gen_candidates(wordlist):
+    candidates = defaultdict(set)
+    for word in wordlist:
+        for ed1 in edits(word):
+            for ed2 in edits(ed1):
+                candidates[ed2].add(word)
+    return candidates
+
+
+def spell(candidates,  word):
+    matches = set(chain.from_iterable(
+        candidates[ed] for ed in edits(word) if ed in candidates
+    ))
+    return matches
+
+
+def spellcheck(objdict, word):
+    if word in objdict:
+        return
+
+    candidates = objdict.get('_candidates')
+    if not candidates:
+        candidates = gen_candidates(list(objdict))
+        objdict._candidates = candidates
+
+    msg = '"%s" not found in %s' % (word, objdict._path)
+    matches = spell(candidates, word)
+    if matches:
+        msg += ', try: %s' % ' or '.join(matches)
+    raise ByrdException(msg)
+
+
+class ObjectDict(dict):
+    """
+    Simple objet sub-class that allows to transform a dict into an
+    object, like: `ObjectDict({'ham': 'spam'}).ham == 'spam'`
+    """
+
+    # Meta allows to hide all the keys starting with an '_'
+    _meta = {}
+
+    def copy(self):
+        res = ObjectDict(super().copy())
+        ObjectDict._meta[id(res)] = ObjectDict._meta.get(id(self), {}).copy()
+        return res
+
+    def __getattr__(self, key):
+        if key.startswith('_'):
+            return ObjectDict._meta[id(self), key]
+
+        if key in self:
+            return self[key]
+        else:
+            return None
+
+    def __setattr__(self, key, value):
+        if key.startswith('_'):
+            ObjectDict._meta[id(self), key] = value
+        else:
+            self[key] = value
+
+
+class DummyClient:
+    '''
+    Dummy Paramiko client, mainly usefull for testing & dry runs
+    '''
+
+    @contextmanager
+    def open_sftp(self):
+        yield None
+
+class Env(ChainMap):
+
+    def __init__(self, *dicts):
+        self.fmt_kind = 'new'
+        return super().__init__(*filter(lambda x: x is not None, dicts))
+
+    def fmt_env(self, child_env, kind=None):
+        new_env = {}
+        for key, val in child_env.items():
+            # env wrap-around!
+            new_val = self.fmt(val, kind=kind)
+            if new_val == val:
+                continue
+            new_env[key] = new_val
+        return Env(new_env, child_env)
+
+    def fmt_string(self, string, kind=None):
+        fmt_kind = kind or self.fmt_kind
+        try:
+            if fmt_kind == 'old':
+                return string % self
+            else:
+                return string.format(**self)
+        except KeyError as exc:
+            msg = 'Unable to format "%s" (missing: "%s")'% (string, exc.args[0])
+            candidates = gen_candidates(self.keys())
+            key = exc.args[0]
+            matches = spell(candidates, key)
+            if matches:
+                msg += ', try: %s' % ' or '.join(matches)
+            raise FmtException(msg )
+        except IndexError:
+            msg = 'Unable to format "%s", positional argument not supported'
+            raise FmtException(msg)
+
+    def fmt(self, what, kind=None):
+        if isinstance(what, str):
+            return self.fmt_string(what, kind=kind)
+        return self.fmt_env(what, kind=kind)

setup.py

@@ -1,9 +1,9 @@
 #!/usr/bin/env python
-from setuptools import setup
+from setuptools import setup, find_packages
 from glob import glob
 import os
 
-import byrd
+from byrd import main
 
 long_description = '''
 
@@ -16,10 +16,10 @@ The name Byrd is a reference to Donald Byrd.
 
 description = ('Simple deployment tool based on Paramiko')
 basedir, _ = os.path.split(__file__)
-pkg_yaml = glob(os.path.join(basedir, 'pkg', '*yaml'))
+pkg_yaml = glob(os.path.join('pkg', '*yaml'))
 
 setup(name='Byrd',
-      version=byrd.__version__,
+      version=main.__version__,
       description=description,
       long_description=long_description,
       author='Bertrand Chenal',
@@ -29,12 +29,11 @@ setup(name='Byrd',
       py_modules=['byrd'],
       entry_points={
           'console_scripts': [
-              'bd = byrd:main',
+              'bd = byrd.main:main',
           ],
       },
-      packages=['pkg'],
-      package_data={'pkg': pkg_yaml},
-      include_package_data=True,
+      packages=['byrd'],
+      package_data={'byrd': ['pkg/*yaml']},
       install_requires=[
           'paramiko',
           'pyyaml',

tests/base_test.py

@@ -3,7 +3,7 @@ from shlex import shlex
 
 import pytest
 
-from byrd import run_batch, load_cli, Env
+from byrd.main import run_batch, load_cli, Env
 
 
 def test_all_conf(test_cfg, log_buff):

tests/conftest.py

@@ -4,7 +4,8 @@ import logging
 
 import pytest
 
-from byrd import logger, log_handler, yaml_load, ObjectDict
+from byrd.utils import logger, log_handler, ObjectDict
+from byrd.config import yaml_load
 
 # Disable default handler
 logger.removeHandler(log_handler)

tests/fmt_file_test.yaml

@@ -1,13 +1,13 @@
-cli: '-c tests/fmt_file.yaml --dry-run -vv fmt-file-new fmt-file-old all'
+cli: '-c tests/fmt_file.yaml --dry-run -qq fmt-file-new fmt-file-old all'
 output: |
   Load config tests/fmt_file.yaml
-  Load config /home/bch/dev/byrd/pkg/os.yaml
-  Load config /home/bch/dev/byrd/pkg/misc.yaml
-  [fmt] /home/bch/dev/byrd/tests/dummy-new-fmt.cfg -> remote.cfg
+  Load config byrd/pkg/os.yaml
+  Load config byrd/pkg/misc.yaml
+  [fmt] tests/dummy-new-fmt.cfg -> remote.cfg
   File head:ham
-  [fmt] /home/bch/dev/byrd/tests/dummy-new-fmt.cfg -> remote.cfg
+  [fmt] tests/dummy-new-fmt.cfg -> remote.cfg
   File head:spam
-  [fmt] /home/bch/dev/byrd/tests/dummy-old-fmt.cfg -> remote.cfg
+  [fmt] tests/dummy-old-fmt.cfg -> remote.cfg
   File head:ham
-  [fmt] /home/bch/dev/byrd/tests/dummy-old-fmt.cfg -> remote.cfg
+  [fmt] tests/dummy-old-fmt.cfg -> remote.cfg
   File head:spam