Add tests for send-tpl

byrd.py

@@ -1,7 +1,9 @@
+from contextlib import contextmanager
 from getpass import getpass
 from hashlib import md5
 from itertools import chain
 from collections import ChainMap, OrderedDict, defaultdict
+from itertools import islice
 from string import Formatter
 import argparse
 import io
@@ -349,6 +351,16 @@ class Env(ChainMap):
         return self.fmt_env(what, kind=kind)
 
 
+class DummyClient:
+    '''
+    Dummy Paramiko client, mainly usefull for testing & dry runs
+    '''
+
+    @contextmanager
+    def open_sftp(self):
+        yield None
+
+
 def get_secret(service, resource, resource_id=None):
     resource_id = resource_id or resource
     secret = keyring.get_password(service, resource_id)
@@ -431,7 +443,7 @@ def run_python(task, env, cli):
         logger.info('[dry-run] ' + code)
         return None
     logger.debug(TAB + TAB.join(code.splitlines()))
-    cmd = 'python -c "import sys;exec(sys.stdin.read())"'
+    cmd = ['python', '-c', 'import sys;exec(sys.stdin.read())']
     if task.sudo:
         user = 'root' if task.sudo is True else task.sudo
         cmd = 'sudo -u {} -- {}'.format(user, cmd)
@@ -480,6 +492,9 @@ def log_stream(stream, buff):
 
 
 def run_helper(client, cmd, env=None, in_buff=None, sudo=False):
+    '''
+    Helper function to run `cmd` command on remote host
+    '''
     chan = client.get_transport().open_session()
     if env:
         chan.update_environment(env)
@@ -530,9 +545,10 @@ def run_remote(task, host, env, cli):
         'host': extract_host(host),
     })
     if cli.dry_run:
-        client = None
+        client = DummyClient()
     else:
         client = connect(host, cli.cfg.auth)
+
     if task.run:
         cmd = env.fmt(task.run)
         prefix = ''
@@ -552,29 +568,11 @@ def run_remote(task, host, env, cli):
     elif task.send:
         local_path = env.fmt(task.send)
         remote_path = env.fmt(task.to)
-        logger.info(f'[send] {local_path} -> {host}:{remote_path}')
         if not os.path.exists(local_path):
-            ByrdException('Path "%s" not found'  % local_path)
-        if cli.dry_run:
-            logger.info('[dry-run]')
-            return
+            raise ByrdException('Path "%s" not found'  % local_path)
         else:
-            with client.open_sftp() as sftp:
-                if os.path.isfile(local_path):
-                    sftp.put(os.path.abspath(local_path), remote_path)
-                elif os.path.isdir(local_path):
-                    for root, subdirs, files in os.walk(local_path):
-                        rel_dir = os.path.relpath(root, local_path)
-                        rel_dirs = os.path.split(rel_dir)
-                        rem_dir = posixpath.join(remote_path, *rel_dirs)
-                        run_helper(client, 'mkdir -p {}'.format(rem_dir))
-                        for f in files:
-                            rel_f = os.path.join(root, f)
-                            rem_file = posixpath.join(rem_dir, f)
-                            sftp.put(os.path.abspath(rel_f), rem_file)
-                else:
-                    msg = 'Unexpected path "%s" (not a file, not a directory)'
-                    ByrdException(msg % local_path)
+            send(client, env, cli, task)
+
     else:
         raise ByrdException('Unable to run task "%s"' % task.name)
 
@@ -583,6 +581,49 @@ def run_remote(task, host, env, cli):
 
     return res
 
+def send(client, env, cli, task):
+    fmt = task.fmt and Env(env, {'fmt': 'new'}).fmt(task.fmt) or None
+    local_path = env.fmt(task.send)
+    remote_path = env.fmt(task.to)
+    dry_run = cli.dry_run
+    with client.open_sftp() as sftp:
+        if os.path.isfile(local_path):
+            send_file(sftp, os.path.abspath(local_path), remote_path, env,
+                      dry_run=dry_run, fmt=fmt)
+        elif os.path.isdir(local_path):
+            for root, subdirs, files in os.walk(local_path):
+                rel_dir = os.path.relpath(root, local_path)
+                rel_dirs = os.path.split(rel_dir)
+                rem_dir = posixpath.join(remote_path, *rel_dirs)
+                run_helper(client, 'mkdir -p {}'.format(rem_dir))
+                for f in files:
+                    rel_f = os.path.join(root, f)
+                    rem_file = posixpath.join(rem_dir, f)
+                    send_file(sftp, os.path.abspath(rel_f), rem_file, env,
+                              dry_run=dry_run, fmt=fmt)
+        else:
+            msg = 'Unexpected path "%s" (not a file, not a directory)'
+            raise ByrdException(msg % local_path)
+
+
+def send_file(sftp, local_path, remote_path, env, dry_run=False, fmt=None):
+    if not fmt:
+        logger.info(f'[send] {local_path} -> {remote_path}')
+        lines = islice(open(local_path), 30)
+        logger.debug('File head:' + TAB.join(lines))
+        if not dry_run:
+            sftp.put(local_path, remote_path)
+        return
+    # Format file content and save it on remote
+    logger.info(f'[fmt] {local_path} -> {remote_path}')
+    content = env.fmt(open(local_path).read(), kind=fmt)
+    lines = islice(content.splitlines(), 30)
+    logger.debug('File head:' + TAB.join(lines))
+    if not dry_run:
+        fh = sftp.open(remote_path, mode='w')
+        fh.write(content)
+        fh.close()
+
 
 def run_task(task, host, cli, env=None):
     '''
@@ -728,7 +769,6 @@ def load_cfg(path, prefix=None):
                 child_prefix, _ = os.path.splitext(rel_path)
 
             child_cfg = load_cfg(child_path, child_prefix.split('/'))
-
             for section in load_sections:
                 if not section in cfg:
                     continue

pkg/os.yaml

@@ -1,3 +1,6 @@
+load:
+  - pkg: misc.yaml
+
 tasks:
   # Generic tasks
   bash:
@@ -15,12 +18,12 @@ tasks:
   mount:
     run: mount | grep ' {path} ' &> /dev/null || mount {path}
   move:
-    desc: Move a file or directory
+    desc: Move a file or directory (if destination does not exists)
     run: |
       if test ! -e {to}
       then mv {from} {to}
       fi
-  copy:
+  copy: # XXX move is lazy but copy is not !?
     desc: Copy a file or directory
     run: |
       if test -f {from}
@@ -48,9 +51,16 @@ tasks:
     desc: Send a file or a directory
     send: "{send}"
     to: "{to}"
+  send-tpl:
+    desc: Format a template and send it (can be file or directory)
+    send: "{send}"
+    to: "{to}"
+    fmt: "{fmt}"
   sudo-send:
     desc: Combine send & sudo-move
     multi:
+      - task: misc/random-string
+        export: tmppath
       - task: send
         env:
           to: "/tmp/{tmppath}"

tests/assert.yaml

@@ -0,0 +1,5 @@
+tasks:
+  one:
+    python: print('one')
+    assert: "stdout == 'one'"
+    once: true

tests/assert_test.yaml

@@ -1,6 +1,6 @@
-cli: '-c examples/assert.yaml one'
+cli: '-c tests/assert.yaml one'
 output: |
-  Load config examples/assert.yaml
+  Load config tests/assert.yaml
   one
   print('one')
   one

tests/dummy-new-fmt.cfg

@@ -0,0 +1 @@
+{host}

tests/dummy-old-fmt.cfg

@@ -0,0 +1 @@
+%(host)s

tests/fmt_file.yaml

@@ -0,0 +1,23 @@
+load:
+  - pkg: os.yaml
+
+networks:
+  all:
+    hosts:
+      - ham
+      - spam
+
+tasks:
+  fmt-file-new:
+    multi:
+      - task: os/send-tpl
+    env:
+      send: tests/dummy-new-fmt.cfg
+      to: remote.cfg
+  fmt-file-old:
+    multi:
+      - task: os/send-tpl
+    env:
+      send: tests/dummy-old-fmt.cfg
+      to: remote.cfg
+      fmt: old

tests/fmt_file_test.yaml

@@ -0,0 +1,13 @@
+cli: '-c tests/fmt_file.yaml --dry-run -vv fmt-file-new fmt-file-old all'
+output: |
+  Load config tests/fmt_file.yaml
+  Load config /home/bch/dev/byrd/pkg/os.yaml
+  Load config /home/bch/dev/byrd/pkg/misc.yaml
+  [fmt] /home/bch/dev/byrd/tests/dummy-new-fmt.cfg -> remote.cfg
+  File head:ham
+  [fmt] /home/bch/dev/byrd/tests/dummy-new-fmt.cfg -> remote.cfg
+  File head:spam
+  [fmt] /home/bch/dev/byrd/tests/dummy-old-fmt.cfg -> remote.cfg
+  File head:ham
+  [fmt] /home/bch/dev/byrd/tests/dummy-old-fmt.cfg -> remote.cfg
+  File head:spam

tests/load.yaml

@@ -0,0 +1,6 @@
+load:
+  - file: network_only.yaml
+    as: net
+tasks:
+  echo-host:
+    local: echo {host}

tests/load_test.yaml

@@ -1,7 +1,7 @@
-cli: '-c examples/load.yaml net/web echo-host --dry-run'
+cli: '-c tests/load.yaml net/web echo-host --dry-run'
 output: |
-  Load config examples/load.yaml
-  Load config examples/network_only.yaml
+  Load config tests/load.yaml
+  Load config tests/network_only.yaml
   echo-host
   [dry-run] echo web1.example.com
   echo-host

tests/multi.yaml

@@ -0,0 +1,28 @@
+tasks:
+  one:
+    python: print('one')
+    once: true
+  two:
+    python: print('two')
+    once: true
+  three:
+    python: print('three')
+    once: true
+  concat:
+    python: |
+      import os
+      print(os.environ['one'], os.environ['two'], os.environ['_'])
+    once: true
+  nested:
+    multi:
+      - python: print('nested')
+        once: true
+  all:
+    multi:
+      - task: one
+        export: one
+      - task: two
+        export: two
+      - task: three
+      - task: concat
+      - task: nested

tests/multi_test.yaml

@@ -1,6 +1,6 @@
-cli: '-c examples/multi.yaml all'
+cli: '-c tests/multi.yaml all'
 output: |
-  Load config examples/multi.yaml
+  Load config tests/multi.yaml
   one
   print('one')
   one

tests/network_only.yaml

@@ -0,0 +1,5 @@
+networks:
+  web:
+    hosts:
+      - web1.example.com
+      - web2.example.com

tests/network_only_test.yaml

@@ -1,2 +1,2 @@
-cli: "--dry-run -c examples/network_only.yaml"
-output: Load config examples/network_only.yaml
+cli: "--dry-run -c tests/network_only.yaml"
+output: Load config tests/network_only.yaml

tests/python.yaml

@@ -0,0 +1,7 @@
+tasks:
+  print:
+    desc: Print True
+    python: |-
+      from random import random
+      print(random() < 1)
+    once: true

tests/python_test.yaml

@@ -1,7 +1,7 @@
-cli: '-c examples/python.yaml print'
+cli: '-c tests/python.yaml print'
 output: |
-  Load config examples/python.yaml
-  Print module type
-  import os
-  print(type(os))
-  <class 'module'>
+  Load config tests/python.yaml
+  Print True
+  from random import random
+  print(random() < 1)
+  True

tests/task_only.yaml

@@ -0,0 +1,5 @@
+tasks:
+  time:
+    desc: Print current time (on local machine)
+    local: date -Iseconds
+    once: true

tests/task_only_test.yaml

@@ -1,5 +1,5 @@
-cli: "--dry-run -c examples/task_only.yaml time"
+cli: "--dry-run -c tests/task_only.yaml time"
 output: |
-  Load config examples/task_only.yaml
+  Load config tests/task_only.yaml
   Print current time (on local machine)
   [dry-run] date -Iseconds