#6145 - Introduce deploy script for websites

modules/adblockplus/files/web/static/deploy_script.py

+#!/usr/bin/env python

[mathias, 2018/06/07 21:52:00]

According to our legal department all source files should contain the Copyright
stanza associated with the project. Also I am missing a __doc__ briefly
explaining the purpose of this script (which you could re-use in the
ArgumentParser epilog).

[f.lopez, 2018/06/18 18:41:30]

Acknowledged.

+
+import argparse
+from filecmp import dircmp
+import hashlib
+import os
+import sys
+import shutil
+import tarfile
+import tempfile
+import urllib
+
+
+def download(url, tmp_dir):
+    file_name = url.split('/')[-1]
+    abs_file_name = os.path.join(tmp_dir, file_name)
+    print 'Downloading: ' + file_name
+    urllib.urlretrieve(url, abs_file_name)
+    return abs_file_name
+
+
+def calculate_md5(file):
+    with open(file) as f:
+        data = f.read()
+        md5_result = hashlib.md5(data).hexdigest()
+    return md5_result.strip()
+
+
+def read_md5(file):
+    with open(file) as f:
+        md5_result = f.readline()
+    return md5_result.strip()
+
+
+def untar(tar_file, tmp_dir):
+    if tarfile.is_tarfile(tar_file):
+        with tarfile.open(tar_file, 'r:gz') as tar:
+            tar.extractall(tmp_dir)
+
+
+def remove_tree(to_remove):
+    if os.path.exists(to_remove):
+        if os.path.isdir(to_remove):
+            shutil.rmtree(to_remove)
+        else:
+            os.remove(to_remove)
+
+
+def deploy_files(dcmp):
+    for name in dcmp.diff_files:
+        copytree(dcmp.right, dcmp.left)
+    for name in dcmp.left_only:
+        remove_tree(os.path.join(dcmp.left, name))
+    for name in dcmp.right_only:
+        copytree(dcmp.right, dcmp.left)
+    for sub_dcmp in dcmp.subdirs.values():
+        deploy_files(sub_dcmp)
+
+
+def copytree(src, dst):

[mathias, 2018/06/07 21:52:00]

Why don't you use shutil.copytree? And why isn't that rationale documented or at
least referred (read: linked) here?

[f.lopez, 2018/06/18 18:41:29]

Acknowledged.

+    if not os.path.exists(dst):
+        os.makedirs(dst)
+        shutil.copystat(src, dst)
+    lst = os.listdir(src)
+    for item in lst:
+        s = os.path.join(src, item)

[mathias, 2018/06/07 21:52:00]

So `s` is an abbreviation for `src` which is of course an abbreviation for
`source`. Aha.

[f.lopez, 2018/06/18 18:41:29]

Acknowledged.

+        d = os.path.join(dst, item)

[mathias, 2018/06/07 21:52:00]

And it continues. At least you are consistent.

[f.lopez, 2018/06/18 18:41:29]

Acknowledged.

+        if os.path.isdir(s):
+            copytree(s, d)
+        else:
+            shutil.copy2(s, d)
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(
+        description="""Fetch a compressed archive in the form of $HASH.tar.gz
+                    and deploy it to /var/www/$WEBSITE folder""",
+        epilog="""--hash must be provided in order to fetch the files,
+               expected files to be fetched are $HASH.tar.gz and $HASH.md5 in
+               order to compare the hashes.
+               --source must be an URL, e.g.
+               https://helpcenter.eyeofiles.com""",
+    )
+    parser.add_argument('--hash', action='store', type=str, required=True,

[mathias, 2018/06/07 21:52:00]

Is this sure to always be a hash? Wouldn't `--revision`, `--version` or `--name`
be a more future-proof choice of name?

[f.lopez, 2018/06/18 18:41:30]

Acknowledged.

+                        help='Hash of the commit to deploy')
+    parser.add_argument('--source', action='store', type=str, required=True,
+                        help='The source where files will be downloaded')
+    parser.add_argument('--website', action='store', type=str,

[mathias, 2018/06/07 21:52:00]

Why does option even exist? IT should not be possible to specify the target
location (or a vital part thereof) from "the outside world" but be bound to the
specific user's website (or set of websites). Otherwise we cant host multiple
websites from different stakeholders on the same host, because one who owns
/var/www/foo could easily overwrite the content of another one who owns
/var/www/bar.

[f.lopez, 2018/06/18 18:41:30]

Acknowledged.

+                        help='The name of the website [e.g. help.eyeo.com]')
+    args = parser.parse_args()
+    hash = args.hash
+    source = args.source
+    url_file = '{0}/{1}.tar.gz'.format(source, hash)
+    url_md5 = '{0}/{1}.md5'.format(source, hash)
+    tmp_dir = tempfile.mkdtemp()
+    try:
+        down_file = download(url_file, tmp_dir)
+        down_md5 = download(url_md5, tmp_dir)
+        if calculate_md5(down_file) == read_md5(down_md5):
+            untar(down_file, tmp_dir)
+            hash_directory = os.path.join(tmp_dir, hash)
+            destination = os.path.join('/var/www/', args.website)
+            dcmp = dircmp(destination, hash_directory)
+            print 'Deploying files'
+            deploy_files(dcmp)
+        else:
+            sys.exit("Hashes don't match")
+    except Exception as e:
+        sys.exit(e)
+    finally:
+        shutil.rmtree(tmp_dir)