#6145 - Introduce deploy script for websites

modules/adblockplus/files/web/static/deploy_script.py

 #!/usr/bin/env python
+#
+# This file is part of the Adblock Plus infrastructure
+# Copyright (C) 2018-present eyeo GmbH
+#
+# Adblock Plus is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# Adblock Plus is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Adblock Plus.  If not, see <http://www.gnu.org/licenses/>.
 
 import argparse
-from contextlib import closing
 from filecmp import dircmp
 import hashlib
 import os
 import sys
 import shutil
 import tarfile
-import urllib2
+import tempfile
+import urllib
 
 
-def download(url):
+__doc__ = """This script MUST be renamed in the form of $WEBSITE, e.g.
+          help.eyeo.com, --name must be provided in order to fetch the
+          files, expected files to be fetched are $NAME.tar.gz and $NAME.md5 in
+          order to compare the hashes. --source must be an URL, e.g.
+          https://helpcenter.eyeofiles.com"""
+
+
+def download(url, temporary_directory):
     file_name = url.split('/')[-1]
-    abs_file_name = os.path.join(os.path.dirname(os.path.realpath(__file__)),

[Vasily Kuznetsov, 2018/05/15 18:09:37]

It seems that you are using the directory of this script as a temporary
directory. Is there a particular reason to not use a real temporary directory
instead? It seems unlikely that the location of this script will be a good place
to stash temporary junk.

[f.lopez, 2018/05/21 22:23:41]

Acknowledged.

-                                 file_name)
+    absolute_file_path = os.path.join(temporary_directory, file_name)
     print 'Downloading: ' + file_name
-    try:
-        with closing(urllib2.urlopen(url)) as page:

[Vasily Kuznetsov, 2018/05/15 18:09:37]

This code is more or less doing `urllib.urlretrieve`
(https://docs.python.org/2/library/urllib.html#urllib.urlretrieve). Have you
thought of using it?

[f.lopez, 2018/05/21 22:23:41]

Acknowledged.

-            block_sz = 8912
-            with open(abs_file_name, 'wb') as f:
-                while True:
-                    buffer = page.read(block_sz)
-                    if not buffer:
-                        break
-                    f.write(buffer)
-        return abs_file_name
-    except urllib2.HTTPError as e:
-        if e.code == 404:
-            sys.exit("File not found on remote source")

[Vasily Kuznetsov, 2018/05/15 18:09:37]

We have a rule for selecting the type of quotes in stings. Basically it's "use
single quotes unless your string contains single quotes". See
https://adblockplus.org/coding-style#python.

[f.lopez, 2018/05/21 22:23:41]

Acknowledged.

-    except Exception as e:
-        sys.exit(e)
+    urllib.urlretrieve(url, absolute_file_path)
+    return absolute_file_path
 
 
 def calculate_md5(file):
-    with open(file) as f:
-        data = f.read()
+    with open(file) as file_handle:
+        data = file_handle.read()
         md5_result = hashlib.md5(data).hexdigest()
     return md5_result.strip()
 
 
 def read_md5(file):
-    with open(file) as f:
-        md5_result = f.readline()
+    with open(file) as file_handle:
+        md5_result = file_handle.readline()
     return md5_result.strip()
 
 
-def untar(tar_file):
+def untar(tar_file, temporary_directory):
     if tarfile.is_tarfile(tar_file):
         with tarfile.open(tar_file, 'r:gz') as tar:
-            tar.extractall(os.path.dirname(os.path.realpath(tar_file)))
-            print 'Extracted in current directory'
-            return os.path.dirname(os.path.abspath(__file__))
+            tar.extractall(temporary_directory)
 
 
 def remove_tree(to_remove):
     if os.path.exists(to_remove):
         if os.path.isdir(to_remove):
             shutil.rmtree(to_remove)
         else:
             os.remove(to_remove)
 
 
-def clean(hash):
-    print "cleaning directory"
-    cwd = os.path.dirname(os.path.abspath(__file__))
-    [remove_tree(os.path.join(cwd, x)) for x in os.listdir(cwd)
-     if x.startswith(hash)]
+def deploy_files(directory_comparison):
+    for name in directory_comparison.diff_files:
+        copytree(directory_comparison.right, directory_comparison.left)
+    for name in directory_comparison.left_only:
+        remove_tree(os.path.join(directory_comparison.left, name))
+    for name in directory_comparison.right_only:
+        copytree(directory_comparison.right, directory_comparison.left)
+    for subdirectory_comparison in directory_comparison.subdirs.values():
+        deploy_files(subdirectory_comparison)
 
 
-def deploy_files(dcmp):
-    for name in dcmp.diff_files:
-        copytree(dcmp.right, dcmp.left)
-    for name in dcmp.left_only:
-        remove_tree(dcmp.left + "/" + name)
-    for name in dcmp.right_only:
-        copytree(dcmp.right, dcmp.left)
-    for sub_dcmp in dcmp.subdirs.values():
-        deploy_files(sub_dcmp)
-
-
-def copytree(src, dst):
-    if not os.path.exists(dst):
-        os.makedirs(dst)
-        shutil.copystat(src, dst)
-    lst = os.listdir(src)
-    for item in lst:
-        s = os.path.join(src, item)
-        d = os.path.join(dst, item)
-        if os.path.isdir(s):
-            copytree(s, d)
+# shutil.copytree copies a tree but the destination directory MUST NOT exist
+# this might break the site for the duration of the files being deployed
+# for more info read: https://docs.python.org/2/library/shutil.html
+def copytree(source, destination):
+    if not os.path.exists(destination):
+        os.makedirs(destination)
+        shutil.copystat(source, destination)
+    source_items = os.listdir(source)
+    for item in source_items:
+        source_path = os.path.join(source, item)
+        destination_path = os.path.join(destination, item)
+        if os.path.isdir(source_path):
+            copytree(source_path, destination_path)
         else:
-            shutil.copy2(s, d)
+            shutil.copy2(source_path, destination_path)
 
 
 if __name__ == '__main__':
+    website = os.path.basename(__file__)
     parser = argparse.ArgumentParser(
-        description='''Fetch a compressed archive in the form of $HASH.tar.gz and

[Vasily Kuznetsov, 2018/05/15 18:09:37]

"Fetch" and "deploys" is inconsistent. It should be either "fetch and deploy" or
"fetches and deploys". First one seems better.

[Vasily Kuznetsov, 2018/05/15 18:09:37]

It's nicer to be consistent with the type of quotes you use for multiline
strings. Normally double quotes are preferred, like in line 101.

[f.lopez, 2018/05/21 22:23:41]

Acknowledged.

[f.lopez, 2018/05/21 22:23:41]

Acknowledged.

-        deploys it to /var/www/$WEBSITE folder''',
-        epilog="""--hash must be provided in order to fetch the files,
-        expected files to be fetched are $HASH.tar.gz and $HASH.md5 in order to

[Vasily Kuznetsov, 2018/05/15 18:09:37]

This indentation is a bit hard to follow. Perhaps it would be better to add
spaces in front of additional lines to make arguments clearly separated.

[f.lopez, 2018/05/21 22:23:42]

Acknowledged.

-        compare the hashes.
-        --url and --domain are mutually exclusive, if url is provided
-        the files will be downloaded from $url/$HASH, otherwise the default
-        value will be fetched from $domain.eyeofiles.com/$HASH""",
+        description="""Fetch a compressed archive in the form of $NAME.tar.gz
+                    and deploy it to /var/www/{0} folder""".format(website),
+        epilog=__doc__,
     )
-    parser.add_argument('--hash', action='store', type=str, nargs='?',

[Vasily Kuznetsov, 2018/05/15 18:09:37]

'store' is the default action, so you don't really need to specify it. I guess
it's ok if you do, but you can also just skip it.

[f.lopez, 2018/05/21 22:23:41]

I know this is the default action, but I think it is nice to see it when reading
the code.

[Vasily Kuznetsov, 2018/05/22 16:38:36]

Acknowledged.

-                        help='Hash of the commit to deploy')
-    parser.add_argument('--url', action='store', type=str,
-                        help='URL where files will be downloaded')
-    parser.add_argument('--domain', action='store', type=str, nargs='?',
-                        help='''The domain to prepend
-                        [eg. https://$domain.eyeofiles.com]''')
-    parser.add_argument('--website', action='store', type=str, nargs='?',
-                        help='The name of the website [e.g. help.eyeo.com]')
-    args = parser.parse_args()
-    hash = args.hash
-    domain = args.domain
-    if args.url:
-        url_file = '{0}/{1}.tar.gz'.format(args.url, hash)
-        url_md5 = '{0}/{1}.md5'.format(args.url, hash)
-    else:
-        url_file = 'https://{0}.eyeofiles.com/{1}.tar.gz'.format(domain, hash)

[Vasily Kuznetsov, 2018/05/15 18:09:37]

What if --domain is not provided?

[f.lopez, 2018/05/21 22:23:41]

You are right, I think it's better if I only provide an argument called 'source'
so instead of having a magical conditional you need to always provide the source
of the files, this way there is no misunderstanding on what's happening behind
the scenes.

[Vasily Kuznetsov, 2018/05/22 16:38:36]

Acknowledged.

-        url_md5 = 'https://{0}.eyeofiles.com/{1}.md5'.format(domain, hash)
-    down_file = download(url_file)
-    down_md5 = download(url_md5)
-    if calculate_md5(down_file) == read_md5(down_md5):
-        tar_directory = untar(down_file)
-        hash_directory = os.path.join(tar_directory, hash)
-        destination = '/var/www/' + args.website
-        dcmp = dircmp(destination, hash_directory)
-        deploy_files(dcmp)
-        clean(hash)

[Vasily Kuznetsov, 2018/05/15 18:09:37]

`clean()` won't be called in cases of errors -- is this intentional?

[f.lopez, 2018/05/21 22:23:42]

Acknowledged.

-    else:
-        sys.exit("Hashes don't match")
+    parser.add_argument('--name', action='store', type=str, required=True,
+                        help='Name of the tarball to deploy')
+    parser.add_argument('--source', action='store', type=str, required=True,
+                        help='The source where files will be downloaded')
+    arguments = parser.parse_args()
+    name = arguments.name
+    source = arguments.source
+    url_file = '{0}/{1}.tar.gz'.format(source, name)
+    url_md5 = '{0}/{1}.md5'.format(source, name)
+    temporary_directory = tempfile.mkdtemp()
+    try:
+        downloaded_file = download(url_file, temporary_directory)
+        downloaded_md5 = download(url_md5, temporary_directory)
+        if calculate_md5(downloaded_file) == read_md5(downloaded_md5):
+            untar(downloaded_file, temporary_directory)
+            tarball_directory = os.path.join(temporary_directory, name)
+            destination = os.path.join('/var/www/', website)
+            directory_comparison = dircmp(destination, tarball_directory)
+            print 'Deploying files'
+            deploy_files(directory_comparison)
+        else:
+            error_message = """{0}.tar.gz md5 computation doesn't match {0}.md5
+                            contents""".format(name)
+            sys.exit(error_message)
+    except Exception as error:
+        sys.exit(error)
+    finally:
+        shutil.rmtree(temporary_directory)