Issue 5250 - Add copyright update script

update-copyright/update_copyright/update_copyright.py

+#!/usr/bin/env python3

[Vasily Kuznetsov, 2017/06/19 16:25:31]

Maybe we should move this file to `update-copyright/update_copyright.py` instead
and remove `update-copyright/update_copyright/__init__.py`?

[rosie, 2017/06/23 14:39:25]

Done.

+
+import os
+import subprocess
+import re
+import datetime
+import shutil
+import urllib.parse
+import urllib.request
+import html.parser
+import argparse
+from posixpath import dirname
+
+
+CURRENT_YEAR = datetime.datetime.now().year
+
+
+def process_repo(url, hg_upstream):
+    repo = os.path.basename(os.path.normpath(url))
+    if repo in {

[Vasily Kuznetsov, 2017/06/19 16:25:31]

This should also not be hardcoded, but it's ok for now, let's land this first
and then deal with these special cases.

[rosie, 2017/06/23 14:39:25]

Acknowledged.

+        # headers are copied from libadblockplus, no need to update seperately
+        'libadblockplus-binaries',
+        # huge and only contains autogenerated builds
+        'downloads',
+    }:
+        return
+
+    try:
+        subprocess.check_call(['hg', 'clone', url, repo])
+        if repo == 'adblockbrowser':
+            # adblockbrowser is a FF fork with its own changes in a
+            #   seperate branch
+            subprocess.check_call(['hg', 'up', '--rev', 'adblockbrowser',

[Vasily Kuznetsov, 2017/06/19 16:25:31]

This should not be hardcoded, but it's ok for now.

[rosie, 2017/06/23 14:39:25]

Acknowledged.

+                                   '--repository', repo])
+        else:
+            # switch to 'master' bookmark if it exists
+            subprocess.call(['hg', 'up', '--rev', 'master',
+                             '--repository', repo])
+        for dirpath, dirnames, filenames in os.walk(repo):
+            if dirpath == repo:
+                dirnames.remove('.hg')
+
+            for filename in filenames:
+                text_replace(dirpath, filename)
+        hg_commit(repo, url)
+
+    finally:
+        shutil.rmtree(repo, ignore_errors=True)
+
+
+def text_replace(dirpath, filename):
+    with open(os.path.join(dirpath, filename), 'r+',
+              encoding='utf-8', newline='') as file:
+        try:
+            text = file.read()
+        except UnicodeDecodeError:
+            print("Error: Couldn't read {}{}".format(dirpath, filename))
+            return
+
+        text = re.sub(
+            r'(copyright.*?\d{4})(?:-\d{4})?\s+eyeo gmbh',
+            r'\1-{} eyeo GmbH'.format(CURRENT_YEAR), text, 0, re.I
+        )
+        file.seek(0)
+        file.write(text)
+        file.truncate()
+
+
+def hg_commit(repo, hg_upstream):
+    try:
+        subprocess.check_call(['hg', 'commit', '-m',
+                               'Noissue - Updated copyright year',
+                               '--repository', repo])
+    except subprocess.CalledProcessError as e:
+        if e.returncode == 1:  # no changes
+            return
+        raise
+
+    # Push changes, or save patch if access denied
+    if 'ssh://hg@hg.adblockplus.org/' in hg_upstream:
+        hg_upstream += repo
+    if subprocess.call(['hg', 'push', '--repository', repo, hg_upstream]) != 0:
+        with open(repo + '.patch', 'wb') as file:
+            print('couldnt push, making patch instead')
+            subprocess.check_call(['hg', 'export', '--repository', repo],
+                                  stdout=file)
+
+
+class Parser(html.parser.HTMLParser):
+    result = []
+    recordtr = False
+    cell = 0
+    current_url = ''
+    dir_path = 'https://hg.adblockplus.org/'
+
+    def handle_starttag(self, tag, attrs):
+        if tag == 'tr':
+            self.recordtr = True
+        if tag == 'td':
+            self.cell += 1
+        if tag == 'a':
+            attrs = dict(attrs)
+            if 'list' in attrs.get('class', '').split():
+                self.current_url = attrs['href']
+
+    def handle_endtag(self, tag):
+        if tag == 'tr':
+            self.recordtr = False
+            self.cell = 0
+
+    def handle_data(self, data):
+        if (self.cell == 2) and self.recordtr is True:
+            self.recordtr = False
+            self.cell = 0
+            # Only process the URL if the description is not Deprecated
+            depr1 = re.search(r'\*DEPRECATED\*', data)
+            depr2 = re.search(r'(Deprecated)', data)
+            if not depr1 and not depr2 and len(self.current_url) > 2:
+                self.result += [self.current_url]
+                return self.result
+
+
+def extract_urls(hg_page):
+    dir_path = dirname(hg_page) + '/'
+    parser = Parser()
+    with urllib.request.urlopen(hg_page) as response:
+        parser.feed(response.read().decode('utf-8'))
+    parser.close()
+    my_result = []
+    for i in range(len(parser.result)):
+        my_result.append(urllib.parse.urljoin(dir_path, parser.result[i]))
+    return my_result
+
+
+def main(hg_page, hg_upstream):
+    repo_list = extract_urls(hg_page)
+    for repo in repo_list:
+        process_repo(repo, hg_upstream)
+
+
+if __name__ == '__main__':
+    arg_parser = argparse.ArgumentParser()
+    arg_parser.add_argument('-u', '--url',
+                            default='https://hg.adblockplus.org/',
+                            help='specify which Mercurial URL site to scrape')
+    arg_parser.add_argument('-p', '--push',
+                            default='ssh://hg@hg.adblockplus.org/',
+                            help='specify where to push the repository')
+    args = arg_parser.parse_args()
+    hg_page = args.url
+    hg_upstream = args.push
+    main(hg_page, hg_upstream)