Issue 5250 - Add copyright update script

update-copyright/update_copyright.py

+#!/usr/bin/env python3

[Sebastian Noack, 2017/07/05 15:46:53]

I suppose we should make this script executable (i.e. the mode above should be
755 instead of 644). Note that if the file is not executable, the shebang (i.e.
#!) here would be useless.

Once you made the script executable, you might also want to adapt the README to
instruct calling ./update_copyright.py instead of "python3 update_copyright.py",
which as a side effect removes some duplication.

[rosie, 2017/07/07 15:55:48]

Done.

+
+import os
+import subprocess
+import re
+import datetime
+import shutil
+import urllib.parse
+import urllib.request
+import html.parser
+import argparse
+
+
+CURRENT_YEAR = datetime.datetime.now().year
+
+
+def process_repo(url, hg_upstream):
+    repo = url.rstrip('/').split('/')[-1]
+
+    if repo in {
+        # headers are copied from libadblockplus, no need to update seperately
+        'libadblockplus-binaries',
+        # huge and only contains autogenerated builds
+        'downloads',
+    }:
+        return
+
+    try:
+        subprocess.check_call(['hg', 'clone', url, repo])
+        if repo == 'adblockbrowser':
+            # adblockbrowser is a FF fork with its own changes in a
+            #   seperate branch
+            subprocess.check_call(['hg', 'up', '--rev', 'adblockbrowser',
+                                   '--repository', repo])
+        else:
+            # switch to 'master' bookmark if it exists
+            subprocess.call(['hg', 'up', '--rev', 'master',
+                             '--repository', repo])
+        for dirpath, dirnames, filenames in os.walk(repo):
+            if dirpath == repo:
+                dirnames.remove('.hg')
+
+            for filename in filenames:
+                text_replace(dirpath, filename)
+        if hg_upstream is None:
+            hg_upstream = url
+        else:
+            hg_upstream += '/' + repo
+        hg_commit(repo, hg_upstream)
+
+    finally:
+        shutil.rmtree(repo, ignore_errors=True)
+
+
+def text_replace(dirpath, filename):
+    with open(os.path.join(dirpath, filename), 'r+',
+              encoding='utf-8', newline='') as file:
+        try:
+            text = file.read()
+        except UnicodeDecodeError:
+            print("Error: Couldn't read {}{}".format(dirpath, filename))

[Sebastian Noack, 2017/07/05 15:46:53]

Failing silently (in the original code) was intended here, as this script is
expected to encounter many binary files (e.g. images). If we output an error
message every time, this will generate a huge amount of noise.

[rosie, 2017/07/07 15:55:49]

Done.

+            return
+
+        text = re.sub(
+            r'(copyright.*?\d{4})(?:-\d{4})?\s+eyeo gmbh',
+            r'\1-{} eyeo GmbH'.format(CURRENT_YEAR), text, 0, re.I
+        )
+        file.seek(0)
+        file.write(text)
+        file.truncate()
+
+
+def hg_commit(repo, hg_upstream):
+    try:
+        subprocess.check_call(['hg', 'commit', '-m',
+                               'Noissue - Updated copyright year',
+                               '--repository', repo])
+    except subprocess.CalledProcessError as e:
+        if e.returncode == 1:  # no changes
+            return
+        raise
+
+    # Push changes, or save patch if access denied
+    if subprocess.call(['hg', 'push', '--repository', repo, hg_upstream]) != 0:
+        with open(repo + '.patch', 'wb') as file:
+            print('couldnt push, making patch instead')
+            subprocess.check_call(['hg', 'export', '--repository', repo],
+                                  stdout=file)
+
+
+class Parser(html.parser.HTMLParser):
+    result = []
+    recordtr = False
+    cell = 0
+    current_url = ''
+
+    def handle_starttag(self, tag, attrs):
+        if tag == 'tr':
+            self.recordtr = True
+        if tag == 'td':
+            self.cell += 1
+        if tag == 'a':
+            attrs = dict(attrs)
+            if 'list' in attrs.get('class', '').split():
+                self.current_url = attrs['href']
+
+    def handle_endtag(self, tag):
+        if tag == 'tr':
+            self.recordtr = False
+            self.cell = 0
+
+    def handle_data(self, data):
+        if self.cell == 2 and self.recordtr is True:
+            self.recordtr = False
+            self.cell = 0
+            # Only process the URL if the description is not Deprecated
+            deprecated = (re.search(r'\*DEPRECATED\*', data) or
+                          re.search(r'(Deprecated)', data))

[Sebastian Noack, 2017/07/05 15:46:53]

This regular expression seems incorrect. The parentheses annotate a group. So it
won't only match substrings like "(Deprecated)" but any occurrence of
"Deprecated", storing the result in the 1st group of the match (which is
discarded here). If you want to match parantheses in the input string, you have
to escape them in the pattern:

  re.search(r'\(Deprecated\)', data)

Anyway, if all you want to do is to find a fixed substring, it would be much
simpler (and less error-prone) to just use the in-operator (instead of regular
expressions):

  '*DEPRECATED*' in data or '(Deprecated)' in data

[rosie, 2017/07/07 15:55:48]

Done.

+            if not deprecated and len(self.current_url) > 2:
+                self.result += [self.current_url]
+                return self.result
+
+
+def extract_urls(hg_page):
+    base_url = os.path.dirname(hg_page) + '/'
+    parser = Parser()
+    with urllib.request.urlopen(hg_page) as response:
+        parser.feed(response.read().decode('utf-8'))
+    parser.close()
+    repo_urls = []
+    for url in parser.result:
+        repo_urls.append(urllib.parse.urljoin(base_url, url))
+    return repo_urls
+
+
+def main(hg_page, hg_upstream):
+    for repo in extract_urls(hg_page):
+        process_repo(repo, hg_upstream)
+
+
+if __name__ == '__main__':
+    arg_parser = argparse.ArgumentParser()
+    arg_parser.add_argument('-u', '--hg-url',
+                            help='specify which Mercurial URL site to scrape',
+                            required=True)
+    arg_parser.add_argument('-p', '--push-url',
+                            default=None,
+                            help='specify where to push the repository')
+    args = arg_parser.parse_args()
+    main(args.hg_url, args.push_url)