Issue 5250 - Add copyright update script

update-copyright/update_copyright.py

 #!/usr/bin/env python3
 
 import os
-import sys
 import subprocess
 import re
 import datetime
 import shutil
 import urllib.parse
 import urllib.request
 import html.parser
 import argparse
-from posixpath import dirname
 
 
 CURRENT_YEAR = datetime.datetime.now().year
 
 
 def process_repo(url, hg_upstream):
     repo = url.rstrip('/').split('/')[-1]
 
     if repo in {
         # headers are copied from libadblockplus, no need to update seperately
@@ skipping 15 matching lines @@
             subprocess.call(['hg', 'up', '--rev', 'master',
                              '--repository', repo])
         for dirpath, dirnames, filenames in os.walk(repo):
             if dirpath == repo:
                 dirnames.remove('.hg')
 
             for filename in filenames:
                 text_replace(dirpath, filename)
         if hg_upstream is None:
             hg_upstream = url
+        else:
+            hg_upstream += '/' + repo
         hg_commit(repo, hg_upstream)
 
     finally:
         shutil.rmtree(repo, ignore_errors=True)
 
 
 def text_replace(dirpath, filename):
     with open(os.path.join(dirpath, filename), 'r+',
               encoding='utf-8', newline='') as file:
         try:
             text = file.read()
         except UnicodeDecodeError:
-            print("Error: Couldn't read {}{}".format(dirpath, filename))
             return
 
         text = re.sub(
             r'(copyright.*?\d{4})(?:-\d{4})?\s+eyeo gmbh',
             r'\1-{} eyeo GmbH'.format(CURRENT_YEAR), text, 0, re.I
         )
         file.seek(0)
         file.write(text)
         file.truncate()
 
 
 def hg_commit(repo, hg_upstream):
     try:
         subprocess.check_call(['hg', 'commit', '-m',
                                'Noissue - Updated copyright year',
                                '--repository', repo])
     except subprocess.CalledProcessError as e:
         if e.returncode == 1:  # no changes
             return
         raise
 
     # Push changes, or save patch if access denied
-    if 'ssh://hg@hg.adblockplus.org/' in hg_upstream:

[Vasily Kuznetsov, 2017/07/03 19:25:47]

Here we still have hardcoded logic related to hg.adblockplus.org. I think a
better way could be to remove this `if` here and add an `else` branch to the
`if` in lines 47-48 so that if `hg_upstream` is given by an option, we consider
it a base url and append the repository name to it. Something like:

    if hg_upstream is None:
        hg_upstream = url
    else:
        hg_upstream += '/' + repo

Seems like it would cover our use case and also work correctly with other
possible values of the -p option.

[rosie, 2017/07/04 13:38:25]

Done.

-        hg_upstream += repo
     if subprocess.call(['hg', 'push', '--repository', repo, hg_upstream]) != 0:
         with open(repo + '.patch', 'wb') as file:
             print('couldnt push, making patch instead')
             subprocess.check_call(['hg', 'export', '--repository', repo],
                                   stdout=file)
 
 
 class Parser(html.parser.HTMLParser):
     result = []
     recordtr = False
@@ skipping 13 matching lines @@
     def handle_endtag(self, tag):
         if tag == 'tr':
             self.recordtr = False
             self.cell = 0
 
     def handle_data(self, data):
         if self.cell == 2 and self.recordtr is True:
             self.recordtr = False
             self.cell = 0
             # Only process the URL if the description is not Deprecated
-            deprecated = (re.search(r'\*DEPRECATED\*', data) or
-                          re.search(r'(Deprecated)', data))
-            if not deprecated and len(self.current_url) > 2:
+            if ('*DEPRECATED*' not in data and
+                    '(Deprecated)' not in data and
+                    len(self.current_url) > 2):
                 self.result += [self.current_url]
                 return self.result
 
 
 def extract_urls(hg_page):
-    base_url = dirname(hg_page) + '/'

[Vasily Kuznetsov, 2017/07/03 19:25:47]

This will probably still break on some non-POSIX systems (like Windows), but I
guess it's ok, many other things wouldn't work either and we can probably live
with this script being POSIX-only.

[rosie, 2017/07/04 13:38:26]

Acknowledged.

+    base_url = os.path.dirname(hg_page) + '/'
     parser = Parser()
     with urllib.request.urlopen(hg_page) as response:
         parser.feed(response.read().decode('utf-8'))
     parser.close()
     repo_urls = []
     for url in parser.result:
         repo_urls.append(urllib.parse.urljoin(base_url, url))
     return repo_urls
 
 
 def main(hg_page, hg_upstream):
     for repo in extract_urls(hg_page):
         process_repo(repo, hg_upstream)
 
 
 if __name__ == '__main__':
     arg_parser = argparse.ArgumentParser()
     arg_parser.add_argument('-u', '--hg-url',
-                            default=None,
                             help='specify which Mercurial URL site to scrape')
     arg_parser.add_argument('-p', '--push-url',
-                            default=None,
                             help='specify where to push the repository')
     args = arg_parser.parse_args()
-    if args.hg_url is None:

[Vasily Kuznetsov, 2017/07/03 19:25:47]

Actually this is not necessary. If you make this option mandatory (check
argparse documentation), argparse will signal errors for you.

[rosie, 2017/07/04 13:38:25]

Done.

-        arg_parser.error('-u HG_URL was not specified')
-        sys.exit(2)
-    hg_page = args.hg_url

[Vasily Kuznetsov, 2017/07/03 19:25:47]

Do you think these intermediate variables add value? I would probably remove
them (and call `main` with `args.hg_url` and `args.push_url` directly), but I
don't feel very strongly about it so whatever seems better to you is cool.

[rosie, 2017/07/04 13:38:25]

Done.

-    hg_upstream = args.push_url
-    main(hg_page, hg_upstream)
+    if args.hg_url and not args.push_url:
+        arg_parser.error('If -u is provided, -p is mandatory')
+    main(args.hg_url or 'https://hg.adblockplus.org/',
+         args.push_url or 'ssh://hg@hg.adblockplus.org/')