Side by Side Diff: publicSuffixListUpdater.py
Issue 29987555:
Issue 7122 - Purge "updatepsl" (Closed)
Patch Set:
Created Jan. 22, 2019, 12:56 p.m.
| Left: | ||
| Right: |
Use n/p to move between diff chunks;
N/P to move between comments.
« no previous file with comments
|
« build.py ('k')
|
no next file »
|
no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Hide Comments ('s')
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Hide Comments ('s')
| OLD | NEW |
|---|---|
| (Empty) | |
| 1 # This Source Code Form is subject to the terms of the Mozilla Public | |
| 2 # License, v. 2.0. If a copy of the MPL was not distributed with this | |
| 3 # file, You can obtain one at http://mozilla.org/MPL/2.0/. | |
| 4 | |
| 5 """ | |
| 6 Update the public suffix list | |
| 7 ============================== | |
| 8 | |
| 9 This script generates a js array of public suffixes (http://publicsuffix.org/) | |
| 10 """ | |
| 11 | |
| 12 import os | |
| 13 import urllib | |
| 14 import json | |
| 15 | |
| 16 | |
| 17 def urlopen(url, attempts=3): | |
| 18 """ | |
| 19 Tries to open a particular URL, retries on failure. | |
| 20 """ | |
| 21 for i in range(attempts): | |
| 22 try: | |
| 23 return urllib.urlopen(url) | |
| 24 except IOError as e: | |
| 25 error = e | |
| 26 time.sleep(5) | |
| 27 raise error | |
| 28 | |
| 29 | |
| 30 def getPublicSuffixList(): | |
| 31 """ | |
| 32 gets download link for a Gecko add-on from the Mozilla Addons site | |
| 33 """ | |
| 34 suffixes = {} | |
| 35 url = 'http://mxr.mozilla.org/mozilla-central/source/netwerk/dns/effective_t ld_names.dat?raw=1' | |
| 36 resource = urlopen(url) | |
| 37 | |
| 38 for line in resource: | |
| 39 line = line.decode('utf-8').rstrip() | |
| 40 if line.startswith('//') or '.' not in line: | |
| 41 continue | |
| 42 | |
| 43 if line.startswith('*.'): | |
| 44 offset = 2 | |
| 45 val = 2 | |
| 46 elif line.startswith('!'): | |
| 47 offset = 1 | |
| 48 val = 0 | |
| 49 else: | |
| 50 offset = 0 | |
| 51 val = 1 | |
| 52 | |
| 53 suffixes[line[offset:].encode('idna').decode('ascii')] = val | |
| 54 | |
| 55 return suffixes | |
| 56 | |
| 57 | |
| 58 def updatePSL(baseDir): | |
| 59 """ | |
| 60 writes the current public suffix list to js file in json format | |
| 61 """ | |
| 62 | |
| 63 psl = getPublicSuffixList() | |
| 64 file = open(os.path.join(baseDir, 'lib', 'publicSuffixList.js'), 'w') | |
| 65 print >>file, 'var publicSuffixes = ' + json.dumps(psl, sort_keys=True, inde nt=4, separators=(',', ': ')) + ';' | |
| 66 file.close() | |
| OLD | NEW |
