Left: | ||
Right: |
OLD | NEW |
---|---|
(Empty) | |
1 # coding: utf-8 | |
Wladimir Palant
2013/06/24 12:35:10
Probably better to have that script under manageme
Thomas Greiner
2013/06/24 14:24:45
Done.
| |
2 | |
3 # This file is part of the Adblock Plus web scripts, | |
4 # Copyright (C) 2006-2013 Eyeo GmbH | |
5 # | |
6 # Adblock Plus is free software: you can redistribute it and/or modify | |
7 # it under the terms of the GNU General Public License version 3 as | |
8 # published by the Free Software Foundation. | |
9 # | |
10 # Adblock Plus is distributed in the hope that it will be useful, | |
11 # but WITHOUT ANY WARRANTY; without even the implied warranty of | |
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
13 # GNU General Public License for more details. | |
14 # | |
15 # You should have received a copy of the GNU General Public License | |
16 # along with Adblock Plus. If not, see <http://www.gnu.org/licenses/>. | |
17 | |
18 import os, subprocess, codecs, urllib, re | |
19 from sitescripts.utils import get_config, setupStderr | |
20 from tempfile import mkdtemp | |
21 from shutil import rmtree | |
22 | |
23 def resolveRepositoryPath(repositories, path): | |
24 repoName = path | |
25 if path.find(':') >= 0: | |
26 repoName, target = path.split(':', 1) | |
27 if not repoName in repositories: | |
28 raise Exception('Cannot resolve path to repository "%s", this repository is unknown' % repoName) | |
29 | |
30 repoPath = repositories[repoName] | |
31 relTargetDir = '' | |
32 if not repoName == path: | |
33 targetDir = os.path.join(repoPath, target) | |
34 relTargetDir = os.path.relpath(targetDir, repoPath) | |
35 | |
36 return repoPath, relTargetDir | |
37 | |
38 | |
39 def updateExternalFilterlists(repositories): | |
40 settings = readSettings() | |
41 for name, setting in settings.iteritems(): | |
42 tempdir = mkdtemp(prefix='adblockplus') | |
Wladimir Palant
2013/06/24 12:35:10
A more generic specific prefix might be better her
Thomas Greiner
2013/06/24 14:24:45
Done.
| |
43 repoPath, targetDir = resolveRepositoryPath(repositories, setting['target']) | |
44 try: | |
Wladimir Palant
2013/06/24 12:35:10
This try statement should immediately follow mkdte
Thomas Greiner
2013/06/24 14:24:45
Done.
| |
45 subprocess.Popen(['hg', 'clone', '-U', repoPath, tempdir], stdout=subproce ss.PIPE).communicate() | |
46 subprocess.Popen(['hg', 'up', '-R', tempdir, '-r', 'default'], stdout=subp rocess.PIPE).communicate() | |
47 | |
48 path = os.path.join(tempdir, targetDir) | |
49 if not os.path.exists(path): | |
50 os.makedirs(path) | |
51 | |
52 filename = name + '.txt' | |
Wladimir Palant
2013/06/24 12:35:10
The settings should really specify the full file n
Thomas Greiner
2013/06/24 14:24:45
Done.
| |
53 path = os.path.join(path, filename) | |
54 exists = os.path.exists(path) | |
55 file = codecs.open(path, 'wb', encoding='utf-8') | |
56 data = urllib.urlopen(setting['source']).read() | |
57 for line in str(data).splitlines(): | |
Wladimir Palant
2013/06/24 12:35:10
Why split lines and write/decode each line separat
Thomas Greiner
2013/06/24 14:24:45
Done.
| |
58 if not line: | |
59 continue | |
60 print >>file, line.strip().decode('iso-8859-1') | |
Wladimir Palant
2013/06/24 12:35:10
The source file encoding should always be UTF-8, d
Thomas Greiner
2013/06/24 14:24:45
Done.
| |
61 file.close(); | |
Wladimir Palant
2013/06/24 12:35:10
No semicolon please.
Thomas Greiner
2013/06/24 14:24:45
Done. :)
| |
62 | |
63 message = 'Updated copy of external filterlist %s' | |
64 if not exists: | |
65 subprocess.Popen(['hg', 'add', '-R', tempdir], stdout=subprocess.PIPE).c ommunicate() | |
Wladimir Palant
2013/06/24 12:35:10
No need to add the file explicitly, use hg commit
Thomas Greiner
2013/06/24 14:24:45
Done.
| |
66 message = 'Added copy of external filterlist %s' | |
67 subprocess.Popen(['hg', 'commit', '-R', tempdir, '-u', 'hgbot', '-m', mess age % filename], stdout=subprocess.PIPE).communicate() | |
68 subprocess.Popen(['hg', 'push', '-R', tempdir], stdout=subprocess.PIPE).co mmunicate() | |
Wladimir Palant
2013/06/24 12:35:10
Please use -q command line parameter for all Mercu
Thomas Greiner
2013/06/24 14:24:45
Done.
| |
69 finally: | |
70 rmtree(tempdir) | |
71 | |
72 def readSettings(): | |
73 result = {} | |
74 for option in get_config().options('externalFilterlists'): | |
Wladimir Palant
2013/06/24 12:35:10
Better:
for option, value in get_config().items('
Thomas Greiner
2013/06/24 14:24:45
Done.
| |
75 if option.find('_') < 0: | |
76 continue | |
77 name, setting = option.rsplit('_', 2) | |
78 if not setting in ('source', 'target'): | |
79 continue | |
80 | |
81 if not name in result: | |
82 result[name] = { | |
83 'source': None, | |
84 'target': None | |
85 } | |
86 if isinstance(result[name][setting], list): | |
Wladimir Palant
2013/06/24 12:35:10
What is that check for? I think we want exactly on
Thomas Greiner
2013/06/24 14:24:45
Done.
| |
87 result[name][setting] = get_config().get('externalFilterlists', option).sp lit(' ') | |
88 else: | |
89 result[name][setting] = get_config().get('externalFilterlists', option) | |
90 return result | |
91 | |
92 if __name__ == '__main__': | |
93 setupStderr() | |
94 | |
95 repositories = {} | |
96 for option, value in get_config().items('subscriptionDownloads'): | |
97 if option.endswith('_repository'): | |
98 repositories[re.sub(r'_repository$', '', option)] = value | |
Wladimir Palant
2013/06/24 12:35:10
This script shouldn't really combine settings from
Thomas Greiner
2013/06/24 14:24:45
Done.
| |
99 | |
100 updateExternalFilterlists(repositories) | |
OLD | NEW |