Modified server-side rules generation to produce prioritized domain lists instead of complete suffi…

updateRules.py

 #!/usr/bin/env python
 # coding: utf-8
  
 # This Source Code is subject to the terms of the Mozilla Public License
 # version 2.0 (the "License"). You can obtain a copy of the License at
 # http://mozilla.org/MPL/2.0/.
 
 """
 Update the dictionaries in the rules
 ====================================
@@ skipping 220 matching lines @@
   'alt.za', 'net.work.za', 'ngo.za', 'tm.za', 'web.za', 'bourse.za',
   'agric.za', 'cybernet.za', 'grondar.za', 'iaccess.za', 'inca.za', 'nis.za', 'olivetti.za', 'pix.za',
 
   # From http://en.wikipedia.org/wiki/.zm
   'ac.zm', 'co.zm', 'com.zm', 'edu.zm', 'gov.zm', 'net.zm', 'org.zm', 'sch.zm',
 
   # From http://en.wikipedia.org/wiki/.zw
   'co.zw', 'ac.zw', 'org.zw',
 ]
 
-def getSuffixes(target, items):
-  suffixes = {}
-  for item, priority in items.iteritems():
-    suffix = item[-1] if len(item) else ''
-    if not suffix in suffixes:
-      suffixes[suffix] = {}
-    suffixes[suffix][item[:-1]] = priority
-  for suffix, items in suffixes.iteritems():
-    if len(items.keys()) == 1:
-      item, priority = items.items()[0]
-      target[suffix] = ''.join(reversed(item)) + ' ' + str(priority)
-    else:
-      target[suffix] = {}
-      getSuffixes(target[suffix], items)
-
 def urlopen(url, attempts=3):
   """
   Tries to open a particular URL, retries on failure.
   """
   for i in range(attempts):
     try:
       return urllib.urlopen(url)
     except IOError, e:
       error = e
       time.sleep(5)
@@ skipping 18 matching lines @@
       tld = line
 
     if tld:
       yield tld
 
 def getTLDs(domains, tldPriority):
   for tld in iterateTLDs():
     if not tld in domains:
       domains[tld] = tldPriority
 
-def updateSchemes(rules):
-  rules['scheme'] = {}
-  getSuffixes(rules['scheme'], schemes)
-
 def updateDomains(rules):
   domains = {}
   reader = codecs.getreader('utf-8')(sys.stdin)
   i = 0
   for domain in itertools.chain(reader.readlines(), additionalDomains):
     domain = domain.rstrip()
     if not domain or domain in domains:
       continue
     domains[domain] = i
     i += 1
 
   maxPriority = i
   for domain in domains.iterkeys():
     domains[domain] = maxPriority - domains[domain]
 
   # Extract TLDs from domain list
   for domain, priority in domains.items():
     while True:
       if not re.search(r'^[^.]+\.+', domain):
         break
       domain = re.sub(r'^[^.]+\.+', '', domain)
       if not domain:
         break
       if not domain in domains or domains[domain] < priority - maxPriority:
         domains[domain] = priority - maxPriority
 
   # Fill up with "official" TLDs
   getTLDs(domains, -maxPriority)
 
-  rules['domain'] = {}
-  getSuffixes(rules['domain'], domains)
+  rules['domain'] = domains
 
 def writeRules(rules):
   path = os.path.join('defaults', 'rules.json')

[Thomas Greiner, 2012/09/25 12:31:37]

Do you really want the file to be located at /defaults/rules.json?

[Wladimir Palant, 2012/09/25 13:26:00]

Actually, it's defaults/rules.json (relative to the current work directory).
This is a bit ugly because it makes the assumption that the script will always
be run from the extension root but other than that it should be fine.

   file = codecs.open(path, 'rb', encoding='utf-8')
   data = file.read()
   file.close()
 
   marker = '// Automatically generated dictionaries'
   markerIndex = data.find(marker)
   if markerIndex < 0:
     raise Exception('Insertion marker not found in %s' % path)
   data = data[0:markerIndex + len(marker)] + '\n'
   data += '  ' + json.dumps(rules, ensure_ascii=False, sort_keys=True, separators = (',', ':'))[1:-1] + '\n}\n'
 
   file = codecs.open(path, 'wb', encoding='utf-8')
   file.write(data)
   file.close()
 
 def updateRules():
   rules = {}
   rules['domainReferrals'] = domainReferrals
-  updateSchemes(rules)
+  rules['scheme'] = schemes
   updateDomains(rules)
   writeRules(rules)
 
 if __name__ == "__main__":
   updateRules()