Issue 5751 - Removing legacy gecko support

localeTools.py

 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import re
 import os
 import sys
 import codecs
 import json
 import urlparse
@@ skipping 143 matching lines @@
 
 def unescapeEntity(value):
     return value.replace('&amp;', '&').replace('&lt;', '<').replace('&gt;', '>').replace('&quot;', '"')
 
 
 def mapLocale(type, locale):
     mapping = langMappingChrome if type == 'ISO-15897' else langMappingGecko
     return mapping.get(locale, locale)
 
 
-def parseDTDString(data, path):
-    result = []
-    currentComment = [None]
-
-    parser = ParserCreate()
-    parser.UseForeignDTD(True)
-    parser.SetParamEntityParsing(XML_PARAM_ENTITY_PARSING_ALWAYS)
-
-    def ExternalEntityRefHandler(context, base, systemId, publicId):
-        subparser = parser.ExternalEntityParserCreate(context, 'utf-8')
-        subparser.Parse(data.encode('utf-8'), True)
-        return 1
-
-    def CommentHandler(data):
-        currentComment[0] = data.strip()
-
-    def EntityDeclHandler(entityName, is_parameter_entity, value, base, systemId, publicId, notationName):
-        result.append((unescapeEntity(entityName), currentComment[0], unescapeEntity(value.strip())))
-        currentComment[0] = None
-
-    parser.ExternalEntityRefHandler = ExternalEntityRefHandler
-    parser.CommentHandler = CommentHandler
-    parser.EntityDeclHandler = EntityDeclHandler
-    parser.Parse('<!DOCTYPE root SYSTEM "foo"><root/>', True)
-
-    for entry in result:
-        yield entry
-
-
 def escapeProperty(value):
     return value.replace('\n', '\\n')
 
 
 def unescapeProperty(value):
     return value.replace('\\n', '\n')
 
 
 def parsePropertiesString(data, path):
     currentComment = None
     for line in data.splitlines():
         match = re.search(r'^\s*[#!]\s*(.*)', line)
         if match:
             currentComment = match.group(1)
         elif '=' in line:
             key, value = line.split('=', 1)
             yield (unescapeProperty(key), currentComment, unescapeProperty(value))
             currentComment = None
         elif re.search(r'\S', line):
             print >>sys.stderr, 'Unrecognized data in file %s: %s' % (path, line)
 
 
 def parseString(data, path):
     result = {'_origData': data}
-    if path.endswith('.dtd'):
-        it = parseDTDString(data, path)
-    elif path.endswith('.properties'):
+    if path.endswith('.properties'):

[Sebastian Noack, 2017/10/03 02:22:39]

Both, the .dtd and the .properties format are specific to legacy Gecko
extensions and can be removed now.

[tlucas, 2017/10/04 11:48:38]

Done.

         it = parsePropertiesString(data, path)
     else:
         return None
 
     for name, comment, value in it:
         result[name] = value
     return result
 
 
 def readFile(path):
     fileHandle = codecs.open(path, 'rb', encoding='utf-8')
     data = fileHandle.read()
     fileHandle.close()
     return parseString(data, path)
 
 
-def generateStringEntry(key, value, path):
-    if path.endswith('.dtd'):
-        return '<!ENTITY %s "%s">\n' % (escapeEntity(key), escapeEntity(value))
-    else:
-        return '%s=%s\n' % (escapeProperty(key), escapeProperty(value))
-
-
 def toJSON(path):
     fileHandle = codecs.open(path, 'rb', encoding='utf-8')
     data = fileHandle.read()
     fileHandle.close()
 
-    if path.endswith('.dtd'):
-        it = parseDTDString(data, path)
-    elif path.endswith('.properties'):
+    if path.endswith('.properties'):
         it = parsePropertiesString(data, path)
     else:
         return None
 
     result = OrderedDict()
     for name, comment, value in it:
         obj = {'message': value}
         if comment == None:
             obj['description'] = name
         else:
             obj['description'] = '%s: %s' % (name, comment)
         result[name] = obj
     return json.dumps(result, ensure_ascii=False, indent=2)
 
 
 def fromJSON(path, data):
     data = json.loads(data)
     if not data:
         if os.path.exists(path):
             os.remove(path)
         return
 
     dir = os.path.dirname(path)
     if not os.path.exists(dir):
         os.makedirs(dir)
     file = codecs.open(path, 'wb', encoding='utf-8')
     for key, value in data.iteritems():
-        file.write(generateStringEntry(key, value['message'], path))
+        file.write('{}={}\n'.format(escapeProperty(key),
+                                    escapeProperty(value['message'])))
     file.close()
 
 
 def preprocessChromeLocale(path, metadata, isMaster):
     fileHandle = codecs.open(path, 'rb', encoding='utf-8')
     data = json.load(fileHandle)
     fileHandle.close()
 
     for key, value in data.iteritems():
         if isMaster:
@@ skipping 28 matching lines @@
 def setupTranslations(localeConfig, projectName, key):
     # Make a new set from the locales list, mapping to Crowdin friendly format
     locales = {mapLocale(localeConfig['name_format'], locale)
                for locale in localeConfig['locales']}
 
     # Fill up with locales that we don't have but the browser supports
     if 'chrome' in localeConfig['target_platforms']:
         for locale in chromeLocales:
             locales.add(mapLocale('ISO-15897', locale))
 
-    if 'gecko' in localeConfig['target_platforms']:
+    if 'gecko-webext' in localeConfig['target_platforms']:
         firefoxLocales = urllib2.urlopen('http://www.mozilla.org/en-US/firefox/all.html').read()
         for match in re.finditer(r'&amp;lang=([\w\-]+)"', firefoxLocales):
             locales.add(mapLocale('BCP-47', match.group(1)))
         langPacks = urllib2.urlopen('https://addons.mozilla.org/en-US/firefox/language-tools/').read()
         for match in re.finditer(r'<tr>.*?</tr>', langPacks, re.S):
             if match.group(0).find('Install Language Pack') >= 0:
                 match2 = re.search(r'lang="([\w\-]+)"', match.group(0))
                 if match2:
                     locales.add(mapLocale('BCP-47', match2.group(1)))
 
@@ skipping 131 matching lines @@
         if not info.filename.endswith('.json'):
             continue
 
         dir, file = os.path.split(info.filename)
         if not re.match(r'^[\w\-]+$', dir) or dir == normalizedDefaultLocale:
             continue
         if localeConfig['file_format'] == 'chrome-json' and file.count('.') == 1:
             origFile = file
         else:
             origFile = re.sub(r'\.json$', '', file)
-        if (localeConfig['file_format'] == 'gecko-dtd' and
-            not origFile.endswith('.dtd') and
-            not origFile.endswith('.properties')):
-            continue
 
         if localeConfig['name_format'] == 'ISO-15897':
             mapping = langMappingChrome
         else:
             mapping = langMappingGecko
 
         for key, value in mapping.iteritems():
             if value == dir:
                 dir = key
         if localeConfig['name_format'] == 'ISO-15897':
@@ skipping 21 matching lines @@
         else:
             fromJSON(path, data)
 
     # Remove any extra files
     for dir, files in dirs.iteritems():
         baseDir = os.path.join(localeConfig['base_path'], dir)
         if not os.path.exists(baseDir):
             continue
         for file in os.listdir(baseDir):
             path = os.path.join(baseDir, file)
-            if os.path.isfile(path) and (file.endswith('.json') or file.endswith('.properties') or file.endswith('.dtd')) and not file in files:
+            valid_extension = os.path.splitext(file)[1] in {'.json',
+                                                            '.properties'}
+            if os.path.isfile(path) and valid_extension and not file in files:
                 os.remove(path)