Issue 5751 - Removing legacy gecko support

localeTools.py

 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import re
 import os
 import sys
 import codecs
 import json
 import urlparse
 import urllib
 import urllib2
 import mimetypes
 from StringIO import StringIO
 from ConfigParser import SafeConfigParser
 from zipfile import ZipFile
 from xml.parsers.expat import ParserCreate, XML_PARAM_ENTITY_PARSING_ALWAYS
 
 CROWDIN_AP_URL = 'https://api.crowdin.com/api/project'
 
 CROWDIN_LANG_MAPPING = {

[tlucas, 2017/10/04 11:48:39]

Note: result of rebasing

     'br': 'br-FR',
     'dsb': 'dsb-DE',
     'es': 'es-ES',
     'fur': 'fur-IT',
     'fy': 'fy-NL',
     'ga': 'ga-IE',
     'gu': 'gu-IN',
     'hsb': 'hsb-DE',
     'hy': 'hy-AM',
     'ml': 'ml-IN',
@@ skipping 23 matching lines @@
     except urllib2.HTTPError as e:
         raise Exception('Server returned HTTP Error {}:\n{}'.format(e.code,
                                                                     e.read()))
 
     if not raw:
         return json.loads(result)
 
     return result
 
 
-class OrderedDict(dict):
-    def __init__(self):
-        self.__order = []
-
-    def __setitem__(self, key, value):
-        self.__order.append(key)
-        dict.__setitem__(self, key, value)
-
-    def iteritems(self):
-        done = set()
-        for key in self.__order:
-            if not key in done and key in self:
-                yield (key, self[key])
-                done.add(key)
-
-
-def escapeEntity(value):
-    return value.replace('&', '&amp;').replace('<', '&lt;').replace('>', '&gt;').replace('"', '&quot;')
-
-
-def unescapeEntity(value):
-    return value.replace('&amp;', '&').replace('&lt;', '<').replace('&gt;', '>').replace('&quot;', '"')
-
-
-def parseDTDString(data, path):
-    result = []
-    currentComment = [None]
-
-    parser = ParserCreate()
-    parser.UseForeignDTD(True)
-    parser.SetParamEntityParsing(XML_PARAM_ENTITY_PARSING_ALWAYS)
-
-    def ExternalEntityRefHandler(context, base, systemId, publicId):
-        subparser = parser.ExternalEntityParserCreate(context, 'utf-8')
-        subparser.Parse(data.encode('utf-8'), True)
-        return 1
-
-    def CommentHandler(data):
-        currentComment[0] = data.strip()
-
-    def EntityDeclHandler(entityName, is_parameter_entity, value, base, systemId, publicId, notationName):
-        result.append((unescapeEntity(entityName), currentComment[0], unescapeEntity(value.strip())))
-        currentComment[0] = None
-
-    parser.ExternalEntityRefHandler = ExternalEntityRefHandler
-    parser.CommentHandler = CommentHandler
-    parser.EntityDeclHandler = EntityDeclHandler
-    parser.Parse('<!DOCTYPE root SYSTEM "foo"><root/>', True)
-
-    for entry in result:
-        yield entry
-
-
-def escapeProperty(value):
-    return value.replace('\n', '\\n')
-
-
-def unescapeProperty(value):
-    return value.replace('\\n', '\n')
-
-
-def parsePropertiesString(data, path):
-    currentComment = None
-    for line in data.splitlines():
-        match = re.search(r'^\s*[#!]\s*(.*)', line)
-        if match:
-            currentComment = match.group(1)
-        elif '=' in line:
-            key, value = line.split('=', 1)
-            yield (unescapeProperty(key), currentComment, unescapeProperty(value))
-            currentComment = None
-        elif re.search(r'\S', line):
-            print >>sys.stderr, 'Unrecognized data in file %s: %s' % (path, line)
-
-
-def parseString(data, path):
-    result = {'_origData': data}
-    if path.endswith('.dtd'):
-        it = parseDTDString(data, path)
-    elif path.endswith('.properties'):
-        it = parsePropertiesString(data, path)
-    else:
-        return None
-
-    for name, comment, value in it:
-        result[name] = value
-    return result
-
-
-def readFile(path):
-    fileHandle = codecs.open(path, 'rb', encoding='utf-8')
-    data = fileHandle.read()
-    fileHandle.close()
-    return parseString(data, path)
-
-
-def generateStringEntry(key, value, path):
-    if path.endswith('.dtd'):
-        return '<!ENTITY %s "%s">\n' % (escapeEntity(key), escapeEntity(value))
-    else:
-        return '%s=%s\n' % (escapeProperty(key), escapeProperty(value))
-
-
-def toJSON(path):
-    fileHandle = codecs.open(path, 'rb', encoding='utf-8')
-    data = fileHandle.read()
-    fileHandle.close()
-
-    if path.endswith('.dtd'):
-        it = parseDTDString(data, path)
-    elif path.endswith('.properties'):
-        it = parsePropertiesString(data, path)
-    else:
-        return None
-
-    result = OrderedDict()
-    for name, comment, value in it:
-        obj = {'message': value}
-        if comment == None:
-            obj['description'] = name
-        else:
-            obj['description'] = '%s: %s' % (name, comment)
-        result[name] = obj
-    return json.dumps(result, ensure_ascii=False, indent=2)
-
-
-def fromJSON(path, data):
-    data = json.loads(data)
-    if not data:
-        if os.path.exists(path):
-            os.remove(path)
-        return
-
-    dir = os.path.dirname(path)
-    if not os.path.exists(dir):
-        os.makedirs(dir)
-    file = codecs.open(path, 'wb', encoding='utf-8')
-    for key, value in data.iteritems():
-        file.write(generateStringEntry(key, value['message'], path))
-    file.close()
-
-
 def preprocessChromeLocale(path, metadata, isMaster):
     fileHandle = codecs.open(path, 'rb', encoding='utf-8')
     data = json.load(fileHandle)
     fileHandle.close()
 
     for key, value in data.iteritems():
         if isMaster:
             # Make sure the key name is listed in the description
             if 'description' in value:
                 value['description'] = '%s: %s' % (key, value['description'])
@@ skipping 44 matching lines @@
         locales.add(match.group(1))
 
     # We don't translate indvidual dialects of languages
     # other than English, Spanish, Portuguese and Chinese.
     for locale in list(locales):
         prefix = locale.split('-')[0]
         if prefix not in {'en', 'es', 'pt', 'zh'}:
             locales.remove(locale)
             locales.add(prefix)
 
-    # Add languages with existing translations. 
+    # Add languages with existing translations.
     locales.update(localeConfig['locales'])
 
     # Don't add the language we translate from as target translation.
     locales.remove(localeConfig['default_locale'].replace('_', '-'))
 
     # Convert to locales understood by Crowdin.
     locales = {CROWDIN_LANG_MAPPING.get(locale, locale) for locale in locales}
     allowed = {locale['crowdin_code'] for locale in
                crowdin_request(projectName, 'supported-languages', key)}

[tlucas, 2017/10/04 11:48:39]

Note: result of rebasing

     if not allowed.issuperset(locales):
         print "Warning, following locales aren't allowed by server: " + ', '.join(locales - allowed)
 
     locales = sorted(locales & allowed)

[tlucas, 2017/10/04 11:48:40]

Note: result of rebasing

     params = urllib.urlencode([('languages[]', locale) for locale in locales])
     crowdin_request(projectName, 'edit-project', key, post_data=params)
 
 
 def crowdin_prepare_upload(files):
     """Create a post body and matching headers, which Crowdin can handle."""
     boundary = '----------ThIs_Is_tHe_bouNdaRY_$'
     body = ''
     for name, data in files:
         body += (
@@ skipping 20 matching lines @@
 
 def updateTranslationMaster(localeConfig, metadata, dir, projectName, key):
     result = crowdin_request(projectName, 'info', key)
 
     existing = set(map(lambda f: f['name'], result['files']))
     add = []
     update = []
     for file in os.listdir(dir):
         path = os.path.join(dir, file)
         if os.path.isfile(path):
-            if localeConfig['file_format'] == 'chrome-json' and file.endswith('.json'):
+            if file.endswith('.json'):
                 data = preprocessChromeLocale(path, metadata, True)
                 newName = file
-            elif localeConfig['file_format'] == 'chrome-json':
+            else:
                 fileHandle = codecs.open(path, 'rb', encoding='utf-8')
                 data = json.dumps({file: {'message': fileHandle.read()}})
                 fileHandle.close()
                 newName = file + '.json'
-            else:
-                data = toJSON(path)
-                newName = file + '.json'
 
             if data:
                 if newName in existing:
                     update.append((newName, data))
                     existing.remove(newName)
                 else:
                     add.append((newName, data))
 
     if len(add):
         query = {'titles[{}]'.format(name): os.path.splitext(name)[0]
                  for name, _ in add}
         query['type'] = 'chrome'
         data, headers = crowdin_prepare_upload(add)
         crowdin_request(projectName, 'add-file', key, query, post_data=data,
                         headers=headers)
     if len(update):
         data, headers = crowdin_prepare_upload(update)
         crowdin_request(projectName, 'update-file', key, post_data=data,
                         headers=headers)
     for file in existing:
         crowdin_request(projectName, 'delete-file', key, {'file': file})
 
 
 def uploadTranslations(localeConfig, metadata, dir, locale, projectName, key):
     files = []
     for file in os.listdir(dir):
         path = os.path.join(dir, file)
         if os.path.isfile(path):
-            if localeConfig['file_format'] == 'chrome-json' and file.endswith('.json'):
+            if file.endswith('.json'):
                 data = preprocessChromeLocale(path, metadata, False)
                 newName = file
-            elif localeConfig['file_format'] == 'chrome-json':
+            else:
                 fileHandle = codecs.open(path, 'rb', encoding='utf-8')
                 data = json.dumps({file: {'message': fileHandle.read()}})
                 fileHandle.close()
                 newName = file + '.json'
-            else:
-                data = toJSON(path)
-                newName = file + '.json'
 
             if data:
                 files.append((newName, data))
     if len(files):
         language = CROWDIN_LANG_MAPPING.get(locale, locale)

[tlucas, 2017/10/04 11:48:39]

Note: result of rebasing

         data, headers = crowdin_prepare_upload(files)
         crowdin_request(projectName, 'upload-translation', key,
                         {'language': language}, post_data=data,
                         headers=headers)
 
 
 def getTranslations(localeConfig, projectName, key):
     """Download all available translations from crowdin.
 
     Trigger crowdin to build the available export, wait for crowdin to
     finish the job and download the generated zip afterwards.
     """
     crowdin_request(projectName, 'export', key)
 
     result = crowdin_request(projectName, 'download/all.zip', key, raw=True)
     zip = ZipFile(StringIO(result))
     dirs = {}
 
-    normalizedDefaultLocale = localeConfig['default_locale']
-    if localeConfig['name_format'] == 'ISO-15897':
-        normalizedDefaultLocale = normalizedDefaultLocale.replace('_', '-')
+    normalizedDefaultLocale = localeConfig['default_locale'].replace('_', '-')
     normalizedDefaultLocale = CROWDIN_LANG_MAPPING.get(normalizedDefaultLocale,
                                                        normalizedDefaultLocale)
 
     for info in zip.infolist():
         if not info.filename.endswith('.json'):
             continue
 
         dir, file = os.path.split(info.filename)
         if not re.match(r'^[\w\-]+$', dir) or dir == normalizedDefaultLocale:
             continue
-        if localeConfig['file_format'] == 'chrome-json' and file.count('.') == 1:
+        if file.count('.') == 1:
             origFile = file
         else:
-            origFile = re.sub(r'\.json$', '', file)
-        if (localeConfig['file_format'] == 'gecko-dtd' and
-            not origFile.endswith('.dtd') and
-            not origFile.endswith('.properties')):
-            continue
+            origFile = os.path.splitext(file)[0]
 
         for key, value in CROWDIN_LANG_MAPPING.iteritems():

[tlucas, 2017/10/04 11:48:39]

Note: result of rebasing

             if value == dir:
                 dir = key
-        if localeConfig['name_format'] == 'ISO-15897':
-            dir = dir.replace('-', '_')
 
         data = zip.open(info.filename).read()
         if data == '[]':
             continue
 
         if not dir in dirs:
             dirs[dir] = set()
         dirs[dir].add(origFile)
 
         path = os.path.join(localeConfig['base_path'], dir, origFile)
         if not os.path.exists(os.path.dirname(path)):
             os.makedirs(os.path.dirname(path))
-        if localeConfig['file_format'] == 'chrome-json' and file.endswith('.json'):
+        if file.endswith('.json'):
             postprocessChromeLocale(path, data)
-        elif localeConfig['file_format'] == 'chrome-json':
+        else:
             data = json.loads(data)
             if origFile in data:
                 fileHandle = codecs.open(path, 'wb', encoding='utf-8')
                 fileHandle.write(data[origFile]['message'])
                 fileHandle.close()
-        else:
-            fromJSON(path, data)
 
     # Remove any extra files
     for dir, files in dirs.iteritems():
         baseDir = os.path.join(localeConfig['base_path'], dir)
         if not os.path.exists(baseDir):
             continue
         for file in os.listdir(baseDir):
             path = os.path.join(baseDir, file)
-            if os.path.isfile(path) and (file.endswith('.json') or file.endswith('.properties') or file.endswith('.dtd')) and not file in files:
+            valid_extension = file.endswith('.json')
+            if os.path.isfile(path) and valid_extension and not file in files:
                 os.remove(path)