Issue 7238 - Fix multi-locale builds

abb-build/build.py

Index: abb-build/build.py
===================================================================
--- a/abb-build/build.py
+++ b/abb-build/build.py
@@ -1,10 +1,9 @@
 #!/usr/bin/env python
-
 # This file is part of Adblock Plus <https://adblockplus.org/>,
 # Copyright (C) 2006-present eyeo GmbH
 #
 # Adblock Plus is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 3 as
 # published by the Free Software Foundation.
 #
 # Adblock Plus is distributed in the hope that it will be useful,
@@ -31,18 +30,22 @@
 _GENERATED_PATH = os.path.join(_BASE_DIR, "generated")
 _GENERATED_MOZCONFIG_PATH = os.path.join(_GENERATED_PATH, "mozconfig")
 _MOZCONFIG_COMMON_PATH = os.path.join(_BASE_DIR, "mozconfig-common")
 _MOZCONFIG_ARM_PATH = os.path.join(_BASE_DIR, "mozconfig-arm")
 _MOZCONFIG_X86_PATH = os.path.join(_BASE_DIR, "mozconfig-x86")
 _MOZCONFIG_STORE_PATH = os.path.join(_BASE_DIR, "mozconfig-store")
 _MOZCONFIG_RELEASE_PATH = os.path.join(_BASE_DIR, "mozconfig-release")
 _MOZCONFIG_CUSTOM_PATH = os.path.join(_BASE_DIR, "mozconfig-custom")
-_MULTI_L10N_PATH = os.path.join(_BASE_DIR, "mozharness", "scripts",
-                                "multil10n.py")
+_MAEMO_LOCALES_PATH = os.path.join(_ABB_PATH, "mobile", "android", "locales",
+                                   "maemo-locales")
+
+_L10N_BASE_PATH = os.path.join(_BASE_DIR, "l10n-central")
+_LOCALE_CHANGESETS_PATH = os.path.join(_ABB_PATH, "mobile", "locales",
+                                       "l10n-changesets.json")
 
 _CMD_BUILD = "build"
 _CMD_SIGN = "sign"
 _CMD_BUILD_SIGN = "build-and-sign"
 _ARCH_ARM = "arm"
 _ARCH_X86 = "x86"
 _ARCH_X86_I386 = "i386"
 _ARCHS = (_ARCH_ARM, _ARCH_X86)
@@ -51,122 +54,186 @@
 _DIST_MODES = (_DIST_STANDALONE, _DIST_STORE)
 _BUILD_DEVBUILD = "devbuild"
 _BUILD_RELEASE = "release"
 _BUILD_MODES = (_BUILD_DEVBUILD, _BUILD_RELEASE)
 _OBJDIR_ARM = "obj-arm-linux-androideabi"
 _OBJDIR_X86 = "obj-i386-linux-android"
 
 
-def print_usage():
+def _print_usage():
     print >>sys.stderr, string.Template("""\
 Usage: $name %s
        $name %s APK_PATH
        $name %s\
 """ % (_CMD_BUILD, _CMD_SIGN,
        _CMD_BUILD_SIGN)).substitute({"name": os.path.basename(sys.argv[0])})
 
 
 def _generate_mozconfig(architecture, distribution_mode, build_mode):
     if not os.path.exists(_GENERATED_PATH):
         os.makedirs(_GENERATED_PATH)
+
     with open(_GENERATED_MOZCONFIG_PATH, "w+") as mozconfig:
         mozconfig.write(". \"%s\"\n" % _MOZCONFIG_COMMON_PATH)
         if architecture == _ARCH_X86:
             mozconfig.write(". \"%s\"\n" % _MOZCONFIG_X86_PATH)
         else:
             mozconfig.write(". \"%s\"\n" % _MOZCONFIG_ARM_PATH)
         if distribution_mode == _DIST_STORE:
             mozconfig.write(". \"%s\"\n" % _MOZCONFIG_STORE_PATH)
         if build_mode == _BUILD_RELEASE:
             mozconfig.write(". \"%s\"\n" % _MOZCONFIG_RELEASE_PATH)
         mozconfig.write(". \"%s\"\n" % _MOZCONFIG_CUSTOM_PATH)
         return _GENERATED_MOZCONFIG_PATH
 
 
-def _build(architecture, distribution_mode, build_mode, sdk_path, ndk_path):
-    build_environment = os.environ.copy()
-    build_environment["ANDROID_SDK_PATH"] = sdk_path
-    build_environment["ANDROID_NDK_PATH"] = ndk_path
-    build_environment["MOZCONFIG"] = _generate_mozconfig(
-        architecture, distribution_mode, build_mode)
-    obj_dir = _OBJDIR_X86 if architecture == _ARCH_X86 else _OBJDIR_ARM
-    build_environment["OBJDIR"] = obj_dir
-    subprocess.check_call([os.path.join(_ABB_PATH, "mach"), "clobber"],
-                          env=build_environment)
-    subprocess.check_call([_MULTI_L10N_PATH, "--cfg", "adblockbrowser-cfg.py"],
-                          env=build_environment)
+def _checkout_l10n():
+    with open(_LOCALE_CHANGESETS_PATH, "r") as fd:
+        changesets = json.load(fd)
 
+    if not os.path.exists(_L10N_BASE_PATH):
+        os.makedirs(_L10N_BASE_PATH)
+
+    l10n_base_repo = "https://hg.mozilla.org/l10n-central/%s/"
+    for locale in changesets.keys():
+            revision = changesets[locale]["revision"]
+            repo_path = os.path.join(_L10N_BASE_PATH, locale)
+            if os.path.exists(repo_path):
+                update_cmd = ["hg", "up", "-r", revision]
+                if subprocess.call(update_cmd, cwd=repo_path):
+                    subprocess.check_call(["hg", "pull"], cwd=repo_path)
+                    subprocess.check_call(update_cmd, cwd=repo_path)
+
+            else:
+                repo = l10n_base_repo % locale
+                subprocess.check_call(["hg", "clone", repo, "-r", revision],
+                                      cwd=_L10N_BASE_PATH)
+
+
+def _read_locales():
+    with open(_MAEMO_LOCALES_PATH, "r") as fd:
+        lines = fd.readlines()
+
+    return [line.strip() for line in lines]
+
+
+def _config_build_env(architecture, distribution_mode, build_mode, sdk_path,
+                      ndk_path, obj_dir, locales):
+    build_env = os.environ.copy()
+    build_env["ANDROID_SDK_PATH"] = sdk_path
+    build_env["ANDROID_NDK_PATH"] = ndk_path
+    mozconfig_path = _generate_mozconfig(architecture, distribution_mode,
+                                         build_mode)
+
+    build_env["MOZCONFIG"] = mozconfig_path
+    build_env["OBJDIR"] = obj_dir
+    build_env["L10NBASEDIR"] = _L10N_BASE_PATH
+    build_env["MOZ_CHROME_MULTILOCALE"] = ' '.join(locales)
+    build_env["AB_CD"] = "multi"
+    return build_env
+
+
+def _create_target_apk(architecture, distribution_mode, build_mode, obj_dir):
     dist_path = os.path.join(_ABB_PATH, obj_dir, "dist")
     arch_suffix = _ARCH_X86_I386 if architecture == _ARCH_X86 else _ARCH_ARM
     [manifest_path] = glob.glob(os.path.join(
-        dist_path, "fennec-*.multi.android-%s.json" % arch_suffix))
+        dist_path, "fennec-*.en-US.android-%s.json" % arch_suffix))
+
     with open(manifest_path) as manifest_file:
         manifest = json.load(manifest_file)
-    apk_name = "fennec-%s.multi.android-%s-unsigned-unaligned.apk" % (
-            manifest["moz_app_version"], arch_suffix)
+
+    apk_name = "fennec-%s.en-US.android-%s-unsigned-unaligned.apk" % (
+        manifest["moz_app_version"], arch_suffix)
+
     apk_path = os.path.join(dist_path, apk_name)
     build_suffix = ("-%s" % manifest["buildid"]
                     if build_mode == _BUILD_DEVBUILD
                     else "")
+
     dist_suffix = ("-%s" % _DIST_STANDALONE
                    if distribution_mode == _DIST_STANDALONE
                    else "")
+
     target_apk_name = "adblockbrowser-%s%s-%s%s.apk" % (
         manifest["moz_app_version"], build_suffix, architecture, dist_suffix)
+
     target_apk_path = os.path.join(dist_path, target_apk_name)
     shutil.copyfile(apk_path, target_apk_path)
-
     target_manifest_path = re.sub(r".apk$", ".json", target_apk_path)
     shutil.copyfile(manifest_path, target_manifest_path)
+    return target_apk_path
 
-    return target_apk_path
+
+def _build(architecture, distribution_mode, build_mode, sdk_path, ndk_path):
+    _checkout_l10n()
+    obj_dir = _OBJDIR_X86 if architecture == _ARCH_X86 else _OBJDIR_ARM
+    locales = _read_locales()
+    build_env = _config_build_env(architecture, distribution_mode, build_mode,
+                                  sdk_path, ndk_path, obj_dir, locales)
+
+    mach_path = os.path.join(_ABB_PATH, "mach")
+    subprocess.check_call([mach_path, "clobber"], env=build_env, cwd=_ABB_PATH)
+    subprocess.check_call([mach_path, "build"], env=build_env, cwd=_ABB_PATH)
+    for locale in locales:
+        chrome_str = "chrome-%s" % locale
+        subprocess.check_call([mach_path, "build", chrome_str], env=build_env,
+                              cwd=_ABB_PATH)
+
+    obj_dir_path = os.path.join(_ABB_PATH, obj_dir)
+    import transform_locales as tl
+    tl.transform_locales(_ABB_PATH, obj_dir_path)
+    subprocess.check_call([mach_path, "package"], env=build_env, cwd=_ABB_PATH)
+    return _create_target_apk(architecture, distribution_mode, build_mode,
+                              obj_dir)
 
 
 def _sign(apk_path, key_store, key_name, sdk_path):
     sys.path.append(os.path.join(_ABB_PATH, "python", "mozboot", "mozboot"))
     from android import ANDROID_BUILD_TOOLS_VERSION
     zipalign_path = os.path.join(sdk_path, "build-tools",
                                  ANDROID_BUILD_TOOLS_VERSION, "zipalign")
+
     temp_apk_path = tempfile.NamedTemporaryFile().name
     shutil.copyfile(apk_path, temp_apk_path)
     try:
         subprocess.check_call(["jarsigner", "-verbose",
                                "-sigalg", "SHA1withRSA",
                                "-digestalg", "SHA1",
                                "-keystore", key_store,
                                temp_apk_path, key_name])
+
         os.remove(apk_path)
         subprocess.check_call([zipalign_path, "-v", "4", temp_apk_path,
                                apk_path])
+
     finally:
         os.remove(temp_apk_path)
 
 if __name__ == "__main__":
     if len(sys.argv) < 2:
-        print_usage()
+        _print_usage()
         sys.exit(1)
 
     mode = sys.argv[1]
     do_build = mode in (_CMD_BUILD, _CMD_BUILD_SIGN)
     do_sign = mode in (_CMD_SIGN, _CMD_BUILD_SIGN)
     if not do_build and not do_sign:
-        print_usage()
+        _print_usage()
         sys.exit(2)
 
     if do_sign:
         if len(sys.argv) < 3:
-            print_usage()
+            _print_usage()
             sys.exit(3)
         apk_path = sys.argv[2]
 
     subprocess.check_call([_ENSURE_DEPENDENCIES_PATH])
+    error_msg = "Invalid %s, check config.py. Supported %s: %s"
     import config
-
-    error_msg = "Invalid %s, check config.py. Supported %s: %s"
     distribution_mode = config.DISTRIBUTION_MODE
     if distribution_mode not in _DIST_MODES:
         print >>sys.stderr, error_msg % (
             "distribution mode", "distribution modes", _DIST_MODES)
         sys.exit(4)
 
     build_mode = config.BUILD_MODE
     if build_mode not in _BUILD_MODES:
@@ -182,9 +249,9 @@
 
     if do_build:
         apk_path = _build(architecture, distribution_mode, build_mode,
                           config.ANDROID_SDK_PATH, config.ANDROID_NDK_PATH)
     if do_sign:
         _sign(apk_path, config.ANDROID_KEYSTORE_PATH, config.ANDROID_KEY_NAME,
               config.ANDROID_SDK_PATH)
     else:
-        print apk_path
+        print(apk_path)