Issue 510 - [Typed objects] Don`t hardcode script load order in unit tests

run_tests.py

 #!/usr/bin/env python
 # coding: utf-8
 
-# This file is part of Adblock Plus <http://adblockplus.org/>,
-# Copyright (C) 2006-2014 Eyeo GmbH
+# This file is part of Adblock Plus <https://adblockplus.org/>,
+# Copyright (C) 2006-2015 Eyeo GmbH
 #
 # Adblock Plus is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 3 as
 # published by the Free Software Foundation.
 #
 # Adblock Plus is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with Adblock Plus.  If not, see <http://www.gnu.org/licenses/>.
 
+import io
+import json
+import os
+import posixpath
+import re
+import urlparse
+
 import flask
-import os
-import re
-import io

[Felix Dahlke, 2015/02/03 06:21:16]

Nit: The import statements seem to be ordered alphabetically.

[Sebastian Noack, 2015/02/03 09:21:10]

Note that according to PEP-8, third-party module imports (like flask here)
should go below core-lib imports.

[Wladimir Palant, 2015/02/03 16:26:04]

Done.

-import urlparse
 
 app = flask.Flask(__name__)
 
-def get_scripts(dir, reldir = ""):

[Felix Dahlke, 2015/02/03 06:21:16]

This would be a bit shorter (and IMHO easier to read) using os.walk and fnmatch,
e.g.:

def get_scripts(dir):
  for root, _, files in os.walk(dir):
    for file in fnmatch.filter(files, "*.js"):
      path = os.path.join(root, file)
      relpath = "/" + os.path.relpath(path, dir)
      yield(path, relpath)

[Sebastian Noack, 2015/02/03 09:21:10]

Even more compact:

import glob

...

def get_scripts(dir):
  for filename in glob.glob(os.path.join(dir, '*.js')):
    ...

[Wladimir Palant, 2015/02/03 16:26:04]

This won't find anything in subdirectories.

Personally, I don't find os.walk() easier to read - the calling convention of
that function is everything but natural. Note also that os.path.relpath() will
produce OS-specific paths which isn't what we need here. However, I've
implemented it this way now.

-  for name in os.listdir(dir):
-    path = os.path.join(dir, name)
-    relpath = reldir + "/" + name
+def get_scripts(dir, reldir=""):
+  for filename in os.listdir(dir):
+    path = os.path.join(dir, filename)
+    # "Relative path" will be the parameter passed in to require(), meaning
+    # that it always has to use forward slashes.
+    relpath = posixpath.join(reldir, filename)
     if os.path.isdir(path):
       for path, relpath in get_scripts(path, relpath):
-        yield (path, relpath)
+        yield path, relpath
     elif os.path.splitext(path)[1] == ".js":
-      yield (path, relpath)
+      yield path, relpath
 
 def js_encode(str):
-  return (str.replace("\\", "\\\\").replace("'", "\\'")
-             .replace("\r", "\\r").replace("\n", "\\n"))

[Felix Dahlke, 2015/02/03 06:21:16]

Makes me wonder if we'll run into trouble when using other control sequences
than \r or \n. Assuming JS supports the same control sequences as C, this should
handle all of them:

re.sub(r"\([a-z0-9])", r"\\\1", ...)

[Wladimir Palant, 2015/02/03 16:26:04]

No, these should be fine within a string. See
http://www.ecma-international.org/ecma-262/5.1/#sec-7.8.4:
So these are the only characters that need special treatment. However,
LineTerminator also includes \u2028 and \u2029.

Either way, using json module would be a cleaner solution.

+  return json.dumps(str)
 
 def script_as_string(path, sourceURL, backcompat):
   if backcompat:
     from jshydra.abp_rewrite import doRewrite
     data = doRewrite([os.path.abspath(path)], []).decode("utf-8")
   else:
     with io.open(path, "r", encoding="utf-8") as handle:
       data = handle.read()
   data += "\n//# sourceURL=%s" % sourceURL
   return js_encode(data)
 
 @app.route("/<path:path>", methods = ["GET"])
 @app.route("/", methods = ["GET"])
 def multiplex(path=""):
-  request_url = urlparse.urlparse(flask.request.url)
+  base_url = flask.request.url
+  request_url = urlparse.urlparse(base_url)
   request_path = request_url.path
   islib = request_path.startswith("/lib/")
   backcompat = request_url.query == "backcompat"
 
   rootdir = os.path.dirname(__file__)
   if request_path == "/lib.js":
-    data = ""
-    for path, relpath in get_scripts(os.path.join(rootdir, "lib")):
-      url = urlparse.urljoin(flask.request.url, "/lib" + relpath)
-      data += "require.modules['%s'] = '%s';\n" % (
-          os.path.splitext(relpath)[0][1:],
-          script_as_string(path, url, backcompat)
-      )
-    return (data, 200, {"Content-Type": "application/javascript; charset=utf-8"})
+    def generate_libs():
+      for path, relpath in get_scripts(os.path.join(rootdir, "lib")):
+        url = urlparse.urljoin(base_url, "/lib/" + relpath)
+        yield "require.sources[%s] = %s;\n" % (
+            js_encode(posixpath.splitext(relpath)[0]),
+            script_as_string(path, url, backcompat)
+        )
+    return flask.Response(generate_libs(), mimetype="application/javascript")
   elif request_path == "/tests.js":
-    data = "var tests = ["
-    for path, relpath in get_scripts(os.path.join(rootdir, "test", "tests")):
-      url = urlparse.urljoin(flask.request.url, "/tests" + relpath)
-      data += "  '%s',\n" % script_as_string(path, url, backcompat)
-    data += "];"
-    return (data, 200, {"Content-Type": "application/javascript; charset=utf-8"})
+    def generate_tests():
+      yield "var tests = ["
+      for path, relpath in get_scripts(os.path.join(rootdir, "test", "tests")):
+        url = urlparse.urljoin(base_url, "/tests/" + relpath)
+        yield "  %s,\n" % script_as_string(path, url, backcompat)
+      yield "];"
+    return flask.Response(generate_tests(), mimetype="application/javascript")
   else:
     if request_path.startswith("/lib/"):
       rootdir = os.path.join(rootdir, "lib")
     else:
       rootdir = os.path.join(rootdir, "test")
     if request_path.endswith("/"):
       request_path += "index.html"
     return flask.send_from_directory(rootdir, request_path.lstrip("/"))
 
 if __name__ == "__main__":
   app.run(debug=True)