-
-
-
-
-
Index: sitescripts/filterhits/static/query.js
===================================================================
deleted file mode 100644
--- a/sitescripts/filterhits/static/query.js
+++ /dev/null
@@ -1,39 +0,0 @@
-(function () {
- var table = $("#results").dataTable({
- serverSide: true,
- bFilter: false,
- "columns": [
- { data: "filter" },
- { data: "domain" },
- { data: "frequency" }
- ],
- order: [[2, "desc"]],
- fnServerData: function (source, data, callback, settings) {
- var sort = settings.aaSorting[0];
- $.ajax({
- dataType: "json",
- type: "GET",
- url: "/query",
- data: {
- echo: settings.iDraw,
- skip: settings._iDisplayStart,
- take: settings._iDisplayLength,
- filter: $("#filter").val(),
- domain: $("#domain").val(),
- order: sort[1],
- order_by: ["filter", "domain", "frequency"][sort[0]]
- },
- success: function(data, status, jqxhr) {
- callback({
- draw: data.echo,
- recordsTotal: data.total,
- recordsFiltered: data.total,
- data: data.results
- }, status, jqxhr);
- }
- });
- }
- });
-
- $("#filter, #domain").on("input", function () { table.fnDraw(); });
-}());
Index: sitescripts/filterhits/test/__init__.py
===================================================================
deleted file mode 100644
Index: sitescripts/filterhits/test/api_tests.py
===================================================================
deleted file mode 100644
--- a/sitescripts/filterhits/test/api_tests.py
+++ /dev/null
@@ -1,172 +0,0 @@
-# coding: utf-8
-
-# This file is part of the Adblock Plus web scripts,
-# Copyright (C) 2006-2016 Eyeo GmbH
-#
-# Adblock Plus is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 3 as
-# published by the Free Software Foundation.
-#
-# Adblock Plus is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Adblock Plus. If not, see .
-
-import json
-import sys
-import StringIO
-import unittest
-from urllib import urlencode
-
-from sitescripts.filterhits.test import test_helpers
-from sitescripts.filterhits import db
-from sitescripts.filterhits.web.query import query_handler
-from sitescripts.filterhits.web.submit import submit as submit_handler
-
-valid_data = """{
- "version": 1,
- "timeSincePush": 12345,
- "addonName": "adblockplus",
- "addonVersion": "2.3.4",
- "application": "firefox",
- "applicationVersion": "31",
- "platform": "gecko",
- "platformVersion": "31",
- "filters": {
- "||example.com^": {
- "firstParty": {
- "example.com": {"hits": 12, "latest": 1414817340948},
- "example.org": {"hits": 4, "latest": 1414859271125}
- },
- "thirdParty": {
- "example.com": {"hits": 5, "latest": 1414916268769}
- },
- "subscriptions": ["EasyList", "EasyList Germany+EasyList"]
- }
- }
-}"""
-
-
-class APITestCase(test_helpers.FilterhitsTestCase):
- def assertResponse(self, handler, expected_response, expected_result=None, expected_headers=None, **environ):
- def check_response(response, headers):
- self.assertEqual(response, expected_response)
- if not expected_headers is None:
- self.assertEqual(headers, expected_headers)
-
- if "body" in environ:
- environ["CONTENT_LENGTH"] = len(environ["body"])
- environ["wsgi.input"] = StringIO.StringIO(environ["body"])
- del environ["body"]
-
- if "get_params" in environ:
- environ["QUERY_STRING"] = urlencode(environ["get_params"])
- del environ["get_params"]
-
- environ["wsgi.errors"] = sys.stderr
- result = handler(environ, check_response)
- if not expected_result is None:
- self.assertEqual(json.loads("".join(result)), expected_result)
-
- def test_submit(self):
- self.assertEqual(len(db.query(self.db, "SELECT * FROM frequencies")), 0)
- self.assertEqual(len(db.query(self.db, "SELECT * FROM filters")), 0)
- # Ensure missing or invalid JSON results in an error
- self.assertResponse(submit_handler, "400 Processing Error",
- REQUEST_METHOD="POST", body="")
- self.assertResponse(submit_handler, "400 Processing Error",
- REQUEST_METHOD="POST", body="Oops...")
- self.assertResponse(submit_handler, "400 Processing Error",
- REQUEST_METHOD="POST", body="{123:]")
- self.assertResponse(submit_handler, "400 Processing Error",
- REQUEST_METHOD="POST", body="1")
- self.assertEqual(len(db.query(self.db, "SELECT * FROM frequencies")), 0)
- self.assertEqual(len(db.query(self.db, "SELECT * FROM filters")), 0)
- # Ensure even an empty object, or one with the wrong fields returns successfully
- self.assertResponse(submit_handler, "204 No Content",
- REQUEST_METHOD="POST", body="{}")
- self.assertResponse(submit_handler, "204 No Content",
- REQUEST_METHOD="POST", body="{\"hello\": \"world\"}")
- self.assertEqual(len(db.query(self.db, "SELECT * FROM frequencies")), 0)
- self.assertEqual(len(db.query(self.db, "SELECT * FROM filters")), 0)
- # Now some actually valid data
- self.assertResponse(submit_handler, "204 No Content",
- REQUEST_METHOD="POST", body=valid_data)
- self.assertEqual(len(db.query(self.db, "SELECT * FROM frequencies")), 2)
- self.assertEqual(len(db.query(self.db, "SELECT * FROM filters")), 1)
- # Now make sure apparently valid data with timestamps that cause geometrical
- # mean calculations to fail with MySQL errors return OK but don't change DB
- invalid_data = json.loads(valid_data)
- invalid_data["filters"]["||example.com^"]["firstParty"]["example.com"]["latest"] = 3
- invalid_data = json.dumps(invalid_data)
- self.assertResponse(submit_handler, "204 No Content",
- REQUEST_METHOD="POST", body=invalid_data)
- self.assertEqual(len(db.query(self.db, "SELECT * FROM frequencies")), 2)
- self.assertEqual(len(db.query(self.db, "SELECT * FROM filters")), 1)
-
- def test_query(self):
- # Basic query with no data, should return successfully
- self.assertResponse(query_handler, "200 OK", {"count": 0, "total": 0, "results": [], "echo": 0})
- # If echo parameter is passed and is integer it should be returned
- self.assertResponse(query_handler, "200 OK", {"count": 0, "total": 0, "results": [], "echo": 1337},
- get_params={"echo": 1337})
- self.assertResponse(query_handler, "200 OK", {"count": 0, "total": 0, "results": [], "echo": 0},
- get_params={"echo": "naughty"})
- # Now let's submit some data so we can query it back out
- test_data = [json.loads(valid_data), json.loads(valid_data), json.loads(valid_data)]
- test_data[1]["filters"]["##Second-Filter|"] = test_data[1]["filters"].pop("||example.com^")
- test_data[2]["filters"]["##Third-Filter|"] = test_data[2]["filters"].pop("||example.com^")
- for data in test_data:
- self.assertResponse(submit_handler, "204 No Content",
- REQUEST_METHOD="POST", body=json.dumps(data))
- # Ordering parameters should be respected
- self.assertResponse(query_handler, "200 OK", {"count": 1, "total": 6,
- "results": [{"domain": "example.com",
- "filter": "||example.com^",
- "frequency": 0}], "echo": 0},
- get_params={"order_by": "filter", "order": "desc", "take": "1"})
- self.assertResponse(query_handler, "200 OK", {"count": 1, "total": 6,
- "results": [{"domain": "example.com",
- "filter": "##Second-Filter|",
- "frequency": 0}], "echo": 0},
- get_params={"order_by": "filter", "order": "asc", "take": "1"})
- # As should filtering parameters
- self.assertResponse(query_handler, "200 OK", {"count": 1, "total": 3,
- "results": [{"domain": "example.com",
- "filter": "##Third-Filter|",
- "frequency": 0}], "echo": 0},
- get_params={"domain": "example.com", "take": "1"})
- self.assertResponse(query_handler, "200 OK", {"count": 1, "total": 2,
- "results": [{"domain": "example.org",
- "filter": "##Third-Filter|",
- "frequency": 4}], "echo": 0},
- get_params={"filter": "Third", "take": 1})
- self.assertResponse(query_handler, "200 OK", {"count": 1, "total": 1,
- "results": [{"domain": "example.com",
- "filter": "##Third-Filter|",
- "frequency": 0}], "echo": 0},
- get_params={"domain": "example.com", "filter": "Third", "take": "1"})
- # ... and pagination parameters
- self.maxDiff = None
- self.assertResponse(query_handler, "200 OK", {"count": 2, "total": 6,
- "results": [{"domain": "example.org",
- "filter": "||example.com^",
- "frequency": 4},
- {"domain": "example.org",
- "filter": "##Second-Filter|",
- "frequency": 4}], "echo": 0},
- get_params={"skip": "1", "take": "2"})
- self.assertResponse(query_handler, "200 OK", {"count": 2, "total": 6,
- "results": [{"domain": "example.org",
- "filter": "##Second-Filter|",
- "frequency": 4},
- {"domain": "example.com",
- "filter": "##Third-Filter|",
- "frequency": 0}], "echo": 0},
- get_params={"skip": "2", "take": "2"})
-
-if __name__ == "__main__":
- unittest.main()
Index: sitescripts/filterhits/test/db_tests.py
===================================================================
deleted file mode 100644
--- a/sitescripts/filterhits/test/db_tests.py
+++ /dev/null
@@ -1,50 +0,0 @@
-# coding: utf-8
-
-# This file is part of the Adblock Plus web scripts,
-# Copyright (C) 2006-2016 Eyeo GmbH
-#
-# Adblock Plus is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 3 as
-# published by the Free Software Foundation.
-#
-# Adblock Plus is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Adblock Plus. If not, see .
-
-import unittest
-
-import MySQLdb
-
-from sitescripts.filterhits.test import test_helpers
-from sitescripts.filterhits import db
-
-
-class DbTestCase(test_helpers.FilterhitsTestCase):
- longMessage = True
- maxDiff = None
-
- def test_query_and_write(self):
- insert_sql = """INSERT INTO `filters` (filter, sha1)
- VALUES (%s, UNHEX(SHA1(filter)))"""
- select_sql = "SELECT filter FROM filters ORDER BY filter ASC"
-
- # Table should be empty to start with
- self.assertEqual(db.query(self.db, select_sql), ())
- # Write some data and query it back
- db.write(self.db, ((insert_sql, "something"),))
- self.assertEqual(db.query(self.db, select_sql), ((u"something",),))
- # Write an array of SQL strings
- db.write(self.db, ((insert_sql, "a"), (insert_sql, "b"), (insert_sql, "c")))
- self.assertEqual(db.query(self.db, select_sql), ((u"a",), (u"b",), (u"c",), (u"something",)))
- # Write a sequence of SQL but roll back when a problem arrises
- with self.assertRaises(MySQLdb.ProgrammingError):
- db.write(self.db, ((insert_sql, "f"), (insert_sql, "g"), (insert_sql, "h"),
- ("GFDGks",)))
- self.assertEqual(db.query(self.db, select_sql), ((u"a",), (u"b",), (u"c",), (u"something",)))
-
-if __name__ == "__main__":
- unittest.main()
Index: sitescripts/filterhits/test/geometrical_mean_tests.py
===================================================================
deleted file mode 100644
--- a/sitescripts/filterhits/test/geometrical_mean_tests.py
+++ /dev/null
@@ -1,167 +0,0 @@
-# coding: utf-8
-
-# This file is part of the Adblock Plus web scripts,
-# Copyright (C) 2006-2016 Eyeo GmbH
-#
-# Adblock Plus is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 3 as
-# published by the Free Software Foundation.
-#
-# Adblock Plus is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Adblock Plus. If not, see .
-
-import unittest
-from datetime import datetime
-
-from sitescripts.filterhits.test import test_helpers
-from sitescripts.filterhits import db, geometrical_mean
-
-test_data = [{
- "filters": {
- "##.top-box-right-ad": {
- "firstParty": {
- "acxiom-online.com": {"hits": 6, "latest": 1414817340948},
- "google.com": {"hits": 50, "latest": 1414849084678},
- "yahoo.com": {"hits": 14, "latest": 1414859271125},
- "doubleclick.net": {"hits": 26, "latest": 1414823430333}
- },
- "thirdParty": {
- "demdex.net": {"hits": 36, "latest": 1414838712373}
- }
- }
- }
-}, {
- "filters": {
- "##.top-box-right-ad": {
- "thirdParty": {
- "adsymptotic.com": {"hits": 49, "latest": 1414953943015},
- "amazon.com": {"hits": 2, "latest": 1414913563746},
- "live.com": {"hits": 34, "latest": 1414916268769},
- "google.com": {"hits": 21, "latest": 1414953920364},
- "yahoo.com": {"hits": 27, "latest": 1414917270343}
- }
- }
- }
-}, {
- "filters": {
- "##.top-box-right-ad": {
- "firstParty": {
- "google.com": {"hits": 14, "latest": 1415008533089},
- "adsymptotic.com": {"hits": 15, "latest": 1414994112862}
- },
- "thirdParty": {
- "yahoo.com": {"hits": 43, "latest": 1415045194098}
- }
- },
- "stevedeace.com##.topAddHolder": {
- "firstParty": {
- "mathtag.com": {"hits": 14, "latest": 1415032601175},
- "amazonaws.com": {"hits": 18, "latest": 1414977342966}
- }
- }
- }
-}]
-
-
-class GeometricalMeanTestCase(test_helpers.FilterhitsTestCase):
- longMessage = True
- maxDiff = None
-
- def geometrical(self, interval, new, new_timestamp, old, old_timestamp):
- delta_divby_interval = (new_timestamp - old_timestamp) / 1000 / float(interval)
- return long(round(old ** (1 - delta_divby_interval) * new ** delta_divby_interval))
-
- def test_calculations(self):
- interval = 86400
-
- # Tables should be empty to start with
- self.assertEqual(db.query(self.db, "SELECT * FROM filters"), ())
- self.assertEqual(db.query(self.db, "SELECT * FROM frequencies"), ())
- # First batch
- db.write(self.db, geometrical_mean.update(interval, test_data[0]))
- self.assertEqual(db.query(self.db, "SELECT * FROM filters"),
- (("8c5ea548436c61f05536e205a29ada6204f603b0".decode("hex"),
- u"##.top-box-right-ad"),))
- self.assertEqual(
- db.query(self.db, "SELECT * FROM frequencies"),
- (("8c5ea548436c61f05536e205a29ada6204f603b0".decode("hex"), u"acxiom-online.com",
- 6L, datetime.utcfromtimestamp(1414817340948 / 1000)),
- ("8c5ea548436c61f05536e205a29ada6204f603b0".decode("hex"), u"demdex.net",
- 36L, datetime.utcfromtimestamp(1414838712373 / 1000)),
- ("8c5ea548436c61f05536e205a29ada6204f603b0".decode("hex"), u"doubleclick.net",
- 26L, datetime.utcfromtimestamp(1414823430333 / 1000)),
- ("8c5ea548436c61f05536e205a29ada6204f603b0".decode("hex"), u"google.com",
- 50L, datetime.utcfromtimestamp(1414849084678 / 1000)),
- ("8c5ea548436c61f05536e205a29ada6204f603b0".decode("hex"), u"yahoo.com",
- 14L, datetime.utcfromtimestamp(1414859271125 / 1000))))
- # Second batch
- db.write(self.db, geometrical_mean.update(interval, test_data[1]))
- self.assertEqual(db.query(self.db, "SELECT * FROM filters"),
- (("8c5ea548436c61f05536e205a29ada6204f603b0".decode("hex"),
- u"##.top-box-right-ad"),))
- self.assertEqual(
- db.query(self.db, "SELECT * FROM frequencies"),
- (("8c5ea548436c61f05536e205a29ada6204f603b0".decode("hex"), u"acxiom-online.com",
- 6L, datetime.utcfromtimestamp(1414817340948 / 1000)),
- ("8c5ea548436c61f05536e205a29ada6204f603b0".decode("hex"), u"adsymptotic.com",
- 49L, datetime.utcfromtimestamp(1414953943015 / 1000)),
- ("8c5ea548436c61f05536e205a29ada6204f603b0".decode("hex"), u"amazon.com",
- 2L, datetime.utcfromtimestamp(1414913563746 / 1000)),
- ("8c5ea548436c61f05536e205a29ada6204f603b0".decode("hex"), u"demdex.net",
- 36L, datetime.utcfromtimestamp(1414838712373 / 1000)),
- ("8c5ea548436c61f05536e205a29ada6204f603b0".decode("hex"), u"doubleclick.net",
- 26L, datetime.utcfromtimestamp(1414823430333 / 1000)),
- ("8c5ea548436c61f05536e205a29ada6204f603b0".decode("hex"), u"google.com",
- self.geometrical(interval, 21, 1414953920364, 50, 1414849084678),
- datetime.utcfromtimestamp(1414953920364 / 1000)),
- ("8c5ea548436c61f05536e205a29ada6204f603b0".decode("hex"), u"live.com",
- 34L, datetime.utcfromtimestamp(1414916268769 / 1000)),
- ("8c5ea548436c61f05536e205a29ada6204f603b0".decode("hex"), u"yahoo.com",
- self.geometrical(interval, 27, 1414917270343, 14, 1414859271125),
- datetime.utcfromtimestamp(1414917270343 / 1000))))
- # Third batch
- db.write(self.db, geometrical_mean.update(interval, test_data[2]))
- self.assertEqual(db.query(self.db, "SELECT * FROM filters"),
- (("22de8d2ba8429eb170a0ece6ea7a426f7b22e574".decode("hex"),
- u"stevedeace.com##.topAddHolder"),
- ("8c5ea548436c61f05536e205a29ada6204f603b0".decode("hex"),
- u"##.top-box-right-ad")))
- self.assertEqual(
- db.query(self.db, "SELECT * FROM frequencies"),
- (("8c5ea548436c61f05536e205a29ada6204f603b0".decode("hex"), u"acxiom-online.com",
- 6L, datetime.utcfromtimestamp(1414817340948 / 1000)),
- ("8c5ea548436c61f05536e205a29ada6204f603b0".decode("hex"), u"adsymptotic.com",
- self.geometrical(interval, 15, 1414994112862, 49, 1414953943015),
- datetime.utcfromtimestamp(1414994112862 / 1000)),
- ("8c5ea548436c61f05536e205a29ada6204f603b0".decode("hex"), u"amazon.com",
- 2L, datetime.utcfromtimestamp(1414913563746 / 1000)),
- ("22de8d2ba8429eb170a0ece6ea7a426f7b22e574".decode("hex"), u"amazonaws.com",
- 18L, datetime.utcfromtimestamp(1414977342966 / 1000)),
- ("8c5ea548436c61f05536e205a29ada6204f603b0".decode("hex"), u"demdex.net",
- 36L, datetime.utcfromtimestamp(1414838712373 / 1000)),
- ("8c5ea548436c61f05536e205a29ada6204f603b0".decode("hex"), u"doubleclick.net",
- 26L, datetime.utcfromtimestamp(1414823430333 / 1000)),
- ("8c5ea548436c61f05536e205a29ada6204f603b0".decode("hex"), u"google.com",
- self.geometrical(interval, 14, 1415008533089,
- self.geometrical(interval, 21, 1414953920364,
- 50, 1414849084678),
- 1414953920364),
- datetime.utcfromtimestamp(1415008533089 / 1000)),
- ("8c5ea548436c61f05536e205a29ada6204f603b0".decode("hex"), u"live.com",
- 34L, datetime.utcfromtimestamp(1414916268769 / 1000)),
- ("22de8d2ba8429eb170a0ece6ea7a426f7b22e574".decode("hex"), u"mathtag.com",
- 14L, datetime.utcfromtimestamp(1415032601175 / 1000)),
- ("8c5ea548436c61f05536e205a29ada6204f603b0".decode("hex"), u"yahoo.com",
- self.geometrical(interval, 43, 1415045194098,
- self.geometrical(interval, 27, 1414917270343,
- 14, 1414859271125),
- 1414917270343),
- datetime.utcfromtimestamp(1415045194098 / 1000))))
-
-if __name__ == "__main__":
- unittest.main()
Index: sitescripts/filterhits/test/log_tests.py
===================================================================
deleted file mode 100644
--- a/sitescripts/filterhits/test/log_tests.py
+++ /dev/null
@@ -1,56 +0,0 @@
-# coding: utf-8
-
-# This file is part of the Adblock Plus web scripts,
-# Copyright (C) 2006-2016 Eyeo GmbH
-#
-# Adblock Plus is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 3 as
-# published by the Free Software Foundation.
-#
-# Adblock Plus is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Adblock Plus. If not, see .
-
-import os
-import shutil
-import time
-import unittest
-
-from sitescripts.filterhits.test import test_helpers
-from sitescripts.filterhits.web import submit
-
-
-class LogTestCase(test_helpers.FilterhitsTestCase):
- def test_log_filterhits(self):
- def list_files(d):
- return filter(os.path.isfile, [os.path.join(d, f) for f in os.listdir(d)])
-
- todays_date = time.strftime("%Y-%m-%d", time.gmtime())
- todays_folder = os.path.join(self.test_dir, todays_date)
-
- # The temporary logging directory is created at the start of all tests but
- # we want to test that the directory is created if it doesn't already exist.
- # So we'll delete the directory here and make sure it's re-created later on.
- shutil.rmtree(self.test_dir)
- self.assertEqual(os.path.exists(self.test_dir), False)
-
- log_file = submit.log_filterhits({"some": "thing"}, self.test_dir, "a=1")
- now = time.strftime("%d/%b/%Y:%H:%M:%S", time.gmtime())
- self.assertEqual(os.path.exists(self.test_dir), True)
- self.assertEqual(os.path.exists(todays_folder), True)
- self.assertEqual(len(list_files(todays_folder)), 1)
- self.assertEqual(os.path.exists(log_file), True)
- with open(list_files(todays_folder)[0], "r") as f:
- self.assertEqual(f.read(), '[%s] a=1\n{"some": "thing"}' % now)
-
- submit.log_filterhits({"some": "thing"}, self.test_dir, "")
- self.assertEqual(os.path.exists(self.test_dir), True)
- self.assertEqual(os.path.exists(todays_folder), True)
- self.assertEqual(len(list_files(todays_folder)), 2)
-
-if __name__ == "__main__":
- unittest.main()
Index: sitescripts/filterhits/test/test_helpers.py
===================================================================
deleted file mode 100644
--- a/sitescripts/filterhits/test/test_helpers.py
+++ /dev/null
@@ -1,71 +0,0 @@
-# coding: utf-8
-
-# This file is part of the Adblock Plus web scripts,
-# Copyright (C) 2006-2016 Eyeo GmbH
-#
-# Adblock Plus is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 3 as
-# published by the Free Software Foundation.
-#
-# Adblock Plus is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Adblock Plus. If not, see .
-
-import tempfile
-import shutil
-import unittest
-
-import MySQLdb
-
-from sitescripts.filterhits import db
-from sitescripts.utils import get_config
-
-
-class FilterhitsTestCase(unittest.TestCase):
- config = get_config()
- _db = None
- _live_config = {
- "dbuser": config.get("filterhitstats", "dbuser"),
- "dbpassword": config.get("filterhitstats", "dbpassword"),
- "database": config.get("filterhitstats", "database"),
- "log_dir": config.get("filterhitstats", "log_dir")
- }
- _test_config = {
- "dbuser": config.get("filterhitstats", "test_dbuser"),
- "dbpassword": config.get("filterhitstats", "test_dbpassword"),
- "database": config.get("filterhitstats", "test_database")
- }
-
- def _clear_database(self):
- db.write(self._db, (("DELETE FROM frequencies",), ("DELETE FROM filters",)))
-
- @property
- def db(self):
- if (not self._db):
- self._db = db.connect()
- self._clear_database()
- return self._db
-
- def setUp(self):
- # Set up a temporary log directory for testing
- self.test_dir = tempfile.mkdtemp()
- # Set up test config
- for k, v in self._test_config.items():
- self.config.set("filterhitstats", k, v)
- self.config.set("filterhitstats", "log_dir", self.test_dir)
-
- def tearDown(self):
- # Clean the database and close our connection
- if self._db:
- self._clear_database()
- self._db.close()
- self._db = None
- # Clean any generated logs
- shutil.rmtree(self.test_dir, ignore_errors=True)
- # Restore the configuration
- for k, v in self._live_config.items():
- self.config.set("filterhitstats", k, v)
Index: sitescripts/filterhits/web/__init__.py
===================================================================
deleted file mode 100644
Index: sitescripts/filterhits/web/common.py
===================================================================
deleted file mode 100644
--- a/sitescripts/filterhits/web/common.py
+++ /dev/null
@@ -1,21 +0,0 @@
-# coding: utf-8
-
-# This file is part of the Adblock Plus web scripts,
-# Copyright (C) 2006-2016 Eyeo GmbH
-#
-# Adblock Plus is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 3 as
-# published by the Free Software Foundation.
-#
-# Adblock Plus is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Adblock Plus. If not, see .
-
-
-def show_error(message, start_response, status="400 Processing Error"):
- start_response(status, [("Content-Type", "text/plain; charset=utf-8")])
- return [message.encode("utf-8")]
Index: sitescripts/filterhits/web/query.py
===================================================================
deleted file mode 100644
--- a/sitescripts/filterhits/web/query.py
+++ /dev/null
@@ -1,86 +0,0 @@
-# coding: utf-8
-
-# This file is part of the Adblock Plus web scripts,
-# Copyright (C) 2006-2016 Eyeo GmbH
-#
-# Adblock Plus is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 3 as
-# published by the Free Software Foundation.
-#
-# Adblock Plus is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Adblock Plus. If not, see .
-
-import json
-import os
-import traceback
-from urlparse import parse_qsl
-
-import MySQLdb
-
-from sitescripts.web import url_handler
-from sitescripts.utils import cached, setupStderr
-from sitescripts.filterhits import db
-from sitescripts.filterhits.web import common
-
-
-def query(domain=None, filter=None, skip=0, take=20, order="DESC", order_by="frequency", **_):
- """
- Returns the SQL and parameters needed to perform a query of the filterhits data.
- """
- sql = """SELECT SQL_CALC_FOUND_ROWS domain, filter, frequency
- FROM frequencies as freq
- LEFT JOIN filters as f ON f.sha1=freq.filter_sha1
- %s
- ORDER BY %s
- LIMIT %%s, %%s"""
-
- where = zip(*[("%s LIKE %%s" % s, "%%%s%%" % p) for s, p in (("domain", domain),
- ("filter", filter)) if p])
- if where:
- where_fields, params = where
- where_sql = "WHERE " + " AND ".join(where_fields)
- else:
- where_sql = ""
- params = []
-
- order = order.upper() if order.upper() in ("ASC", "DESC") else "ASC"
- if order_by not in ["filter", "domain", "frequency"]:
- order_by = "frequency"
- order_by_sql = "`%s` %s" % (order_by, order)
-
- params = list(params) + [int(skip), int(take)]
- return [sql % (where_sql, order_by_sql)] + params
-
-
-@url_handler("/query")
-def query_handler(environ, start_response):
- setupStderr(environ["wsgi.errors"])
- params = dict(parse_qsl(environ.get("QUERY_STRING", "")))
-
- try:
- db_connection = db.connect()
- try:
- results = db.query(db_connection, *query(**params), dict_result=True)
- total = db.query(db_connection, "SELECT FOUND_ROWS()")[0][0]
- finally:
- db_connection.close()
- except MySQLdb.Error:
- traceback.print_exc()
- return common.show_error("Failed to query database!", start_response,
- "500 Database error")
-
- try:
- echo = int(params["echo"])
- except (ValueError, KeyError):
- echo = 0
-
- response_headers = [("Content-type", "application/json; charset=utf-8")]
- start_response("200 OK", response_headers)
- return [json.dumps({"results": results, "echo": echo,
- "total": total, "count": len(results)},
- ensure_ascii=False).encode("utf-8")]
Index: sitescripts/filterhits/web/submit.py
===================================================================
deleted file mode 100644
--- a/sitescripts/filterhits/web/submit.py
+++ /dev/null
@@ -1,110 +0,0 @@
-# coding: utf-8
-
-# This file is part of the Adblock Plus web scripts,
-# Copyright (C) 2006-2016 Eyeo GmbH
-#
-# Adblock Plus is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 3 as
-# published by the Free Software Foundation.
-#
-# Adblock Plus is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Adblock Plus. If not, see .
-
-import json
-import os
-import tempfile
-import time
-import traceback
-from datetime import datetime
-from errno import EEXIST
-
-import MySQLdb
-
-from sitescripts.web import url_handler
-from sitescripts.utils import get_config, setupStderr
-from sitescripts.filterhits import db, geometrical_mean
-from sitescripts.filterhits.web import common
-
-
-def log_filterhits(data, basepath, query_string):
- """
- This logs the provided filterhits data as JSON to a file named after
- the current timestamp in a directory named after the current date.
- """
- now = time.gmtime()
-
- dir_name = time.strftime("%Y-%m-%d", now)
- path = os.path.join(basepath, dir_name)
- try:
- os.makedirs(path)
- except OSError as e:
- if e.errno != EEXIST:
- raise
-
- with tempfile.NamedTemporaryFile(
- prefix=str(int(time.mktime(now))) + "-",
- suffix=".log",
- dir=path,
- delete=False
- ) as f:
- print >> f, "[%s] %s" % (time.strftime("%d/%b/%Y:%H:%M:%S", now), query_string)
- json.dump(data, f)
- return f.name
-
-
-@url_handler("/submit")
-def submit(environ, start_response):
- setupStderr(environ["wsgi.errors"])
- config = get_config()
-
- # Check that this is a POST request
- if environ["REQUEST_METHOD"] != "POST":
- return common.show_error("Unsupported request method", start_response)
-
- # Parse the submitted JSON
- try:
- data = json.loads(environ["wsgi.input"].read(int(environ["CONTENT_LENGTH"])))
- except (KeyError, IOError, ValueError):
- return common.show_error("Error while parsing JSON data.", start_response)
-
- # Make sure the submitted data was contained within an object at least
- if not isinstance(data, dict):
- return common.show_error("Error, data must be contained within an object.", start_response)
-
- # Log the data to a file
- log_dir = config.get("filterhitstats", "log_dir")
- try:
- log_file = log_filterhits(data, log_dir, environ.get("QUERY_STRING", ""))
- except (OSError, IOError):
- traceback.print_exc()
- return common.show_error("Failed to write data to log file!", start_response,
- "500 Logging error")
-
- # Update the geometrical_mean aggregations in the database
- interval = config.get("filterhitstats", "interval")
- try:
- db_connection = db.connect()
- try:
- db.write(db_connection, geometrical_mean.update(interval, data))
- finally:
- db_connection.close()
- except:
- # Updating the aggregations in the database failed for whatever reason,
- # log the details but continue to return 204 to the client to avoid the
- # re-transmission of data.
- processing_error_log = os.path.join(config.get("filterhitstats", "log_dir"),
- "processing-errors.log")
- with open(processing_error_log, "a+") as f:
- message = "Problem processing data file %s:\n%s" % (
- log_file, traceback.format_exc()
- )
- print >> f, "[%s] %s" % (datetime.now().strftime("%d/%b/%Y:%H:%M:%S %z"), message)
-
- # Send back a 204 No Content
- start_response("204 No Content", [])
- return []
Index: tox.ini
===================================================================
--- a/tox.ini
+++ b/tox.ini
@@ -27,7 +27,7 @@
commands =
cp .sitescripts.example .sitescripts.test
pysed \
- -r sitescripts\.(reports|filterhits|testpages|crawler|urlfixer)\.web.* \
+ -r sitescripts\.(reports|testpages|crawler|urlfixer)\.web.* \
'' .sitescripts.test --write
python ensure_dependencies.py
py.test \
@@ -35,5 +35,5 @@
sitescripts/hg/test \
sitescripts/notifications/test \
sitescripts/stats/test
- flake8 --ignore=E129,E501,E704,E711,E712,E713,E714,E731,F401,F841,F821 \
+ flake8 --ignore=E129,E501,E704,E711,E712,E713,E731,F401,F841,F821 \
sitescripts multiplexer.py multiplexer.fcgi