Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code

Delta Between Two Patch Sets: sitescripts/filterhits/test/api_tests.py

Issue 4615801646612480: Issue 395 - Filter hits statistics backend (Closed)
Left Patch Set: Addressed more comments. Created March 2, 2015, 10:37 a.m.
Right Patch Set: Addressed further comments from Sebastian. Created April 2, 2015, 10:13 a.m.
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments.
Jump to:
Left: Side by side diff | Download
Right: Side by side diff | Download
« no previous file with change/comment | « sitescripts/filterhits/test/__init__.py ('k') | sitescripts/filterhits/test/db_tests.py » ('j') | no next file with change/comment »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
LEFTRIGHT
1 # coding: utf-8 1 # coding: utf-8
2 2
3 # This file is part of the Adblock Plus web scripts, 3 # This file is part of the Adblock Plus web scripts,
4 # Copyright (C) 2006-2015 Eyeo GmbH 4 # Copyright (C) 2006-2015 Eyeo GmbH
5 # 5 #
6 # Adblock Plus is free software: you can redistribute it and/or modify 6 # Adblock Plus is free software: you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License version 3 as 7 # it under the terms of the GNU General Public License version 3 as
8 # published by the Free Software Foundation. 8 # published by the Free Software Foundation.
9 # 9 #
10 # Adblock Plus is distributed in the hope that it will be useful, 10 # Adblock Plus is distributed in the hope that it will be useful,
11 # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 # GNU General Public License for more details. 13 # GNU General Public License for more details.
14 # 14 #
15 # You should have received a copy of the GNU General Public License 15 # You should have received a copy of the GNU General Public License
16 # along with Adblock Plus. If not, see <http://www.gnu.org/licenses/>. 16 # along with Adblock Plus. If not, see <http://www.gnu.org/licenses/>.
17 17
18 import MySQLdb, StringIO, json, sys, unittest 18 import json
19 import sys
20 import StringIO
21 import unittest
19 from urllib import urlencode 22 from urllib import urlencode
20 23
24 from sitescripts.filterhits.test import test_helpers
21 from sitescripts.filterhits import db 25 from sitescripts.filterhits import db
22 from sitescripts.filterhits.web.query import query_handler 26 from sitescripts.filterhits.web.query import query_handler
23 from sitescripts.filterhits.web.submit import submit as submit_handler 27 from sitescripts.filterhits.web.submit import submit as submit_handler
24 28
25 valid_data = """{ 29 valid_data = """{
26 "version": 1, 30 "version": 1,
27 "timeSincePush": 12345, 31 "timeSincePush": 12345,
28 "addonName": "adblockplus", 32 "addonName": "adblockplus",
29 "addonVersion": "2.3.4", 33 "addonVersion": "2.3.4",
30 "application": "firefox", 34 "application": "firefox",
31 "applicationVersion": "31", 35 "applicationVersion": "31",
32 "platform": "gecko", 36 "platform": "gecko",
33 "platformVersion": "31", 37 "platformVersion": "31",
34 "filters": { 38 "filters": {
35 "||example.com^": { 39 "||example.com^": {
36 "firstParty": { 40 "firstParty": {
37 "example.com": {"hits": 12, "latest": 1414817340948}, 41 "example.com": {"hits": 12, "latest": 1414817340948},
38 "example.org": {"hits": 4, "latest": 1414859271125} 42 "example.org": {"hits": 4, "latest": 1414859271125}
39 }, 43 },
40 "thirdParty": { 44 "thirdParty": {
41 "example.com": {"hits": 5, "latest": 1414916268769} 45 "example.com": {"hits": 5, "latest": 1414916268769}
42 }, 46 },
43 "subscriptions": ["EasyList", "EasyList Germany+EasyList"] 47 "subscriptions": ["EasyList", "EasyList Germany+EasyList"]
44 } 48 }
45 } 49 }
46 }""" 50 }"""
47 51
48 class APITestCase(unittest.TestCase): 52 class APITestCase(test_helpers.FilterhitsTestCase):
49 def clear_rows(self):
50 if self.db:
51 db.write(self.db, (("DELETE FROM filters",),
52 ("DELETE FROM geometrical_mean",)))
53
54 def setUp(self):
55 try:
56 db.testing = True
57 self.db = db.connect()
58 except MySQLdb.Error:
59 self.db = None
60 self.clear_rows()
61
62 def tearDown(self):
63 if self.db:
64 self.clear_rows()
65 self.db.close()
66 self.db = None
67
68 def assertResponse(self, handler, expected_response, expected_result=None, exp ected_headers=None, **environ): 53 def assertResponse(self, handler, expected_response, expected_result=None, exp ected_headers=None, **environ):
69 def check_response(response, headers): 54 def check_response(response, headers):
70 self.assertEqual(response, expected_response) 55 self.assertEqual(response, expected_response)
71 if not expected_headers is None: 56 if not expected_headers is None:
72 self.assertEqual(headers, expected_headers) 57 self.assertEqual(headers, expected_headers)
73 58
74 if "body" in environ: 59 if "body" in environ:
75 environ["CONTENT_LENGTH"] = len(environ["body"]) 60 environ["CONTENT_LENGTH"] = len(environ["body"])
76 environ["wsgi.input"] = StringIO.StringIO(environ["body"]) 61 environ["wsgi.input"] = StringIO.StringIO(environ["body"])
77 del environ["body"] 62 del environ["body"]
78 63
79 if "get_params" in environ: 64 if "get_params" in environ:
80 environ["QUERY_STRING"] = urlencode(environ["get_params"]) 65 environ["QUERY_STRING"] = urlencode(environ["get_params"])
81 del environ["get_params"] 66 del environ["get_params"]
82 67
83 environ["wsgi.errors"] = sys.stderr 68 environ["wsgi.errors"] = sys.stderr
84 result = handler(environ, check_response) 69 result = handler(environ, check_response)
85 if not expected_result is None: 70 if not expected_result is None:
86 self.assertEqual(json.loads("".join(result)), expected_result) 71 self.assertEqual(json.loads("".join(result)), expected_result)
87 72
88 def test_submit(self): 73 def test_submit(self):
89 self.assertEqual(len(db.query(self.db, "SELECT * FROM geometrical_mean")), 0 ) 74 self.assertEqual(len(db.query(self.db, "SELECT * FROM frequencies")), 0)
90 self.assertEqual(len(db.query(self.db, "SELECT * FROM filters")), 0) 75 self.assertEqual(len(db.query(self.db, "SELECT * FROM filters")), 0)
91 # Ensure missing or invalid JSON results in an error 76 # Ensure missing or invalid JSON results in an error
92 self.assertResponse(submit_handler, "400 Processing Error", 77 self.assertResponse(submit_handler, "400 Processing Error",
93 REQUEST_METHOD="POST", body="") 78 REQUEST_METHOD="POST", body="")
94 self.assertResponse(submit_handler, "400 Processing Error", 79 self.assertResponse(submit_handler, "400 Processing Error",
95 REQUEST_METHOD="POST", body="Oops...") 80 REQUEST_METHOD="POST", body="Oops...")
96 self.assertResponse(submit_handler, "400 Processing Error", 81 self.assertResponse(submit_handler, "400 Processing Error",
97 REQUEST_METHOD="POST", body="{123:]") 82 REQUEST_METHOD="POST", body="{123:]")
98 self.assertResponse(submit_handler, "400 Processing Error", 83 self.assertResponse(submit_handler, "400 Processing Error",
99 REQUEST_METHOD="POST", body="1") 84 REQUEST_METHOD="POST", body="1")
100 self.assertEqual(len(db.query(self.db, "SELECT * FROM geometrical_mean")), 0 ) 85 self.assertEqual(len(db.query(self.db, "SELECT * FROM frequencies")), 0)
101 self.assertEqual(len(db.query(self.db, "SELECT * FROM filters")), 0) 86 self.assertEqual(len(db.query(self.db, "SELECT * FROM filters")), 0)
102 # Ensure even an empty object, or one with the wrong fields returns OK 87 # Ensure even an empty object, or one with the wrong fields returns successf ully
103 self.assertResponse(submit_handler, "200 OK", 88 self.assertResponse(submit_handler, "204 No Content",
104 REQUEST_METHOD="POST", body="{}") 89 REQUEST_METHOD="POST", body="{}")
105 self.assertResponse(submit_handler, "200 OK", 90 self.assertResponse(submit_handler, "204 No Content",
106 REQUEST_METHOD="POST", body="{\"hello\": \"world\"}") 91 REQUEST_METHOD="POST", body="{\"hello\": \"world\"}")
107 self.assertEqual(len(db.query(self.db, "SELECT * FROM geometrical_mean")), 0 ) 92 self.assertEqual(len(db.query(self.db, "SELECT * FROM frequencies")), 0)
108 self.assertEqual(len(db.query(self.db, "SELECT * FROM filters")), 0) 93 self.assertEqual(len(db.query(self.db, "SELECT * FROM filters")), 0)
109 # Now some actually valid data 94 # Now some actually valid data
110 self.assertResponse(submit_handler, "200 OK", 95 self.assertResponse(submit_handler, "204 No Content",
111 REQUEST_METHOD="POST", body=valid_data) 96 REQUEST_METHOD="POST", body=valid_data)
112 self.assertEqual(len(db.query(self.db, "SELECT * FROM geometrical_mean")), 2 ) 97 self.assertEqual(len(db.query(self.db, "SELECT * FROM frequencies")), 2)
113 self.assertEqual(len(db.query(self.db, "SELECT * FROM filters")), 1) 98 self.assertEqual(len(db.query(self.db, "SELECT * FROM filters")), 1)
114 # Now make sure apparently valid data with timestamps that cause geometrical 99 # Now make sure apparently valid data with timestamps that cause geometrical
115 # mean calculations to fail with MySQL errors return OK but don't change DB 100 # mean calculations to fail with MySQL errors return OK but don't change DB
116 invalid_data = json.loads(valid_data) 101 invalid_data = json.loads(valid_data)
117 invalid_data["filters"]["||example.com^"]["firstParty"]["example.com"]["late st"] = 3 102 invalid_data["filters"]["||example.com^"]["firstParty"]["example.com"]["late st"] = 3
118 invalid_data = json.dumps(invalid_data) 103 invalid_data = json.dumps(invalid_data)
119 self.assertResponse(submit_handler, "200 OK", 104 self.assertResponse(submit_handler, "204 No Content",
120 REQUEST_METHOD="POST", body=invalid_data) 105 REQUEST_METHOD="POST", body=invalid_data)
121 self.assertEqual(len(db.query(self.db, "SELECT * FROM geometrical_mean")), 2 ) 106 self.assertEqual(len(db.query(self.db, "SELECT * FROM frequencies")), 2)
122 self.assertEqual(len(db.query(self.db, "SELECT * FROM filters")), 1) 107 self.assertEqual(len(db.query(self.db, "SELECT * FROM filters")), 1)
123 108
124 def test_query(self): 109 def test_query(self):
125 # Basic query with no data, should return OK 110 # Basic query with no data, should return successfully
126 self.assertResponse(query_handler, "200 OK", {"count": 0, "total": 0, "resul ts": [], "echo": 0}) 111 self.assertResponse(query_handler, "200 OK", {"count": 0, "total": 0, "resul ts": [], "echo": 0})
127 # If echo parameter is passed and is integer it should be returned 112 # If echo parameter is passed and is integer it should be returned
128 self.assertResponse(query_handler, "200 OK", {"count": 0, "total": 0, "resul ts": [], "echo": 1337}, 113 self.assertResponse(query_handler, "200 OK", {"count": 0, "total": 0, "resul ts": [], "echo": 1337},
129 get_params={ "echo": 1337 }) 114 get_params={ "echo": 1337 })
130 self.assertResponse(query_handler, "200 OK", {"count": 0, "total": 0, "resul ts": [], "echo": 0}, 115 self.assertResponse(query_handler, "200 OK", {"count": 0, "total": 0, "resul ts": [], "echo": 0},
131 get_params={ "echo": "naughty" }) 116 get_params={ "echo": "naughty" })
132 # Now let's submit some data so we can query it back out 117 # Now let's submit some data so we can query it back out
133 test_data = [json.loads(valid_data), json.loads(valid_data), json.loads(vali d_data)] 118 test_data = [json.loads(valid_data), json.loads(valid_data), json.loads(vali d_data)]
134 test_data[1]["filters"]["##Second-Filter|"] = test_data[1]["filters"].pop("| |example.com^") 119 test_data[1]["filters"]["##Second-Filter|"] = test_data[1]["filters"].pop("| |example.com^")
135 test_data[2]["filters"]["##Third-Filter|"] = test_data[2]["filters"].pop("|| example.com^") 120 test_data[2]["filters"]["##Third-Filter|"] = test_data[2]["filters"].pop("|| example.com^")
136 for data in test_data: 121 for data in test_data:
137 self.assertResponse(submit_handler, "200 OK", 122 self.assertResponse(submit_handler, "204 No Content",
138 REQUEST_METHOD="POST", body=json.dumps(data)) 123 REQUEST_METHOD="POST", body=json.dumps(data))
139 # Ordering parameters should be respected 124 # Ordering parameters should be respected
140 self.assertResponse(query_handler, "200 OK", {"count": 1, "total": 6, 125 self.assertResponse(query_handler, "200 OK", {"count": 1, "total": 6,
141 "results": [{'domain': 'exampl e.com', 126 "results": [{"domain": "exampl e.com",
142 'filter': '||exam ple.com^', 127 "filter": "||exam ple.com^",
143 'hits': 0}], "ech o": 0}, 128 "frequency": 0}], "echo": 0},
144 get_params={ "order_by": "filter", "order": "desc", "tak e": "1" }) 129 get_params={ "order_by": "filter", "order": "desc", "tak e": "1" })
145 self.assertResponse(query_handler, "200 OK", {"count": 1, "total": 6, 130 self.assertResponse(query_handler, "200 OK", {"count": 1, "total": 6,
146 "results": [{'domain': 'exampl e.com', 131 "results": [{"domain": "exampl e.com",
147 'filter': '##Seco nd-Filter|', 132 "filter": "##Seco nd-Filter|",
148 'hits': 0}], "ech o": 0}, 133 "frequency": 0}], "echo": 0},
149 get_params={ "order_by": "filter", "order": "asc", "take ": "1" }) 134 get_params={ "order_by": "filter", "order": "asc", "take ": "1" })
150 # As should filtering parameters 135 # As should filtering parameters
151 self.assertResponse(query_handler, "200 OK", {"count": 1, "total": 3, 136 self.assertResponse(query_handler, "200 OK", {"count": 1, "total": 3,
152 "results": [{'domain': 'exampl e.com', 137 "results": [{"domain": "exampl e.com",
153 'filter': '##Thir d-Filter|', 138 "filter": "##Thir d-Filter|",
154 'hits': 0}], "ech o": 0}, 139 "frequency": 0}], "echo": 0},
155 get_params={ "domain": "example.com", "take": "1" }) 140 get_params={ "domain": "example.com", "take": "1" })
156 self.assertResponse(query_handler, "200 OK", {"count": 1, "total": 2, 141 self.assertResponse(query_handler, "200 OK", {"count": 1, "total": 2,
157 "results": [{'domain': 'exampl e.org', 142 "results": [{"domain": "exampl e.org",
158 'filter': '##Thir d-Filter|', 143 "filter": "##Thir d-Filter|",
159 'hits': 4}], "ech o": 0}, 144 "frequency": 4}], "echo": 0},
160 get_params={ "filter": "Third", "take": 1 }) 145 get_params={ "filter": "Third", "take": 1 })
161 self.assertResponse(query_handler, "200 OK", {"count": 1, "total": 1, 146 self.assertResponse(query_handler, "200 OK", {"count": 1, "total": 1,
162 "results": [{'domain': 'exampl e.com', 147 "results": [{"domain": "exampl e.com",
163 'filter': '##Thir d-Filter|', 148 "filter": "##Thir d-Filter|",
164 'hits': 0}], "ech o": 0}, 149 "frequency": 0}], "echo": 0},
165 get_params={ "domain": "example.com", "filter": "Third", "take": "1" }) 150 get_params={ "domain": "example.com", "filter": "Third", "take": "1" })
166 # ... and pagination parameters 151 # ... and pagination parameters
167 self.maxDiff = None 152 self.maxDiff = None
168 self.assertResponse(query_handler, "200 OK", {"count": 2, "total": 6, 153 self.assertResponse(query_handler, "200 OK", {"count": 2, "total": 6,
169 "results": [{'domain': 'exampl e.org', 154 "results": [{"domain": "exampl e.org",
170 'filter': '||exam ple.com^', 155 "filter": "||exam ple.com^",
171 'hits': 4}, 156 "frequency": 4},
172 {'domain': 'exampl e.org', 157 {"domain": "exampl e.org",
173 'filter': '##Seco nd-Filter|', 158 "filter": "##Seco nd-Filter|",
174 'hits': 4}], "ech o": 0}, 159 "frequency": 4}], "echo": 0},
175 get_params={ "skip": "1", "take": "2" }) 160 get_params={ "skip": "1", "take": "2" })
176 self.assertResponse(query_handler, "200 OK", {"count": 2, "total": 6, 161 self.assertResponse(query_handler, "200 OK", {"count": 2, "total": 6,
177 "results": [{'domain': 'exampl e.org', 162 "results": [{"domain": "exampl e.org",
178 'filter': '##Seco nd-Filter|', 163 "filter": "##Seco nd-Filter|",
179 'hits': 4}, 164 "frequency": 4},
180 {'domain': 'exampl e.com', 165 {"domain": "exampl e.com",
181 'filter': '##Thir d-Filter|', 166 "filter": "##Thir d-Filter|",
182 'hits': 0}], "ech o": 0}, 167 "frequency": 0}], "echo": 0},
183 get_params={ "skip": "2", "take": "2" }) 168 get_params={ "skip": "2", "take": "2" })
184 169
185 if __name__ == '__main__': 170 if __name__ == "__main__":
186 unittest.main() 171 unittest.main()
LEFTRIGHT

Powered by Google App Engine
This is Rietveld