Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code
Issues
All Issues

Side by Side Diff: cms/bin/test_server.py

Issue 29912588: Issue 7019 - [CMS] Refactor `test_server.py` (Closed)
Patch Set: Added timeout option when setting up server Created Oct. 24, 2018, 3:57 p.m.
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments.
Jump to:
View unified diff | Download patch
« no previous file with comments | « .hgignore ('k') | cms/exceptions.py » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 # This file is part of the Adblock Plus web scripts, 1 # This file is part of the Adblock Plus web scripts,
2 # Copyright (C) 2006-present eyeo GmbH 2 # Copyright (C) 2006-present eyeo GmbH
3 # 3 #
4 # Adblock Plus is free software: you can redistribute it and/or modify 4 # Adblock Plus is free software: you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License version 3 as 5 # it under the terms of the GNU General Public License version 3 as
6 # published by the Free Software Foundation. 6 # published by the Free Software Foundation.
7 # 7 #
8 # Adblock Plus is distributed in the hope that it will be useful, 8 # Adblock Plus is distributed in the hope that it will be useful,
9 # but WITHOUT ANY WARRANTY; without even the implied warranty of 9 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # GNU General Public License for more details. 11 # GNU General Public License for more details.
12 # 12 #
13 # You should have received a copy of the GNU General Public License 13 # You should have received a copy of the GNU General Public License
14 # along with Adblock Plus. If not, see <http://www.gnu.org/licenses/>. 14 # along with Adblock Plus. If not, see <http://www.gnu.org/licenses/>.
15 15
16 from __future__ import print_function
17
18 import os
16 import mimetypes 19 import mimetypes
17 import os
18 import sys
19 import argparse 20 import argparse
21 import time
20 22
21 import jinja2 23 import jinja2
22 24
25 from cms.converters import converters
23 from cms.utils import process_page 26 from cms.utils import process_page
24 from cms.sources import create_source 27 from cms.sources import create_source
25 from cms.converters import converters 28 from cms.exceptions import ServerSetupTimeoutException
26
27 source = None
28 address = None
29 port = None
30 29
31 UNICODE_ENCODING = 'utf-8' 30 UNICODE_ENCODING = 'utf-8'
32 31
33 ERROR_TEMPLATE = ''' 32 ERROR_TEMPLATE = '''
34 <html> 33 <html>
35 <head> 34 <head>
36 <title>{{status}}</title> 35 <title>{{status}}</title>
37 </head> 36 </head>
38 <body> 37 <body>
39 <h1>{{status}}</h1> 38 <h1>{{status}}</h1>
40 {% set code = status.split()|first|int %} 39 {% set code = status.split()|first|int %}
41 {% if code == 404 %} 40 {% if code == 404 %}
42 <p>No page found for the address {{uri}}.</p> 41 <p>No page found for the address {{uri}}.</p>
43 {% elif code == 500 %} 42 {% elif code == 500 %}
44 <p>An error occurred while processing the request for {{uri}}:</p> 43 <p>An error occurred while processing the request for {{uri}}:</p>
45 <pre>{{error}}</pre> 44 <pre>{{error}}</pre>
46 {% endif %} 45 {% endif %}
47 </body> 46 </body>
48 </html>''' 47 </html>'''
49 48
50 # Initilize the mimetypes modules manually for consistent behavior, 49 # Initialize the mimetypes modules manually for consistent behavior,
51 # ignoring local files and Windows Registry. 50 # ignoring local files and Windows Registry.
52 mimetypes.init([]) 51 mimetypes.init([])
53 52
54 53
55 def get_page(path): 54 class DynamicServerHandler:
56 path = path.strip('/') 55 """General-purpose WSGI server handler that generates pages on request.
57 if path == '': 56
58 path = source.read_config().get('general', 'defaultlocale') 57 Parameters
59 if '/' in path: 58 ----------
60 locale, page = path.split('/', 1) 59 host: str
61 else: 60 The host where the server will run.
62 locale, page = path, '' 61 port: int
63 62 The TCP port the server will run on.
64 default_page = source.read_config().get('general', 'defaultpage') 63 source_dir: str
65 alternative_page = '/'.join([page, default_page]).lstrip('/') 64 The path to the website contents.
66 65
67 for format in converters.iterkeys(): 66 """
68 for p in (page, alternative_page): 67
69 if source.has_page(p, format): 68 def __init__(self, host, port, source_dir):
70 site_url = 'http://{}:{}'.format(address, port) 69 self.host = host
71 return (p, process_page(source, locale, p, format, site_url)) 70 self.port = port
72 if source.has_localizable_file(locale, page): 71 self.source = create_source(source_dir)
73 return (page, source.read_localizable_file(locale, page)) 72 self.full_url = 'http://{0}:{1}'.format(host, port)
74 73
75 return (None, None) 74 def _get_data(self, path):
76 75 """Read the data corresponding to a website path.
77 76
78 def has_conflicting_pages(page): 77 Parameters
79 pages = [p for p, _ in source.list_pages()] 78 ----------
80 pages.extend(source.list_localizable_files()) 79 path: str
81 80 The path to the page to get the data for.
82 if pages.count(page) > 1: 81
83 return True 82 Returns
84 if any(p.startswith(page + '/') or page.startswith(p + '/') for p in pages): 83 -------
85 return True 84 str or bytes
86 return False 85 The data corresponding to the path we're trying to access.
87 86
88 87 """
89 def get_data(path): 88 if self.source.has_static(path):
90 if source.has_static(path): 89 return self.source.read_static(path)
91 return source.read_static(path) 90
92 91 page, data = self._get_page(path)
93 page, data = get_page(path) 92
94 if page and has_conflicting_pages(page): 93 if page and self._has_conflicts(page):
95 raise Exception('The requested page conflicts with another page') 94 raise Exception('The requested page conflicts with another page.')
96 return data 95
97 96 return data
98 97
99 def show_error(start_response, status, **kwargs): 98 def _get_page(self, path):
100 env = jinja2.Environment(autoescape=True) 99 """Construct a page and return its contents.
101 template = env.from_string(ERROR_TEMPLATE) 100
102 mime = 'text/html; encoding=%s' % UNICODE_ENCODING 101 Parameters
103 start_response(status, [('Content-Type', mime)]) 102 ----------
104 for fragment in template.stream(status=status, **kwargs): 103 path: str
105 yield fragment.encode(UNICODE_ENCODING) 104 The path of the page we want to construct.
106 105
107 106 Returns
108 def handler(environ, start_response): 107 -------
109 path = environ.get('PATH_INFO') 108 (page_name, page_contents): (str, str)
110 109
111 data = get_data(path) 110 """
112 if data is None: 111 path = path.strip('/')
113 return show_error(start_response, '404 Not Found', uri=path) 112 if path == '':
114 113 locale, page = self.source.read_config().get('general',
115 mime = mimetypes.guess_type(path)[0] or 'text/html' 114 'defaultlocale'), ''
116 115 elif '/' in path:
117 if isinstance(data, unicode): 116 locale, page = path.split('/', 1)
118 data = data.encode(UNICODE_ENCODING) 117 else:
119 mime = '%s; charset=%s' % (mime, UNICODE_ENCODING) 118 locale, page = path, ''
120 119
121 start_response('200 OK', [('Content-Type', mime)]) 120 default_page = self.source.read_config().get('general', 'defaultpage')
122 return [data] 121 possible_pages = [page, '/'.join([page, default_page]).lstrip('/')]
122
123 for page_format in converters.iterkeys():
124 for p in possible_pages:
125 if self.source.has_page(p, page_format):
126 return p, process_page(self.source, locale, p, page_format,
127 self.full_url)
128
129 if self.source.has_localizable_file(locale, page):
130 return page, self.source.read_localizable_file(locale, page)
131
132 return None, None
133
134 def _has_conflicts(self, page):
135 """Check if a page has conflicts.
136
137 A page has conflicts if there are other pages with the same name.
138
139 Parameters
140 ----------
141 page: str
142 The path of the page we're checking for conflicts.
143
144 Returns
145 -------
146 bool
147 True - if the page has conflicts
148 False - otherwise
149
150 """
151 pages = [p for p, _ in self.source.list_pages()]
152 pages.extend(self.source.list_localizable_files())
153
154 if pages.count(page) > 1:
155 return True
156 if any(p.startswith(page + '/') or page.startswith(p + '/') for p in
157 pages):
158 return True
159 return False
160
161 def get_error_page(self, start_response, status, **kw):
162 """Create and display an error page.
163
164 Parameters
165 ----------
166 start_response: function
167 It will be called before constructing the error page, to setup
168 things like the status of the response and the headers.
169 status: str
170 The status of the response we're sending the error page with.
171 Needs to have the following format:
172 "<status_code> <status_message>"
173 kw: dict
174 Any additional arguments that will be passed onto the `stream`
175 method of a `jinja2 Template`.
176
177 Returns
178 -------
179 generator of utf8 strings
180 Fragments of the corresponding error HTML template.
181
182 """
183 env = jinja2.Environment(autoescape=True)
184 page_template = env.from_string(ERROR_TEMPLATE)
185 mime = 'text/html; encoding={}'.format(UNICODE_ENCODING)
186
187 start_response(status, [('Content-Type', mime)])
188
189 for fragment in page_template.stream(status=status, **kw):
190 yield fragment.encode(UNICODE_ENCODING)
191
192 def __call__(self, environ, start_response):
193 """Execute the handler, according to the WSGI standards.
194
195 Parameters
196 ---------
197 environ: dict
198 The environment under which the page is requested.
199 The requested page must be under the `PATH_INFO` key.
200 start_response: function
201 Used to initiate a response. Must take two arguments, in this
202 order:
203 - Response status, in the format "<code> <message>".
204 - Response headers, as a list of tuples.
205
206 Returns
207 -------
208 list of str
209 With the data for a specific page.
210
211 """
212 path = environ.get('PATH_INFO')
213
214 data = self._get_data(path)
215
216 if data is None:
217 return self.get_error_page(start_response, '404 Not Found',
218 uri=path)
219
220 mime = mimetypes.guess_type(path)[0] or 'text/html'
221
222 if isinstance(data, unicode):
223 data = data.encode(UNICODE_ENCODING)
224 mime = '{0}; charset={1}'.format(mime, UNICODE_ENCODING)
225
226 start_response('200 OK', [('Content-Type', mime)])
227 return [data]
228
229
230 def parse_arguments():
231 """Set up and parse the arguments required by the script.
232
233 Returns
234 -------
235 argparse.Namespace
236 With the script arguments, as parsed.
237
238 """
239 parser = argparse.ArgumentParser(
240 description='CMS development server created to test pages locally and '
241 'on-the-fly.',
242 )
243
244 parser.add_argument('path', default=os.curdir, nargs='?',
245 help='Path to the website we intend to run. If not '
246 'provided, defaults, to the current directory.')
247 parser.add_argument('--host', default='localhost',
248 help='Address of the host the server will listen on. '
249 'Defaults to "localhost".')
250 parser.add_argument('--port', default=5000, type=int,
251 help='TCP port the server will listen on. Default '
252 '5000.')
253
254 return parser.parse_args()
255
256
257 def wait_for_address(run_fn, handler, timeout=1.0, *args, **kw):
258 """Try to setup the server at a given address.
259
260 If an exception occurs, we sleep for .1 seconds and retry until
261 `timeout` has passed.
262
263 Parameters
264 ----------
265 run_fn: callable
266 Used to initiate the server.
267 handler: DynamicServerHandler
268 Defines the parameters and methods required to handle requests.
269 timeout: float
270 Number of seconds to wait until giving up.
271
272 Raises
273 ------
274 ServerSetupTimeoutException
275 If the server couldn't be setup at the required address after
276 `timeout` passed.
277
278 """
279 start_time = time.time()
280
281 while True:
282 try:
283 with run_fn(handler.host, handler.port, handler, *args, **kw):
284 break
285 except Exception as err:
286 time.sleep(.1)
287 if time.time() - start_time >= timeout:
288 raise ServerSetupTimeoutException(handler.host, handler.port,
289 timeout, err)
290
291
292 def run_werkzeug_server(handler, **kw):
293 """Set up a server that uses `werkzeug`.
294
295 Parameters
296 ----------
297 handler: DynamicServerHandler
298 Defines the parameters and methods required to handle requests.
299
300 Raises
301 ------
302 ImportError
303 If the package `werkzeug` is not installed
304
305 """
306 from werkzeug.serving import run_simple
307 import logging
308
309 def run(*args, **kwargs):
310 # The werkzeug logger must be configured before the
311 # root logger. Also we must prevent it from propagating
312 # messages, otherwise messages are logged twice.
313 logger = logging.getLogger('werkzeug')
314 logger.propagate = False
315 logger.setLevel(logging.INFO)
316 logger.addHandler(logging.StreamHandler())
317
318 run_simple(threaded=True, *args, **kwargs)
319
320 wait_for_address(run, handler, **kw)
321
322
323 def run_builtins_server(handler, **kw):
324 """Configure a server that only uses builtin packages.
325
326 Parameters
327 ----------
328 handler: DynamicServerHandler
329 Defines the parameters and methods required to handle requests.
330
331 """
332 from SocketServer import ThreadingMixIn
333 from wsgiref.simple_server import WSGIServer, make_server
334
335 class ThreadedWSGIServer(ThreadingMixIn, WSGIServer):
336 daemon_threads = True
337
338 def run(host, port, app, **kwargs):
339 def wrapper(environ, start_response):
340 try:
341 return app(environ, start_response)
342 except Exception as e:
343 return handler.get_error_page(
344 start_response, '500 Internal Server Error',
345 uri=environ.get('PATH_INFO'), error=e,
346 )
347
348 server = make_server(host, port, wrapper, ThreadedWSGIServer)
349 print(' * Running on {0}:{1}'.format(*server.server_address))
350 server.serve_forever()
351
352 wait_for_address(run, handler, **kw)
353
354
355 def main():
356 args = parse_arguments()
357 handler = DynamicServerHandler(args.host, args.port, args.path)
358
359 try:
360 run_werkzeug_server(handler, use_reloader=True, use_debugger=True)
361 except ImportError:
362 run_builtins_server(handler)
123 363
124 364
125 if __name__ == '__main__': 365 if __name__ == '__main__':
126 366 main()
127 parser = argparse.ArgumentParser(description='CMS development server created to test pages locally and on-the-fly')
128 parser.add_argument('path', nargs='?', default=os.curdir)
129 parser.add_argument('-a', '--address', default='localhost', help='Address of the interface the server will listen on')
130 parser.add_argument('-p', '--port', type=int, default=5000, help='TCP port t he server will listen on')
131 args = parser.parse_args()
132
133 source = create_source(args.path)
134 address = args.address
135 port = args.port
136
137 try:
138 from werkzeug.serving import ThreadedWSGIServer, run_simple
139
140 # see https://github.com/mitsuhiko/werkzeug/pull/770
141 ThreadedWSGIServer.daemon_threads = True
142
143 def run(*args, **kwargs):
144 # The werkzeug logger must be configured before the
145 # root logger. Also we must prevent it from propagating
146 # messages, otherwise messages are logged twice.
147 import logging
148 logger = logging.getLogger('werkzeug')
149 logger.propagate = False
150 logger.setLevel(logging.INFO)
151 logger.addHandler(logging.StreamHandler())
152
153 run_simple(threaded=True, *args, **kwargs)
154 except ImportError:
155 from SocketServer import ThreadingMixIn
156 from wsgiref.simple_server import WSGIServer, make_server
157
158 class ThreadedWSGIServer(ThreadingMixIn, WSGIServer):
159 daemon_threads = True
160
161 def run(host, port, app, **kwargs):
162 def wrapper(environ, start_response):
163 try:
164 return app(environ, start_response)
165 except Exception as e:
166 return show_error(start_response, '500 Internal Server Error ',
167 uri=environ.get('PATH_INFO'), error=e)
168
169 server = make_server(host, port, wrapper, ThreadedWSGIServer)
170 print ' * Running on http://%s:%i/' % server.server_address
171 server.serve_forever()
172
173 run(address, port, handler, use_reloader=True, use_debugger=True)
OLDNEW
« no previous file with comments | « .hgignore ('k') | cms/exceptions.py » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld