1
2
3
4 """
5 This file is part of the web2py Web Framework
6 Copyrighted by Massimo Di Pierro <mdipierro@cs.depaul.edu>
7 License: LGPLv3 (http://www.gnu.org/licenses/lgpl.html)
8
9 Contains:
10
11 - wsgibase: the gluon wsgi application
12
13 """
14
15 if False: import import_all
16 import gc
17 import cgi
18 import cStringIO
19 import Cookie
20 import os
21 import re
22 import copy
23 import sys
24 import time
25 import datetime
26 import signal
27 import socket
28 import tempfile
29 import random
30 import string
31 import urllib2
32 try:
33 import simplejson as sj
34 except:
35 try:
36 import json as sj
37 except:
38 import contrib.simplejson as sj
39
40 from thread import allocate_lock
41
42 from fileutils import abspath, write_file, parse_version, copystream
43 from settings import global_settings
44 from admin import add_path_first, create_missing_folders, create_missing_app_folders
45 from globals import current
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63 web2py_path = global_settings.applications_parent
64
65 create_missing_folders()
66
67
68 import logging
69 import logging.config
70
71
72
73
74 import gluon.messageboxhandler
75 logging.gluon = gluon
76
77 exists = os.path.exists
78 pjoin = os.path.join
79
80 logpath = abspath("logging.conf")
81 if exists(logpath):
82 logging.config.fileConfig(abspath("logging.conf"))
83 else:
84 logging.basicConfig()
85 logger = logging.getLogger("web2py")
86
87 from restricted import RestrictedError
88 from http import HTTP, redirect
89 from globals import Request, Response, Session
90 from compileapp import build_environment, run_models_in, \
91 run_controller_in, run_view_in
92 from contenttype import contenttype
93 from dal import BaseAdapter
94 from settings import global_settings
95 from validators import CRYPT
96 from cache import CacheInRam
97 from html import URL, xmlescape
98 from utils import is_valid_ip_address, getipaddrinfo
99 from rewrite import load, url_in, THREAD_LOCAL as rwthread, \
100 try_rewrite_on_error, fixup_missing_path_info
101 import newcron
102
103 __all__ = ['wsgibase', 'save_password', 'appfactory', 'HttpServer']
104
105 requests = 0
106
107
108
109
110
111 regex_client = re.compile('[\w\-:]+(\.[\w\-]+)*\.?')
112
113 try:
114 version_info = open(pjoin(global_settings.gluon_parent, 'VERSION'), 'r')
115 raw_version_string = version_info.read().split()[-1].strip()
116 version_info.close()
117 global_settings.web2py_version = raw_version_string
118 web2py_version = global_settings.web2py_version
119 except:
120 raise RuntimeError("Cannot determine web2py version")
121
122 try:
123 import rocket
124 except:
125 if not global_settings.web2py_runtime_gae:
126 logger.warn('unable to import Rocket')
127
128 load()
129
130 HTTPS_SCHEMES = set(('https', 'HTTPS'))
131
132
134 """
135 guess the client address from the environment variables
136
137 first tries 'http_x_forwarded_for', secondly 'remote_addr'
138 if all fails, assume '127.0.0.1' or '::1' (running locally)
139 """
140 g = regex_client.search(env.get('http_x_forwarded_for', ''))
141 client = (g.group() or '').split(',')[0] if g else None
142 if client in (None, '', 'unknown'):
143 g = regex_client.search(env.get('remote_addr', ''))
144 if g:
145 client = g.group()
146 elif env.http_host.startswith('['):
147 client = '::1'
148 else:
149 client = '127.0.0.1'
150 if not is_valid_ip_address(client):
151 raise HTTP(400, "Bad Request (request.client=%s)" % client)
152 return client
153
154
156 """
157 copies request.env.wsgi_input into request.body
158 and stores progress upload status in cache_ram
159 X-Progress-ID:length and X-Progress-ID:uploaded
160 """
161 env = request.env
162 if not env.content_length:
163 return cStringIO.StringIO()
164 source = env.wsgi_input
165 try:
166 size = int(env.content_length)
167 except ValueError:
168 raise HTTP(400, "Invalid Content-Length header")
169 dest = tempfile.TemporaryFile()
170 if not 'X-Progress-ID' in request.vars:
171 copystream(source, dest, size, chunk_size)
172 return dest
173 cache_key = 'X-Progress-ID:' + request.vars['X-Progress-ID']
174 cache_ram = CacheInRam(request)
175 cache_ram(cache_key + ':length', lambda: size, 0)
176 cache_ram(cache_key + ':uploaded', lambda: 0, 0)
177 while size > 0:
178 if size < chunk_size:
179 data = source.read(size)
180 cache_ram.increment(cache_key + ':uploaded', size)
181 else:
182 data = source.read(chunk_size)
183 cache_ram.increment(cache_key + ':uploaded', chunk_size)
184 length = len(data)
185 if length > size:
186 (data, length) = (data[:size], size)
187 size -= length
188 if length == 0:
189 break
190 dest.write(data)
191 if length < chunk_size:
192 break
193 dest.seek(0)
194 cache_ram(cache_key + ':length', None)
195 cache_ram(cache_key + ':uploaded', None)
196 return dest
197
198
200 """
201 this function is used to generate a dynamic page.
202 It first runs all models, then runs the function in the controller,
203 and then tries to render the output using a view/template.
204 this function must run from the [application] folder.
205 A typical example would be the call to the url
206 /[application]/[controller]/[function] that would result in a call
207 to [function]() in applications/[application]/[controller].py
208 rendered by applications/[application]/views/[controller]/[function].html
209 """
210
211
212
213
214
215 environment = build_environment(request, response, session)
216
217
218
219 response.view = '%s/%s.%s' % (request.controller,
220 request.function,
221 request.extension)
222
223
224
225
226
227
228 run_models_in(environment)
229 response._view_environment = copy.copy(environment)
230 page = run_controller_in(request.controller, request.function, environment)
231 if isinstance(page, dict):
232 response._vars = page
233 response._view_environment.update(page)
234 run_view_in(response._view_environment)
235 page = response.body.getvalue()
236
237 global requests
238 requests = ('requests' in globals()) and (requests + 1) % 100 or 0
239 if not requests:
240 gc.collect()
241
242
243
244
245
246
247 default_headers = [
248 ('Content-Type', contenttype('.' + request.extension)),
249 ('Cache-Control',
250 'no-store, no-cache, must-revalidate, post-check=0, pre-check=0'),
251 ('Expires', time.strftime('%a, %d %b %Y %H:%M:%S GMT',
252 time.gmtime())),
253 ('Pragma', 'no-cache')]
254 for key, value in default_headers:
255 response.headers.setdefault(key, value)
256
257 raise HTTP(response.status, page, **response.headers)
258
259
261 """
262 in controller you can use::
263
264 - request.wsgi.environ
265 - request.wsgi.start_response
266
267 to call third party WSGI applications
268 """
269 response.status = str(status).split(' ', 1)[0]
270 response.headers = dict(headers)
271 return lambda *args, **kargs: response.write(escape=False, *args, **kargs)
272
273
275 """
276 In you controller use::
277
278 @request.wsgi.middleware(middleware1, middleware2, ...)
279
280 to decorate actions with WSGI middleware. actions must return strings.
281 uses a simulated environment so it may have weird behavior in some cases
282 """
283 def middleware(f):
284 def app(environ, start_response):
285 data = f()
286 start_response(response.status, response.headers.items())
287 if isinstance(data, list):
288 return data
289 return [data]
290 for item in middleware_apps:
291 app = item(app)
292
293 def caller(app):
294 wsgi = request.wsgi
295 return app(wsgi.environ, wsgi.start_response)
296 return lambda caller=caller, app=app: caller(app)
297 return middleware
298
299
301 new_environ = copy.copy(environ)
302 new_environ['wsgi.input'] = request.body
303 new_environ['wsgi.version'] = 1
304 return new_environ
305
306 ISLE25 = sys.version_info[1] <= 5
307
308 -def parse_get_post_vars(request, environ):
309
310
311 env = request.env
312 dget = cgi.parse_qsl(env.query_string or '', keep_blank_values=1)
313 for (key, value) in dget:
314 if key in request.get_vars:
315 if isinstance(request.get_vars[key], list):
316 request.get_vars[key] += [value]
317 else:
318 request.get_vars[key] = [request.get_vars[key]] + [value]
319 else:
320 request.get_vars[key] = value
321 request.vars[key] = request.get_vars[key]
322
323
324 try:
325 request.body = body = copystream_progress(request)
326 except IOError:
327 raise HTTP(400, "Bad Request - HTTP body is incomplete")
328
329
330 is_json = env.get('http_content_type', '')[:16] == 'application/json'
331
332
333 if is_json:
334 try:
335 json_vars = sj.load(body)
336 body.seek(0)
337 except:
338
339 json_vars = {}
340 pass
341
342 request.get_vars.update(json_vars)
343 request.vars.update(json_vars)
344
345
346
347 if (body and env.request_method in ('POST', 'PUT', 'DELETE', 'BOTH')):
348 dpost = cgi.FieldStorage(fp=body, environ=environ, keep_blank_values=1)
349
350 is_multipart = dpost.type[:10] == 'multipart/'
351 body.seek(0)
352
353
354 def listify(a):
355 return (not isinstance(a, list) and [a]) or a
356 try:
357 keys = sorted(dpost)
358 except TypeError:
359 keys = []
360 for key in keys:
361 if key is None:
362 continue
363 dpk = dpost[key]
364
365 if isinstance(dpk, list):
366 value = []
367 for _dpk in dpk:
368 if not _dpk.filename:
369 value.append(_dpk.value)
370 else:
371 value.append(_dpk)
372 elif not dpk.filename:
373 value = dpk.value
374 else:
375 value = dpk
376 pvalue = listify(value)
377 if key in request.vars:
378 gvalue = listify(request.vars[key])
379 if ISLE25:
380 value = pvalue + gvalue
381 elif is_multipart:
382 pvalue = pvalue[len(gvalue):]
383 else:
384 pvalue = pvalue[:-len(gvalue)]
385 request.vars[key] = value
386 if len(pvalue):
387 request.post_vars[key] = (len(pvalue) >
388 1 and pvalue) or pvalue[0]
389 if is_json:
390
391 request.post_vars.update(json_vars)
392
393
395 """
396 this is the gluon wsgi application. the first function called when a page
397 is requested (static or dynamic). it can be called by paste.httpserver
398 or by apache mod_wsgi.
399
400 - fills request with info
401 - the environment variables, replacing '.' with '_'
402 - adds web2py path and version info
403 - compensates for fcgi missing path_info and query_string
404 - validates the path in url
405
406 The url path must be either:
407
408 1. for static pages:
409
410 - /<application>/static/<file>
411
412 2. for dynamic pages:
413
414 - /<application>[/<controller>[/<function>[/<sub>]]][.<extension>]
415 - (sub may go several levels deep, currently 3 levels are supported:
416 sub1/sub2/sub3)
417
418 The naming conventions are:
419
420 - application, controller, function and extension may only contain
421 [a-zA-Z0-9_]
422 - file and sub may also contain '-', '=', '.' and '/'
423 """
424
425 current.__dict__.clear()
426 request = Request()
427 response = Response()
428 session = Session()
429 env = request.env
430 env.web2py_path = global_settings.applications_parent
431 env.web2py_version = web2py_version
432 env.update(global_settings)
433 static_file = False
434 try:
435 try:
436 try:
437
438
439
440
441
442
443
444
445
446 fixup_missing_path_info(environ)
447 (static_file, version, environ) = url_in(request, environ)
448 response.status = env.web2py_status_code or response.status
449
450 if static_file:
451 if environ.get('QUERY_STRING', '').startswith(
452 'attachment'):
453 response.headers['Content-Disposition'] \
454 = 'attachment'
455 if version:
456 response.headers['Cache-Control'] = 'max-age=315360000'
457 response.headers[
458 'Expires'] = 'Thu, 31 Dec 2037 23:59:59 GMT'
459 response.stream(static_file, request=request)
460
461
462
463
464 app = request.application
465
466 if not global_settings.local_hosts:
467 local_hosts = set(['127.0.0.1', '::ffff:127.0.0.1', '::1'])
468 if not global_settings.web2py_runtime_gae:
469 try:
470 fqdn = socket.getfqdn()
471 local_hosts.add(socket.gethostname())
472 local_hosts.add(fqdn)
473 local_hosts.update([
474 addrinfo[4][0] for addrinfo
475 in getipaddrinfo(fqdn)])
476 if env.server_name:
477 local_hosts.add(env.server_name)
478 local_hosts.update([
479 addrinfo[4][0] for addrinfo
480 in getipaddrinfo(env.server_name)])
481 except (socket.gaierror, TypeError):
482 pass
483 global_settings.local_hosts = list(local_hosts)
484 else:
485 local_hosts = global_settings.local_hosts
486 client = get_client(env)
487 x_req_with = str(env.http_x_requested_with).lower()
488
489 request.update(
490 client = client,
491 folder = abspath('applications', app) + os.sep,
492 ajax = x_req_with == 'xmlhttprequest',
493 cid = env.http_web2py_component_element,
494 is_local = env.remote_addr in local_hosts,
495 is_https = env.wsgi_url_scheme in HTTPS_SCHEMES or \
496 request.env.http_x_forwarded_proto in HTTPS_SCHEMES \
497 or env.https == 'on')
498 request.compute_uuid()
499 request.url = environ['PATH_INFO']
500
501
502
503
504
505 if not exists(request.folder):
506 if app == rwthread.routes.default_application \
507 and app != 'welcome':
508 redirect(URL('welcome', 'default', 'index'))
509 elif rwthread.routes.error_handler:
510 _handler = rwthread.routes.error_handler
511 redirect(URL(_handler['application'],
512 _handler['controller'],
513 _handler['function'],
514 args=app))
515 else:
516 raise HTTP(404, rwthread.routes.error_message
517 % 'invalid request',
518 web2py_error='invalid application')
519 elif not request.is_local and \
520 exists(pjoin(request.folder, 'DISABLED')):
521 raise HTTP(503, "<html><body><h1>Temporarily down for maintenance</h1></body></html>")
522
523
524
525
526
527 create_missing_app_folders(request)
528
529
530
531
532
533 parse_get_post_vars(request, environ)
534
535
536
537
538
539 request.wsgi.environ = environ_aux(environ, request)
540 request.wsgi.start_response = \
541 lambda status='200', headers=[], \
542 exec_info=None, response=response: \
543 start_response_aux(status, headers, exec_info, response)
544 request.wsgi.middleware = \
545 lambda *a: middleware_aux(request, response, *a)
546
547
548
549
550
551 if env.http_cookie:
552 try:
553 request.cookies.load(env.http_cookie)
554 except Cookie.CookieError, e:
555 pass
556
557
558
559
560
561 if not env.web2py_disable_session:
562 session.connect(request, response)
563
564
565
566
567
568 if global_settings.debugging and app != "admin":
569 import gluon.debug
570
571 gluon.debug.dbg.do_debug(mainpyfile=request.folder)
572
573 serve_controller(request, response, session)
574
575 except HTTP, http_response:
576
577 if static_file:
578 return http_response.to(responder, env=env)
579
580 if request.body:
581 request.body.close()
582
583
584
585
586 session._try_store_in_db(request, response)
587
588
589
590
591
592 if response.do_not_commit is True:
593 BaseAdapter.close_all_instances(None)
594
595
596 elif response.custom_commit:
597 BaseAdapter.close_all_instances(response.custom_commit)
598 else:
599 BaseAdapter.close_all_instances('commit')
600
601
602
603
604
605
606 session._try_store_in_cookie_or_file(request, response)
607
608 if request.cid:
609 if response.flash:
610 http_response.headers['web2py-component-flash'] = \
611 urllib2.quote(xmlescape(response.flash)\
612 .replace('\n',''))
613 if response.js:
614 http_response.headers['web2py-component-command'] = \
615 urllib2.quote(response.js.replace('\n',''))
616
617
618
619
620
621 rcookies = response.cookies
622 if session._forget and response.session_id_name in rcookies:
623 del rcookies[response.session_id_name]
624 elif session._secure:
625 rcookies[response.session_id_name]['secure'] = True
626 http_response.cookies2headers(rcookies)
627 ticket = None
628
629 except RestrictedError, e:
630
631 if request.body:
632 request.body.close()
633
634
635
636
637
638 ticket = e.log(request) or 'unknown'
639 if response._custom_rollback:
640 response._custom_rollback()
641 else:
642 BaseAdapter.close_all_instances('rollback')
643
644 http_response = \
645 HTTP(500, rwthread.routes.error_message_ticket %
646 dict(ticket=ticket),
647 web2py_error='ticket %s' % ticket)
648
649 except:
650
651 if request.body:
652 request.body.close()
653
654
655
656
657
658 try:
659 if response._custom_rollback:
660 response._custom_rollback()
661 else:
662 BaseAdapter.close_all_instances('rollback')
663 except:
664 pass
665 e = RestrictedError('Framework', '', '', locals())
666 ticket = e.log(request) or 'unrecoverable'
667 http_response = \
668 HTTP(500, rwthread.routes.error_message_ticket
669 % dict(ticket=ticket),
670 web2py_error='ticket %s' % ticket)
671
672 finally:
673 if response and hasattr(response, 'session_file') \
674 and response.session_file:
675 response.session_file.close()
676
677 session._unlock(response)
678 http_response, new_environ = try_rewrite_on_error(
679 http_response, request, environ, ticket)
680 if not http_response:
681 return wsgibase(new_environ, responder)
682 if global_settings.web2py_crontype == 'soft':
683 newcron.softcron(global_settings.applications_parent).start()
684 return http_response.to(responder, env=env)
685
686
688 """
689 used by main() to save the password in the parameters_port.py file.
690 """
691
692 password_file = abspath('parameters_%i.py' % port)
693 if password == '<random>':
694
695 chars = string.letters + string.digits
696 password = ''.join([random.choice(chars) for i in range(8)])
697 cpassword = CRYPT()(password)[0]
698 print '******************* IMPORTANT!!! ************************'
699 print 'your admin password is "%s"' % password
700 print '*********************************************************'
701 elif password == '<recycle>':
702
703 if exists(password_file):
704 return
705 else:
706 password = ''
707 elif password.startswith('<pam_user:'):
708
709 cpassword = password[1:-1]
710 else:
711
712 cpassword = CRYPT()(password)[0]
713 fp = open(password_file, 'w')
714 if password:
715 fp.write('password="%s"\n' % cpassword)
716 else:
717 fp.write('password=None\n')
718 fp.close()
719
720
721 -def appfactory(wsgiapp=wsgibase,
722 logfilename='httpserver.log',
723 profilerfilename='profiler.log'):
724 """
725 generates a wsgi application that does logging and profiling and calls
726 wsgibase
727
728 .. function:: gluon.main.appfactory(
729 [wsgiapp=wsgibase
730 [, logfilename='httpserver.log'
731 [, profilerfilename='profiler.log']]])
732
733 """
734 if profilerfilename and exists(profilerfilename):
735 os.unlink(profilerfilename)
736 locker = allocate_lock()
737
738 def app_with_logging(environ, responder):
739 """
740 a wsgi app that does logging and profiling and calls wsgibase
741 """
742 status_headers = []
743
744 def responder2(s, h):
745 """
746 wsgi responder app
747 """
748 status_headers.append(s)
749 status_headers.append(h)
750 return responder(s, h)
751
752 time_in = time.time()
753 ret = [0]
754 if not profilerfilename:
755 ret[0] = wsgiapp(environ, responder2)
756 else:
757 import cProfile
758 import pstats
759 logger.warn('profiler is on. this makes web2py slower and serial')
760
761 locker.acquire()
762 cProfile.runctx('ret[0] = wsgiapp(environ, responder2)',
763 globals(), locals(), profilerfilename + '.tmp')
764 stat = pstats.Stats(profilerfilename + '.tmp')
765 stat.stream = cStringIO.StringIO()
766 stat.strip_dirs().sort_stats("time").print_stats(80)
767 profile_out = stat.stream.getvalue()
768 profile_file = open(profilerfilename, 'a')
769 profile_file.write('%s\n%s\n%s\n%s\n\n' %
770 ('=' * 60, environ['PATH_INFO'], '=' * 60, profile_out))
771 profile_file.close()
772 locker.release()
773 try:
774 line = '%s, %s, %s, %s, %s, %s, %f\n' % (
775 environ['REMOTE_ADDR'],
776 datetime.datetime.today().strftime('%Y-%m-%d %H:%M:%S'),
777 environ['REQUEST_METHOD'],
778 environ['PATH_INFO'].replace(',', '%2C'),
779 environ['SERVER_PROTOCOL'],
780 (status_headers[0])[:3],
781 time.time() - time_in,
782 )
783 if not logfilename:
784 sys.stdout.write(line)
785 elif isinstance(logfilename, str):
786 write_file(logfilename, line, 'a')
787 else:
788 logfilename.write(line)
789 except:
790 pass
791 return ret[0]
792
793 return app_with_logging
794
795
797 """
798 the web2py web server (Rocket)
799 """
800
801 - def __init__(
802 self,
803 ip='127.0.0.1',
804 port=8000,
805 password='',
806 pid_filename='httpserver.pid',
807 log_filename='httpserver.log',
808 profiler_filename=None,
809 ssl_certificate=None,
810 ssl_private_key=None,
811 ssl_ca_certificate=None,
812 min_threads=None,
813 max_threads=None,
814 server_name=None,
815 request_queue_size=5,
816 timeout=10,
817 socket_timeout=1,
818 shutdown_timeout=None,
819 path=None,
820 interfaces=None
821 ):
822 """
823 starts the web server.
824 """
825
826 if interfaces:
827
828
829 import types
830 if isinstance(interfaces, types.ListType):
831 for i in interfaces:
832 if not isinstance(i, types.TupleType):
833 raise "Wrong format for rocket interfaces parameter - see http://packages.python.org/rocket/"
834 else:
835 raise "Wrong format for rocket interfaces parameter - see http://packages.python.org/rocket/"
836
837 if path:
838
839
840 global web2py_path
841 path = os.path.normpath(path)
842 web2py_path = path
843 global_settings.applications_parent = path
844 os.chdir(path)
845 [add_path_first(p) for p in (path, abspath('site-packages'), "")]
846 if exists("logging.conf"):
847 logging.config.fileConfig("logging.conf")
848
849 save_password(password, port)
850 self.pid_filename = pid_filename
851 if not server_name:
852 server_name = socket.gethostname()
853 logger.info('starting web server...')
854 rocket.SERVER_NAME = server_name
855 rocket.SOCKET_TIMEOUT = socket_timeout
856 sock_list = [ip, port]
857 if not ssl_certificate or not ssl_private_key:
858 logger.info('SSL is off')
859 elif not rocket.ssl:
860 logger.warning('Python "ssl" module unavailable. SSL is OFF')
861 elif not exists(ssl_certificate):
862 logger.warning('unable to open SSL certificate. SSL is OFF')
863 elif not exists(ssl_private_key):
864 logger.warning('unable to open SSL private key. SSL is OFF')
865 else:
866 sock_list.extend([ssl_private_key, ssl_certificate])
867 if ssl_ca_certificate:
868 sock_list.append(ssl_ca_certificate)
869
870 logger.info('SSL is ON')
871 app_info = {'wsgi_app': appfactory(wsgibase,
872 log_filename,
873 profiler_filename)}
874
875 self.server = rocket.Rocket(interfaces or tuple(sock_list),
876 method='wsgi',
877 app_info=app_info,
878 min_threads=min_threads,
879 max_threads=max_threads,
880 queue_size=int(request_queue_size),
881 timeout=int(timeout),
882 handle_signals=False,
883 )
884
886 """
887 start the web server
888 """
889 try:
890 signal.signal(signal.SIGTERM, lambda a, b, s=self: s.stop())
891 signal.signal(signal.SIGINT, lambda a, b, s=self: s.stop())
892 except:
893 pass
894 write_file(self.pid_filename, str(os.getpid()))
895 self.server.start()
896
897 - def stop(self, stoplogging=False):
898 """
899 stop cron and the web server
900 """
901 newcron.stopcron()
902 self.server.stop(stoplogging)
903 try:
904 os.unlink(self.pid_filename)
905 except:
906 pass
907