1
2
3
4 """
5 This file is part of the web2py Web Framework
6 Copyrighted by Massimo Di Pierro <mdipierro@cs.depaul.edu>
7 License: LGPLv3 (http://www.gnu.org/licenses/lgpl.html)
8
9 Functions required to execute app components
10 ============================================
11
12 FOR INTERNAL USE ONLY
13 """
14
15 import re
16 import sys
17 import fnmatch
18 import os
19 import copy
20 import random
21 import __builtin__
22 from storage import Storage, List
23 from template import parse_template
24 from restricted import restricted, compile2
25 from fileutils import mktree, listdir, read_file, write_file
26 from myregex import regex_expose
27 from languages import translator
28 from dal import BaseAdapter, SQLDB, SQLField, DAL, Field
29 from sqlhtml import SQLFORM, SQLTABLE
30 from cache import Cache
31 from globals import current, Response
32 import settings
33 from cfs import getcfs
34 import html
35 import validators
36 from http import HTTP, redirect
37 import marshal
38 import shutil
39 import imp
40 import logging
41 logger = logging.getLogger("web2py")
42 import rewrite
43 from custom_import import custom_import_install
44
45 try:
46 import py_compile
47 except:
48 logger.warning('unable to import py_compile')
49
50 is_pypy = settings.global_settings.is_pypy
51 is_gae = settings.global_settings.web2py_runtime_gae
52 is_jython = settings.global_settings.is_jython
53
54 pjoin = os.path.join
55
56 TEST_CODE = \
57 r"""
58 def _TEST():
59 import doctest, sys, cStringIO, types, cgi, gluon.fileutils
60 if not gluon.fileutils.check_credentials(request):
61 raise HTTP(401, web2py_error='invalid credentials')
62 stdout = sys.stdout
63 html = '<h2>Testing controller "%s.py" ... done.</h2><br/>\n' \
64 % request.controller
65 for key in sorted([key for key in globals() if not key in __symbols__+['_TEST']]):
66 eval_key = eval(key)
67 if type(eval_key) == types.FunctionType:
68 number_doctests = sum([len(ds.examples) for ds in doctest.DocTestFinder().find(eval_key)])
69 if number_doctests>0:
70 sys.stdout = cStringIO.StringIO()
71 name = '%s/controllers/%s.py in %s.__doc__' \
72 % (request.folder, request.controller, key)
73 doctest.run_docstring_examples(eval_key,
74 globals(), False, name=name)
75 report = sys.stdout.getvalue().strip()
76 if report:
77 pf = 'failed'
78 else:
79 pf = 'passed'
80 html += '<h3 class="%s">Function %s [%s]</h3>\n' \
81 % (pf, key, pf)
82 if report:
83 html += CODE(report, language='web2py', \
84 link='/examples/global/vars/').xml()
85 html += '<br/>\n'
86 else:
87 html += \
88 '<h3 class="nodoctests">Function %s [no doctests]</h3><br/>\n' \
89 % (key)
90 response._vars = html
91 sys.stdout = stdout
92 _TEST()
93 """
94
95 CACHED_REGEXES = {}
96 CACHED_REGEXES_MAX_SIZE = 1000
97
98
107
108
110 """
111 NOTE could simple use a dict and populate it,
112 NOTE not sure if this changes things though if monkey patching import.....
113 """
114
116 try:
117 return getattr(__builtin__, key)
118 except AttributeError:
119 raise KeyError(key)
120
122 setattr(self, key, value)
123
124
125 -def LOAD(c=None, f='index', args=None, vars=None,
126 extension=None, target=None, ajax=False, ajax_trap=False,
127 url=None, user_signature=False, timeout=None, times=1,
128 content='loading...', **attr):
129 """ LOAD a component into the action's document
130
131 Timing options:
132 -times: An integer or string ("infinity"/"continuous")
133 specifies how many times the component is requested
134 -timeout (milliseconds): specifies the time to wait before
135 starting the request or the frequency if times is greater than
136 1 or "infinity".
137 Timing options default to the normal behavior. The component
138 is added on page loading without delay.
139 """
140 from html import TAG, DIV, URL, SCRIPT, XML
141 if args is None:
142 args = []
143 vars = Storage(vars or {})
144 target = target or 'c' + str(random.random())[2:]
145 attr['_id'] = target
146 request = current.request
147 if '.' in f:
148 f, extension = f.rsplit('.', 1)
149 if url or ajax:
150 url = url or URL(request.application, c, f, r=request,
151 args=args, vars=vars, extension=extension,
152 user_signature=user_signature)
153
154 if isinstance(times, basestring):
155 if times.upper() in ("INFINITY", "CONTINUOUS"):
156 times = "Infinity"
157 else:
158 raise TypeError("Unsupported times argument %s" % times)
159 elif isinstance(times, int):
160 if times <= 0:
161 raise ValueError("Times argument must be greater than zero, 'Infinity' or None")
162 else:
163 raise TypeError("Unsupported times argument type %s" % type(times))
164 if timeout is not None:
165 if not isinstance(timeout, (int, long)):
166 raise ValueError("Timeout argument must be an integer or None")
167 elif timeout <= 0:
168 raise ValueError(
169 "Timeout argument must be greater than zero or None")
170 statement = "web2py_component('%s','%s', %s, %s);" \
171 % (url, target, timeout, times)
172 else:
173 statement = "web2py_component('%s','%s');" % (url, target)
174 script = SCRIPT(statement, _type="text/javascript")
175 if not content is None:
176 return TAG[''](script, DIV(content, **attr))
177 else:
178 return TAG[''](script)
179
180 else:
181 if not isinstance(args, (list, tuple)):
182 args = [args]
183 c = c or request.controller
184 other_request = Storage(request)
185 other_request['env'] = Storage(request.env)
186 other_request.controller = c
187 other_request.function = f
188 other_request.extension = extension or request.extension
189 other_request.args = List(args)
190 other_request.vars = vars
191 other_request.get_vars = vars
192 other_request.post_vars = Storage()
193 other_response = Response()
194 other_request.env.path_info = '/' + \
195 '/'.join([request.application, c, f] +
196 map(str, other_request.args))
197 other_request.env.query_string = \
198 vars and URL(vars=vars).split('?')[1] or ''
199 other_request.env.http_web2py_component_location = \
200 request.env.path_info
201 other_request.cid = target
202 other_request.env.http_web2py_component_element = target
203 other_response.view = '%s/%s.%s' % (c, f, other_request.extension)
204
205 other_environment = copy.copy(current.globalenv)
206
207 other_response._view_environment = other_environment
208 other_response.generic_patterns = \
209 copy.copy(current.response.generic_patterns)
210 other_environment['request'] = other_request
211 other_environment['response'] = other_response
212
213
214
215 original_request, current.request = current.request, other_request
216 original_response, current.response = current.response, other_response
217 page = run_controller_in(c, f, other_environment)
218 if isinstance(page, dict):
219 other_response._vars = page
220 other_response._view_environment.update(page)
221 run_view_in(other_response._view_environment)
222 page = other_response.body.getvalue()
223 current.request, current.response = original_request, original_response
224 js = None
225 if ajax_trap:
226 link = URL(request.application, c, f, r=request,
227 args=args, vars=vars, extension=extension,
228 user_signature=user_signature)
229 js = "web2py_trap_form('%s','%s');" % (link, target)
230 script = js and SCRIPT(js, _type="text/javascript") or ''
231 return TAG[''](DIV(XML(page), **attr), script)
232
233
235 """
236 Attention: this helper is new and experimental
237 """
239 self.environment = environment
240
241 - def __call__(self, c=None, f='index', args=None, vars=None,
242 extension=None, target=None, ajax=False, ajax_trap=False,
243 url=None, user_signature=False, content='loading...', **attr):
244 if args is None:
245 args = []
246 vars = Storage(vars or {})
247 import globals
248 target = target or 'c' + str(random.random())[2:]
249 attr['_id'] = target
250 request = self.environment['request']
251 if '.' in f:
252 f, extension = f.rsplit('.', 1)
253 if url or ajax:
254 url = url or html.URL(request.application, c, f, r=request,
255 args=args, vars=vars, extension=extension,
256 user_signature=user_signature)
257 script = html.SCRIPT('web2py_component("%s","%s")' % (url, target),
258 _type="text/javascript")
259 return html.TAG[''](script, html.DIV(content, **attr))
260 else:
261 if not isinstance(args, (list, tuple)):
262 args = [args]
263 c = c or request.controller
264
265 other_request = Storage(request)
266 other_request['env'] = Storage(request.env)
267 other_request.controller = c
268 other_request.function = f
269 other_request.extension = extension or request.extension
270 other_request.args = List(args)
271 other_request.vars = vars
272 other_request.get_vars = vars
273 other_request.post_vars = Storage()
274 other_response = globals.Response()
275 other_request.env.path_info = '/' + \
276 '/'.join([request.application, c, f] +
277 map(str, other_request.args))
278 other_request.env.query_string = \
279 vars and html.URL(vars=vars).split('?')[1] or ''
280 other_request.env.http_web2py_component_location = \
281 request.env.path_info
282 other_request.cid = target
283 other_request.env.http_web2py_component_element = target
284 other_response.view = '%s/%s.%s' % (c, f, other_request.extension)
285 other_environment = copy.copy(self.environment)
286 other_response._view_environment = other_environment
287 other_response.generic_patterns = \
288 copy.copy(current.response.generic_patterns)
289 other_environment['request'] = other_request
290 other_environment['response'] = other_response
291
292
293
294 original_request, current.request = current.request, other_request
295 original_response, current.response = current.response, other_response
296 page = run_controller_in(c, f, other_environment)
297 if isinstance(page, dict):
298 other_response._vars = page
299 other_response._view_environment.update(page)
300 run_view_in(other_response._view_environment)
301 page = other_response.body.getvalue()
302 current.request, current.response = original_request, original_response
303 js = None
304 if ajax_trap:
305 link = html.URL(request.application, c, f, r=request,
306 args=args, vars=vars, extension=extension,
307 user_signature=user_signature)
308 js = "web2py_trap_form('%s','%s');" % (link, target)
309 script = js and html.SCRIPT(js, _type="text/javascript") or ''
310 return html.TAG[''](html.DIV(html.XML(page), **attr), script)
311
312
314 """
315 In apps, instead of importing a local module
316 (in applications/app/modules) with::
317
318 import a.b.c as d
319
320 you should do::
321
322 d = local_import('a.b.c')
323
324 or (to force a reload):
325
326 d = local_import('a.b.c', reload=True)
327
328 This prevents conflict between applications and un-necessary execs.
329 It can be used to import any module, including regular Python modules.
330 """
331 items = name.replace('/', '.')
332 name = "applications.%s.modules.%s" % (app, items)
333 module = __import__(name)
334 for item in name.split(".")[1:]:
335 module = getattr(module, item)
336 if reload_force:
337 reload(module)
338 return module
339
340
341 """
342 OLD IMPLEMENTATION:
343 items = name.replace('/','.').split('.')
344 filename, modulepath = items[-1], pjoin(apath,'modules',*items[:-1])
345 imp.acquire_lock()
346 try:
347 file=None
348 (file,path,desc) = imp.find_module(filename,[modulepath]+sys.path)
349 if not path in sys.modules or reload:
350 if is_gae:
351 module={}
352 execfile(path,{},module)
353 module=Storage(module)
354 else:
355 module = imp.load_module(path,file,path,desc)
356 sys.modules[path] = module
357 else:
358 module = sys.modules[path]
359 except Exception, e:
360 module = None
361 if file:
362 file.close()
363 imp.release_lock()
364 if not module:
365 raise ImportError, "cannot find module %s in %s" % (
366 filename, modulepath)
367 return module
368 """
369
370 _base_environment_ = dict((k, getattr(html, k)) for k in html.__all__)
371 _base_environment_.update(
372 (k, getattr(validators, k)) for k in validators.__all__)
373 _base_environment_['__builtins__'] = __builtins__
374 _base_environment_['HTTP'] = HTTP
375 _base_environment_['redirect'] = redirect
376 _base_environment_['DAL'] = DAL
377 _base_environment_['Field'] = Field
378 _base_environment_['SQLDB'] = SQLDB
379 _base_environment_['SQLField'] = SQLField
380 _base_environment_['SQLFORM'] = SQLFORM
381 _base_environment_['SQLTABLE'] = SQLTABLE
382 _base_environment_['LOAD'] = LOAD
383
384
386 """
387 Build the environment dictionary into which web2py files are executed.
388 """
389
390 environment = dict(_base_environment_)
391
392 if not request.env:
393 request.env = Storage()
394
395
396 response.models_to_run = [r'^\w+\.py$', r'^%s/\w+\.py$' % request.controller,
397 r'^%s/%s/\w+\.py$' % (request.controller, request.function)]
398
399 t = environment['T'] = translator(os.path.join(request.folder,'languages'),
400 request.env.http_accept_language)
401 c = environment['cache'] = Cache(request)
402
403 if store_current:
404 current.globalenv = environment
405 current.request = request
406 current.response = response
407 current.session = session
408 current.T = t
409 current.cache = c
410
411 global __builtins__
412 if is_jython:
413 __builtins__ = mybuiltin()
414 elif is_pypy:
415 __builtins__ = mybuiltin()
416 else:
417 __builtins__['__import__'] = __builtin__.__import__
418 environment['request'] = request
419 environment['response'] = response
420 environment['session'] = session
421 environment['local_import'] = \
422 lambda name, reload=False, app=request.application:\
423 local_import_aux(name, reload, app)
424 BaseAdapter.set_folder(pjoin(request.folder, 'databases'))
425 response._view_environment = copy.copy(environment)
426 custom_import_install()
427 return environment
428
429
431 """
432 Bytecode compiles the file `filename`
433 """
434 py_compile.compile(filename)
435
436
438 """
439 Read the code inside a bytecode compiled file if the MAGIC number is
440 compatible
441
442 :returns: a code object
443 """
444 data = read_file(filename, 'rb')
445 if not is_gae and data[:4] != imp.get_magic():
446 raise SystemError('compiled code is incompatible')
447 return marshal.loads(data[8:])
448
449
451 """
452 Compiles all the views in the application specified by `folder`
453 """
454
455 path = pjoin(folder, 'views')
456 for file in listdir(path, '^[\w/\-]+(\.\w+)*$'):
457 try:
458 data = parse_template(file, path)
459 except Exception, e:
460 raise Exception("%s in %s" % (e, file))
461 filename = ('views/%s.py' % file).replace('/', '_').replace('\\', '_')
462 filename = pjoin(folder, 'compiled', filename)
463 write_file(filename, data)
464 save_pyc(filename)
465 os.unlink(filename)
466
467
469 """
470 Compiles all the models in the application specified by `folder`
471 """
472
473 path = pjoin(folder, 'models')
474 for file in listdir(path, '.+\.py$'):
475 data = read_file(pjoin(path, file))
476 filename = pjoin(folder, 'compiled', 'models', file)
477 mktree(filename)
478 write_file(filename, data)
479 save_pyc(filename)
480 os.unlink(filename)
481
482
484 """
485 Compiles all the controllers in the application specified by `folder`
486 """
487
488 path = pjoin(folder, 'controllers')
489 for file in listdir(path, '.+\.py$'):
490
491 data = read_file(pjoin(path, file))
492 exposed = regex_expose.findall(data)
493 for function in exposed:
494 command = data + "\nresponse._vars=response._caller(%s)\n" % \
495 function
496 filename = pjoin(folder, 'compiled', ('controllers/'
497 + file[:-3]).replace('/', '_')
498 + '_' + function + '.py')
499 write_file(filename, command)
500 save_pyc(filename)
501 os.unlink(filename)
502
503
505 """
506 Runs all models (in the app specified by the current folder)
507 It tries pre-compiled models first before compiling them.
508 """
509
510 folder = environment['request'].folder
511 c = environment['request'].controller
512 f = environment['request'].function
513 cpath = pjoin(folder, 'compiled')
514 if os.path.exists(cpath):
515 for model in listdir(cpath, '^models_\w+\.pyc$', 0):
516 restricted(read_pyc(model), environment, layer=model)
517 path = pjoin(cpath, 'models')
518 models = listdir(path, '^\w+\.pyc$', 0, sort=False)
519 compiled = True
520 else:
521 path = pjoin(folder, 'models')
522 models = listdir(path, '^\w+\.py$', 0, sort=False)
523 compiled = False
524 n = len(path) + 1
525 for model in models:
526 regex = environment['response'].models_to_run
527 if isinstance(regex, list):
528 regex = re_compile('|'.join(regex))
529 file = model[n:].replace(os.path.sep, '/').replace('.pyc', '.py')
530 if not regex.search(file) and c != 'appadmin':
531 continue
532 elif compiled:
533 code = read_pyc(model)
534 elif is_gae:
535 code = getcfs(model, model,
536 lambda: compile2(read_file(model), model))
537 else:
538 code = getcfs(model, model, None)
539 restricted(code, environment, layer=model)
540
541
543 """
544 Runs the controller.function() (for the app specified by
545 the current folder).
546 It tries pre-compiled controller_function.pyc first before compiling it.
547 """
548
549
550 folder = environment['request'].folder
551 path = pjoin(folder, 'compiled')
552 badc = 'invalid controller (%s/%s)' % (controller, function)
553 badf = 'invalid function (%s/%s)' % (controller, function)
554 if os.path.exists(path):
555 filename = pjoin(path, 'controllers_%s_%s.pyc'
556 % (controller, function))
557 if not os.path.exists(filename):
558 raise HTTP(404,
559 rewrite.THREAD_LOCAL.routes.error_message % badf,
560 web2py_error=badf)
561 restricted(read_pyc(filename), environment, layer=filename)
562 elif function == '_TEST':
563
564 from settings import global_settings
565 from admin import abspath, add_path_first
566 paths = (global_settings.gluon_parent, abspath(
567 'site-packages', gluon=True), abspath('gluon', gluon=True), '')
568 [add_path_first(path) for path in paths]
569
570
571 filename = pjoin(folder, 'controllers/%s.py'
572 % controller)
573 if not os.path.exists(filename):
574 raise HTTP(404,
575 rewrite.THREAD_LOCAL.routes.error_message % badc,
576 web2py_error=badc)
577 environment['__symbols__'] = environment.keys()
578 code = read_file(filename)
579 code += TEST_CODE
580 restricted(code, environment, layer=filename)
581 else:
582 filename = pjoin(folder, 'controllers/%s.py'
583 % controller)
584 if not os.path.exists(filename):
585 raise HTTP(404,
586 rewrite.THREAD_LOCAL.routes.error_message % badc,
587 web2py_error=badc)
588 code = read_file(filename)
589 exposed = regex_expose.findall(code)
590 if not function in exposed:
591 raise HTTP(404,
592 rewrite.THREAD_LOCAL.routes.error_message % badf,
593 web2py_error=badf)
594 code = "%s\nresponse._vars=response._caller(%s)\n" % (code, function)
595 if is_gae:
596 layer = filename + ':' + function
597 code = getcfs(layer, filename, lambda: compile2(code, layer))
598 restricted(code, environment, filename)
599 response = environment['response']
600 vars = response._vars
601 if response.postprocessing:
602 vars = reduce(lambda vars, p: p(vars), response.postprocessing, vars)
603 if isinstance(vars, unicode):
604 vars = vars.encode('utf8')
605 elif hasattr(vars, 'xml') and callable(vars.xml):
606 vars = vars.xml()
607 return vars
608
609
611 """
612 Executes the view for the requested action.
613 The view is the one specified in `response.view` or determined by the url
614 or `view/generic.extension`
615 It tries the pre-compiled views_controller_function.pyc before compiling it.
616 """
617 request = environment['request']
618 response = environment['response']
619 view = response.view
620 folder = request.folder
621 path = pjoin(folder, 'compiled')
622 badv = 'invalid view (%s)' % view
623 if response.generic_patterns:
624 patterns = response.generic_patterns
625 regex = re_compile('|'.join(map(fnmatch.translate, patterns)))
626 short_action = '%(controller)s/%(function)s.%(extension)s' % request
627 allow_generic = regex.search(short_action)
628 else:
629 allow_generic = False
630 if not isinstance(view, str):
631 ccode = parse_template(view, pjoin(folder, 'views'),
632 context=environment)
633 restricted(ccode, environment, 'file stream')
634 elif os.path.exists(path):
635 x = view.replace('/', '_')
636 files = ['views_%s.pyc' % x]
637 if allow_generic:
638 files.append('views_generic.%s.pyc' % request.extension)
639
640 if request.extension == 'html':
641 files.append('views_%s.pyc' % x[:-5])
642 if allow_generic:
643 files.append('views_generic.pyc')
644
645 for f in files:
646 filename = pjoin(path, f)
647 if os.path.exists(filename):
648 code = read_pyc(filename)
649 restricted(code, environment, layer=filename)
650 return
651 raise HTTP(404,
652 rewrite.THREAD_LOCAL.routes.error_message % badv,
653 web2py_error=badv)
654 else:
655 filename = pjoin(folder, 'views', view)
656 if not os.path.exists(filename) and allow_generic:
657 view = 'generic.' + request.extension
658 filename = pjoin(folder, 'views', view)
659 if not os.path.exists(filename):
660 raise HTTP(404,
661 rewrite.THREAD_LOCAL.routes.error_message % badv,
662 web2py_error=badv)
663 layer = filename
664 if is_gae:
665 ccode = getcfs(layer, filename,
666 lambda: compile2(parse_template(view,
667 pjoin(folder, 'views'),
668 context=environment), layer))
669 else:
670 ccode = parse_template(view,
671 pjoin(folder, 'views'),
672 context=environment)
673 restricted(ccode, environment, layer)
674
675
677 """
678 Deletes the folder `compiled` containing the compiled application.
679 """
680 try:
681 shutil.rmtree(pjoin(folder, 'compiled'))
682 path = pjoin(folder, 'controllers')
683 for file in listdir(path, '.*\.pyc$', drop=False):
684 os.unlink(file)
685 except OSError:
686 pass
687
688
698
699
701 """
702 Example::
703
704 >>> import traceback, types
705 >>> environment={'x':1}
706 >>> open('a.py', 'w').write('print 1/x')
707 >>> save_pyc('a.py')
708 >>> os.unlink('a.py')
709 >>> if type(read_pyc('a.pyc'))==types.CodeType: print 'code'
710 code
711 >>> exec read_pyc('a.pyc') in environment
712 1
713 """
714
715 return
716
717
718 if __name__ == '__main__':
719 import doctest
720 doctest.testmod()
721