1
2
3
4 """
5 This file is part of the web2py Web Framework
6 Copyrighted by Massimo Di Pierro <mdipierro@cs.depaul.edu>
7 License: LGPLv3 (http://www.gnu.org/licenses/lgpl.html)
8 """
9
10 import sys
11 import storage
12 import os
13 import re
14 import tarfile
15 import glob
16 import time
17 import datetime
18 import logging
19 from http import HTTP
20 from gzip import open as gzopen
21
22
23 __all__ = [
24 'parse_version',
25 'read_file',
26 'write_file',
27 'readlines_file',
28 'up',
29 'abspath',
30 'mktree',
31 'listdir',
32 'recursive_unlink',
33 'cleanpath',
34 'tar',
35 'untar',
36 'tar_compiled',
37 'get_session',
38 'check_credentials',
39 'w2p_pack',
40 'w2p_unpack',
41 'w2p_pack_plugin',
42 'w2p_unpack_plugin',
43 'fix_newlines',
44 'make_fake_file_like_object',
45 ]
46
47
48 -def parse_semantic(version="Version 1.99.0-rc.1+timestamp.2011.09.19.08.23.26"):
49 "http://semver.org/"
50 re_version = re.compile('(\d+)\.(\d+)\.(\d+)(\-(?P<pre>[^\s+]*))?(\+(?P<build>\S*))')
51 m = re_version.match(version.strip().split()[-1])
52 if not m:
53 return None
54 a, b, c = int(m.group(1)), int(m.group(2)), int(m.group(3))
55 pre_release = m.group('pre') or ''
56 build = m.group('build') or ''
57 if build.startswith('timestamp'):
58 build = datetime.datetime.strptime(build.split('.',1)[1], '%Y.%m.%d.%H.%M.%S')
59 return (a, b, c, pre_release, build)
60
61 -def parse_legacy(version="Version 1.99.0 (2011-09-19 08:23:26)"):
62 re_version = re.compile('[^\d]+ (\d+)\.(\d+)\.(\d+)\s*\((?P<datetime>.+?)\)\s*(?P<type>[a-z]+)?')
63 m = re_version.match(version)
64 a, b, c = int(m.group(1)), int(m.group(2)), int(m.group(3)),
65 pre_release = m.group('type') or 'dev'
66 build = datetime.datetime.strptime(m.group('datetime'), '%Y-%m-%d %H:%M:%S')
67 return (a, b, c, pre_release, build)
68
70 version_tuple = parse_semantic(version)
71 if not version_tuple:
72 version_tuple = parse_legacy(version)
73 return version_tuple
74
76 "returns content from filename, making sure to close the file explicitly on exit."
77 f = open(filename, mode)
78 try:
79 return f.read()
80 finally:
81 f.close()
82
83
85 "writes <value> to filename, making sure to close the file explicitly on exit."
86 f = open(filename, mode)
87 try:
88 return f.write(value)
89 finally:
90 f.close()
91
92
94 "applies .split('\n') to the output of read_file()"
95 return read_file(filename, mode).split('\n')
96
97
99 head, tail = os.path.split(path)
100 if head:
101 if tail:
102 mktree(head)
103 if not os.path.exists(head):
104 os.mkdir(head)
105
106
107 -def listdir(
108 path,
109 expression='^.+$',
110 drop=True,
111 add_dirs=False,
112 sort=True,
113 ):
114 """
115 like os.listdir() but you can specify a regex pattern to filter files.
116 if add_dirs is True, the returned items will have the full path.
117 """
118 if path[-1:] != os.path.sep:
119 path = path + os.path.sep
120 if drop:
121 n = len(path)
122 else:
123 n = 0
124 regex = re.compile(expression)
125 items = []
126 for (root, dirs, files) in os.walk(path, topdown=True):
127 for dir in dirs[:]:
128 if dir.startswith('.'):
129 dirs.remove(dir)
130 if add_dirs:
131 items.append(root[n:])
132 for file in sorted(files):
133 if regex.match(file) and not file.startswith('.'):
134 items.append(os.path.join(root, file)[n:])
135 if sort:
136 return sorted(items)
137 else:
138 return items
139
140
148
149
151 """
152 turns any expression/path into a valid filename. replaces / with _ and
153 removes special characters.
154 """
155
156 items = path.split('.')
157 if len(items) > 1:
158 path = re.sub('[^\w\.]+', '_', '_'.join(items[:-1]) + '.'
159 + ''.join(items[-1:]))
160 else:
161 path = re.sub('[^\w\.]+', '_', ''.join(items[-1:]))
162 return path
163
164
166 if not hasattr(tarfile.TarFile, 'extractall'):
167 from tarfile import ExtractError
168
169 class TarFile(tarfile.TarFile):
170
171 def extractall(self, path='.', members=None):
172 """Extract all members from the archive to the current working
173 directory and set owner, modification time and permissions on
174 directories afterwards. `path' specifies a different directory
175 to extract to. `members' is optional and must be a subset of the
176 list returned by getmembers().
177 """
178
179 directories = []
180 if members is None:
181 members = self
182 for tarinfo in members:
183 if tarinfo.isdir():
184
185
186
187
188 try:
189 os.makedirs(os.path.join(path,
190 tarinfo.name), 0777)
191 except EnvironmentError:
192 pass
193 directories.append(tarinfo)
194 else:
195 self.extract(tarinfo, path)
196
197
198
199 directories.sort(lambda a, b: cmp(a.name, b.name))
200 directories.reverse()
201
202
203
204 for tarinfo in directories:
205 path = os.path.join(path, tarinfo.name)
206 try:
207 self.chown(tarinfo, path)
208 self.utime(tarinfo, path)
209 self.chmod(tarinfo, path)
210 except ExtractError, e:
211 if self.errorlevel > 1:
212 raise
213 else:
214 self._dbg(1, 'tarfile: %s' % e)
215
216 _cls = TarFile
217 else:
218 _cls = tarfile.TarFile
219
220 tar = _cls(filename, 'r')
221 ret = tar.extractall(path, members)
222 tar.close()
223 return ret
224
225
226 -def tar(file, dir, expression='^.+$', filenames=None):
227 """
228 tars dir into file, only tars file that match expression
229 """
230
231 tar = tarfile.TarFile(file, 'w')
232 try:
233 if filenames is None:
234 filenames = listdir(dir, expression, add_dirs=True)
235 for file in filenames:
236 tar.add(os.path.join(dir, file), file, False)
237 finally:
238 tar.close()
239
241 """
242 untar file into dir
243 """
244
245 _extractall(file, dir)
246
247
248 -def w2p_pack(filename, path, compiled=False, filenames=None):
249 filename = abspath(filename)
250 path = abspath(path)
251 tarname = filename + '.tar'
252 if compiled:
253 tar_compiled(tarname, path, '^[\w\.\-]+$')
254 else:
255 tar(tarname, path, '^[\w\.\-]+$', filenames=filenames)
256 w2pfp = gzopen(filename, 'wb')
257 tarfp = open(tarname, 'rb')
258 w2pfp.write(tarfp.read())
259 w2pfp.close()
260 tarfp.close()
261 os.unlink(tarname)
262
264 if not os.path.exists('welcome.w2p') or os.path.exists('NEWINSTALL'):
265 try:
266 w2p_pack('welcome.w2p', 'applications/welcome')
267 os.unlink('NEWINSTALL')
268 logging.info("New installation: created welcome.w2p file")
269 except:
270 logging.error("New installation error: unable to create welcome.w2p file")
271
272
274
275 if filename=='welcome.w2p':
276 create_welcome_w2p()
277 filename = abspath(filename)
278 path = abspath(path)
279 if filename[-4:] == '.w2p' or filename[-3:] == '.gz':
280 if filename[-4:] == '.w2p':
281 tarname = filename[:-4] + '.tar'
282 else:
283 tarname = filename[:-3] + '.tar'
284 fgzipped = gzopen(filename, 'rb')
285 tarfile = open(tarname, 'wb')
286 tarfile.write(fgzipped.read())
287 tarfile.close()
288 fgzipped.close()
289 else:
290 tarname = filename
291 untar(tarname, path)
292 if delete_tar:
293 os.unlink(tarname)
294
295
297 """Pack the given plugin into a w2p file.
298 Will match files at:
299 <path>/*/plugin_[name].*
300 <path>/*/plugin_[name]/*
301 """
302 filename = abspath(filename)
303 path = abspath(path)
304 if not filename.endswith('web2py.plugin.%s.w2p' % plugin_name):
305 raise Exception("Not a web2py plugin name")
306 plugin_tarball = tarfile.open(filename, 'w:gz')
307 try:
308 app_dir = path
309 while app_dir[-1] == '/':
310 app_dir = app_dir[:-1]
311 files1 = glob.glob(
312 os.path.join(app_dir, '*/plugin_%s.*' % plugin_name))
313 files2 = glob.glob(
314 os.path.join(app_dir, '*/plugin_%s/*' % plugin_name))
315 for file in files1 + files2:
316 plugin_tarball.add(file, arcname=file[len(app_dir) + 1:])
317 finally:
318 plugin_tarball.close()
319
320
327
328
330 """
331 used to tar a compiled application.
332 the content of models, views, controllers is not stored in the tar file.
333 """
334
335 tar = tarfile.TarFile(file, 'w')
336 for file in listdir(dir, expression, add_dirs=True):
337 filename = os.path.join(dir, file)
338 if os.path.islink(filename):
339 continue
340 if os.path.isfile(filename) and file[-4:] != '.pyc':
341 if file[:6] == 'models':
342 continue
343 if file[:5] == 'views':
344 continue
345 if file[:11] == 'controllers':
346 continue
347 if file[:7] == 'modules':
348 continue
349 tar.add(filename, file, False)
350 tar.close()
351
352
354 return os.path.dirname(os.path.normpath(path))
355
356
358 """ checks that user is authorized to access other_application"""
359 if request.application == other_application:
360 raise KeyError
361 try:
362 session_id = request.cookies['session_id_' + other_application].value
363 osession = storage.load_storage(os.path.join(
364 up(request.folder), other_application, 'sessions', session_id))
365 except Exception, e:
366 osession = storage.Storage()
367 return osession
368
369
370 -def check_credentials(request, other_application='admin',
371 expiration=60 * 60, gae_login=True):
372 """ checks that user is authorized to access other_application"""
373 if request.env.web2py_runtime_gae:
374 from google.appengine.api import users
375 if users.is_current_user_admin():
376 return True
377 elif gae_login:
378 login_html = '<a href="%s">Sign in with your google account</a>.' \
379 % users.create_login_url(request.env.path_info)
380 raise HTTP(200, '<html><body>%s</body></html>' % login_html)
381 else:
382 return False
383 else:
384 dt = time.time() - expiration
385 s = get_session(request, other_application)
386 return (s.authorized and s.last_time and s.last_time > dt)
387
388
390 regex = re.compile(r'''(\r
391 |\r|
392 )''')
393 for filename in listdir(path, '.*\.(py|html)$', drop=False):
394 rdata = read_file(filename, 'rb')
395 wdata = regex.sub('\n', rdata)
396 if wdata != rdata:
397 write_file(filename, wdata, 'wb')
398
399
400 -def copystream(
401 src,
402 dest,
403 size,
404 chunk_size=10 ** 5,
405 ):
406 """
407 this is here because I think there is a bug in shutil.copyfileobj
408 """
409 while size > 0:
410 if size < chunk_size:
411 data = src.read(size)
412 else:
413 data = src.read(chunk_size)
414 length = len(data)
415 if length > size:
416 (data, length) = (data[:size], size)
417 size -= length
418 if length == 0:
419 break
420 dest.write(data)
421 if length < chunk_size:
422 break
423 dest.seek(0)
424 return
425
426
428 class LogFile(object):
429 def write(self, value):
430 pass
431
432 def close(self):
433 pass
434 return LogFile()
435
436
437 from settings import global_settings
438
439
440
442 "convert relative path to absolute path based (by default) on applications_parent"
443 path = os.path.join(*relpath)
444 gluon = base.get('gluon', False)
445 if os.path.isabs(path):
446 return path
447 if gluon:
448 return os.path.join(global_settings.gluon_parent, path)
449 return os.path.join(global_settings.applications_parent, path)
450