gluon.newcron

1 #!/usr/bin/env python 2 # -*- coding: utf-8 -*- 3 4 """ 5 Created by Attila Csipa <web2py@csipa.in.rs> 6 Modified by Massimo Di Pierro <mdipierro@cs.depaul.edu> 7 """ 8 9 import sys 10 import os 11 import threading 12 import logging 13 import time 14 import sched 15 import re 16 import datetime 17 import platform 18 import portalocker 19 import fileutils 20 import cPickle 21 from settings import global_settings 22 23 logger = logging.getLogger("web2py.cron") 24 _cron_stopping = False 25 _cron_subprocs = [] 26 27

28 -def absolute_path_link(path):

29 """ 30 Return an absolute path for the destination of a symlink 31 32 """ 33 if os.path.islink(path): 34 link = os.readlink(path) 35 if not os.path.isabs(link): 36 link = os.path.join(os.path.dirname(path), link) 37 else: 38 link = os.path.abspath(path) 39 return link

40 41

42 -def stopcron():

43 "graceful shutdown of cron" 44 global _cron_stopping 45 _cron_stopping = True 46 while _cron_subprocs: 47 _cron_subprocs.pop().terminate()

48

49 -class extcron(threading.Thread):

50

51 - def __init__(self, applications_parent, apps=None):

52 threading.Thread.__init__(self) 53 self.setDaemon(False) 54 self.path = applications_parent 55 self.apps = apps

56 # crondance(self.path, 'external', startup=True, apps=self.apps) 57

58 - def run(self):

59 if not _cron_stopping: 60 logger.debug('external cron invocation') 61 crondance(self.path, 'external', startup=False, apps=self.apps)

62 63

64 -class hardcron(threading.Thread):

65

66 - def __init__(self, applications_parent):

67 threading.Thread.__init__(self) 68 self.setDaemon(True) 69 self.path = applications_parent 70 crondance(self.path, 'hard', startup=True)

71

72 - def launch(self):

73 if not _cron_stopping: 74 logger.debug('hard cron invocation') 75 crondance(self.path, 'hard', startup=False)

76

77 - def run(self):

78 s = sched.scheduler(time.time, time.sleep) 79 logger.info('Hard cron daemon started') 80 while not _cron_stopping: 81 now = time.time() 82 s.enter(60 - now % 60, 1, self.launch, ()) 83 s.run()

84 85

86 -class softcron(threading.Thread):

87

88 - def __init__(self, applications_parent):

89 threading.Thread.__init__(self) 90 self.path = applications_parent

91 # crondance(self.path, 'soft', startup=True) 92

93 - def run(self):

94 if not _cron_stopping: 95 logger.debug('soft cron invocation') 96 crondance(self.path, 'soft', startup=False)

97 98

99 -class Token(object):

100

101 - def __init__(self, path):

102 self.path = os.path.join(path, 'cron.master') 103 if not os.path.exists(self.path): 104 fileutils.write_file(self.path, '', 'wb') 105 self.master = None 106 self.now = time.time()

107

108 - def acquire(self, startup=False):

109 """ 110 returns the time when the lock is acquired or 111 None if cron already running 112 113 lock is implemented by writing a pickle (start, stop) in cron.master 114 start is time when cron job starts and stop is time when cron completed 115 stop == 0 if job started but did not yet complete 116 if a cron job started within less than 60 seconds, acquire returns None 117 if a cron job started before 60 seconds and did not stop, 118 a warning is issue "Stale cron.master detected" 119 """ 120 if portalocker.LOCK_EX is None: 121 logger.warning('WEB2PY CRON: Disabled because no file locking') 122 return None 123 self.master = open(self.path, 'rb+') 124 try: 125 ret = None 126 portalocker.lock(self.master, portalocker.LOCK_EX) 127 try: 128 (start, stop) = cPickle.load(self.master) 129 except: 130 (start, stop) = (0, 1) 131 if startup or self.now - start > 59.99: 132 ret = self.now 133 if not stop: 134 # this happens if previous cron job longer than 1 minute 135 logger.warning('WEB2PY CRON: Stale cron.master detected') 136 logger.debug('WEB2PY CRON: Acquiring lock') 137 self.master.seek(0) 138 cPickle.dump((self.now, 0), self.master) 139 finally: 140 portalocker.unlock(self.master) 141 if not ret: 142 # do this so no need to release 143 self.master.close() 144 return ret

145

146 - def release(self):

147 """ 148 this function writes into cron.master the time when cron job 149 was completed 150 """ 151 if not self.master.closed: 152 portalocker.lock(self.master, portalocker.LOCK_EX) 153 logger.debug('WEB2PY CRON: Releasing cron lock') 154 self.master.seek(0) 155 (start, stop) = cPickle.load(self.master) 156 if start == self.now: # if this is my lock 157 self.master.seek(0) 158 cPickle.dump((self.now, time.time()), self.master) 159 portalocker.unlock(self.master) 160 self.master.close()

161 162

163 -def rangetolist(s, period='min'):

164 retval = [] 165 if s.startswith('*'): 166 if period == 'min': 167 s = s.replace('*', '0-59', 1) 168 elif period == 'hr': 169 s = s.replace('*', '0-23', 1) 170 elif period == 'dom': 171 s = s.replace('*', '1-31', 1) 172 elif period == 'mon': 173 s = s.replace('*', '1-12', 1) 174 elif period == 'dow': 175 s = s.replace('*', '0-6', 1) 176 m = re.compile(r'(\d+)-(\d+)/(\d+)') 177 match = m.match(s) 178 if match: 179 for i in range(int(match.group(1)), int(match.group(2)) + 1): 180 if i % int(match.group(3)) == 0: 181 retval.append(i) 182 return retval

183 184

185 -def parsecronline(line):

186 task = {} 187 if line.startswith('@reboot'): 188 line = line.replace('@reboot', '-1 * * * *') 189 elif line.startswith('@yearly'): 190 line = line.replace('@yearly', '0 0 1 1 *') 191 elif line.startswith('@annually'): 192 line = line.replace('@annually', '0 0 1 1 *') 193 elif line.startswith('@monthly'): 194 line = line.replace('@monthly', '0 0 1 * *') 195 elif line.startswith('@weekly'): 196 line = line.replace('@weekly', '0 0 * * 0') 197 elif line.startswith('@daily'): 198 line = line.replace('@daily', '0 0 * * *') 199 elif line.startswith('@midnight'): 200 line = line.replace('@midnight', '0 0 * * *') 201 elif line.startswith('@hourly'): 202 line = line.replace('@hourly', '0 * * * *') 203 params = line.strip().split(None, 6) 204 if len(params) < 7: 205 return None 206 daysofweek = {'sun': 0, 'mon': 1, 'tue': 2, 'wed': 3, 'thu': 4, 207 'fri': 5, 'sat': 6} 208 for (s, id) in zip(params[:5], ['min', 'hr', 'dom', 'mon', 'dow']): 209 if not s in [None, '*']: 210 task[id] = [] 211 vals = s.split(',') 212 for val in vals: 213 if val != '-1' and '-' in val and '/' not in val: 214 val = '%s/1' % val 215 if '/' in val: 216 task[id] += rangetolist(val, id) 217 elif val.isdigit() or val == '-1': 218 task[id].append(int(val)) 219 elif id == 'dow' and val[:3].lower() in daysofweek: 220 task[id].append(daysofweek(val[:3].lower())) 221 task['user'] = params[5] 222 task['cmd'] = params[6] 223 return task

224 225

226 -class cronlauncher(threading.Thread):

227

228 - def __init__(self, cmd, shell=True):

229 threading.Thread.__init__(self) 230 if platform.system() == 'Windows': 231 shell = False 232 self.cmd = cmd 233 self.shell = shell

234

235 - def run(self):

236 import subprocess 237 global _cron_subprocs 238 if isinstance(self.cmd, (list, tuple)): 239 cmd = self.cmd 240 else: 241 cmd = self.cmd.split() 242 proc = subprocess.Popen(cmd, 243 stdin=subprocess.PIPE, 244 stdout=subprocess.PIPE, 245 stderr=subprocess.PIPE, 246 shell=self.shell) 247 _cron_subprocs.append(proc) 248 (stdoutdata, stderrdata) = proc.communicate() 249 if proc.returncode != 0: 250 logger.warning( 251 'WEB2PY CRON Call returned code %s:\n%s' % 252 (proc.returncode, stdoutdata + stderrdata)) 253 else: 254 logger.debug('WEB2PY CRON Call returned success:\n%s' 255 % stdoutdata)

256 257

258 -def crondance(applications_parent, ctype='soft', startup=False, apps=None):

259 apppath = os.path.join(applications_parent, 'applications') 260 cron_path = os.path.join(applications_parent) 261 token = Token(cron_path) 262 cronmaster = token.acquire(startup=startup) 263 if not cronmaster: 264 return 265 now_s = time.localtime() 266 checks = (('min', now_s.tm_min), 267 ('hr', now_s.tm_hour), 268 ('mon', now_s.tm_mon), 269 ('dom', now_s.tm_mday), 270 ('dow', (now_s.tm_wday + 1) % 7)) 271 272 if apps is None: 273 apps = [x for x in os.listdir(apppath) 274 if os.path.isdir(os.path.join(apppath, x))] 275 276 full_apath_links = set() 277 278 for app in apps: 279 if _cron_stopping: 280 break 281 apath = os.path.join(apppath, app) 282 283 # if app is a symbolic link to other app, skip it 284 full_apath_link = absolute_path_link(apath) 285 if full_apath_link in full_apath_links: 286 continue 287 else: 288 full_apath_links.add(full_apath_link) 289 290 cronpath = os.path.join(apath, 'cron') 291 crontab = os.path.join(cronpath, 'crontab') 292 if not os.path.exists(crontab): 293 continue 294 try: 295 cronlines = fileutils.readlines_file(crontab, 'rt') 296 lines = [x.strip() for x in cronlines if x.strip( 297 ) and not x.strip().startswith('#')] 298 tasks = [parsecronline(cline) for cline in lines] 299 except Exception, e: 300 logger.error('WEB2PY CRON: crontab read error %s' % e) 301 continue 302 303 for task in tasks: 304 if _cron_stopping: 305 break 306 commands = [sys.executable] 307 w2p_path = fileutils.abspath('web2py.py', gluon=True) 308 if os.path.exists(w2p_path): 309 commands.append(w2p_path) 310 if global_settings.applications_parent != global_settings.gluon_parent: 311 commands.extend(('-f', global_settings.applications_parent)) 312 citems = [(k in task and not v in task[k]) for k, v in checks] 313 task_min = task.get('min', []) 314 if not task: 315 continue 316 elif not startup and task_min == [-1]: 317 continue 318 elif task_min != [-1] and reduce(lambda a, b: a or b, citems): 319 continue 320 logger.info('WEB2PY CRON (%s): %s executing %s in %s at %s' 321 % (ctype, app, task.get('cmd'), 322 os.getcwd(), datetime.datetime.now())) 323 action, command, models = False, task['cmd'], '' 324 if command.startswith('**'): 325 (action, models, command) = (True, '', command[2:]) 326 elif command.startswith('*'): 327 (action, models, command) = (True, '-M', command[1:]) 328 else: 329 action = False 330 331 if action and command.endswith('.py'): 332 commands.extend(('-J', # cron job 333 models, # import models? 334 '-S', app, # app name 335 '-a', '"<recycle>"', # password 336 '-R', command)) # command 337 elif action: 338 commands.extend(('-J', # cron job 339 models, # import models? 340 '-S', app + '/' + command, # app name 341 '-a', '"<recycle>"')) # password 342 else: 343 commands = command 344 345 # from python docs: 346 # You do not need shell=True to run a batch file or 347 # console-based executable. 348 shell = False 349 350 try: 351 cronlauncher(commands, shell=shell).start() 352 except Exception, e: 353 logger.warning( 354 'WEB2PY CRON: Execution error for %s: %s' 355 % (task.get('cmd'), e)) 356 token.release()

357

Source Code for Module gluon.newcron