gluon.languages

1 #!/usr/bin/env python 2 # -*- coding: utf-8 -*- 3 4 """ 5 This file is part of the web2py Web Framework 6 Copyrighted by Massimo Di Pierro <mdipierro@cs.depaul.edu> 7 License: LGPLv3 (http://www.gnu.org/licenses/lgpl.html) 8 9 Plural subsystem is created by Vladyslav Kozlovskyy (Ukraine) 10 <dbdevelop@gmail.com> 11 """ 12 13 import os 14 import re 15 import sys 16 import pkgutil 17 import logging 18 import marshal 19 from cgi import escape 20 from threading import RLock 21 22 try: 23 import copyreg as copy_reg # python 3 24 except ImportError: 25 import copy_reg # python 2 26 27 from portalocker import read_locked, LockedFile 28 from utf8 import Utf8 29 30 from fileutils import listdir 31 import settings 32 from cfs import getcfs 33 from html import XML, xmlescape 34 from contrib.markmin.markmin2html import render, markmin_escape 35 from string import maketrans 36 37 __all__ = ['translator', 'findT', 'update_all_languages'] 38 39 ostat = os.stat 40 oslistdir = os.listdir 41 pjoin = os.path.join 42 pexists = os.path.exists 43 pdirname = os.path.dirname 44 isdir = os.path.isdir 45 is_gae = False # settings.global_settings.web2py_runtime_gae 46 47 DEFAULT_LANGUAGE = 'en' 48 DEFAULT_LANGUAGE_NAME = 'English' 49 50 # DEFAULT PLURAL-FORMS RULES: 51 # language doesn't use plural forms 52 DEFAULT_NPLURALS = 1 53 # only one singular/plural form is used 54 DEFAULT_GET_PLURAL_ID = lambda n: 0 55 # word is unchangeable 56 DEFAULT_CONSTRUCT_PLURAL_FORM = lambda word, plural_id: word 57 58 NUMBERS = (int, long, float) 59 60 # pattern to find T(blah blah blah) expressions 61 PY_STRING_LITERAL_RE = r'(?<=[^\w]T$)(?P<name>'\ 62 + r"[uU]?[rR]?(?:'''(?:[^']|'{1,2}(?!'))*''')|"\ 63 + r"(?:'(?:[^'\\]|\\.)*')|" + r'(?:"""(?:[^"]|"{1,2}(?!"))*""")|'\ 64 + r'(?:"(?:[^"\\]|\\.)*"))' 65 66 regex_translate = re.compile(PY_STRING_LITERAL_RE, re.DOTALL) 67 regex_param = re.compile(r'{(?P<s>.+?)}') 68 69 # pattern for a valid accept_language 70 regex_language = \ 71 re.compile('([a-z]{2}(?:\-[a-z]{2})?(?:\-[a-z]{2})?)(?:[,;]|$)') 72 regex_langfile = re.compile('^[a-z]{2}(-[a-z]{2})?\.py$') 73 regex_backslash = re.compile(r"\\([\\{}%])") 74 regex_plural = re.compile('%({.+?})') 75 regex_plural_dict = re.compile('^{(?P<w>[^()[\]][^()[\]]*?)\((?P<n>[^()\[\]]+)$}$') # %%{word(varname or number)} 76 regex_plural_tuple = re.compile( 77 '^{(?P<w>[^[\]()]+)(?:\[(?P<i>\d+)\])?}$') # %%{word[index]} or %%{word} 78 regex_plural_file = re.compile('^plural-[a-zA-Z]{2}(-[a-zA-Z]{2})?\.py$') 79 80

81 -def safe_eval(text):

82 if text.strip(): 83 try: 84 import ast 85 return ast.literal_eval(text) 86 except ImportError: 87 return eval(text, {}, {}) 88 return None

89 90 # used as default filter in translator.M() 91 92

93 -def markmin(s):

94 def markmin_aux(m): 95 return '{%s}' % markmin_escape(m.group('s'))

96 return render(regex_param.sub(markmin_aux, s), 97 sep='br', autolinks=None, id_prefix='') 98 99 # UTF8 helper functions 100 101

102 -def upper_fun(s):

103 return unicode(s, 'utf-8').upper().encode('utf-8')

104 105

106 -def title_fun(s):

107 return unicode(s, 'utf-8').title().encode('utf-8')

108 109

110 -def cap_fun(s):

111 return unicode(s, 'utf-8').capitalize().encode('utf-8')

112 ttab_in = maketrans("\\%{}", '\x1c\x1d\x1e\x1f') 113 ttab_out = maketrans('\x1c\x1d\x1e\x1f', "\\%{}") 114 115 # cache of translated messages: 116 # global_language_cache: 117 # { 'languages/xx.py': 118 # ( {"def-message": "xx-message", 119 # ... 120 # "def-message": "xx-message"}, lock_object ) 121 # 'languages/yy.py': ( {dict}, lock_object ) 122 # ... 123 # } 124 125 global_language_cache = {} 126 127

128 -def get_from_cache(cache, val, fun):

129 lang_dict, lock = cache 130 lock.acquire() 131 try: 132 result = lang_dict.get(val) 133 finally: 134 lock.release() 135 if result: 136 return result 137 lock.acquire() 138 try: 139 result = lang_dict.setdefault(val, fun()) 140 finally: 141 lock.release() 142 return result

143 144

145 -def clear_cache(filename):

146 cache = global_language_cache.setdefault( 147 filename, ({}, RLock())) 148 lang_dict, lock = cache 149 lock.acquire() 150 try: 151 lang_dict.clear() 152 finally: 153 lock.release()

154 155

156 -def read_dict_aux(filename):

157 lang_text = read_locked(filename).replace('\r\n', '\n') 158 clear_cache(filename) 159 try: 160 return safe_eval(lang_text) or {} 161 except Exception: 162 e = sys.exc_info()[1] 163 status = 'Syntax error in %s (%s)' % (filename, e) 164 logging.error(status) 165 return {'__corrupted__': status}

166 167

168 -def read_dict(filename):

169 """ return dictionary with translation messages 170 """ 171 return getcfs('lang:' + filename, filename, 172 lambda: read_dict_aux(filename))

173 174

175 -def read_possible_plural_rules():

176 """ 177 create list of all possible plural rules files 178 result is cached in PLURAL_RULES dictionary to increase speed 179 """ 180 plurals = {} 181 try: 182 import contrib.plural_rules as package 183 for importer, modname, ispkg in pkgutil.iter_modules(package.__path__): 184 if len(modname) == 2: 185 module = __import__(package.__name__ + '.' + modname, 186 fromlist=[modname]) 187 lang = modname 188 pname = modname + '.py' 189 nplurals = getattr(module, 'nplurals', DEFAULT_NPLURALS) 190 get_plural_id = getattr( 191 module, 'get_plural_id', 192 DEFAULT_GET_PLURAL_ID) 193 construct_plural_form = getattr( 194 module, 'construct_plural_form', 195 DEFAULT_CONSTRUCT_PLURAL_FORM) 196 plurals[lang] = (lang, nplurals, get_plural_id, 197 construct_plural_form) 198 except ImportError: 199 e = sys.exc_info()[1] 200 logging.warn('Unable to import plural rules: %s' % e) 201 return plurals

202 203 PLURAL_RULES = read_possible_plural_rules() 204 205

206 -def read_possible_languages_aux(langdir):

207 def get_lang_struct(lang, langcode, langname, langfile_mtime): 208 if lang == 'default': 209 real_lang = langcode.lower() 210 else: 211 real_lang = lang 212 (prules_langcode, 213 nplurals, 214 get_plural_id, 215 construct_plural_form 216 ) = PLURAL_RULES.get(real_lang[:2], ('default', 217 DEFAULT_NPLURALS, 218 DEFAULT_GET_PLURAL_ID, 219 DEFAULT_CONSTRUCT_PLURAL_FORM)) 220 if prules_langcode != 'default': 221 (pluraldict_fname, 222 pluraldict_mtime) = plurals.get(real_lang, 223 plurals.get(real_lang[:2], 224 ('plural-%s.py' % real_lang, 0))) 225 else: 226 pluraldict_fname = None 227 pluraldict_mtime = 0 228 return (langcode, # language code from !langcode! 229 langname, 230 # language name in national spelling from !langname! 231 langfile_mtime, # m_time of language file 232 pluraldict_fname, # name of plural dictionary file or None (when default.py is not exist) 233 pluraldict_mtime, # m_time of plural dictionary file or 0 if file is not exist 234 prules_langcode, # code of plural rules language or 'default' 235 nplurals, # nplurals for current language 236 get_plural_id, # get_plural_id() for current language 237 construct_plural_form) # construct_plural_form() for current language

238 239 plurals = {} 240 flist = oslistdir(langdir) if isdir(langdir) else [] 241 242 # scan languages directory for plural dict files: 243 for pname in flist: 244 if regex_plural_file.match(pname): 245 plurals[pname[7:-3]] = (pname, 246 ostat(pjoin(langdir, pname)).st_mtime) 247 langs = {} 248 # scan languages directory for langfiles: 249 for fname in flist: 250 if regex_langfile.match(fname) or fname == 'default.py': 251 fname_with_path = pjoin(langdir, fname) 252 d = read_dict(fname_with_path) 253 lang = fname[:-3] 254 langcode = d.get('!langcode!', lang if lang != 'default' 255 else DEFAULT_LANGUAGE) 256 langname = d.get('!langname!', langcode) 257 langfile_mtime = ostat(fname_with_path).st_mtime 258 langs[lang] = get_lang_struct(lang, langcode, 259 langname, langfile_mtime) 260 if 'default' not in langs: 261 # if default.py is not found, 262 # add DEFAULT_LANGUAGE as default language: 263 langs['default'] = get_lang_struct('default', DEFAULT_LANGUAGE, 264 DEFAULT_LANGUAGE_NAME, 0) 265 deflang = langs['default'] 266 deflangcode = deflang[0] 267 if deflangcode not in langs: 268 # create language from default.py: 269 langs[deflangcode] = deflang[:2] + (0,) + deflang[3:] 270 271 return langs 272 273

274 -def read_possible_languages(langpath):

275 return getcfs('langs:' + langpath, langpath, 276 lambda: read_possible_languages_aux(langpath))

277 278

279 -def read_plural_dict_aux(filename):

280 lang_text = read_locked(filename).replace('\r\n', '\n') 281 try: 282 return eval(lang_text) or {} 283 except Exception: 284 e = sys.exc_info()[1] 285 status = 'Syntax error in %s (%s)' % (filename, e) 286 logging.error(status) 287 return {'__corrupted__': status}

288 289

290 -def read_plural_dict(filename):

291 return getcfs('plurals:' + filename, filename, 292 lambda: read_plural_dict_aux(filename))

293 294

295 -def write_plural_dict(filename, contents):

296 if '__corrupted__' in contents: 297 return 298 try: 299 fp = LockedFile(filename, 'w') 300 fp.write('#!/usr/bin/env python\n{\n# "singular form (0)": ["first plural form (1)", "second plural form (2)", ...],\n') 301 # coding: utf8\n{\n') 302 for key in sorted(contents, lambda x, y: cmp(unicode(x, 'utf-8').lower(), unicode(y, 'utf-8').lower())): 303 forms = '[' + ','.join([repr(Utf8(form)) 304 for form in contents[key]]) + ']' 305 fp.write('%s: %s,\n' % (repr(Utf8(key)), forms)) 306 fp.write('}\n') 307 except (IOError, OSError): 308 if not is_gae: 309 logging.warning('Unable to write to file %s' % filename) 310 return 311 finally: 312 fp.close()

313 314

315 -def write_dict(filename, contents):

316 if '__corrupted__' in contents: 317 return 318 try: 319 fp = LockedFile(filename, 'w') 320 except (IOError, OSError): 321 if not settings.global_settings.web2py_runtime_gae: 322 logging.warning('Unable to write to file %s' % filename) 323 return 324 fp.write('# coding: utf8\n{\n') 325 for key in sorted(contents, lambda x, y: cmp(unicode(x, 'utf-8').lower(), unicode(y, 'utf-8').lower())): 326 fp.write('%s: %s,\n' % (repr(Utf8(key)), repr(Utf8(contents[key])))) 327 fp.write('}\n') 328 fp.close()

329 330

331 -class lazyT(object):

332 """ 333 never to be called explicitly, returned by 334 translator.__call__() or translator.M() 335 """ 336 m = s = T = f = t = None 337 M = is_copy = False 338

339 - def __init__( 340 self, 341 message, 342 symbols={}, 343 T=None, 344 filter=None, 345 ftag=None, 346 M=False 347 ):

348 if isinstance(message, lazyT): 349 self.m = message.m 350 self.s = message.s 351 self.T = message.T 352 self.f = message.f 353 self.t = message.t 354 self.M = message.M 355 self.is_copy = True 356 else: 357 self.m = message 358 self.s = symbols 359 self.T = T 360 self.f = filter 361 self.t = ftag 362 self.M = M 363 self.is_copy = False

364

365 - def __repr__(self):

366 return "<lazyT %s>" % (repr(Utf8(self.m)), )

367

368 - def __str__(self):

369 return str(self.T.apply_filter(self.m, self.s, self.f, self.t) if self.M else 370 self.T.translate(self.m, self.s))

371

372 - def __eq__(self, other):

373 return str(self) == str(other)

374

375 - def __ne__(self, other):

376 return str(self) != str(other)

377

378 - def __add__(self, other):

379 return '%s%s' % (self, other)

380

381 - def __radd__(self, other):

382 return '%s%s' % (other, self)

383

384 - def __mul__(self, other):

385 return str(self) * other

386

387 - def __cmp__(self, other):

388 return cmp(str(self), str(other))

389

390 - def __hash__(self):

391 return hash(str(self))

392

393 - def __getattr__(self, name):

394 return getattr(str(self), name)

395

396 - def __getitem__(self, i):

397 return str(self)[i]

398

399 - def __getslice__(self, i, j):

400 return str(self)[i:j]

401

402 - def __iter__(self):

403 for c in str(self): 404 yield c

405

406 - def __len__(self):

407 return len(str(self))

408

409 - def xml(self):

410 return str(self) if self.M else escape(str(self))

411

412 - def encode(self, *a, **b):

413 return str(self).encode(*a, **b)

414

415 - def decode(self, *a, **b):

416 return str(self).decode(*a, **b)

417

418 - def read(self):

419 return str(self)

420

421 - def __mod__(self, symbols):

422 if self.is_copy: 423 return lazyT(self) 424 return lazyT(self.m, symbols, self.T, self.f, self.t, self.M)

425 426

427 -class translator(object):

428 """ 429 this class is instantiated by gluon.compileapp.build_environment 430 as the T object 431 :: 432 T.force(None) # turns off translation 433 T.force('fr, it') # forces web2py to translate using fr.py or it.py 434 435 T(\"Hello World\") # translates \"Hello World\" using the selected file 436 437 notice 1: there is no need to force since, by default, T uses 438 http_accept_language to determine a translation file. 439 notice 2: 440 en and en-en are considered different languages! 441 notice 3: 442 if language xx-yy is not found force() probes other similar 443 languages using such algorithm: 444 xx-yy.py -> xx.py -> xx-yy*.py -> xx*.py 445 """ 446

447 - def __init__(self, langpath, http_accept_language):

448 self.langpath = langpath 449 self.http_accept_language = http_accept_language 450 self.is_writable = not is_gae 451 # filled in self.force(): 452 #------------------------ 453 # self.cache 454 # self.accepted_language 455 # self.language_file 456 # self.plural_language 457 # self.nplurals 458 # self.get_plural_id 459 # self.construct_plural_form 460 # self.plural_file 461 # self.plural_dict 462 # self.requested_languages 463 #---------------------------------------- 464 # filled in self.set_current_languages(): 465 #---------------------------------------- 466 # self.default_language_file 467 # self.default_t 468 # self.current_languages 469 self.set_current_languages() 470 self.lazy = True 471 self.otherTs = {} 472 self.filter = markmin 473 self.ftag = 'markmin'

474

475 - def get_possible_languages_info(self, lang=None):

476 """ 477 return info for selected language or dictionary with all 478 possible languages info from APP/languages/*.py 479 args: 480 *lang* (str): language 481 returns: 482 if *lang* is defined: 483 return tuple(langcode, langname, langfile_mtime, 484 pluraldict_fname, pluraldict_mtime, 485 prules_langcode, nplurals, 486 get_plural_id, construct_plural_form) 487 or None 488 489 if *lang* is NOT defined: 490 returns dictionary with all possible languages: 491 { langcode(from filename): 492 ( langcode, # language code from !langcode! 493 langname, 494 # language name in national spelling from !langname! 495 langfile_mtime, # m_time of language file 496 pluraldict_fname,# name of plural dictionary file or None (when default.py is not exist) 497 pluraldict_mtime,# m_time of plural dictionary file or 0 if file is not exist 498 prules_langcode, # code of plural rules language or 'default' 499 nplurals, # nplurals for current language 500 get_plural_id, # get_plural_id() for current language 501 construct_plural_form) # construct_plural_form() for current language 502 } 503 """ 504 info = read_possible_languages(self.langpath) 505 if lang: 506 info = info.get(lang) 507 return info

508

509 - def get_possible_languages(self):

510 """ get list of all possible languages for current applications """ 511 return list(set(self.current_languages + 512 [lang for lang in read_possible_languages(self.langpath).iterkeys() 513 if lang != 'default']))

514

515 - def set_current_languages(self, *languages):

516 """ 517 set current AKA "default" languages 518 setting one of this languages makes force() function 519 turn translation off to use default language 520 """ 521 if len(languages) == 1 and isinstance( 522 languages[0], (tuple, list)): 523 languages = languages[0] 524 if not languages or languages[0] is None: 525 # set default language from default.py/DEFAULT_LANGUAGE 526 pl_info = self.get_possible_languages_info('default') 527 if pl_info[2] == 0: # langfile_mtime 528 # if languages/default.py is not found 529 self.default_language_file = self.langpath 530 self.default_t = {} 531 self.current_languages = [DEFAULT_LANGUAGE] 532 else: 533 self.default_language_file = pjoin(self.langpath, 534 'default.py') 535 self.default_t = read_dict(self.default_language_file) 536 self.current_languages = [pl_info[0]] # !langcode! 537 else: 538 self.current_languages = list(languages) 539 self.force(self.http_accept_language)

540

541 - def plural(self, word, n):

542 """ get plural form of word for number *n* 543 NOTE: *word" MUST be defined in current language 544 (T.accepted_language) 545 546 invoked from T()/T.M() in %%{} tag 547 args: 548 word (str): word in singular 549 n (numeric): number plural form created for 550 551 returns: 552 (str): word in appropriate singular/plural form 553 """ 554 if int(n) == 1: 555 return word 556 elif word: 557 id = self.get_plural_id(abs(int(n))) 558 # id = 0 singular form 559 # id = 1 first plural form 560 # id = 2 second plural form 561 # etc. 562 if id != 0: 563 forms = self.plural_dict.get(word, []) 564 if len(forms) >= id: 565 # have this plural form: 566 return forms[id - 1] 567 else: 568 # guessing this plural form 569 forms += [''] * (self.nplurals - len(forms) - 1) 570 form = self.construct_plural_form(word, id) 571 forms[id - 1] = form 572 self.plural_dict[word] = forms 573 if self.is_writable and self.plural_file: 574 write_plural_dict(self.plural_file, 575 self.plural_dict) 576 return form 577 return word

578

579 - def force(self, *languages):

580 """ 581 582 select language(s) for translation 583 584 if a list of languages is passed as a parameter, 585 first language from this list that matches the ones 586 from the possible_languages dictionary will be 587 selected 588 589 default language will be selected if none 590 of them matches possible_languages. 591 """ 592 pl_info = read_possible_languages(self.langpath) 593 594 def set_plural(language): 595 """ 596 initialize plural forms subsystem 597 """ 598 lang_info = pl_info.get(language) 599 if lang_info: 600 (pname, 601 pmtime, 602 self.plural_language, 603 self.nplurals, 604 self.get_plural_id, 605 self.construct_plural_form 606 ) = lang_info[3:] 607 pdict = {} 608 if pname: 609 pname = pjoin(self.langpath, pname) 610 if pmtime != 0: 611 pdict = read_plural_dict(pname) 612 self.plural_file = pname 613 self.plural_dict = pdict 614 else: 615 self.plural_language = 'default' 616 self.nplurals = DEFAULT_NPLURALS 617 self.get_plural_id = DEFAULT_GET_PLURAL_ID 618 self.construct_plural_form = DEFAULT_CONSTRUCT_PLURAL_FORM 619 self.plural_file = None 620 self.plural_dict = {}

621 language = '' 622 if len(languages) == 1 and isinstance(languages[0], str): 623 languages = regex_language.findall(languages[0].lower()) 624 elif not languages or languages[0] is None: 625 languages = [] 626 self.requested_languages = languages = tuple(languages) 627 if languages: 628 all_languages = set(lang for lang in pl_info.iterkeys() 629 if lang != 'default') \ 630 | set(self.current_languages) 631 for lang in languages: 632 # compare "aa-bb" | "aa" from *language* parameter 633 # with strings from langlist using such alghorythm: 634 # xx-yy.py -> xx.py -> xx*.py 635 lang5 = lang[:5] 636 if lang5 in all_languages: 637 language = lang5 638 else: 639 lang2 = lang[:2] 640 if len(lang5) > 2 and lang2 in all_languages: 641 language = lang2 642 else: 643 for l in all_languages: 644 if l[:2] == lang2: 645 language = l 646 if language: 647 if language in self.current_languages: 648 break 649 self.language_file = pjoin(self.langpath, language + '.py') 650 self.t = read_dict(self.language_file) 651 self.cache = global_language_cache.setdefault( 652 self.language_file, 653 ({}, RLock())) 654 set_plural(language) 655 self.accepted_language = language 656 return languages 657 self.accepted_language = language or self.current_languages[0] 658 self.language_file = self.default_language_file 659 self.cache = global_language_cache.setdefault(self.language_file, 660 ({}, RLock())) 661 self.t = self.default_t 662 set_plural(self.accepted_language) 663 return languages

664

665 - def __call__(self, message, symbols={}, language=None, lazy=None):

666 """ 667 get cached translated plain text message with inserted parameters(symbols) 668 if lazy==True lazyT object is returned 669 """ 670 if lazy is None: 671 lazy = self.lazy 672 if not language: 673 if lazy: 674 return lazyT(message, symbols, self) 675 else: 676 return self.translate(message, symbols) 677 else: 678 try: 679 otherT = self.otherTs[language] 680 except KeyError: 681 otherT = self.otherTs[language] = translator( 682 self.langpath, self.http_accept_language) 683 otherT.force(language) 684 return otherT(message, symbols, lazy=lazy)

685

686 - def apply_filter(self, message, symbols={}, filter=None, ftag=None):

687 def get_tr(message, prefix, filter): 688 s = self.get_t(message, prefix) 689 return filter(s) if filter else self.filter(s)

690 if filter: 691 prefix = '@' + (ftag or 'userdef') + '\x01' 692 else: 693 prefix = '@' + self.ftag + '\x01' 694 message = get_from_cache( 695 self.cache, prefix + message, 696 lambda: get_tr(message, prefix, filter)) 697 if symbols or symbols == 0 or symbols == "": 698 if isinstance(symbols, dict): 699 symbols.update( 700 (key, xmlescape(value).translate(ttab_in)) 701 for key, value in symbols.iteritems() 702 if not isinstance(value, NUMBERS)) 703 else: 704 if not isinstance(symbols, tuple): 705 symbols = (symbols,) 706 symbols = tuple( 707 value if isinstance(value, NUMBERS) 708 else xmlescape(value).translate(ttab_in) 709 for value in symbols) 710 message = self.params_substitution(message, symbols) 711 return XML(message.translate(ttab_out)) 712

713 - def M(self, message, symbols={}, language=None, 714 lazy=None, filter=None, ftag=None):

715 """ 716 get cached translated markmin-message with inserted parametes 717 if lazy==True lazyT object is returned 718 """ 719 if lazy is None: 720 lazy = self.lazy 721 if not language: 722 if lazy: 723 return lazyT(message, symbols, self, filter, ftag, True) 724 else: 725 return self.apply_filter(message, symbols, filter, ftag) 726 else: 727 try: 728 otherT = self.otherTs[language] 729 except KeyError: 730 otherT = self.otherTs[language] = translator(self.request) 731 otherT.force(language) 732 return otherT.M(message, symbols, lazy=lazy)

733

734 - def get_t(self, message, prefix=''):

735 """ 736 user ## to add a comment into a translation string 737 the comment can be useful do discriminate different possible 738 translations for the same string (for example different locations) 739 740 T(' hello world ') -> ' hello world ' 741 T(' hello world ## token') -> ' hello world ' 742 T('hello ## world## token') -> 'hello ## world' 743 744 the ## notation is ignored in multiline strings and strings that 745 start with ##. this is to allow markmin syntax to be translated 746 """ 747 if isinstance(message, unicode): 748 message = message.encode('utf8') 749 if isinstance(prefix, unicode): 750 prefix = prefix.encode('utf8') 751 key = prefix + message 752 mt = self.t.get(key, None) 753 if mt is not None: 754 return mt 755 # we did not find a translation 756 if message.find('##') > 0 and not '\n' in message: 757 # remove comments 758 message = message.rsplit('##', 1)[0] 759 # guess translation same as original 760 self.t[key] = mt = self.default_t.get(key, message) 761 # update language file for latter translation 762 if self.is_writable and self.language_file != self.default_language_file: 763 write_dict(self.language_file, self.t) 764 return regex_backslash.sub( 765 lambda m: m.group(1).translate(ttab_in), mt)

766

767 - def params_substitution(self, message, symbols):

768 """ 769 substitute parameters from symbols into message using %. 770 also parse %%{} placeholders for plural-forms processing. 771 returns: string with parameters 772 NOTE: *symbols* MUST BE OR tuple OR dict of parameters! 773 """ 774 def sub_plural(m): 775 """string in %{} is transformed by this rules: 776 If string starts with \\, ! or ? such transformations 777 take place: 778 779 "!string of words" -> "String of word" (Capitalize) 780 "!!string of words" -> "String Of Word" (Title) 781 "!!!string of words" -> "STRING OF WORD" (Upper) 782 "\\!string of words" -> "!string of word" 783 (remove \\ and disable transformations) 784 "?word?number" -> "word" (return word, if number == 1) 785 "?number" or "??number" -> "" (remove number, 786 if number == 1) 787 "?word?number" -> "number" (if number != 1) 788 """ 789 def sub_tuple(m): 790 """ word[number], !word[number], !!word[number], !!!word[number] 791 word, !word, !!word, !!!word, ?word?number, ??number, ?number 792 ?word?word[number], ?word?[number], ??word[number] 793 """ 794 w, i = m.group('w', 'i') 795 c = w[0] 796 if c not in '!?': 797 return self.plural(w, symbols[int(i or 0)]) 798 elif c == '?': 799 (p1, sep, p2) = w[1:].partition("?") 800 part1 = p1 if sep else "" 801 (part2, sep, part3) = (p2 if sep else p1).partition("?") 802 if not sep: 803 part3 = part2 804 if i is None: 805 # ?[word]?number[?number] or ?number 806 if not part2: 807 return m.group(0) 808 num = int(part2) 809 else: 810 # ?[word]?word2[?word3][number] 811 num = int(symbols[int(i or 0)]) 812 return part1 if num == 1 else part3 if num == 0 else part2 813 elif w.startswith('!!!'): 814 word = w[3:] 815 fun = upper_fun 816 elif w.startswith('!!'): 817 word = w[2:] 818 fun = title_fun 819 else: 820 word = w[1:] 821 fun = cap_fun 822 if i is not None: 823 return fun(self.plural(word, symbols[int(i)])) 824 return fun(word)

825 826 def sub_dict(m): 827 """ word(var), !word(var), !!word(var), !!!word(var) 828 word(num), !word(num), !!word(num), !!!word(num) 829 ?word2(var), ?word1?word2(var), ?word1?word2?word0(var) 830 ?word2(num), ?word1?word2(num), ?word1?word2?word0(num) 831 """ 832 w, n = m.group('w', 'n') 833 c = w[0] 834 n = int(n) if n.isdigit() else symbols[n] 835 if c not in '!?': 836 return self.plural(w, n) 837 elif c == '?': 838 # ?[word1]?word2[?word0](var or num), ?[word1]?word2(var or num) or ?word2(var or num) 839 (p1, sep, p2) = w[1:].partition("?") 840 part1 = p1 if sep else "" 841 (part2, sep, part3) = (p2 if sep else p1).partition("?") 842 if not sep: 843 part3 = part2 844 num = int(n) 845 return part1 if num == 1 else part3 if num == 0 else part2 846 elif w.startswith('!!!'): 847 word = w[3:] 848 fun = upper_fun 849 elif w.startswith('!!'): 850 word = w[2:] 851 fun = title_fun 852 else: 853 word = w[1:] 854 fun = cap_fun 855 return fun(self.plural(word, n)) 856 857 s = m.group(1) 858 part = regex_plural_tuple.sub(sub_tuple, s) 859 if part == s: 860 part = regex_plural_dict.sub(sub_dict, s) 861 if part == s: 862 return m.group(0) 863 return part 864 message = message % symbols 865 message = regex_plural.sub(sub_plural, message) 866 return message 867

868 - def translate(self, message, symbols):

869 """ 870 get cached translated message with inserted parameters(symbols) 871 """ 872 message = get_from_cache(self.cache, message, 873 lambda: self.get_t(message)) 874 if symbols or symbols == 0 or symbols == "": 875 if isinstance(symbols, dict): 876 symbols.update( 877 (key, str(value).translate(ttab_in)) 878 for key, value in symbols.iteritems() 879 if not isinstance(value, NUMBERS)) 880 else: 881 if not isinstance(symbols, tuple): 882 symbols = (symbols,) 883 symbols = tuple( 884 value if isinstance(value, NUMBERS) 885 else str(value).translate(ttab_in) 886 for value in symbols) 887 message = self.params_substitution(message, symbols) 888 return message.translate(ttab_out)

889 890

891 -def findT(path, language=DEFAULT_LANGUAGE):

892 """ 893 must be run by the admin app 894 """ 895 lang_file = pjoin(path, 'languages', language + '.py') 896 sentences = read_dict(lang_file) 897 mp = pjoin(path, 'models') 898 cp = pjoin(path, 'controllers') 899 vp = pjoin(path, 'views') 900 mop = pjoin(path, 'modules') 901 for filename in \ 902 listdir(mp, '^.+\.py$', 0) + listdir(cp, '^.+\.py$', 0)\ 903 + listdir(vp, '^.+\.html$', 0) + listdir(mop, '^.+\.py$', 0): 904 data = read_locked(filename) 905 items = regex_translate.findall(data) 906 for item in items: 907 try: 908 message = safe_eval(item) 909 except: 910 continue # silently ignore inproperly formatted strings 911 if not message.startswith('#') and not '\n' in message: 912 tokens = message.rsplit('##', 1) 913 else: 914 # this allows markmin syntax in translations 915 tokens = [message] 916 if len(tokens) == 2: 917 message = tokens[0].strip() + '##' + tokens[1].strip() 918 if message and not message in sentences: 919 sentences[message] = message 920 if not '!langcode!' in sentences: 921 sentences['!langcode!'] = ( 922 DEFAULT_LANGUAGE if language in ('default', DEFAULT_LANGUAGE) else language) 923 if not '!langname!' in sentences: 924 sentences['!langname!'] = ( 925 DEFAULT_LANGUAGE_NAME if language in ('default', DEFAULT_LANGUAGE) 926 else sentences['!langcode!']) 927 write_dict(lang_file, sentences)

928 929 ### important to allow safe session.flash=T(....) 930 931

932 -def lazyT_unpickle(data):

933 return marshal.loads(data)

934 935

936 -def lazyT_pickle(data):

937 return lazyT_unpickle, (marshal.dumps(str(data)),)

938 copy_reg.pickle(lazyT, lazyT_pickle, lazyT_unpickle) 939 940

941 -def update_all_languages(application_path):

942 path = pjoin(application_path, 'languages/') 943 for language in oslistdir(path): 944 if regex_langfile.match(language): 945 findT(application_path, language[:-3])

946 947 948 if __name__ == '__main__': 949 import doctest 950 doctest.testmod() 951

Source Code for Module gluon.languages