Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code | Sign in
(68)

Delta Between Two Patch Sets: translate/storage/cpo.py

Issue 81: Fix CPO memory leak SVN Base: https://translate.svn.sourceforge.net/svnroot/translate/src/trunk/
Left Patch Set: Added tests Created 1 year, 3 months ago
Right Patch Set: Fix addlocation method Created 1 year, 3 months ago
Use n/p to move between diff chunks; N/P to move between comments. Please Sign in to add in-line comments.
Jump to:
Left: View regular side by side diff
Right: View regular side by side diff
LEFTRIGHT
1 #!/usr/bin/env python 1 #!/usr/bin/env python
2 # -*- coding: utf-8 -*- 2 # -*- coding: utf-8 -*-
3 # 3 #
4 # Copyright 2002-2007 Zuza Software Foundation 4 # Copyright 2002-2007 Zuza Software Foundation
5 # 5 #
6 # This file is part of translate. 6 # This file is part of translate.
7 # 7 #
8 # translate is free software; you can redistribute it and/or modify 8 # translate is free software; you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by 9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation; either version 2 of the License, or 10 # the Free Software Foundation; either version 2 of the License, or
11 # (at your option) any later version. 11 # (at your option) any later version.
12 # 12 #
13 # translate is distributed in the hope that it will be useful, 13 # translate is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details. 16 # GNU General Public License for more details.
17 # 17 #
18 # You should have received a copy of the GNU General Public License 18 # You should have received a copy of the GNU General Public License
19 # along with translate; if not, write to the Free Software 19 # along with translate; if not, write to the Free Software
20 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 20 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
21 21
22 """Classes that hold units of .po files (pounit) or entire files (pofile). 22 """Classes that hold units of .po files (pounit) or entire files (pofile).
23 23
24 Gettext-style .po (or .pot) files are used in translations for KDE, GNOME and 24 Gettext-style .po (or .pot) files are used in translations for KDE, GNOME and
25 many other projects. 25 many other projects.
26 26
27 This uses libgettextpo from the gettext package. Any version before 0.17 will 27 This uses libgettextpo from the gettext package. Any version before 0.17 will
28 at least cause some subtle bugs or may not work at all. Developers might want 28 at least cause some subtle bugs or may not work at all. Developers might want
29 to have a look at gettext-tools/libgettextpo/gettext-po.h from the gettext 29 to have a look at gettext-tools/libgettextpo/gettext-po.h from the gettext
30 package for the public API of the library. 30 package for the public API of the library.
31 """ 31 """
32 32
33 from translate.misc.multistring import multistring 33 from translate.misc.multistring import multistring
34 from translate.storage import pocommon 34 from translate.storage import pocommon
35 from translate.misc import quote 35 from translate.misc import quote
36 from translate.lang import data 36 from translate.lang import data
37 from ctypes import * 37 from ctypes import *
38 import ctypes.util 38 import ctypes.util
39 try: 39 try:
40 import cStringIO as StringIO 40 import cStringIO as StringIO
41 except ImportError: 41 except ImportError:
42 import StringIO 42 import StringIO
43 import os 43 import os
44 import pypo 44 import pypo
45 import re 45 import re
46 import sys 46 import sys
47 import tempfile 47 import tempfile
48 48
49 lsep = " " 49 lsep = " "
50 """Seperator for #: entries""" 50 """Seperator for #: entries"""
51 51
52 STRING = c_char_p 52 STRING = c_char_p
53 53
54 # Structures 54 # Structures
55 class po_message(Structure): 55 class po_message(Structure):
56 _fields_ = [] 56 _fields_ = []
57 57
58 # Function prototypes 58 # Function prototypes
59 xerror_prototype = CFUNCTYPE(None, c_int, POINTER(po_message), STRING, c_uint, c _uint, c_int, STRING) 59 xerror_prototype = CFUNCTYPE(None, c_int, POINTER(po_message), STRING, c_uint, c _uint, c_int, STRING)
60 xerror2_prototype = CFUNCTYPE(None, c_int, POINTER(po_message), STRING, c_uint, c_uint, c_int, STRING, POINTER(po_message), STRING, c_uint, c_uint, c_int, STRIN G) 60 xerror2_prototype = CFUNCTYPE(None, c_int, POINTER(po_message), STRING, c_uint, c_uint, c_int, STRING, POINTER(po_message), STRING, c_uint, c_uint, c_int, STRIN G)
61 61
62 62
63 # Structures (error handler) 63 # Structures (error handler)
64 class po_xerror_handler(Structure): 64 class po_xerror_handler(Structure):
65 _fields_ = [('xerror', xerror_prototype), 65 _fields_ = [('xerror', xerror_prototype),
66 ('xerror2', xerror2_prototype)] 66 ('xerror2', xerror2_prototype)]
67 67
68 class po_error_handler(Structure): 68 class po_error_handler(Structure):
69 _fields_ = [ 69 _fields_ = [
70 ('error', CFUNCTYPE(None, c_int, c_int, STRING)), 70 ('error', CFUNCTYPE(None, c_int, c_int, STRING)),
71 ('error_at_line', CFUNCTYPE(None, c_int, c_int, STRING, c_uint, STRING)), 71 ('error_at_line', CFUNCTYPE(None, c_int, c_int, STRING, c_uint, STRING)),
72 ('multiline_warning', CFUNCTYPE(None, STRING, STRING)), 72 ('multiline_warning', CFUNCTYPE(None, STRING, STRING)),
73 ('multiline_error', CFUNCTYPE(None, STRING, STRING)), 73 ('multiline_error', CFUNCTYPE(None, STRING, STRING)),
74 ] 74 ]
75 75
76 # Callback functions for po_xerror_handler 76 # Callback functions for po_xerror_handler
77 def xerror_cb(severity, message, filename, lineno, column, multilint_p, message_ text): 77 def xerror_cb(severity, message, filename, lineno, column, multilint_p, message_ text):
78 print >> sys.stderr, "xerror_cb", severity, message, filename, lineno, colum n, multilint_p, message_text 78 print >> sys.stderr, "xerror_cb", severity, message, filename, lineno, colum n, multilint_p, message_text
79 if severity >= 1: 79 if severity >= 1:
80 raise ValueError(message_text) 80 raise ValueError(message_text)
81 81
82 def xerror2_cb(severity, message1, filename1, lineno1, column1, multiline_p1, me ssage_text1, message2, filename2, lineno2, column2, multiline_p2, message_text2) : 82 def xerror2_cb(severity, message1, filename1, lineno1, column1, multiline_p1, me ssage_text1, message2, filename2, lineno2, column2, multiline_p2, message_text2) :
83 print >> sys.stderr, "xerror2_cb", severity, message1, filename1, lineno1, c olumn1, multiline_p1, message_text1, message2, filename2, lineno2, column2, mult iline_p2, message_text2 83 print >> sys.stderr, "xerror2_cb", severity, message1, filename1, lineno1, c olumn1, multiline_p1, message_text1, message2, filename2, lineno2, column2, mult iline_p2, message_text2
84 if severity >= 1: 84 if severity >= 1:
85 raise ValueError(message_text1) 85 raise ValueError(message_text1)
86 86
87 87
88 88
89 # Load libgettextpo 89 # Load libgettextpo
90 gpo = None 90 gpo = None
91 # 'gettextpo' is recognised on Unix, while only 'libgettextpo' is recognised on 91 # 'gettextpo' is recognised on Unix, while only 'libgettextpo' is recognised on
92 # windows. Therefore we test both. 92 # windows. Therefore we test both.
93 names = ['gettextpo', 'libgettextpo'] 93 names = ['gettextpo', 'libgettextpo']
94 for name in names: 94 for name in names:
95 lib_location = ctypes.util.find_library(name) 95 lib_location = ctypes.util.find_library(name)
96 if lib_location: 96 if lib_location:
97 gpo = cdll.LoadLibrary(lib_location) 97 gpo = cdll.LoadLibrary(lib_location)
98 if gpo: 98 if gpo:
99 break 99 break
100 else: 100 else:
101 # Now we are getting desperate, so let's guess a unix type DLL that might 101 # Now we are getting desperate, so let's guess a unix type DLL that might
102 # be in LD_LIBRARY_PATH or loaded with LD_PRELOAD 102 # be in LD_LIBRARY_PATH or loaded with LD_PRELOAD
103 try: 103 try:
104 gpo = cdll.LoadLibrary('libgettextpo.so') 104 gpo = cdll.LoadLibrary('libgettextpo.so')
105 except OSError, e: 105 except OSError, e:
106 raise ImportError("gettext PO library not found") 106 raise ImportError("gettext PO library not found")
107 107
108 # Setup return and paramater types 108 # Setup return and paramater types
109 # File access 109 # File access
110 gpo.po_file_read_v3.argtypes = [STRING, POINTER(po_xerror_handler)] 110 gpo.po_file_read_v3.argtypes = [STRING, POINTER(po_xerror_handler)]
111 gpo.po_file_write_v2.argtypes = [c_int, STRING, POINTER(po_xerror_handler)] 111 gpo.po_file_write_v2.argtypes = [c_int, STRING, POINTER(po_xerror_handler)]
112 gpo.po_file_write_v2.retype = c_int 112 gpo.po_file_write_v2.retype = c_int
113 113
114 # Header 114 # Header
115 gpo.po_file_domain_header.restype = STRING 115 gpo.po_file_domain_header.restype = STRING
116 gpo.po_header_field.restype = STRING 116 gpo.po_header_field.restype = STRING
117 gpo.po_header_field.argtypes = [STRING, STRING] 117 gpo.po_header_field.argtypes = [STRING, STRING]
118 118
119 # Locations (filepos) 119 # Locations (filepos)
120 gpo.po_filepos_file.restype = STRING 120 gpo.po_filepos_file.restype = STRING
121 gpo.po_message_filepos.restype = c_int 121 gpo.po_message_filepos.restype = c_int
122 gpo.po_message_filepos.argtypes = [c_int, c_int] 122 gpo.po_message_filepos.argtypes = [c_int, c_int]
123 gpo.po_message_add_filepos.argtypes = [c_int, STRING, c_int] 123 gpo.po_message_add_filepos.argtypes = [c_int, STRING, c_size_t]
124 124
125 # Message (get methods) 125 # Message (get methods)
126 gpo.po_message_comments.restype = STRING 126 gpo.po_message_comments.restype = STRING
127 gpo.po_message_extracted_comments.restype = STRING 127 gpo.po_message_extracted_comments.restype = STRING
128 gpo.po_message_prev_msgctxt.restype = STRING 128 gpo.po_message_prev_msgctxt.restype = STRING
129 gpo.po_message_prev_msgid.restype = STRING 129 gpo.po_message_prev_msgid.restype = STRING
130 gpo.po_message_prev_msgid_plural.restype = STRING 130 gpo.po_message_prev_msgid_plural.restype = STRING
131 gpo.po_message_is_format.restype = c_int 131 gpo.po_message_is_format.restype = c_int
132 gpo.po_message_msgctxt.restype = STRING 132 gpo.po_message_msgctxt.restype = STRING
133 gpo.po_message_msgid.restype = STRING 133 gpo.po_message_msgid.restype = STRING
134 gpo.po_message_msgid_plural.restype = STRING 134 gpo.po_message_msgid_plural.restype = STRING
135 gpo.po_message_msgstr.restype = STRING 135 gpo.po_message_msgstr.restype = STRING
136 gpo.po_message_msgstr_plural.restype = STRING 136 gpo.po_message_msgstr_plural.restype = STRING
137 137
138 # Message (set methods) 138 # Message (set methods)
139 gpo.po_message_set_comments.argtypes = [c_int, STRING] 139 gpo.po_message_set_comments.argtypes = [c_int, STRING]
140 gpo.po_message_set_extracted_comments.argtypes = [c_int, STRING] 140 gpo.po_message_set_extracted_comments.argtypes = [c_int, STRING]
141 gpo.po_message_set_fuzzy.argtypes = [c_int, c_int] 141 gpo.po_message_set_fuzzy.argtypes = [c_int, c_int]
142 gpo.po_message_set_msgctxt.argtypes = [c_int, STRING] 142 gpo.po_message_set_msgctxt.argtypes = [c_int, STRING]
143 gpo.po_message_set_format.argtypese = [c_int, STRING, c_int] 143 gpo.po_message_set_format.argtypes = [c_int, STRING, c_int]
dwaynebailey 2008/08/29 13:19:25 Not sure what this has to do with addlocations? Y
144 144
145 # Setup the po_xerror_handler 145 # Setup the po_xerror_handler
146 xerror_handler = po_xerror_handler() 146 xerror_handler = po_xerror_handler()
147 xerror_handler.xerror = xerror_prototype(xerror_cb) 147 xerror_handler.xerror = xerror_prototype(xerror_cb)
148 xerror_handler.xerror2 = xerror2_prototype(xerror2_cb) 148 xerror_handler.xerror2 = xerror2_prototype(xerror2_cb)
149 149
150 def escapeforpo(text): 150 def escapeforpo(text):
151 return pypo.escapeforpo(text) 151 return pypo.escapeforpo(text)
152 152
153 def quoteforpo(text): 153 def quoteforpo(text):
154 return pypo.quoteforpo(text) 154 return pypo.quoteforpo(text)
155 155
156 def unquotefrompo(postr, joinwithlinebreak=False): 156 def unquotefrompo(postr, joinwithlinebreak=False):
157 return pypo.unquotefrompo(postr, joinwithlinebreak) 157 return pypo.unquotefrompo(postr, joinwithlinebreak)
158 158
159 def encodingToUse(encoding): 159 def encodingToUse(encoding):
160 return pypo.encodingToUse(encoding) 160 return pypo.encodingToUse(encoding)
161 161
162 def get_libgettextpo_version(): 162 def get_libgettextpo_version():
163 """Returns the libgettextpo version 163 """Returns the libgettextpo version
164 164
165 @return: a three-value tuple containing the libgettextpo version in the 165 @return: a three-value tuple containing the libgettextpo version in the
166 following format: 166 following format:
167 (major version, minor version, subminor version) 167 (major version, minor version, subminor version)
168 """ 168 """
169 libversion = c_long.in_dll(gpo, 'libgettextpo_version') 169 libversion = c_long.in_dll(gpo, 'libgettextpo_version')
170 major = libversion.value >> 16 170 major = libversion.value >> 16
171 minor = libversion.value >> 8 171 minor = libversion.value >> 8
172 subminor = libversion.value - (major << 16) - (minor << 8) 172 subminor = libversion.value - (major << 16) - (minor << 8)
173 return major, minor, subminor 173 return major, minor, subminor
174 174
175 175
176 class pounit(pocommon.pounit): 176 class pounit(pocommon.pounit):
177 def __init__(self, source=None, encoding='utf-8', gpo_message=None): 177 def __init__(self, source=None, encoding='utf-8', gpo_message=None):
178 self._encoding = encoding 178 self._encoding = encoding
179 if not gpo_message: 179 if not gpo_message:
180 self._gpo_message = gpo.po_message_create() 180 self._gpo_message = gpo.po_message_create()
181 if source or source == "": 181 if source or source == "":
182 self.source = source 182 self.source = source
183 self.target = "" 183 self.target = ""
184 elif gpo_message: 184 elif gpo_message:
185 self._gpo_message = gpo_message 185 self._gpo_message = gpo_message
186 186
187 def setmsgidcomment(self, msgidcomment): 187 def setmsgidcomment(self, msgidcomment):
188 if msgidcomment: 188 if msgidcomment:
189 newsource = "_: " + msgidcomment + "\n" + self.source 189 newsource = "_: " + msgidcomment + "\n" + self.source
190 self.source = newsource 190 self.source = newsource
191 msgidcomment = property(None, setmsgidcomment) 191 msgidcomment = property(None, setmsgidcomment)
192 192
193 def setmsgid_plural(self, msgid_plural): 193 def setmsgid_plural(self, msgid_plural):
(...skipping 142 matching lines...) Show 10 above Show 10 below
336 return 336 return
337 text = data.forceunicode(text) 337 text = data.forceunicode(text)
338 oldnotes = self.getnotes(origin) 338 oldnotes = self.getnotes(origin)
339 newnotes = None 339 newnotes = None
340 if oldnotes: 340 if oldnotes:
341 if position == "append": 341 if position == "append":
342 newnotes = oldnotes + "\n" + text 342 newnotes = oldnotes + "\n" + text
343 elif position == "merge": 343 elif position == "merge":
344 if oldnotes != text: 344 if oldnotes != text:
345 oldnoteslist = oldnotes.split("\n") 345 oldnoteslist = oldnotes.split("\n")
346 for newline in text.split("\n"): 346 for newline in text.split("\n"):
347 newline = newline.rstrip() 347 newline = newline.rstrip()
348 # avoid duplicate comment lines (this might cause some p roblems) 348 # avoid duplicate comment lines (this might cause some p roblems)
349 if newline not in oldnotes or len(newline) < 5: 349 if newline not in oldnotes or len(newline) < 5:
350 oldnoteslist.append(newline) 350 oldnoteslist.append(newline)
351 newnotes = "\n".join(oldnoteslist) 351 newnotes = "\n".join(oldnoteslist)
352 else: 352 else:
353 newnotes = text + '\n' + oldnotes 353 newnotes = text + '\n' + oldnotes
354 else: 354 else:
355 newnotes = "\n".join([line.rstrip() for line in text.split("\n")]) 355 newnotes = "\n".join([line.rstrip() for line in text.split("\n")])
356 356
357 if newnotes: 357 if newnotes:
358 newlines = [] 358 newlines = []
359 needs_space = get_libgettextpo_version() < (0, 17, 0) 359 needs_space = get_libgettextpo_version() < (0, 17, 0)
360 for line in newnotes.split("\n"): 360 for line in newnotes.split("\n"):
361 if line and needs_space: 361 if line and needs_space:
362 newlines.append(" " + line) 362 newlines.append(" " + line)
363 else: 363 else:
364 newlines.append(line) 364 newlines.append(line)
365 newnotes = "\n".join(newlines) 365 newnotes = "\n".join(newlines)
366 if origin in ["programmer", "developer", "source code"]: 366 if origin in ["programmer", "developer", "source code"]:
367 gpo.po_message_set_extracted_comments(self._gpo_message, newnote s) 367 gpo.po_message_set_extracted_comments(self._gpo_message, newnote s)
368 else: 368 else:
369 gpo.po_message_set_comments(self._gpo_message, newnotes) 369 gpo.po_message_set_comments(self._gpo_message, newnotes)
370 370
371 def removenotes(self): 371 def removenotes(self):
372 gpo.po_message_set_comments(self._gpo_message, "") 372 gpo.po_message_set_comments(self._gpo_message, "")
373 373
374 def copy(self): 374 def copy(self):
375 """Returns a copy of the this unit that can be used independently. 375 """Returns a copy of the this unit that can be used independently.
376 """ 376 """
377 unit = pounit() 377 unit = pounit()
378 unit.source = self.source 378 unit.source = self.source
379 unit.target = self.target 379 unit.target = self.target
380 unit.markfuzzy(self.isfuzzy()) 380 unit.markfuzzy(self.isfuzzy())
381 if self.isobsolete(): 381 if self.isobsolete():
382 unit.makeobsolete() 382 unit.makeobsolete()
383 for location in self.getlocations(): 383 for location in self.getlocations():
384 unit.addlocation(location) 384 unit.addlocation(location)
385 notes = [] 385 notes = []
386 origins = ['translator', 'developer', 'programmer', 'source code'] 386 origins = ['translator', 'developer', 'programmer', 'source code']
dwaynebailey 2008/08/29 13:19:25 Maybe call them note_types
387 for origin in origins: 387 for origin in origins:
388 note = self.getnotes(origin=origin) 388 note = self.getnotes(origin=origin)
389 if note in notes: 389 if note in notes:
390 continue 390 continue
391 else: 391 else:
392 unit.addnote(note, origin=origin) 392 unit.addnote(note, origin=origin)
393 notes.append(note) 393 notes.append(note)
394 context = gpo.po_message_msgctxt(self._gpo_message) 394 context = gpo.po_message_msgctxt(self._gpo_message)
395 if context: 395 if context:
396 unit.setcontext(context) 396 unit.setcontext(context)
397 comments = self._extract_msgidcomments() 397 comments = self._extract_msgidcomments()
398 if comments: 398 if comments:
399 unit.setmsgidcomment(comments) 399 unit.setmsgidcomment(comments)
400 for typecomment in self.gettypecomments(): 400 for typecomment in self.gettypecomments():
401 unit.addtypecomment(typecomment) 401 unit.addtypecomment(typecomment)
402 return unit 402 return unit
dwaynebailey 2008/08/29 13:19:25 Generally I think this part looks good
403 403
404 def merge(self, otherpo, overwrite=False, comments=True, authoritative=False ): 404 def merge(self, otherpo, overwrite=False, comments=True, authoritative=False ):
405 """Merges the otherpo (with the same msgid) into this one. 405 """Merges the otherpo (with the same msgid) into this one.
406 406
407 Overwrite non-blank self.msgstr only if overwrite is True 407 Overwrite non-blank self.msgstr only if overwrite is True
408 merge comments only if comments is True 408 merge comments only if comments is True
409 409
410 """ 410 """
411 411
412 if not isinstance(otherpo, pounit): 412 if not isinstance(otherpo, pounit):
413 super(pounit, self).merge(otherpo, overwrite, comments) 413 super(pounit, self).merge(otherpo, overwrite, comments)
414 return 414 return
415 if comments: 415 if comments:
416 self.addnote(otherpo.getnotes("translator"), origin="translator", po sition="merge") 416 self.addnote(otherpo.getnotes("translator"), origin="translator", po sition="merge")
417 # FIXME mergelists(self.typecomments, otherpo.typecomments) 417 # FIXME mergelists(self.typecomments, otherpo.typecomments)
418 if not authoritative: 418 if not authoritative:
419 # We don't bring across otherpo.automaticcomments as we consider ourself 419 # We don't bring across otherpo.automaticcomments as we consider ourself
420 # to be the the authority. Same applies to otherpo.msgidcomment s 420 # to be the the authority. Same applies to otherpo.msgidcomment s
421 self.addnote(otherpo.getnotes("developer"), origin="developer", position="merge") 421 self.addnote(otherpo.getnotes("developer"), origin="developer", position="merge")
422 self.msgidcomment = otherpo._extract_msgidcomments() or None 422 self.msgidcomment = otherpo._extract_msgidcomments() or None
423 self.addlocations(otherpo.getlocations()) 423 self.addlocations(otherpo.getlocations())
424 if not self.istranslated() or overwrite: 424 if not self.istranslated() or overwrite:
425 # Remove kde-style comments from the translation (if any). 425 # Remove kde-style comments from the translation (if any).
426 if self._extract_msgidcomments(otherpo.target): 426 if self._extract_msgidcomments(otherpo.target):
427 otherpo.target = otherpo.target.replace('_: ' + otherpo._extract _msgidcomments()+ '\n', '') 427 otherpo.target = otherpo.target.replace('_: ' + otherpo._extract _msgidcomments()+ '\n', '')
428 self.target = otherpo.target 428 self.target = otherpo.target
429 if self.source != otherpo.source: 429 if self.source != otherpo.source:
430 self.markfuzzy() 430 self.markfuzzy()
431 else: 431 else:
432 self.markfuzzy(otherpo.isfuzzy()) 432 self.markfuzzy(otherpo.isfuzzy())
433 elif not otherpo.istranslated(): 433 elif not otherpo.istranslated():
434 if self.source != otherpo.source: 434 if self.source != otherpo.source:
435 self.markfuzzy() 435 self.markfuzzy()
436 else: 436 else:
437 if self.target != otherpo.target: 437 if self.target != otherpo.target:
438 self.markfuzzy() 438 self.markfuzzy()
439 439
440 def isheader(self): 440 def isheader(self):
441 #return self.source == u"" and self.target != u"" 441 #return self.source == u"" and self.target != u""
442 # we really want to make sure that there is no msgidcomment or msgctxt 442 # we really want to make sure that there is no msgidcomment or msgctxt
443 return self.getid() == "" and len(self.target) > 0 443 return self.getid() == "" and len(self.target) > 0
444 444
445 def isblank(self): 445 def isblank(self):
446 return len(self.source) == 0 and len(self.target) == 0 446 return len(self.source) == 0 and len(self.target) == 0
447 447
448 def hastypecomment(self, typecomment): 448 def hastypecomment(self, typecomment):
449 return gpo.po_message_is_format(self._gpo_message, typecomment) 449 return gpo.po_message_is_format(self._gpo_message, typecomment)
450 450
451 def gettypecomments(self): 451 def gettypecomments(self):
452 """Returns the type comments of a unit, if any. 452 """Returns the type comments of a unit, if any.
453 453
454 @return: a list containing the type comments of the unit 454 @return: a list containing the type comments of the unit
455 """ 455 """
456 #XXX: brute force 456 #XXX: brute force
457 formats = [ 457 formats = [
458 "c", "objc", "sh", "python", "lisp", "elisp", "librep", "scheme", 458 "c", "objc", "sh", "python", "lisp", "elisp", "librep", "scheme",
459 "smalltalk", "java", "csharp", "awk", "object-pascal", "ycp", "tcl", 459 "smalltalk", "java", "csharp", "awk", "object-pascal", "ycp", "tcl",
460 "perl", "perl-brace", "php", "gcc-internal", "qt", "kde", "boost", 460 "perl", "perl-brace", "php", "gcc-internal", "qt", "kde", "boost",
461 ] 461 ]
dwaynebailey 2008/08/29 13:19:25 I thought we could get a list of valid formats fro
462 comments = [] 462 comments = []
463 for format in formats: 463 for format in formats:
464 format += '-format' 464 format += '-format'
465 if self.hastypecomment(format): 465 if self.hastypecomment(format):
466 comments.append(format) 466 comments.append(format)
467 return comments 467 return comments
468 468
469 def addtypecomment(self, typecomment): 469 def addtypecomment(self, typecomment):
470 """Adds a type comment to the unit 470 """Adds a type comment to the unit
471 471
472 @oaram typecomment: the type comment to be added, for example 472 @oaram typecomment: the type comment to be added, for example
473 "c-format". 473 "c-format".
474 """ 474 """
475 gpo.po_message_set_format(self._gpo_message, typecomment, 1) 475 gpo.po_message_set_format(self._gpo_message, typecomment, 1)
476 476
477 def hasmarkedcomment(self, commentmarker): 477 def hasmarkedcomment(self, commentmarker):
478 commentmarker = "(%s)" % commentmarker 478 commentmarker = "(%s)" % commentmarker
479 for comment in self.getnotes("translator").split("\n"): 479 for comment in self.getnotes("translator").split("\n"):
480 if comment.startswith(commentmarker): 480 if comment.startswith(commentmarker):
481 return True 481 return True
482 return False 482 return False
483 483
484 def istranslated(self): 484 def istranslated(self):
485 return super(pounit, self).istranslated() and not self.isobsolete() 485 return super(pounit, self).istranslated() and not self.isobsolete()
486 486
487 def istranslatable(self): 487 def istranslatable(self):
488 return not (self.isheader() or self.isblank() or self.isobsolete()) 488 return not (self.isheader() or self.isblank() or self.isobsolete())
489 489
490 def isfuzzy(self): 490 def isfuzzy(self):
491 return gpo.po_message_is_fuzzy(self._gpo_message) 491 return gpo.po_message_is_fuzzy(self._gpo_message)
492 492
493 def markfuzzy(self, present=True): 493 def markfuzzy(self, present=True):
494 gpo.po_message_set_fuzzy(self._gpo_message, present) 494 gpo.po_message_set_fuzzy(self._gpo_message, present)
495 495
496 def isreview(self): 496 def isreview(self):
497 return self.hasmarkedcomment("review") or self.hasmarkedcomment("pofilte r") 497 return self.hasmarkedcomment("review") or self.hasmarkedcomment("pofilte r")
498 498
499 def isobsolete(self): 499 def isobsolete(self):
500 return gpo.po_message_is_obsolete(self._gpo_message) 500 return gpo.po_message_is_obsolete(self._gpo_message)
501 501
502 def makeobsolete(self): 502 def makeobsolete(self):
503 # FIXME: libgettexpo currently does not reset other data, we probably wa nt to do that 503 # FIXME: libgettexpo currently does not reset other data, we probably wa nt to do that
504 # but a better solution would be for libgettextpo to output correct data on serialisation 504 # but a better solution would be for libgettextpo to output correct data on serialisation
505 gpo.po_message_set_obsolete(self._gpo_message, True) 505 gpo.po_message_set_obsolete(self._gpo_message, True)
506 506
507 def resurrect(self): 507 def resurrect(self):
508 gpo.po_message_set_obsolete(self._gpo_message, False) 508 gpo.po_message_set_obsolete(self._gpo_message, False)
509 509
510 def hasplural(self): 510 def hasplural(self):
511 return gpo.po_message_msgid_plural(self._gpo_message) is not None 511 return gpo.po_message_msgid_plural(self._gpo_message) is not None
512 512
513 def _extract_msgidcomments(self, text=None): 513 def _extract_msgidcomments(self, text=None):
514 """Extract KDE style msgid comments from the unit. 514 """Extract KDE style msgid comments from the unit.
515 515
516 @rtype: String 516 @rtype: String
517 @return: Returns the extracted msgidcomments found in this unit's msgid. 517 @return: Returns the extracted msgidcomments found in this unit's msgid.
518 518
519 """ 519 """
520 520
521 if not text: 521 if not text:
522 text = gpo.po_message_msgid(self._gpo_message) 522 text = gpo.po_message_msgid(self._gpo_message)
523 if text: 523 if text:
524 msgidcomment = re.search("_: (.*)\n", text) 524 msgidcomment = re.search("_: (.*)\n", text)
525 if msgidcomment: 525 if msgidcomment:
526 return msgidcomment.group(1).decode(self._encoding) 526 return msgidcomment.group(1).decode(self._encoding)
527 return u"" 527 return u""
528 528
529 def __str__(self): 529 def __str__(self):
530 pf = pofile() 530 pf = pofile()
531 unit = self.copy() 531 unit = self.copy()
532 pf.addunit(unit) 532 pf.addunit(unit)
533 return str(pf) 533 return str(pf)
534 534
535 def getlocations(self): 535 def getlocations(self):
536 locations = [] 536 locations = []
537 i = 0 537 i = 0
538 location = gpo.po_message_filepos(self._gpo_message, i) 538 location = gpo.po_message_filepos(self._gpo_message, i)
539 while location: 539 while location:
540 locname = gpo.po_filepos_file(location) 540 locname = gpo.po_filepos_file(location)
541 locline = gpo.po_filepos_start_line(location) 541 locline = gpo.po_filepos_start_line(location)
542 if locline == -1: 542 if locline == -1:
543 locstring = locname 543 locstring = locname
544 else: 544 else:
545 locstring = locname + ":" + str(locline) 545 locstring = locname + ":" + str(locline)
546 locations.append(locstring) 546 locations.append(locstring)
547 i += 1 547 i += 1
548 location = gpo.po_message_filepos(self._gpo_message, i) 548 location = gpo.po_message_filepos(self._gpo_message, i)
549 return locations 549 return locations
550 550
551 def addlocation(self, location): 551 def addlocation(self, location):
552 for loc in location.split(): 552 for loc in location.split():
553 parts = loc.split(":") 553 parts = loc.split(":")
554 file = parts[0] 554 file = parts[0]
555 if len(parts) == 2: 555 if len(parts) == 2:
556 line = int(parts[1]) 556 line = int(parts[1])
557 else: 557 else:
558 line = -1 558 # libgettextpo requires a (size_t)(-1) if no file
559 # location is available
560 line = c_size_t(-1)
559 gpo.po_message_add_filepos(self._gpo_message, file, line) 561 gpo.po_message_add_filepos(self._gpo_message, file, line)
560 562
561 def setcontext(self, context): 563 def setcontext(self, context):
562 """Sets the context message""" 564 """Sets the context message"""
563 gpo.po_message_set_msgctxt(self._gpo_message, context) 565 gpo.po_message_set_msgctxt(self._gpo_message, context)
dwaynebailey 2008/08/29 13:19:25 Again one piece at a time. If you can separate ou
564 566
565 def getcontext(self): 567 def getcontext(self):
566 msgctxt = gpo.po_message_msgctxt(self._gpo_message) 568 msgctxt = gpo.po_message_msgctxt(self._gpo_message)
567 msgidcomment = self._extract_msgidcomments() 569 msgidcomment = self._extract_msgidcomments()
568 if msgctxt: 570 if msgctxt:
569 return msgctxt + msgidcomment 571 return msgctxt + msgidcomment
570 else: 572 else:
571 return msgidcomment 573 return msgidcomment
572 574
573 class pofile(pocommon.pofile): 575 class pofile(pocommon.pofile):
574 UnitClass = pounit 576 UnitClass = pounit
575 def __init__(self, inputfile=None, encoding=None, unitclass=pounit): 577 def __init__(self, inputfile=None, encoding=None, unitclass=pounit):
576 self.UnitClass = unitclass 578 self.UnitClass = unitclass
577 pocommon.pofile.__init__(self, unitclass=unitclass) 579 pocommon.pofile.__init__(self, unitclass=unitclass)
578 self._gpo_memory_file = None 580 self._gpo_memory_file = None
579 self._gpo_message_iterator = None 581 self._gpo_message_iterator = None
580 self._encoding = encodingToUse(encoding) 582 self._encoding = encodingToUse(encoding)
581 if inputfile is not None: 583 if inputfile is not None:
582 self.parse(inputfile) 584 self.parse(inputfile)
583 else: 585 else:
584 self._gpo_memory_file = gpo.po_file_create() 586 self._gpo_memory_file = gpo.po_file_create()
585 self._gpo_message_iterator = gpo.po_message_iterator(self._gpo_memor y_file, None) 587 self._gpo_message_iterator = gpo.po_message_iterator(self._gpo_memor y_file, None)
586 588
587 def addunit(self, unit): 589 def addunit(self, unit):
588 unitcopy = unit.copy() 590 unitcopy = unit.copy()
589 gpo.po_message_insert(self._gpo_message_iterator, unitcopy._gpo_message) 591 gpo.po_message_insert(self._gpo_message_iterator, unitcopy._gpo_message)
590 self.units.append(unitcopy) 592 self.units.append(unitcopy)
591 593
592 def addsourceunit(self, source): 594 def addsourceunit(self, source):
593 self.addunit(self.UnitClass(source)) 595 self.addunit(self.UnitClass(source))
594 unit = self.findunit(source) 596 unit = self.findunit(source)
595 return unit 597 return unit
596 598
597 def removeduplicates(self, duplicatestyle="merge"): 599 def removeduplicates(self, duplicatestyle="merge"):
598 """make sure each msgid is unique ; merge comments etc from duplicates i nto original""" 600 """make sure each msgid is unique ; merge comments etc from duplicates i nto original"""
599 msgiddict = {} 601 msgiddict = {}
600 uniqueunits = [] 602 uniqueunits = []
601 # we sometimes need to keep track of what has been marked 603 # we sometimes need to keep track of what has been marked
602 # TODO: this is using a list as the pos aren't hashable, but this is slo w... 604 # TODO: this is using a list as the pos aren't hashable, but this is slo w...
603 markedpos = [] 605 markedpos = []
604 def addcomment(thepo): 606 def addcomment(thepo):
605 thepo.msgidcomment = " ".join(thepo.getlocations()) 607 thepo.msgidcomment = " ".join(thepo.getlocations())
606 markedpos.append(thepo) 608 markedpos.append(thepo)
607 for thepo in self.units: 609 for thepo in self.units:
608 if thepo.isheader(): 610 if thepo.isheader():
609 uniqueunits.append(thepo) 611 uniqueunits.append(thepo)
610 continue 612 continue
611 if duplicatestyle.startswith("msgid_comment"): 613 if duplicatestyle.startswith("msgid_comment"):
612 msgid = thepo._extract_msgidcomments() + thepo.source 614 msgid = thepo._extract_msgidcomments() + thepo.source
613 else: 615 else:
614 msgid = thepo.source 616 msgid = thepo.source
615 if duplicatestyle == "msgid_comment_all": 617 if duplicatestyle == "msgid_comment_all":
616 addcomment(thepo) 618 addcomment(thepo)
617 uniqueunits.append(thepo) 619 uniqueunits.append(thepo)
618 elif msgid in msgiddict: 620 elif msgid in msgiddict:
619 if duplicatestyle == "merge": 621 if duplicatestyle == "merge":
620 if msgid: 622 if msgid:
621 msgiddict[msgid].merge(thepo) 623 msgiddict[msgid].merge(thepo)
622 else: 624 else:
623 addcomment(thepo) 625 addcomment(thepo)
624 uniqueunits.append(thepo) 626 uniqueunits.append(thepo)
625 elif duplicatestyle == "keep": 627 elif duplicatestyle == "keep":
626 uniqueunits.append(thepo) 628 uniqueunits.append(thepo)
627 elif duplicatestyle == "msgid_comment": 629 elif duplicatestyle == "msgid_comment":
628 origpo = msgiddict[msgid] 630 origpo = msgiddict[msgid]
629 if origpo not in markedpos: 631 if origpo not in markedpos:
630 addcomment(origpo) 632 addcomment(origpo)
631 addcomment(thepo) 633 addcomment(thepo)
632 uniqueunits.append(thepo) 634 uniqueunits.append(thepo)
633 elif duplicatestyle == "msgctxt": 635 elif duplicatestyle == "msgctxt":
634 origpo = msgiddict[msgid] 636 origpo = msgiddict[msgid]
635 if origpo not in markedpos: 637 if origpo not in markedpos:
636 gpo.po_message_set_msgctxt(origpo._gpo_message, " ".join (origpo.getlocations())) 638 gpo.po_message_set_msgctxt(origpo._gpo_message, " ".join (origpo.getlocations()))
637 markedpos.append(thepo) 639 markedpos.append(thepo)
638 gpo.po_message_set_msgctxt(thepo._gpo_message, " ".join(thep o.getlocations())) 640 gpo.po_message_set_msgctxt(thepo._gpo_message, " ".join(thep o.getlocations()))
639 uniqueunits.append(thepo) 641 uniqueunits.append(thepo)
640 else: 642 else:
641 if not msgid and duplicatestyle != "keep": 643 if not msgid and duplicatestyle != "keep":
642 addcomment(thepo) 644 addcomment(thepo)
643 msgiddict[msgid] = thepo 645 msgiddict[msgid] = thepo
644 uniqueunits.append(thepo) 646 uniqueunits.append(thepo)
645 new_gpo_memory_file = gpo.po_file_create() 647 new_gpo_memory_file = gpo.po_file_create()
646 new_gpo_message_iterator = gpo.po_message_iterator(new_gpo_memory_file, None) 648 new_gpo_message_iterator = gpo.po_message_iterator(new_gpo_memory_file, None)
647 for unit in uniqueunits: 649 for unit in uniqueunits:
648 gpo.po_message_insert(new_gpo_message_iterator, unit._gpo_message) 650 gpo.po_message_insert(new_gpo_message_iterator, unit._gpo_message)
649 gpo.po_message_iterator_free(self._gpo_message_iterator) 651 gpo.po_message_iterator_free(self._gpo_message_iterator)
650 self._gpo_message_iterator = new_gpo_message_iterator 652 self._gpo_message_iterator = new_gpo_message_iterator
651 self._gpo_memory_file = new_gpo_memory_file 653 self._gpo_memory_file = new_gpo_memory_file
652 self.units = uniqueunits 654 self.units = uniqueunits
653 655
654 def __str__(self): 656 def __str__(self):
655 def obsolete_workaround(): 657 def obsolete_workaround():
656 # Remove all items that are not output by msgmerge when a unit is ob solete. This is a work 658 # Remove all items that are not output by msgmerge when a unit is ob solete. This is a work
657 # around for bug in libgettextpo 659 # around for bug in libgettextpo
658 # FIXME Do version test in case they fix this bug 660 # FIXME Do version test in case they fix this bug
659 for unit in self.units: 661 for unit in self.units:
660 if unit.isobsolete(): 662 if unit.isobsolete():
661 gpo.po_message_set_extracted_comments(unit._gpo_message, "") 663 gpo.po_message_set_extracted_comments(unit._gpo_message, "")
662 location = gpo.po_message_filepos(unit._gpo_message, 0) 664 location = gpo.po_message_filepos(unit._gpo_message, 0)
663 while location: 665 while location:
664 gpo.po_message_remove_filepos(unit._gpo_message, 0) 666 gpo.po_message_remove_filepos(unit._gpo_message, 0)
665 location = gpo.po_message_filepos(unit._gpo_message, 0) 667 location = gpo.po_message_filepos(unit._gpo_message, 0)
666
667 outputstring = "" 668 outputstring = ""
668 if self._gpo_memory_file: 669 if self._gpo_memory_file:
669 obsolete_workaround() 670 obsolete_workaround()
670 f = tempfile.NamedTemporaryFile(prefix='translate', suffix='.po') 671 f = tempfile.NamedTemporaryFile(prefix='translate', suffix='.po')
671 self._gpo_memory_file = gpo.po_file_write_v2(self._gpo_memory_file, f.name, xerror_handler) 672 self._gpo_memory_file = gpo.po_file_write_v2(self._gpo_memory_file, f.name, xerror_handler)
672 f.seek(0) 673 f.seek(0)
673 outputstring = f.read() 674 outputstring = f.read()
674 f.close() 675 f.close()
675 return outputstring 676 return outputstring
676 677
677 def isempty(self): 678 def isempty(self):
678 """Returns True if the object doesn't contain any translation units.""" 679 """Returns True if the object doesn't contain any translation units."""
679 if len(self.units) == 0: 680 if len(self.units) == 0:
680 return True 681 return True
681 # Skip the first unit if it is a header. 682 # Skip the first unit if it is a header.
682 if self.units[0].isheader(): 683 if self.units[0].isheader():
683 units = self.units[1:] 684 units = self.units[1:]
684 else: 685 else:
685 units = self.units 686 units = self.units
686 687
687 for unit in units: 688 for unit in units:
688 if not unit.isblank() and not unit.isobsolete(): 689 if not unit.isblank() and not unit.isobsolete():
689 return False 690 return False
690 return True 691 return True
691 692
692 def parse(self, input): 693 def parse(self, input):
693 if hasattr(input, 'name'): 694 if hasattr(input, 'name'):
694 self.filename = input.name 695 self.filename = input.name
695 elif not getattr(self, 'filename', ''): 696 elif not getattr(self, 'filename', ''):
696 self.filename = '' 697 self.filename = ''
697 698
698 if hasattr(input, "read"): 699 if hasattr(input, "read"):
699 posrc = input.read() 700 posrc = input.read()
700 input.close() 701 input.close()
701 input = posrc 702 input = posrc
702 703
703 needtmpfile = not os.path.isfile(input) 704 needtmpfile = not os.path.isfile(input)
704 if needtmpfile: 705 if needtmpfile:
705 # This is not a file - we write the string to a temporary file 706 # This is not a file - we write the string to a temporary file
706 fd, fname = tempfile.mkstemp(prefix='translate', suffix='.po') 707 fd, fname = tempfile.mkstemp(prefix='translate', suffix='.po')
707 os.write(fd, input) 708 os.write(fd, input)
708 input = fname 709 input = fname
709 os.close(fd) 710 os.close(fd)
710 711
711 self._gpo_memory_file = gpo.po_file_read_v3(input, xerror_handler) 712 self._gpo_memory_file = gpo.po_file_read_v3(input, xerror_handler)
712 if self._gpo_memory_file is None: 713 if self._gpo_memory_file is None:
713 print >> sys.stderr, "Error:" 714 print >> sys.stderr, "Error:"
714 715
715 if needtmpfile: 716 if needtmpfile:
716 os.remove(input) 717 os.remove(input)
717 718
718 # Handle xerrors here 719 # Handle xerrors here
719 self._header = gpo.po_file_domain_header(self._gpo_memory_file, None) 720 self._header = gpo.po_file_domain_header(self._gpo_memory_file, None)
720 if self._header: 721 if self._header:
721 charset = gpo.po_header_field(self._header, "Content-Type") 722 charset = gpo.po_header_field(self._header, "Content-Type")
722 if charset: 723 if charset:
723 charset = re.search("charset=([^\\s]+)", charset).group(1) 724 charset = re.search("charset=([^\\s]+)", charset).group(1)
724 self._encoding = encodingToUse(charset) 725 self._encoding = encodingToUse(charset)
725 self._gpo_message_iterator = gpo.po_message_iterator(self._gpo_memory_fi le, None) 726 self._gpo_message_iterator = gpo.po_message_iterator(self._gpo_memory_fi le, None)
726 newmessage = gpo.po_next_message(self._gpo_message_iterator) 727 newmessage = gpo.po_next_message(self._gpo_message_iterator)
727 while newmessage: 728 while newmessage:
728 newunit = pounit(gpo_message=newmessage) 729 newunit = pounit(gpo_message=newmessage)
729 self.units.append(newunit) 730 self.units.append(newunit)
730 newmessage = gpo.po_next_message(self._gpo_message_iterator) 731 newmessage = gpo.po_next_message(self._gpo_message_iterator)
731 732
732 def __del__(self): 733 def __del__(self):
733 # We currently disable this while we still get segmentation faults. 734 # We currently disable this while we still get segmentation faults.
734 # Note that this is definitely leaking memory because of this. 735 # Note that this is definitely leaking memory because of this.
735 self._free_iterator() 736 self._free_iterator()
736 if self._gpo_memory_file is not None: 737 if self._gpo_memory_file is not None:
737 gpo.po_file_free(self._gpo_memory_file) 738 gpo.po_file_free(self._gpo_memory_file)
738 self._gpo_memory_file = None 739 self._gpo_memory_file = None
739 740
740 def _free_iterator(self): 741 def _free_iterator(self):
741 # We currently disable this while we still get segmentation faults. 742 # We currently disable this while we still get segmentation faults.
742 # Note that this is definitely leaking memory because of this. 743 # Note that this is definitely leaking memory because of this.
743 if self._gpo_message_iterator is not None: 744 if self._gpo_message_iterator is not None:
744 gpo.po_message_iterator_free(self._gpo_message_iterator) 745 gpo.po_message_iterator_free(self._gpo_message_iterator)
745 self._gpo_message_iterator = None 746 self._gpo_message_iterator = None
LEFTRIGHT

Powered by Google App Engine
This is Rietveld r159