Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code | Sign in
(9)

Delta Between Two Patch Sets: translate/storage/cpo.py

Issue 81: Fix CPO memory leak SVN Base: https://translate.svn.sourceforge.net/svnroot/translate/src/trunk/
Left Patch Set: Created 1 year, 4 months ago
Right Patch Set: Added tests Created 1 year, 4 months ago
Use n/p to move between diff chunks; N/P to move between comments. Please Sign in to add in-line comments.
Jump to:
Left: View regular side by side diff
Right: View regular side by side diff
LEFTRIGHT
1 #!/usr/bin/env python 1 #!/usr/bin/env python
2 # -*- coding: utf-8 -*- 2 # -*- coding: utf-8 -*-
3 # 3 #
4 # Copyright 2002-2007 Zuza Software Foundation 4 # Copyright 2002-2007 Zuza Software Foundation
5 # 5 #
6 # This file is part of translate. 6 # This file is part of translate.
7 # 7 #
8 # translate is free software; you can redistribute it and/or modify 8 # translate is free software; you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by 9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation; either version 2 of the License, or 10 # the Free Software Foundation; either version 2 of the License, or
11 # (at your option) any later version. 11 # (at your option) any later version.
12 # 12 #
13 # translate is distributed in the hope that it will be useful, 13 # translate is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details. 16 # GNU General Public License for more details.
17 # 17 #
18 # You should have received a copy of the GNU General Public License 18 # You should have received a copy of the GNU General Public License
19 # along with translate; if not, write to the Free Software 19 # along with translate; if not, write to the Free Software
20 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 20 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
21 21
22 """Classes that hold units of .po files (pounit) or entire files (pofile). 22 """Classes that hold units of .po files (pounit) or entire files (pofile).
23 23
24 Gettext-style .po (or .pot) files are used in translations for KDE, GNOME and 24 Gettext-style .po (or .pot) files are used in translations for KDE, GNOME and
25 many other projects. 25 many other projects.
26 26
27 This uses libgettextpo from the gettext package. Any version before 0.17 will 27 This uses libgettextpo from the gettext package. Any version before 0.17 will
28 at least cause some subtle bugs or may not work at all. Developers might want 28 at least cause some subtle bugs or may not work at all. Developers might want
29 to have a look at gettext-tools/libgettextpo/gettext-po.h from the gettext 29 to have a look at gettext-tools/libgettextpo/gettext-po.h from the gettext
30 package for the public API of the library. 30 package for the public API of the library.
31 """ 31 """
32 32
33 from translate.misc.multistring import multistring 33 from translate.misc.multistring import multistring
34 from translate.storage import pocommon 34 from translate.storage import pocommon
35 from translate.misc import quote 35 from translate.misc import quote
36 from translate.lang import data 36 from translate.lang import data
37 from ctypes import * 37 from ctypes import *
38 import ctypes.util 38 import ctypes.util
39 try: 39 try:
40 import cStringIO as StringIO 40 import cStringIO as StringIO
41 except ImportError: 41 except ImportError:
42 import StringIO 42 import StringIO
43 import os 43 import os
44 import pypo 44 import pypo
45 import re 45 import re
46 import sys 46 import sys
47 import tempfile 47 import tempfile
48 48
49 lsep = " " 49 lsep = " "
50 """Seperator for #: entries""" 50 """Seperator for #: entries"""
51 51
52 STRING = c_char_p 52 STRING = c_char_p
53 53
54 # Structures 54 # Structures
55 class po_message(Structure): 55 class po_message(Structure):
56 _fields_ = [] 56 _fields_ = []
57 57
58 # Function prototypes 58 # Function prototypes
59 xerror_prototype = CFUNCTYPE(None, c_int, POINTER(po_message), STRING, c_uint, c _uint, c_int, STRING) 59 xerror_prototype = CFUNCTYPE(None, c_int, POINTER(po_message), STRING, c_uint, c _uint, c_int, STRING)
60 xerror2_prototype = CFUNCTYPE(None, c_int, POINTER(po_message), STRING, c_uint, c_uint, c_int, STRING, POINTER(po_message), STRING, c_uint, c_uint, c_int, STRIN G) 60 xerror2_prototype = CFUNCTYPE(None, c_int, POINTER(po_message), STRING, c_uint, c_uint, c_int, STRING, POINTER(po_message), STRING, c_uint, c_uint, c_int, STRIN G)
61 61
62 62
63 # Structures (error handler) 63 # Structures (error handler)
64 class po_xerror_handler(Structure): 64 class po_xerror_handler(Structure):
65 _fields_ = [('xerror', xerror_prototype), 65 _fields_ = [('xerror', xerror_prototype),
66 ('xerror2', xerror2_prototype)] 66 ('xerror2', xerror2_prototype)]
67 67
68 class po_error_handler(Structure): 68 class po_error_handler(Structure):
69 _fields_ = [ 69 _fields_ = [
70 ('error', CFUNCTYPE(None, c_int, c_int, STRING)), 70 ('error', CFUNCTYPE(None, c_int, c_int, STRING)),
71 ('error_at_line', CFUNCTYPE(None, c_int, c_int, STRING, c_uint, STRING)), 71 ('error_at_line', CFUNCTYPE(None, c_int, c_int, STRING, c_uint, STRING)),
72 ('multiline_warning', CFUNCTYPE(None, STRING, STRING)), 72 ('multiline_warning', CFUNCTYPE(None, STRING, STRING)),
73 ('multiline_error', CFUNCTYPE(None, STRING, STRING)), 73 ('multiline_error', CFUNCTYPE(None, STRING, STRING)),
74 ] 74 ]
75 75
76 # Callback functions for po_xerror_handler 76 # Callback functions for po_xerror_handler
77 def xerror_cb(severity, message, filename, lineno, column, multilint_p, message_ text): 77 def xerror_cb(severity, message, filename, lineno, column, multilint_p, message_ text):
78 print >> sys.stderr, "xerror_cb", severity, message, filename, lineno, colum n, multilint_p, message_text 78 print >> sys.stderr, "xerror_cb", severity, message, filename, lineno, colum n, multilint_p, message_text
79 if severity >= 1: 79 if severity >= 1:
80 raise ValueError(message_text) 80 raise ValueError(message_text)
81 81
82 def xerror2_cb(severity, message1, filename1, lineno1, column1, multiline_p1, me ssage_text1, message2, filename2, lineno2, column2, multiline_p2, message_text2) : 82 def xerror2_cb(severity, message1, filename1, lineno1, column1, multiline_p1, me ssage_text1, message2, filename2, lineno2, column2, multiline_p2, message_text2) :
83 print >> sys.stderr, "xerror2_cb", severity, message1, filename1, lineno1, c olumn1, multiline_p1, message_text1, message2, filename2, lineno2, column2, mult iline_p2, message_text2 83 print >> sys.stderr, "xerror2_cb", severity, message1, filename1, lineno1, c olumn1, multiline_p1, message_text1, message2, filename2, lineno2, column2, mult iline_p2, message_text2
84 if severity >= 1: 84 if severity >= 1:
85 raise ValueError(message_text1) 85 raise ValueError(message_text1)
86 86
87 87
88 88
89 # Load libgettextpo 89 # Load libgettextpo
90 gpo = None 90 gpo = None
91 # 'gettextpo' is recognised on Unix, while only 'libgettextpo' is recognised on 91 # 'gettextpo' is recognised on Unix, while only 'libgettextpo' is recognised on
92 # windows. Therefore we test both. 92 # windows. Therefore we test both.
93 names = ['gettextpo', 'libgettextpo'] 93 names = ['gettextpo', 'libgettextpo']
94 for name in names: 94 for name in names:
95 lib_location = ctypes.util.find_library(name) 95 lib_location = ctypes.util.find_library(name)
96 if lib_location: 96 if lib_location:
97 gpo = cdll.LoadLibrary(lib_location) 97 gpo = cdll.LoadLibrary(lib_location)
98 if gpo: 98 if gpo:
99 break 99 break
100 else: 100 else:
101 # Now we are getting desperate, so let's guess a unix type DLL that might 101 # Now we are getting desperate, so let's guess a unix type DLL that might
102 # be in LD_LIBRARY_PATH or loaded with LD_PRELOAD 102 # be in LD_LIBRARY_PATH or loaded with LD_PRELOAD
103 try: 103 try:
104 gpo = cdll.LoadLibrary('libgettextpo.so') 104 gpo = cdll.LoadLibrary('libgettextpo.so')
105 except OSError, e: 105 except OSError, e:
106 raise ImportError("gettext PO library not found") 106 raise ImportError("gettext PO library not found")
107 107
108 # Setup return and paramater types 108 # Setup return and paramater types
109 # File access 109 # File access
110 gpo.po_file_read_v3.argtypes = [STRING, POINTER(po_xerror_handler)] 110 gpo.po_file_read_v3.argtypes = [STRING, POINTER(po_xerror_handler)]
111 gpo.po_file_write_v2.argtypes = [c_int, STRING, POINTER(po_xerror_handler)] 111 gpo.po_file_write_v2.argtypes = [c_int, STRING, POINTER(po_xerror_handler)]
112 gpo.po_file_write_v2.retype = c_int 112 gpo.po_file_write_v2.retype = c_int
113 113
114 # Header 114 # Header
115 gpo.po_file_domain_header.restype = STRING 115 gpo.po_file_domain_header.restype = STRING
116 gpo.po_header_field.restype = STRING 116 gpo.po_header_field.restype = STRING
117 gpo.po_header_field.argtypes = [STRING, STRING] 117 gpo.po_header_field.argtypes = [STRING, STRING]
118 118
119 # Locations (filepos) 119 # Locations (filepos)
120 gpo.po_filepos_file.restype = STRING 120 gpo.po_filepos_file.restype = STRING
121 gpo.po_message_filepos.restype = c_int 121 gpo.po_message_filepos.restype = c_int
122 gpo.po_message_filepos.argtypes = [c_int, c_int] 122 gpo.po_message_filepos.argtypes = [c_int, c_int]
123 gpo.po_message_add_filepos.argtypes = [c_int, STRING, c_int] 123 gpo.po_message_add_filepos.argtypes = [c_int, STRING, c_int]
124 124
125 # Message (get methods) 125 # Message (get methods)
126 gpo.po_message_comments.restype = STRING 126 gpo.po_message_comments.restype = STRING
127 gpo.po_message_extracted_comments.restype = STRING 127 gpo.po_message_extracted_comments.restype = STRING
128 gpo.po_message_prev_msgctxt.restype = STRING 128 gpo.po_message_prev_msgctxt.restype = STRING
129 gpo.po_message_prev_msgid.restype = STRING 129 gpo.po_message_prev_msgid.restype = STRING
130 gpo.po_message_prev_msgid_plural.restype = STRING 130 gpo.po_message_prev_msgid_plural.restype = STRING
131 gpo.po_message_is_format.restype = c_int 131 gpo.po_message_is_format.restype = c_int
132 gpo.po_message_msgctxt.restype = STRING 132 gpo.po_message_msgctxt.restype = STRING
133 gpo.po_message_msgid.restype = STRING 133 gpo.po_message_msgid.restype = STRING
134 gpo.po_message_msgid_plural.restype = STRING 134 gpo.po_message_msgid_plural.restype = STRING
135 gpo.po_message_msgstr.restype = STRING 135 gpo.po_message_msgstr.restype = STRING
136 gpo.po_message_msgstr_plural.restype = STRING 136 gpo.po_message_msgstr_plural.restype = STRING
137 137
138 # Message (set methods) 138 # Message (set methods)
139 gpo.po_message_set_comments.argtypes = [c_int, STRING] 139 gpo.po_message_set_comments.argtypes = [c_int, STRING]
140 gpo.po_message_set_extracted_comments.argtypes = [c_int, STRING] 140 gpo.po_message_set_extracted_comments.argtypes = [c_int, STRING]
141 gpo.po_message_set_fuzzy.argtypes = [c_int, c_int] 141 gpo.po_message_set_fuzzy.argtypes = [c_int, c_int]
142 gpo.po_message_set_msgctxt.argtypes = [c_int, STRING] 142 gpo.po_message_set_msgctxt.argtypes = [c_int, STRING]
143 gpo.po_message_set_format.argtypese = [c_int, STRING, c_int]
143 144
144 # Setup the po_xerror_handler 145 # Setup the po_xerror_handler
145 xerror_handler = po_xerror_handler() 146 xerror_handler = po_xerror_handler()
146 xerror_handler.xerror = xerror_prototype(xerror_cb) 147 xerror_handler.xerror = xerror_prototype(xerror_cb)
147 xerror_handler.xerror2 = xerror2_prototype(xerror2_cb) 148 xerror_handler.xerror2 = xerror2_prototype(xerror2_cb)
148 149
149 def escapeforpo(text): 150 def escapeforpo(text):
150 return pypo.escapeforpo(text) 151 return pypo.escapeforpo(text)
151 152
152 def quoteforpo(text): 153 def quoteforpo(text):
153 return pypo.quoteforpo(text) 154 return pypo.quoteforpo(text)
154 155
155 def unquotefrompo(postr, joinwithlinebreak=False): 156 def unquotefrompo(postr, joinwithlinebreak=False):
156 return pypo.unquotefrompo(postr, joinwithlinebreak) 157 return pypo.unquotefrompo(postr, joinwithlinebreak)
157 158
158 def encodingToUse(encoding): 159 def encodingToUse(encoding):
159 return pypo.encodingToUse(encoding) 160 return pypo.encodingToUse(encoding)
160 161
161 def get_libgettextpo_version(): 162 def get_libgettextpo_version():
162 """Returns the libgettextpo version 163 """Returns the libgettextpo version
163 164
164 @return: a three-value tuple containing the libgettextpo version in the 165 @return: a three-value tuple containing the libgettextpo version in the
165 following format: 166 following format:
166 (major version, minor version, subminor version) 167 (major version, minor version, subminor version)
167 """ 168 """
168 libversion = c_long.in_dll(gpo, 'libgettextpo_version') 169 libversion = c_long.in_dll(gpo, 'libgettextpo_version')
169 major = libversion.value >> 16 170 major = libversion.value >> 16
170 minor = libversion.value >> 8 171 minor = libversion.value >> 8
171 subminor = libversion.value - (major << 16) - (minor << 8) 172 subminor = libversion.value - (major << 16) - (minor << 8)
172 return major, minor, subminor 173 return major, minor, subminor
173 174
174 175
175 class pounit(pocommon.pounit): 176 class pounit(pocommon.pounit):
176 def __init__(self, source=None, encoding='utf-8', gpo_message=None): 177 def __init__(self, source=None, encoding='utf-8', gpo_message=None):
177 self._encoding = encoding 178 self._encoding = encoding
178 if not gpo_message: 179 if not gpo_message:
179 self._gpo_message = gpo.po_message_create() 180 self._gpo_message = gpo.po_message_create()
180 if source or source == "": 181 if source or source == "":
181 self.source = source 182 self.source = source
182 self.target = "" 183 self.target = ""
183 elif gpo_message: 184 elif gpo_message:
184 self._gpo_message = gpo_message 185 self._gpo_message = gpo_message
185 186
186 def setmsgidcomment(self, msgidcomment): 187 def setmsgidcomment(self, msgidcomment):
187 if msgidcomment: 188 if msgidcomment:
188 newsource = "_: " + msgidcomment + "\n" + self.source 189 newsource = "_: " + msgidcomment + "\n" + self.source
189 self.source = newsource 190 self.source = newsource
190 msgidcomment = property(None, setmsgidcomment) 191 msgidcomment = property(None, setmsgidcomment)
191 192
192 def setmsgid_plural(self, msgid_plural): 193 def setmsgid_plural(self, msgid_plural):
(...skipping 131 matching lines...) Show 10 above Show 10 below
324 else: 325 else:
325 raise ValueError("Comment type not valid") 326 raise ValueError("Comment type not valid")
326 327
327 if comments and get_libgettextpo_version() < (0, 17, 0): 328 if comments and get_libgettextpo_version() < (0, 17, 0):
328 comments = "\n".join([line.strip() for line in comments.split("\n")] ) 329 comments = "\n".join([line.strip() for line in comments.split("\n")] )
329 # Let's drop the last newline 330 # Let's drop the last newline
330 return comments[:-1].decode(self._encoding) 331 return comments[:-1].decode(self._encoding)
331 332
332 def addnote(self, text, origin=None, position="append"): 333 def addnote(self, text, origin=None, position="append"):
333 # ignore empty strings and strings without non-space characters 334 # ignore empty strings and strings without non-space characters
334 if (not text) or (not text.strip()): 335 if (not text) or (not text.strip()):
335 return 336 return
336 text = data.forceunicode(text) 337 text = data.forceunicode(text)
337 oldnotes = self.getnotes(origin) 338 oldnotes = self.getnotes(origin)
338 newnotes = None 339 newnotes = None
339 if oldnotes: 340 if oldnotes:
340 if position == "append": 341 if position == "append":
341 newnotes = oldnotes + "\n" + text 342 newnotes = oldnotes + "\n" + text
342 elif position == "merge": 343 elif position == "merge":
343 if oldnotes != text: 344 if oldnotes != text:
344 oldnoteslist = oldnotes.split("\n") 345 oldnoteslist = oldnotes.split("\n")
345 for newline in text.split("\n"): 346 for newline in text.split("\n"):
346 newline = newline.rstrip() 347 newline = newline.rstrip()
347 # avoid duplicate comment lines (this might cause some p roblems) 348 # avoid duplicate comment lines (this might cause some p roblems)
348 if newline not in oldnotes or len(newline) < 5: 349 if newline not in oldnotes or len(newline) < 5:
349 oldnoteslist.append(newline) 350 oldnoteslist.append(newline)
350 newnotes = "\n".join(oldnoteslist) 351 newnotes = "\n".join(oldnoteslist)
351 else: 352 else:
352 newnotes = text + '\n' + oldnotes 353 newnotes = text + '\n' + oldnotes
353 else: 354 else:
354 newnotes = "\n".join([line.rstrip() for line in text.split("\n")]) 355 newnotes = "\n".join([line.rstrip() for line in text.split("\n")])
355 356
356 if newnotes: 357 if newnotes:
357 newlines = [] 358 newlines = []
358 needs_space = get_libgettextpo_version() < (0, 17, 0) 359 needs_space = get_libgettextpo_version() < (0, 17, 0)
359 for line in newnotes.split("\n"): 360 for line in newnotes.split("\n"):
360 if line and needs_space: 361 if line and needs_space:
361 newlines.append(" " + line) 362 newlines.append(" " + line)
362 else: 363 else:
363 newlines.append(line) 364 newlines.append(line)
364 newnotes = "\n".join(newlines) 365 newnotes = "\n".join(newlines)
365 if origin in ["programmer", "developer", "source code"]: 366 if origin in ["programmer", "developer", "source code"]:
366 gpo.po_message_set_extracted_comments(self._gpo_message, newnote s) 367 gpo.po_message_set_extracted_comments(self._gpo_message, newnote s)
367 else: 368 else:
368 gpo.po_message_set_comments(self._gpo_message, newnotes) 369 gpo.po_message_set_comments(self._gpo_message, newnotes)
369 370
370 def removenotes(self): 371 def removenotes(self):
371 gpo.po_message_set_comments(self._gpo_message, "") 372 gpo.po_message_set_comments(self._gpo_message, "")
372 373
373 def copy(self): 374 def copy(self):
374 unit = self.buildfromunit(self) 375 """Returns a copy of the this unit that can be used independently.
375 #XXX: adding missing data 376 """
377 unit = pounit()
378 unit.source = self.source
379 unit.target = self.target
380 unit.markfuzzy(self.isfuzzy())
376 if self.isobsolete(): 381 if self.isobsolete():
377 unit.makeobsolete() 382 unit.makeobsolete()
383 for location in self.getlocations():
384 unit.addlocation(location)
385 notes = []
386 origins = ['translator', 'developer', 'programmer', 'source code']
387 for origin in origins:
388 note = self.getnotes(origin=origin)
389 if note in notes:
390 continue
391 else:
392 unit.addnote(note, origin=origin)
393 notes.append(note)
378 context = gpo.po_message_msgctxt(self._gpo_message) 394 context = gpo.po_message_msgctxt(self._gpo_message)
379 if context: 395 if context:
380 unit.setcontext(context) 396 unit.setcontext(context)
381 comments = self._extract_msgidcomments() 397 comments = self._extract_msgidcomments()
382 if comments: 398 if comments:
383 print 'HAS COMMENTS', comments
384 unit.setmsgidcomment(comments) 399 unit.setmsgidcomment(comments)
400 for typecomment in self.gettypecomments():
401 unit.addtypecomment(typecomment)
385 return unit 402 return unit
386 403
387 def merge(self, otherpo, overwrite=False, comments=True, authoritative=False ): 404 def merge(self, otherpo, overwrite=False, comments=True, authoritative=False ):
388 """Merges the otherpo (with the same msgid) into this one. 405 """Merges the otherpo (with the same msgid) into this one.
389 406
390 Overwrite non-blank self.msgstr only if overwrite is True 407 Overwrite non-blank self.msgstr only if overwrite is True
391 merge comments only if comments is True 408 merge comments only if comments is True
392 409
393 """ 410 """
394 411
395 if not isinstance(otherpo, pounit): 412 if not isinstance(otherpo, pounit):
396 super(pounit, self).merge(otherpo, overwrite, comments) 413 super(pounit, self).merge(otherpo, overwrite, comments)
397 return 414 return
398 if comments: 415 if comments:
399 self.addnote(otherpo.getnotes("translator"), origin="translator", po sition="merge") 416 self.addnote(otherpo.getnotes("translator"), origin="translator", po sition="merge")
400 # FIXME mergelists(self.typecomments, otherpo.typecomments) 417 # FIXME mergelists(self.typecomments, otherpo.typecomments)
401 if not authoritative: 418 if not authoritative:
402 # We don't bring across otherpo.automaticcomments as we consider ourself 419 # We don't bring across otherpo.automaticcomments as we consider ourself
403 # to be the the authority. Same applies to otherpo.msgidcomment s 420 # to be the the authority. Same applies to otherpo.msgidcomment s
404 self.addnote(otherpo.getnotes("developer"), origin="developer", position="merge") 421 self.addnote(otherpo.getnotes("developer"), origin="developer", position="merge")
405 self.msgidcomment = otherpo._extract_msgidcomments() or None 422 self.msgidcomment = otherpo._extract_msgidcomments() or None
406 self.addlocations(otherpo.getlocations()) 423 self.addlocations(otherpo.getlocations())
407 if not self.istranslated() or overwrite: 424 if not self.istranslated() or overwrite:
408 # Remove kde-style comments from the translation (if any). 425 # Remove kde-style comments from the translation (if any).
409 if self._extract_msgidcomments(otherpo.target): 426 if self._extract_msgidcomments(otherpo.target):
410 otherpo.target = otherpo.target.replace('_: ' + otherpo._extract _msgidcomments()+ '\n', '') 427 otherpo.target = otherpo.target.replace('_: ' + otherpo._extract _msgidcomments()+ '\n', '')
411 self.target = otherpo.target 428 self.target = otherpo.target
412 if self.source != otherpo.source: 429 if self.source != otherpo.source:
413 self.markfuzzy() 430 self.markfuzzy()
414 else: 431 else:
415 self.markfuzzy(otherpo.isfuzzy()) 432 self.markfuzzy(otherpo.isfuzzy())
416 elif not otherpo.istranslated(): 433 elif not otherpo.istranslated():
417 if self.source != otherpo.source: 434 if self.source != otherpo.source:
418 self.markfuzzy() 435 self.markfuzzy()
419 else: 436 else:
420 if self.target != otherpo.target: 437 if self.target != otherpo.target:
421 self.markfuzzy() 438 self.markfuzzy()
422 439
423 def isheader(self): 440 def isheader(self):
424 #return self.source == u"" and self.target != u"" 441 #return self.source == u"" and self.target != u""
425 # we really want to make sure that there is no msgidcomment or msgctxt 442 # we really want to make sure that there is no msgidcomment or msgctxt
426 return self.getid() == "" and len(self.target) > 0 443 return self.getid() == "" and len(self.target) > 0
427 444
428 def isblank(self): 445 def isblank(self):
429 return len(self.source) == 0 and len(self.target) == 0 446 return len(self.source) == 0 and len(self.target) == 0
430 447
431 def hastypecomment(self, typecomment): 448 def hastypecomment(self, typecomment):
432 return gpo.po_message_is_format(self._gpo_message, typecomment) 449 return gpo.po_message_is_format(self._gpo_message, typecomment)
433 450
451 def gettypecomments(self):
452 """Returns the type comments of a unit, if any.
453
454 @return: a list containing the type comments of the unit
455 """
456 #XXX: brute force
457 formats = [
458 "c", "objc", "sh", "python", "lisp", "elisp", "librep", "scheme",
459 "smalltalk", "java", "csharp", "awk", "object-pascal", "ycp", "tcl",
460 "perl", "perl-brace", "php", "gcc-internal", "qt", "kde", "boost",
461 ]
462 comments = []
463 for format in formats:
464 format += '-format'
465 if self.hastypecomment(format):
466 comments.append(format)
467 return comments
468
469 def addtypecomment(self, typecomment):
470 """Adds a type comment to the unit
471
472 @oaram typecomment: the type comment to be added, for example
473 "c-format".
474 """
475 gpo.po_message_set_format(self._gpo_message, typecomment, 1)
476
434 def hasmarkedcomment(self, commentmarker): 477 def hasmarkedcomment(self, commentmarker):
435 commentmarker = "(%s)" % commentmarker 478 commentmarker = "(%s)" % commentmarker
436 for comment in self.getnotes("translator").split("\n"): 479 for comment in self.getnotes("translator").split("\n"):
437 if comment.startswith(commentmarker): 480 if comment.startswith(commentmarker):
438 return True 481 return True
439 return False 482 return False
440 483
441 def istranslated(self): 484 def istranslated(self):
442 return super(pounit, self).istranslated() and not self.isobsolete() 485 return super(pounit, self).istranslated() and not self.isobsolete()
443 486
444 def istranslatable(self): 487 def istranslatable(self):
445 return not (self.isheader() or self.isblank() or self.isobsolete()) 488 return not (self.isheader() or self.isblank() or self.isobsolete())
446 489
447 def isfuzzy(self): 490 def isfuzzy(self):
448 return gpo.po_message_is_fuzzy(self._gpo_message) 491 return gpo.po_message_is_fuzzy(self._gpo_message)
449 492
450 def markfuzzy(self, present=True): 493 def markfuzzy(self, present=True):
451 gpo.po_message_set_fuzzy(self._gpo_message, present) 494 gpo.po_message_set_fuzzy(self._gpo_message, present)
452 495
453 def isreview(self): 496 def isreview(self):
454 return self.hasmarkedcomment("review") or self.hasmarkedcomment("pofilte r") 497 return self.hasmarkedcomment("review") or self.hasmarkedcomment("pofilte r")
455 498
456 def isobsolete(self): 499 def isobsolete(self):
457 return gpo.po_message_is_obsolete(self._gpo_message) 500 return gpo.po_message_is_obsolete(self._gpo_message)
458 501
459 def makeobsolete(self): 502 def makeobsolete(self):
460 # FIXME: libgettexpo currently does not reset other data, we probably wa nt to do that 503 # FIXME: libgettexpo currently does not reset other data, we probably wa nt to do that
461 # but a better solution would be for libgettextpo to output correct data on serialisation 504 # but a better solution would be for libgettextpo to output correct data on serialisation
462 gpo.po_message_set_obsolete(self._gpo_message, True) 505 gpo.po_message_set_obsolete(self._gpo_message, True)
463 506
464 def resurrect(self): 507 def resurrect(self):
465 gpo.po_message_set_obsolete(self._gpo_message, False) 508 gpo.po_message_set_obsolete(self._gpo_message, False)
466 509
467 def hasplural(self): 510 def hasplural(self):
468 return gpo.po_message_msgid_plural(self._gpo_message) is not None 511 return gpo.po_message_msgid_plural(self._gpo_message) is not None
469 512
470 def _extract_msgidcomments(self, text=None): 513 def _extract_msgidcomments(self, text=None):
471 """Extract KDE style msgid comments from the unit. 514 """Extract KDE style msgid comments from the unit.
472 515
473 @rtype: String 516 @rtype: String
474 @return: Returns the extracted msgidcomments found in this unit's msgid. 517 @return: Returns the extracted msgidcomments found in this unit's msgid.
475 518
476 """ 519 """
477 520
478 if not text: 521 if not text:
479 text = gpo.po_message_msgid(self._gpo_message) 522 text = gpo.po_message_msgid(self._gpo_message)
480 if text: 523 if text:
481 msgidcomment = re.search("_: (.*)\n", text) 524 msgidcomment = re.search("_: (.*)\n", text)
482 if msgidcomment: 525 if msgidcomment:
483 return msgidcomment.group(1).decode(self._encoding) 526 return msgidcomment.group(1).decode(self._encoding)
484 return u"" 527 return u""
485 528
486 def __str__(self): 529 def __str__(self):
487 pf = pofile() 530 pf = pofile()
488 unit = self.copy() 531 unit = self.copy()
489 pf.addunit(unit) 532 pf.addunit(unit)
490 return str(pf) 533 return str(pf)
491 534
492 def getlocations(self): 535 def getlocations(self):
493 locations = [] 536 locations = []
494 i = 0 537 i = 0
495 location = gpo.po_message_filepos(self._gpo_message, i) 538 location = gpo.po_message_filepos(self._gpo_message, i)
496 while location: 539 while location:
497 locname = gpo.po_filepos_file(location) 540 locname = gpo.po_filepos_file(location)
498 locline = gpo.po_filepos_start_line(location) 541 locline = gpo.po_filepos_start_line(location)
499 if locline == -1: 542 if locline == -1:
500 locstring = locname 543 locstring = locname
501 else: 544 else:
502 locstring = locname + ":" + str(locline) 545 locstring = locname + ":" + str(locline)
503 locations.append(locstring) 546 locations.append(locstring)
504 i += 1 547 i += 1
505 location = gpo.po_message_filepos(self._gpo_message, i) 548 location = gpo.po_message_filepos(self._gpo_message, i)
506 return locations 549 return locations
507 550
508 def addlocation(self, location): 551 def addlocation(self, location):
509 for loc in location.split(): 552 for loc in location.split():
510 parts = loc.split(":") 553 parts = loc.split(":")
511 file = parts[0] 554 file = parts[0]
512 if len(parts) == 2: 555 if len(parts) == 2:
513 line = int(parts[1]) 556 line = int(parts[1])
514 else: 557 else:
515 line = -1 558 line = -1
516 gpo.po_message_add_filepos(self._gpo_message, file, line) 559 gpo.po_message_add_filepos(self._gpo_message, file, line)
517 560
518 def setcontext(self, context): 561 def setcontext(self, context):
519 """Sets the context message""" 562 """Sets the context message"""
520 #FIXME: not complete
521 gpo.po_message_set_msgctxt(self._gpo_message, context) 563 gpo.po_message_set_msgctxt(self._gpo_message, context)
522 564
523 def getcontext(self): 565 def getcontext(self):
524 msgctxt = gpo.po_message_msgctxt(self._gpo_message) 566 msgctxt = gpo.po_message_msgctxt(self._gpo_message)
525 msgidcomment = self._extract_msgidcomments() 567 msgidcomment = self._extract_msgidcomments()
526 if msgctxt: 568 if msgctxt:
527 return msgctxt + msgidcomment 569 return msgctxt + msgidcomment
528 else: 570 else:
529 return msgidcomment 571 return msgidcomment
530 572
531 class pofile(pocommon.pofile): 573 class pofile(pocommon.pofile):
532 UnitClass = pounit 574 UnitClass = pounit
533 def __init__(self, inputfile=None, encoding=None, unitclass=pounit): 575 def __init__(self, inputfile=None, encoding=None, unitclass=pounit):
534 self.UnitClass = unitclass 576 self.UnitClass = unitclass
535 pocommon.pofile.__init__(self, unitclass=unitclass) 577 pocommon.pofile.__init__(self, unitclass=unitclass)
536 self._gpo_memory_file = None 578 self._gpo_memory_file = None
537 self._gpo_message_iterator = None 579 self._gpo_message_iterator = None
538 self._encoding = encodingToUse(encoding) 580 self._encoding = encodingToUse(encoding)
539 if inputfile is not None: 581 if inputfile is not None:
540 self.parse(inputfile) 582 self.parse(inputfile)
541 else: 583 else:
542 self._gpo_memory_file = gpo.po_file_create() 584 self._gpo_memory_file = gpo.po_file_create()
543 self._gpo_message_iterator = gpo.po_message_iterator(self._gpo_memor y_file, None) 585 self._gpo_message_iterator = gpo.po_message_iterator(self._gpo_memor y_file, None)
544 586
545 def addunit(self, unit): 587 def addunit(self, unit):
546 unitcopy = unit.copy() 588 unitcopy = unit.copy()
547 gpo.po_message_insert(self._gpo_message_iterator, unitcopy._gpo_message) 589 gpo.po_message_insert(self._gpo_message_iterator, unitcopy._gpo_message)
548 self.units.append(unitcopy) 590 self.units.append(unitcopy)
591
592 def addsourceunit(self, source):
593 self.addunit(self.UnitClass(source))
594 unit = self.findunit(source)
595 return unit
549 596
550 def removeduplicates(self, duplicatestyle="merge"): 597 def removeduplicates(self, duplicatestyle="merge"):
551 """make sure each msgid is unique ; merge comments etc from duplicates i nto original""" 598 """make sure each msgid is unique ; merge comments etc from duplicates i nto original"""
552 msgiddict = {} 599 msgiddict = {}
553 uniqueunits = [] 600 uniqueunits = []
554 # we sometimes need to keep track of what has been marked 601 # we sometimes need to keep track of what has been marked
555 # TODO: this is using a list as the pos aren't hashable, but this is slo w... 602 # TODO: this is using a list as the pos aren't hashable, but this is slo w...
556 markedpos = [] 603 markedpos = []
557 def addcomment(thepo): 604 def addcomment(thepo):
558 thepo.msgidcomment = " ".join(thepo.getlocations()) 605 thepo.msgidcomment = " ".join(thepo.getlocations())
559 markedpos.append(thepo) 606 markedpos.append(thepo)
560 for thepo in self.units: 607 for thepo in self.units:
561 if thepo.isheader(): 608 if thepo.isheader():
562 uniqueunits.append(thepo) 609 uniqueunits.append(thepo)
563 continue 610 continue
564 if duplicatestyle.startswith("msgid_comment"): 611 if duplicatestyle.startswith("msgid_comment"):
565 msgid = thepo._extract_msgidcomments() + thepo.source 612 msgid = thepo._extract_msgidcomments() + thepo.source
566 else: 613 else:
567 msgid = thepo.source 614 msgid = thepo.source
568 if duplicatestyle == "msgid_comment_all": 615 if duplicatestyle == "msgid_comment_all":
569 addcomment(thepo) 616 addcomment(thepo)
570 uniqueunits.append(thepo) 617 uniqueunits.append(thepo)
571 elif msgid in msgiddict: 618 elif msgid in msgiddict:
572 if duplicatestyle == "merge": 619 if duplicatestyle == "merge":
573 if msgid: 620 if msgid:
574 msgiddict[msgid].merge(thepo) 621 msgiddict[msgid].merge(thepo)
575 else: 622 else:
576 addcomment(thepo) 623 addcomment(thepo)
577 uniqueunits.append(thepo) 624 uniqueunits.append(thepo)
578 elif duplicatestyle == "keep": 625 elif duplicatestyle == "keep":
579 uniqueunits.append(thepo) 626 uniqueunits.append(thepo)
580 elif duplicatestyle == "msgid_comment": 627 elif duplicatestyle == "msgid_comment":
581 origpo = msgiddict[msgid] 628 origpo = msgiddict[msgid]
582 if origpo not in markedpos: 629 if origpo not in markedpos:
583 addcomment(origpo) 630 addcomment(origpo)
584 addcomment(thepo) 631 addcomment(thepo)
585 uniqueunits.append(thepo) 632 uniqueunits.append(thepo)
586 elif duplicatestyle == "msgctxt": 633 elif duplicatestyle == "msgctxt":
587 origpo = msgiddict[msgid] 634 origpo = msgiddict[msgid]
588 if origpo not in markedpos: 635 if origpo not in markedpos:
589 gpo.po_message_set_msgctxt(origpo._gpo_message, " ".join (origpo.getlocations())) 636 gpo.po_message_set_msgctxt(origpo._gpo_message, " ".join (origpo.getlocations()))
590 markedpos.append(thepo) 637 markedpos.append(thepo)
591 gpo.po_message_set_msgctxt(thepo._gpo_message, " ".join(thep o.getlocations())) 638 gpo.po_message_set_msgctxt(thepo._gpo_message, " ".join(thep o.getlocations()))
592 uniqueunits.append(thepo) 639 uniqueunits.append(thepo)
593 else: 640 else:
594 if not msgid and duplicatestyle != "keep": 641 if not msgid and duplicatestyle != "keep":
595 addcomment(thepo) 642 addcomment(thepo)
596 msgiddict[msgid] = thepo 643 msgiddict[msgid] = thepo
597 uniqueunits.append(thepo) 644 uniqueunits.append(thepo)
598 new_gpo_memory_file = gpo.po_file_create() 645 new_gpo_memory_file = gpo.po_file_create()
599 new_gpo_message_iterator = gpo.po_message_iterator(new_gpo_memory_file, None) 646 new_gpo_message_iterator = gpo.po_message_iterator(new_gpo_memory_file, None)
600 for unit in uniqueunits: 647 for unit in uniqueunits:
601 gpo.po_message_insert(new_gpo_message_iterator, unit._gpo_message) 648 gpo.po_message_insert(new_gpo_message_iterator, unit._gpo_message)
602 gpo.po_message_iterator_free(self._gpo_message_iterator) 649 gpo.po_message_iterator_free(self._gpo_message_iterator)
603 self._gpo_message_iterator = new_gpo_message_iterator 650 self._gpo_message_iterator = new_gpo_message_iterator
604 self._gpo_memory_file = new_gpo_memory_file 651 self._gpo_memory_file = new_gpo_memory_file
605 self.units = uniqueunits 652 self.units = uniqueunits
606 653
607 def __str__(self): 654 def __str__(self):
608 def obsolete_workaround(): 655 def obsolete_workaround():
609 # Remove all items that are not output by msgmerge when a unit is ob solete. This is a work 656 # Remove all items that are not output by msgmerge when a unit is ob solete. This is a work
610 # around for bug in libgettextpo 657 # around for bug in libgettextpo
611 # FIXME Do version test in case they fix this bug 658 # FIXME Do version test in case they fix this bug
612 for unit in self.units: 659 for unit in self.units:
613 if unit.isobsolete(): 660 if unit.isobsolete():
614 gpo.po_message_set_extracted_comments(unit._gpo_message, "") 661 gpo.po_message_set_extracted_comments(unit._gpo_message, "")
615 location = gpo.po_message_filepos(unit._gpo_message, 0) 662 location = gpo.po_message_filepos(unit._gpo_message, 0)
616 while location: 663 while location:
617 gpo.po_message_remove_filepos(unit._gpo_message, 0) 664 gpo.po_message_remove_filepos(unit._gpo_message, 0)
618 location = gpo.po_message_filepos(unit._gpo_message, 0) 665 location = gpo.po_message_filepos(unit._gpo_message, 0)
666
619 outputstring = "" 667 outputstring = ""
620 if self._gpo_memory_file: 668 if self._gpo_memory_file:
621 obsolete_workaround() 669 obsolete_workaround()
622 f = tempfile.NamedTemporaryFile(prefix='translate', suffix='.po') 670 f = tempfile.NamedTemporaryFile(prefix='translate', suffix='.po')
623 self._gpo_memory_file = gpo.po_file_write_v2(self._gpo_memory_file, f.name, xerror_handler) 671 self._gpo_memory_file = gpo.po_file_write_v2(self._gpo_memory_file, f.name, xerror_handler)
624 f.seek(0) 672 f.seek(0)
625 outputstring = f.read() 673 outputstring = f.read()
626 f.close() 674 f.close()
627 return outputstring 675 return outputstring
628 676
629 def isempty(self): 677 def isempty(self):
630 """Returns True if the object doesn't contain any translation units.""" 678 """Returns True if the object doesn't contain any translation units."""
631 if len(self.units) == 0: 679 if len(self.units) == 0:
632 return True 680 return True
633 # Skip the first unit if it is a header. 681 # Skip the first unit if it is a header.
634 if self.units[0].isheader(): 682 if self.units[0].isheader():
635 units = self.units[1:] 683 units = self.units[1:]
636 else: 684 else:
637 units = self.units 685 units = self.units
638 686
639 for unit in units: 687 for unit in units:
640 if not unit.isblank() and not unit.isobsolete(): 688 if not unit.isblank() and not unit.isobsolete():
641 return False 689 return False
642 return True 690 return True
643 691
644 def parse(self, input): 692 def parse(self, input):
645 if hasattr(input, 'name'): 693 if hasattr(input, 'name'):
646 self.filename = input.name 694 self.filename = input.name
647 elif not getattr(self, 'filename', ''): 695 elif not getattr(self, 'filename', ''):
648 self.filename = '' 696 self.filename = ''
649 697
650 if hasattr(input, "read"): 698 if hasattr(input, "read"):
651 posrc = input.read() 699 posrc = input.read()
652 input.close() 700 input.close()
653 input = posrc 701 input = posrc
654 702
655 needtmpfile = not os.path.isfile(input) 703 needtmpfile = not os.path.isfile(input)
656 if needtmpfile: 704 if needtmpfile:
657 # This is not a file - we write the string to a temporary file 705 # This is not a file - we write the string to a temporary file
658 fd, fname = tempfile.mkstemp(prefix='translate', suffix='.po') 706 fd, fname = tempfile.mkstemp(prefix='translate', suffix='.po')
659 os.write(fd, input) 707 os.write(fd, input)
660 input = fname 708 input = fname
661 os.close(fd) 709 os.close(fd)
662 710
663 self._gpo_memory_file = gpo.po_file_read_v3(input, xerror_handler) 711 self._gpo_memory_file = gpo.po_file_read_v3(input, xerror_handler)
664 if self._gpo_memory_file is None: 712 if self._gpo_memory_file is None:
665 print >> sys.stderr, "Error:" 713 print >> sys.stderr, "Error:"
666 714
667 if needtmpfile: 715 if needtmpfile:
668 os.remove(input) 716 os.remove(input)
669 717
670 # Handle xerrors here 718 # Handle xerrors here
671 self._header = gpo.po_file_domain_header(self._gpo_memory_file, None) 719 self._header = gpo.po_file_domain_header(self._gpo_memory_file, None)
672 if self._header: 720 if self._header:
673 charset = gpo.po_header_field(self._header, "Content-Type") 721 charset = gpo.po_header_field(self._header, "Content-Type")
674 if charset: 722 if charset:
675 charset = re.search("charset=([^\\s]+)", charset).group(1) 723 charset = re.search("charset=([^\\s]+)", charset).group(1)
676 self._encoding = encodingToUse(charset) 724 self._encoding = encodingToUse(charset)
677 self._gpo_message_iterator = gpo.po_message_iterator(self._gpo_memory_fi le, None) 725 self._gpo_message_iterator = gpo.po_message_iterator(self._gpo_memory_fi le, None)
678 newmessage = gpo.po_next_message(self._gpo_message_iterator) 726 newmessage = gpo.po_next_message(self._gpo_message_iterator)
679 while newmessage: 727 while newmessage:
680 newunit = pounit(gpo_message=newmessage) 728 newunit = pounit(gpo_message=newmessage)
681 self.units.append(newunit) 729 self.units.append(newunit)
682 newmessage = gpo.po_next_message(self._gpo_message_iterator) 730 newmessage = gpo.po_next_message(self._gpo_message_iterator)
683 #self._free_iterator()
684 731
685 def __del__(self): 732 def __del__(self):
686 #print 'DELL CALLED', id(self)
687 # We currently disable this while we still get segmentation faults. 733 # We currently disable this while we still get segmentation faults.
688 # Note that this is definitely leaking memory because of this. 734 # Note that this is definitely leaking memory because of this.
689 #return
690 self._free_iterator() 735 self._free_iterator()
691 if self._gpo_memory_file is not None: 736 if self._gpo_memory_file is not None:
692 gpo.po_file_free(self._gpo_memory_file) 737 gpo.po_file_free(self._gpo_memory_file)
693 self._gpo_memory_file = None 738 self._gpo_memory_file = None
694 739
695 def _free_iterator(self): 740 def _free_iterator(self):
696 #print 'ITERATOR CALLED', id(self)
697 # We currently disable this while we still get segmentation faults. 741 # We currently disable this while we still get segmentation faults.
698 # Note that this is definitely leaking memory because of this. 742 # Note that this is definitely leaking memory because of this.
699 #return
700 if self._gpo_message_iterator is not None: 743 if self._gpo_message_iterator is not None:
701 gpo.po_message_iterator_free(self._gpo_message_iterator) 744 gpo.po_message_iterator_free(self._gpo_message_iterator)
702 self._gpo_message_iterator = None 745 self._gpo_message_iterator = None
LEFTRIGHT

Powered by Google App Engine
This is Rietveld r159