Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code | Sign in
(35)

Side by Side Diff: translate/storage/cpo.py

Issue 81: Fix CPO memory leak SVN Base: https://translate.svn.sourceforge.net/svnroot/translate/src/trunk/
Patch Set: Added some tests workarounds Created 1 year, 3 months ago
Use n/p to move between diff chunks; N/P to move between comments. Please Sign in to add in-line comments.
Jump to:
View unified diff
OLDNEW
1 #!/usr/bin/env python 1 #!/usr/bin/env python
2 # -*- coding: utf-8 -*- 2 # -*- coding: utf-8 -*-
3 # 3 #
4 # Copyright 2002-2007 Zuza Software Foundation 4 # Copyright 2002-2007 Zuza Software Foundation
5 # 5 #
6 # This file is part of translate. 6 # This file is part of translate.
7 # 7 #
8 # translate is free software; you can redistribute it and/or modify 8 # translate is free software; you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by 9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation; either version 2 of the License, or 10 # the Free Software Foundation; either version 2 of the License, or
11 # (at your option) any later version. 11 # (at your option) any later version.
12 # 12 #
13 # translate is distributed in the hope that it will be useful, 13 # translate is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details. 16 # GNU General Public License for more details.
17 # 17 #
18 # You should have received a copy of the GNU General Public License 18 # You should have received a copy of the GNU General Public License
19 # along with translate; if not, write to the Free Software 19 # along with translate; if not, write to the Free Software
20 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 20 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
21 21
22 """Classes that hold units of .po files (pounit) or entire files (pofile). 22 """Classes that hold units of .po files (pounit) or entire files (pofile).
23 23
24 Gettext-style .po (or .pot) files are used in translations for KDE, GNOME and 24 Gettext-style .po (or .pot) files are used in translations for KDE, GNOME and
25 many other projects. 25 many other projects.
26 26
27 This uses libgettextpo from the gettext package. Any version before 0.17 will 27 This uses libgettextpo from the gettext package. Any version before 0.17 will
28 at least cause some subtle bugs or may not work at all. Developers might want 28 at least cause some subtle bugs or may not work at all. Developers might want
29 to have a look at gettext-tools/libgettextpo/gettext-po.h from the gettext 29 to have a look at gettext-tools/libgettextpo/gettext-po.h from the gettext
30 package for the public API of the library. 30 package for the public API of the library.
31 """ 31 """
32 32
33 from translate.misc.multistring import multistring 33 from translate.misc.multistring import multistring
34 from translate.storage import pocommon 34 from translate.storage import pocommon
35 from translate.misc import quote 35 from translate.misc import quote
36 from translate.lang import data 36 from translate.lang import data
37 from ctypes import * 37 from ctypes import *
38 import ctypes.util 38 import ctypes.util
39 try: 39 try:
40 import cStringIO as StringIO 40 import cStringIO as StringIO
41 except ImportError: 41 except ImportError:
42 import StringIO 42 import StringIO
43 import os 43 import os
44 import pypo 44 import pypo
45 import re 45 import re
46 import sys 46 import sys
47 import tempfile 47 import tempfile
48 48
49 lsep = " " 49 lsep = " "
50 """Seperator for #: entries""" 50 """Seperator for #: entries"""
51 51
52 STRING = c_char_p 52 STRING = c_char_p
53 53
54 # Structures 54 # Structures
55 class po_message(Structure): 55 class po_message(Structure):
56 _fields_ = [] 56 _fields_ = []
57 57
58 # Function prototypes 58 # Function prototypes
59 xerror_prototype = CFUNCTYPE(None, c_int, POINTER(po_message), STRING, c_uint, c _uint, c_int, STRING) 59 xerror_prototype = CFUNCTYPE(None, c_int, POINTER(po_message), STRING, c_uint, c _uint, c_int, STRING)
60 xerror2_prototype = CFUNCTYPE(None, c_int, POINTER(po_message), STRING, c_uint, c_uint, c_int, STRING, POINTER(po_message), STRING, c_uint, c_uint, c_int, STRIN G) 60 xerror2_prototype = CFUNCTYPE(None, c_int, POINTER(po_message), STRING, c_uint, c_uint, c_int, STRING, POINTER(po_message), STRING, c_uint, c_uint, c_int, STRIN G)
61 61
62 62
63 # Structures (error handler) 63 # Structures (error handler)
64 class po_xerror_handler(Structure): 64 class po_xerror_handler(Structure):
65 _fields_ = [('xerror', xerror_prototype), 65 _fields_ = [('xerror', xerror_prototype),
66 ('xerror2', xerror2_prototype)] 66 ('xerror2', xerror2_prototype)]
67 67
68 class po_error_handler(Structure): 68 class po_error_handler(Structure):
69 _fields_ = [ 69 _fields_ = [
70 ('error', CFUNCTYPE(None, c_int, c_int, STRING)), 70 ('error', CFUNCTYPE(None, c_int, c_int, STRING)),
71 ('error_at_line', CFUNCTYPE(None, c_int, c_int, STRING, c_uint, STRING)), 71 ('error_at_line', CFUNCTYPE(None, c_int, c_int, STRING, c_uint, STRING)),
72 ('multiline_warning', CFUNCTYPE(None, STRING, STRING)), 72 ('multiline_warning', CFUNCTYPE(None, STRING, STRING)),
73 ('multiline_error', CFUNCTYPE(None, STRING, STRING)), 73 ('multiline_error', CFUNCTYPE(None, STRING, STRING)),
74 ] 74 ]
75 75
76 # Callback functions for po_xerror_handler 76 # Callback functions for po_xerror_handler
77 def xerror_cb(severity, message, filename, lineno, column, multilint_p, message_ text): 77 def xerror_cb(severity, message, filename, lineno, column, multilint_p, message_ text):
78 print >> sys.stderr, "xerror_cb", severity, message, filename, lineno, colum n, multilint_p, message_text 78 print >> sys.stderr, "xerror_cb", severity, message, filename, lineno, colum n, multilint_p, message_text
79 if severity >= 1: 79 if severity >= 1:
80 raise ValueError(message_text) 80 raise ValueError(message_text)
81 81
82 def xerror2_cb(severity, message1, filename1, lineno1, column1, multiline_p1, me ssage_text1, message2, filename2, lineno2, column2, multiline_p2, message_text2) : 82 def xerror2_cb(severity, message1, filename1, lineno1, column1, multiline_p1, me ssage_text1, message2, filename2, lineno2, column2, multiline_p2, message_text2) :
83 print >> sys.stderr, "xerror2_cb", severity, message1, filename1, lineno1, c olumn1, multiline_p1, message_text1, message2, filename2, lineno2, column2, mult iline_p2, message_text2 83 print >> sys.stderr, "xerror2_cb", severity, message1, filename1, lineno1, c olumn1, multiline_p1, message_text1, message2, filename2, lineno2, column2, mult iline_p2, message_text2
84 if severity >= 1: 84 if severity >= 1:
85 raise ValueError(message_text1) 85 raise ValueError(message_text1)
86 86
87 87
88 88
89 # Load libgettextpo 89 # Load libgettextpo
90 gpo = None 90 gpo = None
91 # 'gettextpo' is recognised on Unix, while only 'libgettextpo' is recognised on 91 # 'gettextpo' is recognised on Unix, while only 'libgettextpo' is recognised on
92 # windows. Therefore we test both. 92 # windows. Therefore we test both.
93 names = ['gettextpo', 'libgettextpo'] 93 names = ['gettextpo', 'libgettextpo']
94 for name in names: 94 for name in names:
95 lib_location = ctypes.util.find_library(name) 95 lib_location = ctypes.util.find_library(name)
96 if lib_location: 96 if lib_location:
97 gpo = cdll.LoadLibrary(lib_location) 97 gpo = cdll.LoadLibrary(lib_location)
98 if gpo: 98 if gpo:
99 break 99 break
100 else: 100 else:
101 # Now we are getting desperate, so let's guess a unix type DLL that might 101 # Now we are getting desperate, so let's guess a unix type DLL that might
102 # be in LD_LIBRARY_PATH or loaded with LD_PRELOAD 102 # be in LD_LIBRARY_PATH or loaded with LD_PRELOAD
103 try: 103 try:
104 gpo = cdll.LoadLibrary('libgettextpo.so') 104 gpo = cdll.LoadLibrary('libgettextpo.so')
105 except OSError, e: 105 except OSError, e:
106 raise ImportError("gettext PO library not found") 106 raise ImportError("gettext PO library not found")
107 107
108 # Setup return and paramater types 108 # Setup return and paramater types
109 # File access 109 # File access
110 gpo.po_file_read_v3.argtypes = [STRING, POINTER(po_xerror_handler)] 110 gpo.po_file_read_v3.argtypes = [STRING, POINTER(po_xerror_handler)]
111 gpo.po_file_write_v2.argtypes = [c_int, STRING, POINTER(po_xerror_handler)] 111 gpo.po_file_write_v2.argtypes = [c_int, STRING, POINTER(po_xerror_handler)]
112 gpo.po_file_write_v2.retype = c_int 112 gpo.po_file_write_v2.retype = c_int
113 113
114 # Header 114 # Header
115 gpo.po_file_domain_header.restype = STRING 115 gpo.po_file_domain_header.restype = STRING
116 gpo.po_header_field.restype = STRING 116 gpo.po_header_field.restype = STRING
117 gpo.po_header_field.argtypes = [STRING, STRING] 117 gpo.po_header_field.argtypes = [STRING, STRING]
118 118
119 # Locations (filepos) 119 # Locations (filepos)
120 gpo.po_filepos_file.restype = STRING 120 gpo.po_filepos_file.restype = STRING
121 gpo.po_message_filepos.restype = c_int 121 gpo.po_message_filepos.restype = c_int
122 gpo.po_message_filepos.argtypes = [c_int, c_int] 122 gpo.po_message_filepos.argtypes = [c_int, c_int]
123 gpo.po_message_add_filepos.argtypes = [c_int, STRING, c_int] 123 gpo.po_message_add_filepos.argtypes = [c_int, STRING, c_int]
124 124
125 # Message (get methods) 125 # Message (get methods)
126 gpo.po_message_comments.restype = STRING 126 gpo.po_message_comments.restype = STRING
127 gpo.po_message_extracted_comments.restype = STRING 127 gpo.po_message_extracted_comments.restype = STRING
128 gpo.po_message_prev_msgctxt.restype = STRING 128 gpo.po_message_prev_msgctxt.restype = STRING
129 gpo.po_message_prev_msgid.restype = STRING 129 gpo.po_message_prev_msgid.restype = STRING
130 gpo.po_message_prev_msgid_plural.restype = STRING 130 gpo.po_message_prev_msgid_plural.restype = STRING
131 gpo.po_message_is_format.restype = c_int 131 gpo.po_message_is_format.restype = c_int
132 gpo.po_message_msgctxt.restype = STRING 132 gpo.po_message_msgctxt.restype = STRING
133 gpo.po_message_msgid.restype = STRING 133 gpo.po_message_msgid.restype = STRING
134 gpo.po_message_msgid_plural.restype = STRING 134 gpo.po_message_msgid_plural.restype = STRING
135 gpo.po_message_msgstr.restype = STRING 135 gpo.po_message_msgstr.restype = STRING
136 gpo.po_message_msgstr_plural.restype = STRING 136 gpo.po_message_msgstr_plural.restype = STRING
137 137
138 # Message (set methods) 138 # Message (set methods)
139 gpo.po_message_set_comments.argtypes = [c_int, STRING] 139 gpo.po_message_set_comments.argtypes = [c_int, STRING]
140 gpo.po_message_set_extracted_comments.argtypes = [c_int, STRING] 140 gpo.po_message_set_extracted_comments.argtypes = [c_int, STRING]
141 gpo.po_message_set_fuzzy.argtypes = [c_int, c_int] 141 gpo.po_message_set_fuzzy.argtypes = [c_int, c_int]
142 gpo.po_message_set_msgctxt.argtypes = [c_int, STRING] 142 gpo.po_message_set_msgctxt.argtypes = [c_int, STRING]
143 gpo.po_message_set_format.argtypese = [c_int, STRING, c_int]
143 144
144 # Setup the po_xerror_handler 145 # Setup the po_xerror_handler
145 xerror_handler = po_xerror_handler() 146 xerror_handler = po_xerror_handler()
146 xerror_handler.xerror = xerror_prototype(xerror_cb) 147 xerror_handler.xerror = xerror_prototype(xerror_cb)
147 xerror_handler.xerror2 = xerror2_prototype(xerror2_cb) 148 xerror_handler.xerror2 = xerror2_prototype(xerror2_cb)
148 149
149 def escapeforpo(text): 150 def escapeforpo(text):
150 return pypo.escapeforpo(text) 151 return pypo.escapeforpo(text)
151 152
152 def quoteforpo(text): 153 def quoteforpo(text):
153 return pypo.quoteforpo(text) 154 return pypo.quoteforpo(text)
154 155
155 def unquotefrompo(postr, joinwithlinebreak=False): 156 def unquotefrompo(postr, joinwithlinebreak=False):
156 return pypo.unquotefrompo(postr, joinwithlinebreak) 157 return pypo.unquotefrompo(postr, joinwithlinebreak)
157 158
158 def encodingToUse(encoding): 159 def encodingToUse(encoding):
159 return pypo.encodingToUse(encoding) 160 return pypo.encodingToUse(encoding)
160 161
161 def get_libgettextpo_version(): 162 def get_libgettextpo_version():
162 """Returns the libgettextpo version 163 """Returns the libgettextpo version
163 164
164 @return: a three-value tuple containing the libgettextpo version in the 165 @return: a three-value tuple containing the libgettextpo version in the
165 following format: 166 following format:
166 (major version, minor version, subminor version) 167 (major version, minor version, subminor version)
167 """ 168 """
168 libversion = c_long.in_dll(gpo, 'libgettextpo_version') 169 libversion = c_long.in_dll(gpo, 'libgettextpo_version')
169 major = libversion.value >> 16 170 major = libversion.value >> 16
170 minor = libversion.value >> 8 171 minor = libversion.value >> 8
171 subminor = libversion.value - (major << 16) - (minor << 8) 172 subminor = libversion.value - (major << 16) - (minor << 8)
172 return major, minor, subminor 173 return major, minor, subminor
173 174
174 175
175 class pounit(pocommon.pounit): 176 class pounit(pocommon.pounit):
176 def __init__(self, source=None, encoding='utf-8', gpo_message=None): 177 def __init__(self, source=None, encoding='utf-8', gpo_message=None):
177 self._encoding = encoding 178 self._encoding = encoding
178 if not gpo_message: 179 if not gpo_message:
179 self._gpo_message = gpo.po_message_create() 180 self._gpo_message = gpo.po_message_create()
180 if source or source == "": 181 if source or source == "":
181 self.source = source 182 self.source = source
182 self.target = "" 183 self.target = ""
183 elif gpo_message: 184 elif gpo_message:
184 self._gpo_message = gpo_message 185 self._gpo_message = gpo_message
185 186
186 def setmsgidcomment(self, msgidcomment): 187 def setmsgidcomment(self, msgidcomment):
187 if msgidcomment: 188 if msgidcomment:
188 newsource = "_: " + msgidcomment + "\n" + self.source 189 newsource = "_: " + msgidcomment + "\n" + self.source
189 self.source = newsource 190 self.source = newsource
190 msgidcomment = property(None, setmsgidcomment) 191 msgidcomment = property(None, setmsgidcomment)
191 192
192 def setmsgid_plural(self, msgid_plural): 193 def setmsgid_plural(self, msgid_plural):
(...skipping 131 matching lines...) Show 10 above Show 10 below
324 else: 325 else:
325 raise ValueError("Comment type not valid") 326 raise ValueError("Comment type not valid")
326 327
327 if comments and get_libgettextpo_version() < (0, 17, 0): 328 if comments and get_libgettextpo_version() < (0, 17, 0):
328 comments = "\n".join([line.strip() for line in comments.split("\n")] ) 329 comments = "\n".join([line.strip() for line in comments.split("\n")] )
329 # Let's drop the last newline 330 # Let's drop the last newline
330 return comments[:-1].decode(self._encoding) 331 return comments[:-1].decode(self._encoding)
331 332
332 def addnote(self, text, origin=None, position="append"): 333 def addnote(self, text, origin=None, position="append"):
333 # ignore empty strings and strings without non-space characters 334 # ignore empty strings and strings without non-space characters
334 if (not text) or (not text.strip()): 335 if (not text) or (not text.strip()):
335 return 336 return
336 text = data.forceunicode(text) 337 text = data.forceunicode(text)
337 oldnotes = self.getnotes(origin) 338 oldnotes = self.getnotes(origin)
338 newnotes = None 339 newnotes = None
339 if oldnotes: 340 if oldnotes:
340 if position == "append": 341 if position == "append":
341 newnotes = oldnotes + "\n" + text 342 newnotes = oldnotes + "\n" + text
342 elif position == "merge": 343 elif position == "merge":
343 if oldnotes != text: 344 if oldnotes != text:
344 oldnoteslist = oldnotes.split("\n") 345 oldnoteslist = oldnotes.split("\n")
345 for newline in text.split("\n"): 346 for newline in text.split("\n"):
346 newline = newline.rstrip() 347 newline = newline.rstrip()
347 # avoid duplicate comment lines (this might cause some p roblems) 348 # avoid duplicate comment lines (this might cause some p roblems)
348 if newline not in oldnotes or len(newline) < 5: 349 if newline not in oldnotes or len(newline) < 5:
349 oldnoteslist.append(newline) 350 oldnoteslist.append(newline)
350 newnotes = "\n".join(oldnoteslist) 351 newnotes = "\n".join(oldnoteslist)
351 else: 352 else:
352 newnotes = text + '\n' + oldnotes 353 newnotes = text + '\n' + oldnotes
353 else: 354 else:
354 newnotes = "\n".join([line.rstrip() for line in text.split("\n")]) 355 newnotes = "\n".join([line.rstrip() for line in text.split("\n")])
355 356
356 if newnotes: 357 if newnotes:
357 newlines = [] 358 newlines = []
358 needs_space = get_libgettextpo_version() < (0, 17, 0) 359 needs_space = get_libgettextpo_version() < (0, 17, 0)
359 for line in newnotes.split("\n"): 360 for line in newnotes.split("\n"):
360 if line and needs_space: 361 if line and needs_space:
361 newlines.append(" " + line) 362 newlines.append(" " + line)
362 else: 363 else:
363 newlines.append(line) 364 newlines.append(line)
364 newnotes = "\n".join(newlines) 365 newnotes = "\n".join(newlines)
365 if origin in ["programmer", "developer", "source code"]: 366 if origin in ["programmer", "developer", "source code"]:
366 gpo.po_message_set_extracted_comments(self._gpo_message, newnote s) 367 gpo.po_message_set_extracted_comments(self._gpo_message, newnote s)
367 else: 368 else:
368 gpo.po_message_set_comments(self._gpo_message, newnotes) 369 gpo.po_message_set_comments(self._gpo_message, newnotes)
369 370
370 def removenotes(self): 371 def removenotes(self):
371 gpo.po_message_set_comments(self._gpo_message, "") 372 gpo.po_message_set_comments(self._gpo_message, "")
372 373
373 def copy(self): 374 def copy(self):
374 newpo = self.__class__() 375 """Returns a copy of the this unit that can be used independently.
375 newpo._gpo_message = self._gpo_message 376 """
376 return newpo 377 unit = pounit()
378 unit.source = self.source
379 unit.target = self.target
380 unit.markfuzzy(self.isfuzzy())
381 if self.isobsolete():
382 unit.makeobsolete()
383 for location in self.getlocations():
384 unit.addlocation(location)
385 notes = []
386 origins = ['translator', 'developer', 'programmer', 'source code']
387 for origin in origins:
388 note = self.getnotes(origin=origin)
389 if note in notes:
390 continue
391 if note:
392 unit.addnote(note, origin=origin)
393 notes.append(note)
394 context = gpo.po_message_msgctxt(self._gpo_message)
395 if context:
396 unit.setcontext(context)
397 comments = self._extract_msgidcomments()
398 if comments:
399 unit.setmsgidcomment(comments)
400 for typecomment in self.gettypecomments():
401 unit.addtypecomment(typecomment)
402 return unit
377 403
378 def merge(self, otherpo, overwrite=False, comments=True, authoritative=False ): 404 def merge(self, otherpo, overwrite=False, comments=True, authoritative=False ):
379 """Merges the otherpo (with the same msgid) into this one. 405 """Merges the otherpo (with the same msgid) into this one.
380 406
381 Overwrite non-blank self.msgstr only if overwrite is True 407 Overwrite non-blank self.msgstr only if overwrite is True
382 merge comments only if comments is True 408 merge comments only if comments is True
383 409
384 """ 410 """
385 411
386 if not isinstance(otherpo, pounit): 412 if not isinstance(otherpo, pounit):
387 super(pounit, self).merge(otherpo, overwrite, comments) 413 super(pounit, self).merge(otherpo, overwrite, comments)
388 return 414 return
389 if comments: 415 if comments:
390 self.addnote(otherpo.getnotes("translator"), origin="translator", po sition="merge") 416 self.addnote(otherpo.getnotes("translator"), origin="translator", po sition="merge")
391 # FIXME mergelists(self.typecomments, otherpo.typecomments) 417 # FIXME mergelists(self.typecomments, otherpo.typecomments)
392 if not authoritative: 418 if not authoritative:
393 # We don't bring across otherpo.automaticcomments as we consider ourself 419 # We don't bring across otherpo.automaticcomments as we consider ourself
394 # to be the the authority. Same applies to otherpo.msgidcomment s 420 # to be the the authority. Same applies to otherpo.msgidcomment s
395 self.addnote(otherpo.getnotes("developer"), origin="developer", position="merge") 421 self.addnote(otherpo.getnotes("developer"), origin="developer", position="merge")
396 self.msgidcomment = otherpo._extract_msgidcomments() or None 422 self.msgidcomment = otherpo._extract_msgidcomments() or None
397 self.addlocations(otherpo.getlocations()) 423 self.addlocations(otherpo.getlocations())
398 if not self.istranslated() or overwrite: 424 if not self.istranslated() or overwrite:
399 # Remove kde-style comments from the translation (if any). 425 # Remove kde-style comments from the translation (if any).
400 if self._extract_msgidcomments(otherpo.target): 426 if self._extract_msgidcomments(otherpo.target):
401 otherpo.target = otherpo.target.replace('_: ' + otherpo._extract _msgidcomments()+ '\n', '') 427 otherpo.target = otherpo.target.replace('_: ' + otherpo._extract _msgidcomments()+ '\n', '')
402 self.target = otherpo.target 428 self.target = otherpo.target
403 if self.source != otherpo.source: 429 if self.source != otherpo.source:
404 self.markfuzzy() 430 self.markfuzzy()
405 else: 431 else:
406 self.markfuzzy(otherpo.isfuzzy()) 432 self.markfuzzy(otherpo.isfuzzy())
407 elif not otherpo.istranslated(): 433 elif not otherpo.istranslated():
408 if self.source != otherpo.source: 434 if self.source != otherpo.source:
409 self.markfuzzy() 435 self.markfuzzy()
410 else: 436 else:
411 if self.target != otherpo.target: 437 if self.target != otherpo.target:
412 self.markfuzzy() 438 self.markfuzzy()
413 439
414 def isheader(self): 440 def isheader(self):
415 #return self.source == u"" and self.target != u"" 441 #return self.source == u"" and self.target != u""
416 # we really want to make sure that there is no msgidcomment or msgctxt 442 # we really want to make sure that there is no msgidcomment or msgctxt
417 return self.getid() == "" and len(self.target) > 0 443 return self.getid() == "" and len(self.target) > 0
418 444
419 def isblank(self): 445 def isblank(self):
420 return len(self.source) == 0 and len(self.target) == 0 446 return len(self.source) == 0 and len(self.target) == 0
421 447
422 def hastypecomment(self, typecomment): 448 def hastypecomment(self, typecomment):
423 return gpo.po_message_is_format(self._gpo_message, typecomment) 449 return gpo.po_message_is_format(self._gpo_message, typecomment)
450
451 def gettypecomments(self):
452 #XXX: brute force hack
453 formats = [
454 "c", "objc", "sh", "python", "lisp", "elisp", "librep", "scheme",
455 "smalltalk", "java", "csharp", "awk", "object-pascal", "ycp", "tcl",
456 "perl", "perl-brace", "php", "gcc-internal", "qt", "kde", "boost",
457 ]
458 comments = []
459 for format in formats:
460 format += '-format'
461 if self.hastypecomment(format):
462 comments.append(format)
463 return comments
464
465 def addtypecomment(self, typecomment):
466 gpo.po_message_set_format(self._gpo_message, typecomment, 1)
424 467
425 def hasmarkedcomment(self, commentmarker): 468 def hasmarkedcomment(self, commentmarker):
426 commentmarker = "(%s)" % commentmarker 469 commentmarker = "(%s)" % commentmarker
427 for comment in self.getnotes("translator").split("\n"): 470 for comment in self.getnotes("translator").split("\n"):
428 if comment.startswith(commentmarker): 471 if comment.startswith(commentmarker):
429 return True 472 return True
430 return False 473 return False
431 474
432 def istranslated(self): 475 def istranslated(self):
433 return super(pounit, self).istranslated() and not self.isobsolete() 476 return super(pounit, self).istranslated() and not self.isobsolete()
434 477
435 def istranslatable(self): 478 def istranslatable(self):
436 return not (self.isheader() or self.isblank() or self.isobsolete()) 479 return not (self.isheader() or self.isblank() or self.isobsolete())
437 480
438 def isfuzzy(self): 481 def isfuzzy(self):
439 return gpo.po_message_is_fuzzy(self._gpo_message) 482 return gpo.po_message_is_fuzzy(self._gpo_message)
440 483
441 def markfuzzy(self, present=True): 484 def markfuzzy(self, present=True):
442 gpo.po_message_set_fuzzy(self._gpo_message, present) 485 gpo.po_message_set_fuzzy(self._gpo_message, present)
443 486
444 def isreview(self): 487 def isreview(self):
445 return self.hasmarkedcomment("review") or self.hasmarkedcomment("pofilte r") 488 return self.hasmarkedcomment("review") or self.hasmarkedcomment("pofilte r")
446 489
447 def isobsolete(self): 490 def isobsolete(self):
448 return gpo.po_message_is_obsolete(self._gpo_message) 491 return gpo.po_message_is_obsolete(self._gpo_message)
449 492
450 def makeobsolete(self): 493 def makeobsolete(self):
451 # FIXME: libgettexpo currently does not reset other data, we probably wa nt to do that 494 # FIXME: libgettexpo currently does not reset other data, we probably wa nt to do that
452 # but a better solution would be for libgettextpo to output correct data on serialisation 495 # but a better solution would be for libgettextpo to output correct data on serialisation
453 gpo.po_message_set_obsolete(self._gpo_message, True) 496 gpo.po_message_set_obsolete(self._gpo_message, True)
454 497
455 def resurrect(self): 498 def resurrect(self):
456 gpo.po_message_set_obsolete(self._gpo_message, False) 499 gpo.po_message_set_obsolete(self._gpo_message, False)
457 500
458 def hasplural(self): 501 def hasplural(self):
459 return gpo.po_message_msgid_plural(self._gpo_message) is not None 502 return gpo.po_message_msgid_plural(self._gpo_message) is not None
460 503
461 def _extract_msgidcomments(self, text=None): 504 def _extract_msgidcomments(self, text=None):
462 """Extract KDE style msgid comments from the unit. 505 """Extract KDE style msgid comments from the unit.
463 506
464 @rtype: String 507 @rtype: String
465 @return: Returns the extracted msgidcomments found in this unit's msgid. 508 @return: Returns the extracted msgidcomments found in this unit's msgid.
466 509
467 """ 510 """
468 511
469 if not text: 512 if not text:
470 text = gpo.po_message_msgid(self._gpo_message) 513 text = gpo.po_message_msgid(self._gpo_message)
471 if text: 514 if text:
472 msgidcomment = re.search("_: (.*)\n", text) 515 msgidcomment = re.search("_: (.*)\n", text)
473 if msgidcomment: 516 if msgidcomment:
474 return msgidcomment.group(1).decode(self._encoding) 517 return msgidcomment.group(1).decode(self._encoding)
475 return u"" 518 return u""
476 519
477 def __str__(self): 520 def __str__(self):
478 pf = pofile() 521 pf = pofile()
479 pf.addunit(self) 522 unit = self.copy()
523 pf.addunit(unit)
480 return str(pf) 524 return str(pf)
481 525
482 def getlocations(self): 526 def getlocations(self):
483 locations = [] 527 locations = []
484 i = 0 528 i = 0
485 location = gpo.po_message_filepos(self._gpo_message, i) 529 location = gpo.po_message_filepos(self._gpo_message, i)
486 while location: 530 while location:
487 locname = gpo.po_filepos_file(location) 531 locname = gpo.po_filepos_file(location)
488 locline = gpo.po_filepos_start_line(location) 532 locline = gpo.po_filepos_start_line(location)
489 if locline == -1: 533 if locline == -1:
490 locstring = locname 534 locstring = locname
491 else: 535 else:
492 locstring = locname + ":" + str(locline) 536 locstring = locname + ":" + str(locline)
493 locations.append(locstring) 537 locations.append(locstring)
494 i += 1 538 i += 1
495 location = gpo.po_message_filepos(self._gpo_message, i) 539 location = gpo.po_message_filepos(self._gpo_message, i)
496 return locations 540 return locations
497 541
498 def addlocation(self, location): 542 def addlocation(self, location):
499 for loc in location.split(): 543 for loc in location.split():
500 parts = loc.split(":") 544 parts = loc.split(":")
501 file = parts[0] 545 file = parts[0]
502 if len(parts) == 2: 546 if len(parts) == 2:
503 line = int(parts[1]) 547 line = int(parts[1])
504 else: 548 else:
505 line = -1 549 line = -1
506 gpo.po_message_add_filepos(self._gpo_message, file, line) 550 gpo.po_message_add_filepos(self._gpo_message, file, line)
551
552 def setcontext(self, context):
553 """Sets the context message"""
554 #FIXME: not complete
555 gpo.po_message_set_msgctxt(self._gpo_message, context)
507 556
508 def getcontext(self): 557 def getcontext(self):
509 msgctxt = gpo.po_message_msgctxt(self._gpo_message) 558 msgctxt = gpo.po_message_msgctxt(self._gpo_message)
510 msgidcomment = self._extract_msgidcomments() 559 msgidcomment = self._extract_msgidcomments()
511 if msgctxt: 560 if msgctxt:
512 return msgctxt + msgidcomment 561 return msgctxt + msgidcomment
513 else: 562 else:
514 return msgidcomment 563 return msgidcomment
515 564
516 class pofile(pocommon.pofile): 565 class pofile(pocommon.pofile):
517 UnitClass = pounit 566 UnitClass = pounit
518 def __init__(self, inputfile=None, encoding=None, unitclass=pounit): 567 def __init__(self, inputfile=None, encoding=None, unitclass=pounit):
519 self.UnitClass = unitclass 568 self.UnitClass = unitclass
520 pocommon.pofile.__init__(self, unitclass=unitclass) 569 pocommon.pofile.__init__(self, unitclass=unitclass)
521 self._gpo_memory_file = None 570 self._gpo_memory_file = None
522 self._gpo_message_iterator = None 571 self._gpo_message_iterator = None
523 self._encoding = encodingToUse(encoding) 572 self._encoding = encodingToUse(encoding)
524 if inputfile is not None: 573 if inputfile is not None:
525 self.parse(inputfile) 574 self.parse(inputfile)
526 else: 575 else:
527 self._gpo_memory_file = gpo.po_file_create() 576 self._gpo_memory_file = gpo.po_file_create()
528 self._gpo_message_iterator = gpo.po_message_iterator(self._gpo_memor y_file, None) 577 self._gpo_message_iterator = gpo.po_message_iterator(self._gpo_memor y_file, None)
529 578
530 def addunit(self, unit): 579 def addunit(self, unit):
531 gpo.po_message_insert(self._gpo_message_iterator, unit._gpo_message) 580 unitcopy = unit.copy()
532 self.units.append(unit) 581 gpo.po_message_insert(self._gpo_message_iterator, unitcopy._gpo_message)
582 self.units.append(unitcopy)
533 583
534 def removeduplicates(self, duplicatestyle="merge"): 584 def removeduplicates(self, duplicatestyle="merge"):
535 """make sure each msgid is unique ; merge comments etc from duplicates i nto original""" 585 """make sure each msgid is unique ; merge comments etc from duplicates i nto original"""
536 msgiddict = {} 586 msgiddict = {}
537 uniqueunits = [] 587 uniqueunits = []
538 # we sometimes need to keep track of what has been marked 588 # we sometimes need to keep track of what has been marked
539 # TODO: this is using a list as the pos aren't hashable, but this is slo w... 589 # TODO: this is using a list as the pos aren't hashable, but this is slo w...
540 markedpos = [] 590 markedpos = []
541 def addcomment(thepo): 591 def addcomment(thepo):
542 thepo.msgidcomment = " ".join(thepo.getlocations()) 592 thepo.msgidcomment = " ".join(thepo.getlocations())
543 markedpos.append(thepo) 593 markedpos.append(thepo)
544 for thepo in self.units: 594 for thepo in self.units:
545 if thepo.isheader(): 595 if thepo.isheader():
546 uniqueunits.append(thepo) 596 uniqueunits.append(thepo)
547 continue 597 continue
548 if duplicatestyle.startswith("msgid_comment"): 598 if duplicatestyle.startswith("msgid_comment"):
549 msgid = thepo._extract_msgidcomments() + thepo.source 599 msgid = thepo._extract_msgidcomments() + thepo.source
550 else: 600 else:
551 msgid = thepo.source 601 msgid = thepo.source
552 if duplicatestyle == "msgid_comment_all": 602 if duplicatestyle == "msgid_comment_all":
553 addcomment(thepo) 603 addcomment(thepo)
554 uniqueunits.append(thepo) 604 uniqueunits.append(thepo)
555 elif msgid in msgiddict: 605 elif msgid in msgiddict:
556 if duplicatestyle == "merge": 606 if duplicatestyle == "merge":
557 if msgid: 607 if msgid:
558 msgiddict[msgid].merge(thepo) 608 msgiddict[msgid].merge(thepo)
559 else: 609 else:
560 addcomment(thepo) 610 addcomment(thepo)
561 uniqueunits.append(thepo) 611 uniqueunits.append(thepo)
562 elif duplicatestyle == "keep": 612 elif duplicatestyle == "keep":
563 uniqueunits.append(thepo) 613 uniqueunits.append(thepo)
564 elif duplicatestyle == "msgid_comment": 614 elif duplicatestyle == "msgid_comment":
565 origpo = msgiddict[msgid] 615 origpo = msgiddict[msgid]
566 if origpo not in markedpos: 616 if origpo not in markedpos:
567 addcomment(origpo) 617 addcomment(origpo)
568 addcomment(thepo) 618 addcomment(thepo)
569 uniqueunits.append(thepo) 619 uniqueunits.append(thepo)
570 elif duplicatestyle == "msgctxt": 620 elif duplicatestyle == "msgctxt":
571 origpo = msgiddict[msgid] 621 origpo = msgiddict[msgid]
572 if origpo not in markedpos: 622 if origpo not in markedpos:
573 gpo.po_message_set_msgctxt(origpo._gpo_message, " ".join (origpo.getlocations())) 623 gpo.po_message_set_msgctxt(origpo._gpo_message, " ".join (origpo.getlocations()))
574 markedpos.append(thepo) 624 markedpos.append(thepo)
575 gpo.po_message_set_msgctxt(thepo._gpo_message, " ".join(thep o.getlocations())) 625 gpo.po_message_set_msgctxt(thepo._gpo_message, " ".join(thep o.getlocations()))
576 uniqueunits.append(thepo) 626 uniqueunits.append(thepo)
577 else: 627 else:
578 if not msgid and duplicatestyle != "keep": 628 if not msgid and duplicatestyle != "keep":
579 addcomment(thepo) 629 addcomment(thepo)
580 msgiddict[msgid] = thepo 630 msgiddict[msgid] = thepo
581 uniqueunits.append(thepo) 631 uniqueunits.append(thepo)
582 new_gpo_memory_file = gpo.po_file_create() 632 new_gpo_memory_file = gpo.po_file_create()
583 new_gpo_message_iterator = gpo.po_message_iterator(new_gpo_memory_file, None) 633 new_gpo_message_iterator = gpo.po_message_iterator(new_gpo_memory_file, None)
584 for unit in uniqueunits: 634 for unit in uniqueunits:
585 gpo.po_message_insert(new_gpo_message_iterator, unit._gpo_message) 635 gpo.po_message_insert(new_gpo_message_iterator, unit._gpo_message)
586 gpo.po_message_iterator_free(self._gpo_message_iterator) 636 gpo.po_message_iterator_free(self._gpo_message_iterator)
587 self._gpo_message_iterator = new_gpo_message_iterator 637 self._gpo_message_iterator = new_gpo_message_iterator
588 self._gpo_memory_file = new_gpo_memory_file 638 self._gpo_memory_file = new_gpo_memory_file
589 self.units = uniqueunits 639 self.units = uniqueunits
590 640
591 def __str__(self): 641 def __str__(self):
592 def obsolete_workaround(): 642 def obsolete_workaround():
593 # Remove all items that are not output by msgmerge when a unit is ob solete. This is a work 643 # Remove all items that are not output by msgmerge when a unit is ob solete. This is a work
594 # around for bug in libgettextpo 644 # around for bug in libgettextpo
595 # FIXME Do version test in case they fix this bug 645 # FIXME Do version test in case they fix this bug
596 for unit in self.units: 646 for unit in self.units:
597 if unit.isobsolete(): 647 if unit.isobsolete():
598 gpo.po_message_set_extracted_comments(unit._gpo_message, "") 648 gpo.po_message_set_extracted_comments(unit._gpo_message, "")
599 location = gpo.po_message_filepos(unit._gpo_message, 0) 649 location = gpo.po_message_filepos(unit._gpo_message, 0)
600 while location: 650 while location:
601 gpo.po_message_remove_filepos(unit._gpo_message, 0) 651 gpo.po_message_remove_filepos(unit._gpo_message, 0)
602 location = gpo.po_message_filepos(unit._gpo_message, 0) 652 location = gpo.po_message_filepos(unit._gpo_message, 0)
603 outputstring = "" 653 outputstring = ""
604 if self._gpo_memory_file: 654 if self._gpo_memory_file:
605 obsolete_workaround() 655 obsolete_workaround()
606 f = tempfile.NamedTemporaryFile(prefix='translate', suffix='.po') 656 f = tempfile.NamedTemporaryFile(prefix='translate', suffix='.po')
607 self._gpo_memory_file = gpo.po_file_write_v2(self._gpo_memory_file, f.name, xerror_handler) 657 self._gpo_memory_file = gpo.po_file_write_v2(self._gpo_memory_file, f.name, xerror_handler)
608 f.seek(0) 658 f.seek(0)
609 outputstring = f.read() 659 outputstring = f.read()
610 f.close() 660 f.close()
611 return outputstring 661 return outputstring
612 662
613 def isempty(self): 663 def isempty(self):
614 """Returns True if the object doesn't contain any translation units.""" 664 """Returns True if the object doesn't contain any translation units."""
615 if len(self.units) == 0: 665 if len(self.units) == 0:
616 return True 666 return True
617 # Skip the first unit if it is a header. 667 # Skip the first unit if it is a header.
618 if self.units[0].isheader(): 668 if self.units[0].isheader():
619 units = self.units[1:] 669 units = self.units[1:]
620 else: 670 else:
621 units = self.units 671 units = self.units
622 672
623 for unit in units: 673 for unit in units:
624 if not unit.isblank() and not unit.isobsolete(): 674 if not unit.isblank() and not unit.isobsolete():
625 return False 675 return False
626 return True 676 return True
627 677
628 def parse(self, input): 678 def parse(self, input):
629 if hasattr(input, 'name'): 679 if hasattr(input, 'name'):
630 self.filename = input.name 680 self.filename = input.name
631 elif not getattr(self, 'filename', ''): 681 elif not getattr(self, 'filename', ''):
632 self.filename = '' 682 self.filename = ''
633 683
634 if hasattr(input, "read"): 684 if hasattr(input, "read"):
635 posrc = input.read() 685 posrc = input.read()
636 input.close() 686 input.close()
637 input = posrc 687 input = posrc
638 688
639 needtmpfile = not os.path.isfile(input) 689 needtmpfile = not os.path.isfile(input)
640 if needtmpfile: 690 if needtmpfile:
641 # This is not a file - we write the string to a temporary file 691 # This is not a file - we write the string to a temporary file
642 fd, fname = tempfile.mkstemp(prefix='translate', suffix='.po') 692 fd, fname = tempfile.mkstemp(prefix='translate', suffix='.po')
643 os.write(fd, input) 693 os.write(fd, input)
644 input = fname 694 input = fname
645 os.close(fd) 695 os.close(fd)
646 696
647 self._gpo_memory_file = gpo.po_file_read_v3(input, xerror_handler) 697 self._gpo_memory_file = gpo.po_file_read_v3(input, xerror_handler)
648 if self._gpo_memory_file is None: 698 if self._gpo_memory_file is None:
649 print >> sys.stderr, "Error:" 699 print >> sys.stderr, "Error:"
650 700
651 if needtmpfile: 701 if needtmpfile:
652 os.remove(input) 702 os.remove(input)
653 703
654 # Handle xerrors here 704 # Handle xerrors here
655 self._header = gpo.po_file_domain_header(self._gpo_memory_file, None) 705 self._header = gpo.po_file_domain_header(self._gpo_memory_file, None)
656 if self._header: 706 if self._header:
657 charset = gpo.po_header_field(self._header, "Content-Type") 707 charset = gpo.po_header_field(self._header, "Content-Type")
658 if charset: 708 if charset:
659 charset = re.search("charset=([^\\s]+)", charset).group(1) 709 charset = re.search("charset=([^\\s]+)", charset).group(1)
660 self._encoding = encodingToUse(charset) 710 self._encoding = encodingToUse(charset)
661 self._gpo_message_iterator = gpo.po_message_iterator(self._gpo_memory_fi le, None) 711 self._gpo_message_iterator = gpo.po_message_iterator(self._gpo_memory_fi le, None)
662 newmessage = gpo.po_next_message(self._gpo_message_iterator) 712 newmessage = gpo.po_next_message(self._gpo_message_iterator)
663 while newmessage: 713 while newmessage:
664 newunit = pounit(gpo_message=newmessage) 714 newunit = pounit(gpo_message=newmessage)
665 self.units.append(newunit) 715 self.units.append(newunit)
666 newmessage = gpo.po_next_message(self._gpo_message_iterator) 716 newmessage = gpo.po_next_message(self._gpo_message_iterator)
667 self._free_iterator()
668 717
669 def __del__(self): 718 def __del__(self):
670 # We currently disable this while we still get segmentation faults. 719 # We currently disable this while we still get segmentation faults.
671 # Note that this is definitely leaking memory because of this. 720 # Note that this is definitely leaking memory because of this.
672 return
673 self._free_iterator() 721 self._free_iterator()
674 if self._gpo_memory_file is not None: 722 if self._gpo_memory_file is not None:
675 gpo.po_file_free(self._gpo_memory_file) 723 gpo.po_file_free(self._gpo_memory_file)
676 self._gpo_memory_file = None 724 self._gpo_memory_file = None
677 725
678 def _free_iterator(self): 726 def _free_iterator(self):
679 # We currently disable this while we still get segmentation faults. 727 # We currently disable this while we still get segmentation faults.
680 # Note that this is definitely leaking memory because of this. 728 # Note that this is definitely leaking memory because of this.
681 return
682 if self._gpo_message_iterator is not None: 729 if self._gpo_message_iterator is not None:
683 gpo.po_message_iterator_free(self._gpo_message_iterator) 730 gpo.po_message_iterator_free(self._gpo_message_iterator)
684 self._gpo_message_iterator = None 731 self._gpo_message_iterator = None
OLDNEW

Powered by Google App Engine
This is Rietveld r159