Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code | Sign in
(37)

Delta Between Two Patch Sets: translate/storage/cpo.py

Issue 81: Fix CPO memory leak SVN Base: https://translate.svn.sourceforge.net/svnroot/translate/src/trunk/
Left Patch Set: Fix addlocation method Created 1 year, 3 months ago
Right Patch Set: Using low leve accessors Created 1 year, 4 months ago
Use n/p to move between diff chunks; N/P to move between comments. Please Sign in to add in-line comments.
Jump to:
Left: View regular side by side diff
Right: View regular side by side diff
LEFTRIGHT
1 #!/usr/bin/env python 1 #!/usr/bin/env python
2 # -*- coding: utf-8 -*- 2 # -*- coding: utf-8 -*-
3 # 3 #
4 # Copyright 2002-2007 Zuza Software Foundation 4 # Copyright 2002-2007 Zuza Software Foundation
5 # 5 #
6 # This file is part of translate. 6 # This file is part of translate.
7 # 7 #
8 # translate is free software; you can redistribute it and/or modify 8 # translate is free software; you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by 9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation; either version 2 of the License, or 10 # the Free Software Foundation; either version 2 of the License, or
11 # (at your option) any later version. 11 # (at your option) any later version.
12 # 12 #
13 # translate is distributed in the hope that it will be useful, 13 # translate is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details. 16 # GNU General Public License for more details.
17 # 17 #
18 # You should have received a copy of the GNU General Public License 18 # You should have received a copy of the GNU General Public License
19 # along with translate; if not, write to the Free Software 19 # along with translate; if not, write to the Free Software
20 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 20 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
21 21
22 """Classes that hold units of .po files (pounit) or entire files (pofile). 22 """Classes that hold units of .po files (pounit) or entire files (pofile).
23 23
24 Gettext-style .po (or .pot) files are used in translations for KDE, GNOME and 24 Gettext-style .po (or .pot) files are used in translations for KDE, GNOME and
25 many other projects. 25 many other projects.
26 26
27 This uses libgettextpo from the gettext package. Any version before 0.17 will 27 This uses libgettextpo from the gettext package. Any version before 0.17 will
28 at least cause some subtle bugs or may not work at all. Developers might want 28 at least cause some subtle bugs or may not work at all. Developers might want
29 to have a look at gettext-tools/libgettextpo/gettext-po.h from the gettext 29 to have a look at gettext-tools/libgettextpo/gettext-po.h from the gettext
30 package for the public API of the library. 30 package for the public API of the library.
31 """ 31 """
32 32
33 from translate.misc.multistring import multistring 33 from translate.misc.multistring import multistring
34 from translate.storage import pocommon 34 from translate.storage import pocommon
35 from translate.misc import quote 35 from translate.misc import quote
36 from translate.lang import data 36 from translate.lang import data
37 from ctypes import * 37 from ctypes import *
38 import ctypes.util 38 import ctypes.util
39 try: 39 try:
40 import cStringIO as StringIO 40 import cStringIO as StringIO
41 except ImportError: 41 except ImportError:
42 import StringIO 42 import StringIO
43 import os 43 import os
44 import pypo 44 import pypo
45 import re 45 import re
46 import sys 46 import sys
47 import tempfile 47 import tempfile
48 48
49 lsep = " " 49 lsep = " "
50 """Seperator for #: entries""" 50 """Seperator for #: entries"""
51 51
52 STRING = c_char_p 52 STRING = c_char_p
53 53
54 # Structures 54 # Structures
55 class po_message(Structure): 55 class po_message(Structure):
56 _fields_ = [] 56 _fields_ = []
57 57
58 # Function prototypes 58 # Function prototypes
59 xerror_prototype = CFUNCTYPE(None, c_int, POINTER(po_message), STRING, c_uint, c _uint, c_int, STRING) 59 xerror_prototype = CFUNCTYPE(None, c_int, POINTER(po_message), STRING, c_uint, c _uint, c_int, STRING)
60 xerror2_prototype = CFUNCTYPE(None, c_int, POINTER(po_message), STRING, c_uint, c_uint, c_int, STRING, POINTER(po_message), STRING, c_uint, c_uint, c_int, STRIN G) 60 xerror2_prototype = CFUNCTYPE(None, c_int, POINTER(po_message), STRING, c_uint, c_uint, c_int, STRING, POINTER(po_message), STRING, c_uint, c_uint, c_int, STRIN G)
61 61
62 62
63 # Structures (error handler) 63 # Structures (error handler)
64 class po_xerror_handler(Structure): 64 class po_xerror_handler(Structure):
65 _fields_ = [('xerror', xerror_prototype), 65 _fields_ = [('xerror', xerror_prototype),
66 ('xerror2', xerror2_prototype)] 66 ('xerror2', xerror2_prototype)]
67 67
68 class po_error_handler(Structure): 68 class po_error_handler(Structure):
69 _fields_ = [ 69 _fields_ = [
70 ('error', CFUNCTYPE(None, c_int, c_int, STRING)), 70 ('error', CFUNCTYPE(None, c_int, c_int, STRING)),
71 ('error_at_line', CFUNCTYPE(None, c_int, c_int, STRING, c_uint, STRING)), 71 ('error_at_line', CFUNCTYPE(None, c_int, c_int, STRING, c_uint, STRING)),
72 ('multiline_warning', CFUNCTYPE(None, STRING, STRING)), 72 ('multiline_warning', CFUNCTYPE(None, STRING, STRING)),
73 ('multiline_error', CFUNCTYPE(None, STRING, STRING)), 73 ('multiline_error', CFUNCTYPE(None, STRING, STRING)),
74 ] 74 ]
75 75
76 # Callback functions for po_xerror_handler 76 # Callback functions for po_xerror_handler
77 def xerror_cb(severity, message, filename, lineno, column, multilint_p, message_ text): 77 def xerror_cb(severity, message, filename, lineno, column, multilint_p, message_ text):
78 print >> sys.stderr, "xerror_cb", severity, message, filename, lineno, colum n, multilint_p, message_text 78 print >> sys.stderr, "xerror_cb", severity, message, filename, lineno, colum n, multilint_p, message_text
79 if severity >= 1: 79 if severity >= 1:
80 raise ValueError(message_text) 80 raise ValueError(message_text)
81 81
82 def xerror2_cb(severity, message1, filename1, lineno1, column1, multiline_p1, me ssage_text1, message2, filename2, lineno2, column2, multiline_p2, message_text2) : 82 def xerror2_cb(severity, message1, filename1, lineno1, column1, multiline_p1, me ssage_text1, message2, filename2, lineno2, column2, multiline_p2, message_text2) :
83 print >> sys.stderr, "xerror2_cb", severity, message1, filename1, lineno1, c olumn1, multiline_p1, message_text1, message2, filename2, lineno2, column2, mult iline_p2, message_text2 83 print >> sys.stderr, "xerror2_cb", severity, message1, filename1, lineno1, c olumn1, multiline_p1, message_text1, message2, filename2, lineno2, column2, mult iline_p2, message_text2
84 if severity >= 1: 84 if severity >= 1:
85 raise ValueError(message_text1) 85 raise ValueError(message_text1)
86 86
87 87
88 88
89 # Load libgettextpo 89 # Load libgettextpo
90 gpo = None 90 gpo = None
91 # 'gettextpo' is recognised on Unix, while only 'libgettextpo' is recognised on 91 # 'gettextpo' is recognised on Unix, while only 'libgettextpo' is recognised on
92 # windows. Therefore we test both. 92 # windows. Therefore we test both.
93 names = ['gettextpo', 'libgettextpo'] 93 names = ['gettextpo', 'libgettextpo']
94 for name in names: 94 for name in names:
95 lib_location = ctypes.util.find_library(name) 95 lib_location = ctypes.util.find_library(name)
96 if lib_location: 96 if lib_location:
97 gpo = cdll.LoadLibrary(lib_location) 97 gpo = cdll.LoadLibrary(lib_location)
98 if gpo: 98 if gpo:
99 break 99 break
100 else: 100 else:
101 # Now we are getting desperate, so let's guess a unix type DLL that might 101 # Now we are getting desperate, so let's guess a unix type DLL that might
102 # be in LD_LIBRARY_PATH or loaded with LD_PRELOAD 102 # be in LD_LIBRARY_PATH or loaded with LD_PRELOAD
103 try: 103 try:
104 gpo = cdll.LoadLibrary('libgettextpo.so') 104 gpo = cdll.LoadLibrary('libgettextpo.so')
105 except OSError, e: 105 except OSError, e:
106 raise ImportError("gettext PO library not found") 106 raise ImportError("gettext PO library not found")
107 107
108 # Setup return and paramater types 108 # Setup return and paramater types
109 # File access 109 # File access
110 gpo.po_file_read_v3.argtypes = [STRING, POINTER(po_xerror_handler)] 110 gpo.po_file_read_v3.argtypes = [STRING, POINTER(po_xerror_handler)]
111 gpo.po_file_write_v2.argtypes = [c_int, STRING, POINTER(po_xerror_handler)] 111 gpo.po_file_write_v2.argtypes = [c_int, STRING, POINTER(po_xerror_handler)]
112 gpo.po_file_write_v2.retype = c_int 112 gpo.po_file_write_v2.retype = c_int
113 113
114 # Header 114 # Header
115 gpo.po_file_domain_header.restype = STRING 115 gpo.po_file_domain_header.restype = STRING
116 gpo.po_header_field.restype = STRING 116 gpo.po_header_field.restype = STRING
117 gpo.po_header_field.argtypes = [STRING, STRING] 117 gpo.po_header_field.argtypes = [STRING, STRING]
118 118
119 # Locations (filepos) 119 # Locations (filepos)
120 gpo.po_filepos_file.restype = STRING 120 gpo.po_filepos_file.restype = STRING
121 gpo.po_message_filepos.restype = c_int 121 gpo.po_message_filepos.restype = c_int
122 gpo.po_message_filepos.argtypes = [c_int, c_int] 122 gpo.po_message_filepos.argtypes = [c_int, c_int]
123 gpo.po_message_add_filepos.argtypes = [c_int, STRING, c_size_t] 123 gpo.po_message_add_filepos.argtypes = [c_int, STRING, c_int]
124 124
125 # Message (get methods) 125 # Message (get methods)
126 gpo.po_message_comments.restype = STRING 126 gpo.po_message_comments.restype = STRING
127 gpo.po_message_extracted_comments.restype = STRING 127 gpo.po_message_extracted_comments.restype = STRING
128 gpo.po_message_prev_msgctxt.restype = STRING 128 gpo.po_message_prev_msgctxt.restype = STRING
129 gpo.po_message_prev_msgid.restype = STRING 129 gpo.po_message_prev_msgid.restype = STRING
130 gpo.po_message_prev_msgid_plural.restype = STRING 130 gpo.po_message_prev_msgid_plural.restype = STRING
131 gpo.po_message_is_format.restype = c_int 131 gpo.po_message_is_format.restype = c_int
132 gpo.po_message_msgctxt.restype = STRING 132 gpo.po_message_msgctxt.restype = STRING
133 gpo.po_message_msgid.restype = STRING 133 gpo.po_message_msgid.restype = STRING
134 gpo.po_message_msgid_plural.restype = STRING 134 gpo.po_message_msgid_plural.restype = STRING
135 gpo.po_message_msgstr.restype = STRING 135 gpo.po_message_msgstr.restype = STRING
136 gpo.po_message_msgstr_plural.restype = STRING 136 gpo.po_message_msgstr_plural.restype = STRING
137 137
138 # Message (set methods) 138 # Message (set methods)
139 gpo.po_message_set_comments.argtypes = [c_int, STRING] 139 gpo.po_message_set_comments.argtypes = [c_int, STRING]
140 gpo.po_message_set_extracted_comments.argtypes = [c_int, STRING] 140 gpo.po_message_set_extracted_comments.argtypes = [c_int, STRING]
141 gpo.po_message_set_fuzzy.argtypes = [c_int, c_int] 141 gpo.po_message_set_fuzzy.argtypes = [c_int, c_int]
142 gpo.po_message_set_msgctxt.argtypes = [c_int, STRING] 142 gpo.po_message_set_msgctxt.argtypes = [c_int, STRING]
143 gpo.po_message_set_format.argtypes = [c_int, STRING, c_int]
dwaynebailey 2008/08/29 13:19:25 Not sure what this has to do with addlocations? Y
144 143
145 # Setup the po_xerror_handler 144 # Setup the po_xerror_handler
146 xerror_handler = po_xerror_handler() 145 xerror_handler = po_xerror_handler()
147 xerror_handler.xerror = xerror_prototype(xerror_cb) 146 xerror_handler.xerror = xerror_prototype(xerror_cb)
148 xerror_handler.xerror2 = xerror2_prototype(xerror2_cb) 147 xerror_handler.xerror2 = xerror2_prototype(xerror2_cb)
149 148
150 def escapeforpo(text): 149 def escapeforpo(text):
151 return pypo.escapeforpo(text) 150 return pypo.escapeforpo(text)
152 151
153 def quoteforpo(text): 152 def quoteforpo(text):
154 return pypo.quoteforpo(text) 153 return pypo.quoteforpo(text)
155 154
156 def unquotefrompo(postr, joinwithlinebreak=False): 155 def unquotefrompo(postr, joinwithlinebreak=False):
157 return pypo.unquotefrompo(postr, joinwithlinebreak) 156 return pypo.unquotefrompo(postr, joinwithlinebreak)
158 157
159 def encodingToUse(encoding): 158 def encodingToUse(encoding):
160 return pypo.encodingToUse(encoding) 159 return pypo.encodingToUse(encoding)
161 160
162 def get_libgettextpo_version(): 161 def get_libgettextpo_version():
163 """Returns the libgettextpo version 162 """Returns the libgettextpo version
164 163
165 @return: a three-value tuple containing the libgettextpo version in the 164 @return: a three-value tuple containing the libgettextpo version in the
166 following format: 165 following format:
167 (major version, minor version, subminor version) 166 (major version, minor version, subminor version)
168 """ 167 """
169 libversion = c_long.in_dll(gpo, 'libgettextpo_version') 168 libversion = c_long.in_dll(gpo, 'libgettextpo_version')
170 major = libversion.value >> 16 169 major = libversion.value >> 16
171 minor = libversion.value >> 8 170 minor = libversion.value >> 8
172 subminor = libversion.value - (major << 16) - (minor << 8) 171 subminor = libversion.value - (major << 16) - (minor << 8)
173 return major, minor, subminor 172 return major, minor, subminor
174 173
175 174
176 class pounit(pocommon.pounit): 175 class pounit(pocommon.pounit):
177 def __init__(self, source=None, encoding='utf-8', gpo_message=None): 176 def __init__(self, source=None, encoding='utf-8', gpo_message=None):
178 self._encoding = encoding 177 self._encoding = encoding
179 if not gpo_message: 178 if not gpo_message:
180 self._gpo_message = gpo.po_message_create() 179 self._gpo_message = gpo.po_message_create()
181 if source or source == "": 180 if source or source == "":
182 self.source = source 181 self.source = source
183 self.target = "" 182 self.target = ""
184 elif gpo_message: 183 elif gpo_message:
185 self._gpo_message = gpo_message 184 self._gpo_message = gpo_message
186 185
187 def setmsgidcomment(self, msgidcomment): 186 def setmsgidcomment(self, msgidcomment):
188 if msgidcomment: 187 if msgidcomment:
189 newsource = "_: " + msgidcomment + "\n" + self.source 188 newsource = "_: " + msgidcomment + "\n" + self.source
190 self.source = newsource 189 self.source = newsource
191 msgidcomment = property(None, setmsgidcomment) 190 msgidcomment = property(None, setmsgidcomment)
192 191
193 def setmsgid_plural(self, msgid_plural): 192 def setmsgid_plural(self, msgid_plural):
(...skipping 142 matching lines...) Show 10 above Show 10 below
336 return 335 return
337 text = data.forceunicode(text) 336 text = data.forceunicode(text)
338 oldnotes = self.getnotes(origin) 337 oldnotes = self.getnotes(origin)
339 newnotes = None 338 newnotes = None
340 if oldnotes: 339 if oldnotes:
341 if position == "append": 340 if position == "append":
342 newnotes = oldnotes + "\n" + text 341 newnotes = oldnotes + "\n" + text
343 elif position == "merge": 342 elif position == "merge":
344 if oldnotes != text: 343 if oldnotes != text:
345 oldnoteslist = oldnotes.split("\n") 344 oldnoteslist = oldnotes.split("\n")
346 for newline in text.split("\n"): 345 for newline in text.split("\n"):
347 newline = newline.rstrip() 346 newline = newline.rstrip()
348 # avoid duplicate comment lines (this might cause some p roblems) 347 # avoid duplicate comment lines (this might cause some p roblems)
349 if newline not in oldnotes or len(newline) < 5: 348 if newline not in oldnotes or len(newline) < 5:
350 oldnoteslist.append(newline) 349 oldnoteslist.append(newline)
351 newnotes = "\n".join(oldnoteslist) 350 newnotes = "\n".join(oldnoteslist)
352 else: 351 else:
353 newnotes = text + '\n' + oldnotes 352 newnotes = text + '\n' + oldnotes
354 else: 353 else:
355 newnotes = "\n".join([line.rstrip() for line in text.split("\n")]) 354 newnotes = "\n".join([line.rstrip() for line in text.split("\n")])
356 355
357 if newnotes: 356 if newnotes:
358 newlines = [] 357 newlines = []
359 needs_space = get_libgettextpo_version() < (0, 17, 0) 358 needs_space = get_libgettextpo_version() < (0, 17, 0)
360 for line in newnotes.split("\n"): 359 for line in newnotes.split("\n"):
361 if line and needs_space: 360 if line and needs_space:
362 newlines.append(" " + line) 361 newlines.append(" " + line)
363 else: 362 else:
364 newlines.append(line) 363 newlines.append(line)
365 newnotes = "\n".join(newlines) 364 newnotes = "\n".join(newlines)
366 if origin in ["programmer", "developer", "source code"]: 365 if origin in ["programmer", "developer", "source code"]:
367 gpo.po_message_set_extracted_comments(self._gpo_message, newnote s) 366 gpo.po_message_set_extracted_comments(self._gpo_message, newnote s)
368 else: 367 else:
369 gpo.po_message_set_comments(self._gpo_message, newnotes) 368 gpo.po_message_set_comments(self._gpo_message, newnotes)
370 369
371 def removenotes(self): 370 def removenotes(self):
372 gpo.po_message_set_comments(self._gpo_message, "") 371 gpo.po_message_set_comments(self._gpo_message, "")
373 372
374 def copy(self): 373 def copy(self):
375 """Returns a copy of the this unit that can be used independently. 374 """Returns a copy of the this unit that can be used independently.
376 """ 375 """
377 unit = pounit() 376 unit = pounit()
378 unit.source = self.source 377 unit.source = self.source
379 unit.target = self.target 378 unit.target = self.target
380 unit.markfuzzy(self.isfuzzy()) 379 unit.markfuzzy(self.isfuzzy())
381 if self.isobsolete(): 380 if self.isobsolete():
382 unit.makeobsolete() 381 unit.makeobsolete()
383 for location in self.getlocations(): 382 for location in self.getlocations():
384 unit.addlocation(location) 383 unit.addlocation(location)
385 notes = [] 384 notes = []
386 origins = ['translator', 'developer', 'programmer', 'source code'] 385 origins = ['translator', 'developer', 'programmer', 'source code']
dwaynebailey 2008/08/29 13:19:25 Maybe call them note_types
387 for origin in origins: 386 for origin in origins:
388 note = self.getnotes(origin=origin) 387 note = self.getnotes(origin=origin)
389 if note in notes: 388 if note in notes:
390 continue 389 continue
391 else: 390 if note:
392 unit.addnote(note, origin=origin) 391 unit.addnote(note, origin=origin)
393 notes.append(note) 392 notes.append(note)
394 context = gpo.po_message_msgctxt(self._gpo_message) 393 context = gpo.po_message_msgctxt(self._gpo_message)
395 if context: 394 if context:
396 unit.setcontext(context) 395 unit.setcontext(context)
397 comments = self._extract_msgidcomments() 396 comments = self._extract_msgidcomments()
398 if comments: 397 if comments:
399 unit.setmsgidcomment(comments) 398 unit.setmsgidcomment(comments)
400 for typecomment in self.gettypecomments():
401 unit.addtypecomment(typecomment)
402 return unit 399 return unit
dwaynebailey 2008/08/29 13:19:25 Generally I think this part looks good
403 400
404 def merge(self, otherpo, overwrite=False, comments=True, authoritative=False ): 401 def merge(self, otherpo, overwrite=False, comments=True, authoritative=False ):
405 """Merges the otherpo (with the same msgid) into this one. 402 """Merges the otherpo (with the same msgid) into this one.
406 403
407 Overwrite non-blank self.msgstr only if overwrite is True 404 Overwrite non-blank self.msgstr only if overwrite is True
408 merge comments only if comments is True 405 merge comments only if comments is True
409 406
410 """ 407 """
411 408
412 if not isinstance(otherpo, pounit): 409 if not isinstance(otherpo, pounit):
413 super(pounit, self).merge(otherpo, overwrite, comments) 410 super(pounit, self).merge(otherpo, overwrite, comments)
414 return 411 return
415 if comments: 412 if comments:
416 self.addnote(otherpo.getnotes("translator"), origin="translator", po sition="merge") 413 self.addnote(otherpo.getnotes("translator"), origin="translator", po sition="merge")
417 # FIXME mergelists(self.typecomments, otherpo.typecomments) 414 # FIXME mergelists(self.typecomments, otherpo.typecomments)
418 if not authoritative: 415 if not authoritative:
419 # We don't bring across otherpo.automaticcomments as we consider ourself 416 # We don't bring across otherpo.automaticcomments as we consider ourself
420 # to be the the authority. Same applies to otherpo.msgidcomment s 417 # to be the the authority. Same applies to otherpo.msgidcomment s
421 self.addnote(otherpo.getnotes("developer"), origin="developer", position="merge") 418 self.addnote(otherpo.getnotes("developer"), origin="developer", position="merge")
422 self.msgidcomment = otherpo._extract_msgidcomments() or None 419 self.msgidcomment = otherpo._extract_msgidcomments() or None
423 self.addlocations(otherpo.getlocations()) 420 self.addlocations(otherpo.getlocations())
424 if not self.istranslated() or overwrite: 421 if not self.istranslated() or overwrite:
425 # Remove kde-style comments from the translation (if any). 422 # Remove kde-style comments from the translation (if any).
426 if self._extract_msgidcomments(otherpo.target): 423 if self._extract_msgidcomments(otherpo.target):
427 otherpo.target = otherpo.target.replace('_: ' + otherpo._extract _msgidcomments()+ '\n', '') 424 otherpo.target = otherpo.target.replace('_: ' + otherpo._extract _msgidcomments()+ '\n', '')
428 self.target = otherpo.target 425 self.target = otherpo.target
429 if self.source != otherpo.source: 426 if self.source != otherpo.source:
430 self.markfuzzy() 427 self.markfuzzy()
431 else: 428 else:
432 self.markfuzzy(otherpo.isfuzzy()) 429 self.markfuzzy(otherpo.isfuzzy())
433 elif not otherpo.istranslated(): 430 elif not otherpo.istranslated():
434 if self.source != otherpo.source: 431 if self.source != otherpo.source:
435 self.markfuzzy() 432 self.markfuzzy()
436 else: 433 else:
437 if self.target != otherpo.target: 434 if self.target != otherpo.target:
438 self.markfuzzy() 435 self.markfuzzy()
439 436
440 def isheader(self): 437 def isheader(self):
441 #return self.source == u"" and self.target != u"" 438 #return self.source == u"" and self.target != u""
442 # we really want to make sure that there is no msgidcomment or msgctxt 439 # we really want to make sure that there is no msgidcomment or msgctxt
443 return self.getid() == "" and len(self.target) > 0 440 return self.getid() == "" and len(self.target) > 0
444 441
445 def isblank(self): 442 def isblank(self):
446 return len(self.source) == 0 and len(self.target) == 0 443 return len(self.source) == 0 and len(self.target) == 0
447 444
448 def hastypecomment(self, typecomment): 445 def hastypecomment(self, typecomment):
449 return gpo.po_message_is_format(self._gpo_message, typecomment) 446 return gpo.po_message_is_format(self._gpo_message, typecomment)
450 447
451 def gettypecomments(self):
452 """Returns the type comments of a unit, if any.
453
454 @return: a list containing the type comments of the unit
455 """
456 #XXX: brute force
457 formats = [
458 "c", "objc", "sh", "python", "lisp", "elisp", "librep", "scheme",
459 "smalltalk", "java", "csharp", "awk", "object-pascal", "ycp", "tcl",
460 "perl", "perl-brace", "php", "gcc-internal", "qt", "kde", "boost",
461 ]
dwaynebailey 2008/08/29 13:19:25 I thought we could get a list of valid formats fro
462 comments = []
463 for format in formats:
464 format += '-format'
465 if self.hastypecomment(format):
466 comments.append(format)
467 return comments
468
469 def addtypecomment(self, typecomment):
470 """Adds a type comment to the unit
471
472 @oaram typecomment: the type comment to be added, for example
473 "c-format".
474 """
475 gpo.po_message_set_format(self._gpo_message, typecomment, 1)
476
477 def hasmarkedcomment(self, commentmarker): 448 def hasmarkedcomment(self, commentmarker):
478 commentmarker = "(%s)" % commentmarker 449 commentmarker = "(%s)" % commentmarker
479 for comment in self.getnotes("translator").split("\n"): 450 for comment in self.getnotes("translator").split("\n"):
480 if comment.startswith(commentmarker): 451 if comment.startswith(commentmarker):
481 return True 452 return True
482 return False 453 return False
483 454
484 def istranslated(self): 455 def istranslated(self):
485 return super(pounit, self).istranslated() and not self.isobsolete() 456 return super(pounit, self).istranslated() and not self.isobsolete()
486 457
487 def istranslatable(self): 458 def istranslatable(self):
488 return not (self.isheader() or self.isblank() or self.isobsolete()) 459 return not (self.isheader() or self.isblank() or self.isobsolete())
489 460
490 def isfuzzy(self): 461 def isfuzzy(self):
491 return gpo.po_message_is_fuzzy(self._gpo_message) 462 return gpo.po_message_is_fuzzy(self._gpo_message)
492 463
493 def markfuzzy(self, present=True): 464 def markfuzzy(self, present=True):
494 gpo.po_message_set_fuzzy(self._gpo_message, present) 465 gpo.po_message_set_fuzzy(self._gpo_message, present)
495 466
496 def isreview(self): 467 def isreview(self):
497 return self.hasmarkedcomment("review") or self.hasmarkedcomment("pofilte r") 468 return self.hasmarkedcomment("review") or self.hasmarkedcomment("pofilte r")
498 469
499 def isobsolete(self): 470 def isobsolete(self):
500 return gpo.po_message_is_obsolete(self._gpo_message) 471 return gpo.po_message_is_obsolete(self._gpo_message)
501 472
502 def makeobsolete(self): 473 def makeobsolete(self):
503 # FIXME: libgettexpo currently does not reset other data, we probably wa nt to do that 474 # FIXME: libgettexpo currently does not reset other data, we probably wa nt to do that
504 # but a better solution would be for libgettextpo to output correct data on serialisation 475 # but a better solution would be for libgettextpo to output correct data on serialisation
505 gpo.po_message_set_obsolete(self._gpo_message, True) 476 gpo.po_message_set_obsolete(self._gpo_message, True)
506 477
507 def resurrect(self): 478 def resurrect(self):
508 gpo.po_message_set_obsolete(self._gpo_message, False) 479 gpo.po_message_set_obsolete(self._gpo_message, False)
509 480
510 def hasplural(self): 481 def hasplural(self):
511 return gpo.po_message_msgid_plural(self._gpo_message) is not None 482 return gpo.po_message_msgid_plural(self._gpo_message) is not None
512 483
513 def _extract_msgidcomments(self, text=None): 484 def _extract_msgidcomments(self, text=None):
514 """Extract KDE style msgid comments from the unit. 485 """Extract KDE style msgid comments from the unit.
515 486
516 @rtype: String 487 @rtype: String
517 @return: Returns the extracted msgidcomments found in this unit's msgid. 488 @return: Returns the extracted msgidcomments found in this unit's msgid.
518 489
519 """ 490 """
520 491
521 if not text: 492 if not text:
522 text = gpo.po_message_msgid(self._gpo_message) 493 text = gpo.po_message_msgid(self._gpo_message)
523 if text: 494 if text:
524 msgidcomment = re.search("_: (.*)\n", text) 495 msgidcomment = re.search("_: (.*)\n", text)
525 if msgidcomment: 496 if msgidcomment:
526 return msgidcomment.group(1).decode(self._encoding) 497 return msgidcomment.group(1).decode(self._encoding)
527 return u"" 498 return u""
528 499
529 def __str__(self): 500 def __str__(self):
530 pf = pofile() 501 pf = pofile()
531 unit = self.copy() 502 unit = self.copy()
532 pf.addunit(unit) 503 pf.addunit(unit)
533 return str(pf) 504 return str(pf)
534 505
535 def getlocations(self): 506 def getlocations(self):
536 locations = [] 507 locations = []
537 i = 0 508 i = 0
538 location = gpo.po_message_filepos(self._gpo_message, i) 509 location = gpo.po_message_filepos(self._gpo_message, i)
539 while location: 510 while location:
540 locname = gpo.po_filepos_file(location) 511 locname = gpo.po_filepos_file(location)
541 locline = gpo.po_filepos_start_line(location) 512 locline = gpo.po_filepos_start_line(location)
542 if locline == -1: 513 if locline == -1:
543 locstring = locname 514 locstring = locname
544 else: 515 else:
545 locstring = locname + ":" + str(locline) 516 locstring = locname + ":" + str(locline)
546 locations.append(locstring) 517 locations.append(locstring)
547 i += 1 518 i += 1
548 location = gpo.po_message_filepos(self._gpo_message, i) 519 location = gpo.po_message_filepos(self._gpo_message, i)
549 return locations 520 return locations
550 521
551 def addlocation(self, location): 522 def addlocation(self, location):
552 for loc in location.split(): 523 for loc in location.split():
553 parts = loc.split(":") 524 parts = loc.split(":")
554 file = parts[0] 525 file = parts[0]
555 if len(parts) == 2: 526 if len(parts) == 2:
556 line = int(parts[1]) 527 line = int(parts[1])
557 else: 528 else:
558 # libgettextpo requires a (size_t)(-1) if no file 529 line = -1
559 # location is available
560 line = c_size_t(-1)
561 gpo.po_message_add_filepos(self._gpo_message, file, line) 530 gpo.po_message_add_filepos(self._gpo_message, file, line)
562 531
563 def setcontext(self, context): 532 def setcontext(self, context):
564 """Sets the context message""" 533 """Sets the context message"""
534 #FIXME: not complete
565 gpo.po_message_set_msgctxt(self._gpo_message, context) 535 gpo.po_message_set_msgctxt(self._gpo_message, context)
dwaynebailey 2008/08/29 13:19:25 Again one piece at a time. If you can separate ou
566 536
567 def getcontext(self): 537 def getcontext(self):
568 msgctxt = gpo.po_message_msgctxt(self._gpo_message) 538 msgctxt = gpo.po_message_msgctxt(self._gpo_message)
569 msgidcomment = self._extract_msgidcomments() 539 msgidcomment = self._extract_msgidcomments()
570 if msgctxt: 540 if msgctxt:
571 return msgctxt + msgidcomment 541 return msgctxt + msgidcomment
572 else: 542 else:
573 return msgidcomment 543 return msgidcomment
574 544
575 class pofile(pocommon.pofile): 545 class pofile(pocommon.pofile):
576 UnitClass = pounit 546 UnitClass = pounit
577 def __init__(self, inputfile=None, encoding=None, unitclass=pounit): 547 def __init__(self, inputfile=None, encoding=None, unitclass=pounit):
578 self.UnitClass = unitclass 548 self.UnitClass = unitclass
579 pocommon.pofile.__init__(self, unitclass=unitclass) 549 pocommon.pofile.__init__(self, unitclass=unitclass)
580 self._gpo_memory_file = None 550 self._gpo_memory_file = None
581 self._gpo_message_iterator = None 551 self._gpo_message_iterator = None
582 self._encoding = encodingToUse(encoding) 552 self._encoding = encodingToUse(encoding)
583 if inputfile is not None: 553 if inputfile is not None:
584 self.parse(inputfile) 554 self.parse(inputfile)
585 else: 555 else:
586 self._gpo_memory_file = gpo.po_file_create() 556 self._gpo_memory_file = gpo.po_file_create()
587 self._gpo_message_iterator = gpo.po_message_iterator(self._gpo_memor y_file, None) 557 self._gpo_message_iterator = gpo.po_message_iterator(self._gpo_memor y_file, None)
588 558
589 def addunit(self, unit): 559 def addunit(self, unit):
590 unitcopy = unit.copy() 560 unitcopy = unit.copy()
591 gpo.po_message_insert(self._gpo_message_iterator, unitcopy._gpo_message) 561 gpo.po_message_insert(self._gpo_message_iterator, unitcopy._gpo_message)
592 self.units.append(unitcopy) 562 self.units.append(unitcopy)
593
594 def addsourceunit(self, source):
595 self.addunit(self.UnitClass(source))
596 unit = self.findunit(source)
597 return unit
598 563
599 def removeduplicates(self, duplicatestyle="merge"): 564 def removeduplicates(self, duplicatestyle="merge"):
600 """make sure each msgid is unique ; merge comments etc from duplicates i nto original""" 565 """make sure each msgid is unique ; merge comments etc from duplicates i nto original"""
601 msgiddict = {} 566 msgiddict = {}
602 uniqueunits = [] 567 uniqueunits = []
603 # we sometimes need to keep track of what has been marked 568 # we sometimes need to keep track of what has been marked
604 # TODO: this is using a list as the pos aren't hashable, but this is slo w... 569 # TODO: this is using a list as the pos aren't hashable, but this is slo w...
605 markedpos = [] 570 markedpos = []
606 def addcomment(thepo): 571 def addcomment(thepo):
607 thepo.msgidcomment = " ".join(thepo.getlocations()) 572 thepo.msgidcomment = " ".join(thepo.getlocations())
608 markedpos.append(thepo) 573 markedpos.append(thepo)
609 for thepo in self.units: 574 for thepo in self.units:
610 if thepo.isheader(): 575 if thepo.isheader():
611 uniqueunits.append(thepo) 576 uniqueunits.append(thepo)
612 continue 577 continue
613 if duplicatestyle.startswith("msgid_comment"): 578 if duplicatestyle.startswith("msgid_comment"):
614 msgid = thepo._extract_msgidcomments() + thepo.source 579 msgid = thepo._extract_msgidcomments() + thepo.source
615 else: 580 else:
616 msgid = thepo.source 581 msgid = thepo.source
617 if duplicatestyle == "msgid_comment_all": 582 if duplicatestyle == "msgid_comment_all":
618 addcomment(thepo) 583 addcomment(thepo)
619 uniqueunits.append(thepo) 584 uniqueunits.append(thepo)
620 elif msgid in msgiddict: 585 elif msgid in msgiddict:
621 if duplicatestyle == "merge": 586 if duplicatestyle == "merge":
622 if msgid: 587 if msgid:
623 msgiddict[msgid].merge(thepo) 588 msgiddict[msgid].merge(thepo)
624 else: 589 else:
625 addcomment(thepo) 590 addcomment(thepo)
626 uniqueunits.append(thepo) 591 uniqueunits.append(thepo)
627 elif duplicatestyle == "keep": 592 elif duplicatestyle == "keep":
628 uniqueunits.append(thepo) 593 uniqueunits.append(thepo)
629 elif duplicatestyle == "msgid_comment": 594 elif duplicatestyle == "msgid_comment":
630 origpo = msgiddict[msgid] 595 origpo = msgiddict[msgid]
631 if origpo not in markedpos: 596 if origpo not in markedpos:
632 addcomment(origpo) 597 addcomment(origpo)
633 addcomment(thepo) 598 addcomment(thepo)
634 uniqueunits.append(thepo) 599 uniqueunits.append(thepo)
635 elif duplicatestyle == "msgctxt": 600 elif duplicatestyle == "msgctxt":
636 origpo = msgiddict[msgid] 601 origpo = msgiddict[msgid]
637 if origpo not in markedpos: 602 if origpo not in markedpos:
638 gpo.po_message_set_msgctxt(origpo._gpo_message, " ".join (origpo.getlocations())) 603 gpo.po_message_set_msgctxt(origpo._gpo_message, " ".join (origpo.getlocations()))
639 markedpos.append(thepo) 604 markedpos.append(thepo)
640 gpo.po_message_set_msgctxt(thepo._gpo_message, " ".join(thep o.getlocations())) 605 gpo.po_message_set_msgctxt(thepo._gpo_message, " ".join(thep o.getlocations()))
641 uniqueunits.append(thepo) 606 uniqueunits.append(thepo)
642 else: 607 else:
643 if not msgid and duplicatestyle != "keep": 608 if not msgid and duplicatestyle != "keep":
644 addcomment(thepo) 609 addcomment(thepo)
645 msgiddict[msgid] = thepo 610 msgiddict[msgid] = thepo
646 uniqueunits.append(thepo) 611 uniqueunits.append(thepo)
647 new_gpo_memory_file = gpo.po_file_create() 612 new_gpo_memory_file = gpo.po_file_create()
648 new_gpo_message_iterator = gpo.po_message_iterator(new_gpo_memory_file, None) 613 new_gpo_message_iterator = gpo.po_message_iterator(new_gpo_memory_file, None)
649 for unit in uniqueunits: 614 for unit in uniqueunits:
650 gpo.po_message_insert(new_gpo_message_iterator, unit._gpo_message) 615 gpo.po_message_insert(new_gpo_message_iterator, unit._gpo_message)
651 gpo.po_message_iterator_free(self._gpo_message_iterator) 616 gpo.po_message_iterator_free(self._gpo_message_iterator)
652 self._gpo_message_iterator = new_gpo_message_iterator 617 self._gpo_message_iterator = new_gpo_message_iterator
653 self._gpo_memory_file = new_gpo_memory_file 618 self._gpo_memory_file = new_gpo_memory_file
654 self.units = uniqueunits 619 self.units = uniqueunits
655 620
656 def __str__(self): 621 def __str__(self):
657 def obsolete_workaround(): 622 def obsolete_workaround():
658 # Remove all items that are not output by msgmerge when a unit is ob solete. This is a work 623 # Remove all items that are not output by msgmerge when a unit is ob solete. This is a work
659 # around for bug in libgettextpo 624 # around for bug in libgettextpo
660 # FIXME Do version test in case they fix this bug 625 # FIXME Do version test in case they fix this bug
661 for unit in self.units: 626 for unit in self.units:
662 if unit.isobsolete(): 627 if unit.isobsolete():
663 gpo.po_message_set_extracted_comments(unit._gpo_message, "") 628 gpo.po_message_set_extracted_comments(unit._gpo_message, "")
664 location = gpo.po_message_filepos(unit._gpo_message, 0) 629 location = gpo.po_message_filepos(unit._gpo_message, 0)
665 while location: 630 while location:
666 gpo.po_message_remove_filepos(unit._gpo_message, 0) 631 gpo.po_message_remove_filepos(unit._gpo_message, 0)
667 location = gpo.po_message_filepos(unit._gpo_message, 0) 632 location = gpo.po_message_filepos(unit._gpo_message, 0)
668 outputstring = "" 633 outputstring = ""
669 if self._gpo_memory_file: 634 if self._gpo_memory_file:
670 obsolete_workaround() 635 obsolete_workaround()
671 f = tempfile.NamedTemporaryFile(prefix='translate', suffix='.po') 636 f = tempfile.NamedTemporaryFile(prefix='translate', suffix='.po')
672 self._gpo_memory_file = gpo.po_file_write_v2(self._gpo_memory_file, f.name, xerror_handler) 637 self._gpo_memory_file = gpo.po_file_write_v2(self._gpo_memory_file, f.name, xerror_handler)
673 f.seek(0) 638 f.seek(0)
674 outputstring = f.read() 639 outputstring = f.read()
675 f.close() 640 f.close()
676 return outputstring 641 return outputstring
677 642
678 def isempty(self): 643 def isempty(self):
679 """Returns True if the object doesn't contain any translation units.""" 644 """Returns True if the object doesn't contain any translation units."""
680 if len(self.units) == 0: 645 if len(self.units) == 0:
681 return True 646 return True
682 # Skip the first unit if it is a header. 647 # Skip the first unit if it is a header.
683 if self.units[0].isheader(): 648 if self.units[0].isheader():
684 units = self.units[1:] 649 units = self.units[1:]
685 else: 650 else:
686 units = self.units 651 units = self.units
687 652
688 for unit in units: 653 for unit in units:
689 if not unit.isblank() and not unit.isobsolete(): 654 if not unit.isblank() and not unit.isobsolete():
690 return False 655 return False
691 return True 656 return True
692 657
693 def parse(self, input): 658 def parse(self, input):
694 if hasattr(input, 'name'): 659 if hasattr(input, 'name'):
695 self.filename = input.name 660 self.filename = input.name
696 elif not getattr(self, 'filename', ''): 661 elif not getattr(self, 'filename', ''):
697 self.filename = '' 662 self.filename = ''
698 663
699 if hasattr(input, "read"): 664 if hasattr(input, "read"):
700 posrc = input.read() 665 posrc = input.read()
701 input.close() 666 input.close()
702 input = posrc 667 input = posrc
703 668
704 needtmpfile = not os.path.isfile(input) 669 needtmpfile = not os.path.isfile(input)
705 if needtmpfile: 670 if needtmpfile:
706 # This is not a file - we write the string to a temporary file 671 # This is not a file - we write the string to a temporary file
707 fd, fname = tempfile.mkstemp(prefix='translate', suffix='.po') 672 fd, fname = tempfile.mkstemp(prefix='translate', suffix='.po')
708 os.write(fd, input) 673 os.write(fd, input)
709 input = fname 674 input = fname
710 os.close(fd) 675 os.close(fd)
711 676
712 self._gpo_memory_file = gpo.po_file_read_v3(input, xerror_handler) 677 self._gpo_memory_file = gpo.po_file_read_v3(input, xerror_handler)
713 if self._gpo_memory_file is None: 678 if self._gpo_memory_file is None:
714 print >> sys.stderr, "Error:" 679 print >> sys.stderr, "Error:"
715 680
716 if needtmpfile: 681 if needtmpfile:
717 os.remove(input) 682 os.remove(input)
718 683
719 # Handle xerrors here 684 # Handle xerrors here
720 self._header = gpo.po_file_domain_header(self._gpo_memory_file, None) 685 self._header = gpo.po_file_domain_header(self._gpo_memory_file, None)
721 if self._header: 686 if self._header:
722 charset = gpo.po_header_field(self._header, "Content-Type") 687 charset = gpo.po_header_field(self._header, "Content-Type")
723 if charset: 688 if charset:
724 charset = re.search("charset=([^\\s]+)", charset).group(1) 689 charset = re.search("charset=([^\\s]+)", charset).group(1)
725 self._encoding = encodingToUse(charset) 690 self._encoding = encodingToUse(charset)
726 self._gpo_message_iterator = gpo.po_message_iterator(self._gpo_memory_fi le, None) 691 self._gpo_message_iterator = gpo.po_message_iterator(self._gpo_memory_fi le, None)
727 newmessage = gpo.po_next_message(self._gpo_message_iterator) 692 newmessage = gpo.po_next_message(self._gpo_message_iterator)
728 while newmessage: 693 while newmessage:
729 newunit = pounit(gpo_message=newmessage) 694 newunit = pounit(gpo_message=newmessage)
730 self.units.append(newunit) 695 self.units.append(newunit)
731 newmessage = gpo.po_next_message(self._gpo_message_iterator) 696 newmessage = gpo.po_next_message(self._gpo_message_iterator)
732 697
733 def __del__(self): 698 def __del__(self):
734 # We currently disable this while we still get segmentation faults. 699 # We currently disable this while we still get segmentation faults.
735 # Note that this is definitely leaking memory because of this. 700 # Note that this is definitely leaking memory because of this.
736 self._free_iterator() 701 self._free_iterator()
737 if self._gpo_memory_file is not None: 702 if self._gpo_memory_file is not None:
738 gpo.po_file_free(self._gpo_memory_file) 703 gpo.po_file_free(self._gpo_memory_file)
739 self._gpo_memory_file = None 704 self._gpo_memory_file = None
740 705
741 def _free_iterator(self): 706 def _free_iterator(self):
742 # We currently disable this while we still get segmentation faults. 707 # We currently disable this while we still get segmentation faults.
743 # Note that this is definitely leaking memory because of this. 708 # Note that this is definitely leaking memory because of this.
744 if self._gpo_message_iterator is not None: 709 if self._gpo_message_iterator is not None:
745 gpo.po_message_iterator_free(self._gpo_message_iterator) 710 gpo.po_message_iterator_free(self._gpo_message_iterator)
746 self._gpo_message_iterator = None 711 self._gpo_message_iterator = None
LEFTRIGHT

Powered by Google App Engine
This is Rietveld r159