source: trunk/tools/linguist/shared/po.cpp@ 385

Last change on this file since 385 was 2, checked in by Dmitry A. Kuminov, 16 years ago

Initially imported qt-all-opensource-src-4.5.1 from Trolltech.
File size: 25.0 KB
Line 
1/****************************************************************************
2**
3** Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies).
4** Contact: Qt Software Information (qt-info@nokia.com)
5**
6** This file is part of the Qt Linguist of the Qt Toolkit.
7**
8** $QT_BEGIN_LICENSE:LGPL$
9** Commercial Usage
10** Licensees holding valid Qt Commercial licenses may use this file in
11** accordance with the Qt Commercial License Agreement provided with the
12** Software or, alternatively, in accordance with the terms contained in
13** a written agreement between you and Nokia.
14**
15** GNU Lesser General Public License Usage
16** Alternatively, this file may be used under the terms of the GNU Lesser
17** General Public License version 2.1 as published by the Free Software
18** Foundation and appearing in the file LICENSE.LGPL included in the
19** packaging of this file. Please review the following information to
20** ensure the GNU Lesser General Public License version 2.1 requirements
21** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
22**
23** In addition, as a special exception, Nokia gives you certain
24** additional rights. These rights are described in the Nokia Qt LGPL
25** Exception version 1.0, included in the file LGPL_EXCEPTION.txt in this
26** package.
27**
28** GNU General Public License Usage
29** Alternatively, this file may be used under the terms of the GNU
30** General Public License version 3.0 as published by the Free Software
31** Foundation and appearing in the file LICENSE.GPL included in the
32** packaging of this file. Please review the following information to
33** ensure the GNU General Public License version 3.0 requirements will be
34** met: http://www.gnu.org/copyleft/gpl.html.
35**
36** If you are unsure which license is appropriate for your use, please
37** contact the sales department at qt-sales@nokia.com.
38** $QT_END_LICENSE$
39**
40****************************************************************************/
41
42#include "translator.h"
43
44#include <QtCore/QDebug>
45#include <QtCore/QIODevice>
46#include <QtCore/QHash>
47#include <QtCore/QString>
48#include <QtCore/QTextStream>
49
50#include <ctype.h>
51
52#define MAGIC_OBSOLETE_REFERENCE "Obsolete_PO_entries"
53
54// Uncomment if you wish to hard wrap long lines in .po files. Note that this
55// affects only msg strings, not comments.
56//#define HARD_WRAP_LONG_WORDS
57
58QT_BEGIN_NAMESPACE
59
60static const int MAX_LEN = 79;
61
62static QString poEscapedString(const QString &prefix, const QString &keyword,
63 bool noWrap, const QString &ba)
64{
65 QStringList lines;
66 int off = 0;
67 QString res;
68 while (off < ba.length()) {
69 ushort c = ba[off++].unicode();
70 switch (c) {
71 case '\n':
72 res += QLatin1String("\\n");
73 lines.append(res);
74 res.clear();
75 break;
76 case '\r':
77 res += QLatin1String("\\r");
78 break;
79 case '\t':
80 res += QLatin1String("\\t");
81 break;
82 case '\v':
83 res += QLatin1String("\\v");
84 break;
85 case '\a':
86 res += QLatin1String("\\a");
87 break;
88 case '\b':
89 res += QLatin1String("\\b");
90 break;
91 case '\f':
92 res += QLatin1String("\\f");
93 break;
94 case '"':
95 res += QLatin1String("\\\"");
96 break;
97 case '\\':
98 res += QLatin1String("\\\\");
99 break;
100 default:
101 if (c < 32) {
102 res += QLatin1String("\\x");
103 res += QString::number(c, 16);
104 if (off < ba.length() && isxdigit(ba[off].unicode()))
105 res += QLatin1String("\"\"");
106 } else {
107 res += QChar(c);
108 }
109 break;
110 }
111 }
112 if (!res.isEmpty())
113 lines.append(res);
114 if (!lines.isEmpty()) {
115 if (!noWrap) {
116 if (lines.count() != 1 ||
117 lines.first().length() > MAX_LEN - keyword.length() - prefix.length() - 3)
118 {
119 QStringList olines = lines;
120 lines = QStringList(QString());
121 const int maxlen = MAX_LEN - prefix.length() - 2;
122 foreach (const QString &line, olines) {
123 int off = 0;
124 while (off + maxlen < line.length()) {
125 int idx = line.lastIndexOf(QLatin1Char(' '), off + maxlen - 1) + 1;
126 if (idx == off) {
127#ifdef HARD_WRAP_LONG_WORDS
128 // This doesn't seem too nice, but who knows ...
129 idx = off + maxlen;
130#else
131 idx = line.indexOf(QLatin1Char(' '), off + maxlen) + 1;
132 if (!idx)
133 break;
134#endif
135 }
136 lines.append(line.mid(off, idx - off));
137 off = idx;
138 }
139 lines.append(line.mid(off));
140 }
141 }
142 } else if (lines.count() > 1) {
143 lines.prepend(QString());
144 }
145 }
146 return prefix + keyword + QLatin1String(" \"") +
147 lines.join(QLatin1String("\"\n") + prefix + QLatin1Char('"')) +
148 QLatin1String("\"\n");
149}
150
151static QString poEscapedLines(const QString &prefix, bool addSpace, const QStringList &lines)
152{
153 QString out;
154 foreach (const QString &line, lines) {
155 out += prefix;
156 if (addSpace && !line.isEmpty())
157 out += QLatin1Char(' ' );
158 out += line;
159 out += QLatin1Char('\n');
160 }
161 return out;
162}
163
164static QString poEscapedLines(const QString &prefix, bool addSpace, const QString &in0)
165{
166 QString in = in0;
167 if (in.endsWith(QLatin1Char('\n')))
168 in.chop(1);
169 return poEscapedLines(prefix, addSpace, in.split(QLatin1Char('\n')));
170}
171
172static QString poWrappedEscapedLines(const QString &prefix, bool addSpace, const QString &line)
173{
174 const int maxlen = MAX_LEN - prefix.length();
175 QStringList lines;
176 int off = 0;
177 while (off + maxlen < line.length()) {
178 int idx = line.lastIndexOf(QLatin1Char(' '), off + maxlen - 1);
179 if (idx < off) {
180#if 0 //def HARD_WRAP_LONG_WORDS
181 // This cannot work without messing up semantics, so do not even try.
182#else
183 idx = line.indexOf(QLatin1Char(' '), off + maxlen);
184 if (idx < 0)
185 break;
186#endif
187 }
188 lines.append(line.mid(off, idx - off));
189 off = idx + 1;
190 }
191 lines.append(line.mid(off));
192 return poEscapedLines(prefix, addSpace, lines);
193}
194
195struct PoItem
196{
197public:
198 PoItem()
199 : isPlural(false), isFuzzy(false)
200 {}
201
202
203public:
204 QString id;
205 QString context;
206 QString tscomment;
207 QString oldTscomment;
208 QString lineNumber;
209 QString fileName;
210 QString references;
211 QString translatorComments;
212 QString automaticComments;
213 QString msgId;
214 QString oldMsgId;
215 QStringList msgStr;
216 bool isPlural;
217 bool isFuzzy;
218 QHash<QString, QString> extra;
219};
220
221
222static bool isTranslationLine(const QString &line)
223{
224 return line.startsWith(QLatin1String("#~ msgstr"))
225 || line.startsWith(QLatin1String("msgstr"));
226}
227
228static QString slurpEscapedString(const QStringList &lines, int & l,
229 int offset, const QString &prefix, ConversionData &cd)
230{
231 QString msg;
232 int stoff;
233
234 for (; l < lines.size(); ++l) {
235 const QString &line = lines.at(l);
236 if (line.isEmpty() || !line.startsWith(prefix))
237 break;
238 while (line[offset].isSpace()) // No length check, as string has no trailing spaces.
239 offset++;
240 if (line[offset].unicode() != '"')
241 break;
242 offset++;
243 forever {
244 if (offset == line.length())
245 goto premature_eol;
246 ushort c = line[offset++].unicode();
247 if (c == '"') {
248 if (offset == line.length())
249 break;
250 while (line[offset].isSpace())
251 offset++;
252 if (line[offset++].unicode() != '"') {
253 cd.appendError(QString::fromLatin1(
254 "PO parsing error: extra characters on line %1.")
255 .arg(l + 1));
256 break;
257 }
258 continue;
259 }
260 if (c == '\\') {
261 if (offset == line.length())
262 goto premature_eol;
263 c = line[offset++].unicode();
264 switch (c) {
265 case 'r':
266 msg += QLatin1Char('\r'); // Maybe just throw it away?
267 break;
268 case 'n':
269 msg += QLatin1Char('\n');
270 break;
271 case 't':
272 msg += QLatin1Char('\t');
273 break;
274 case 'v':
275 msg += QLatin1Char('\v');
276 break;
277 case 'a':
278 msg += QLatin1Char('\a');
279 break;
280 case 'b':
281 msg += QLatin1Char('\b');
282 break;
283 case 'f':
284 msg += QLatin1Char('\f');
285 break;
286 case '"':
287 msg += QLatin1Char('"');
288 break;
289 case '\\':
290 msg += QLatin1Char('\\');
291 break;
292 case '0':
293 case '1':
294 case '2':
295 case '3':
296 case '4':
297 case '5':
298 case '6':
299 case '7':
300 stoff = offset - 1;
301 while ((c = line[offset].unicode()) >= '0' && c <= '7')
302 if (++offset == line.length())
303 goto premature_eol;
304 msg += QChar(line.mid(stoff, offset - stoff).toUInt(0, 8));
305 break;
306 case 'x':
307 stoff = offset;
308 while (isxdigit(line[offset].unicode()))
309 if (++offset == line.length())
310 goto premature_eol;
311 msg += QChar(line.mid(stoff, offset - stoff).toUInt(0, 16));
312 break;
313 default:
314 cd.appendError(QString::fromLatin1(
315 "PO parsing error: invalid escape '\\%1' (line %2).")
316 .arg(QChar(c)).arg(l + 1));
317 msg += QLatin1Char('\\');
318 msg += QChar(c);
319 break;
320 }
321 } else {
322 msg += QChar(c);
323 }
324 }
325 offset = prefix.size();
326 }
327 --l;
328 return msg;
329
330premature_eol:
331 cd.appendError(QString::fromLatin1(
332 "PO parsing error: premature end of line %1.").arg(l + 1));
333 return QString();
334}
335
336static void slurpComment(QString &msg, const QStringList &lines, int & l)
337{
338 const QChar newline = QLatin1Char('\n');
339 QString prefix = lines.at(l);
340 for (int i = 1; ; i++) {
341 if (prefix.at(i).unicode() != ' ') {
342 prefix.truncate(i);
343 break;
344 }
345 }
346 for (; l < lines.size(); ++l) {
347 const QString &line = lines.at(l);
348 if (line.startsWith(prefix))
349 msg += line.mid(prefix.size());
350 else if (line != QLatin1String("#"))
351 break;
352 msg += newline;
353 }
354 --l;
355}
356
357bool loadPO(Translator &translator, QIODevice &dev, ConversionData &cd)
358{
359 const QChar quote = QLatin1Char('"');
360 const QChar newline = QLatin1Char('\n');
361 QTextStream in(&dev);
362 bool error = false;
363
364 // format of a .po file entry:
365 // white-space
366 // # translator-comments
367 // #. automatic-comments
368 // #: reference...
369 // #, flag...
370 // #~ msgctxt, msgid*, msgstr - used for obsoleted messages
371 // #| msgctxt, msgid* previous untranslated-string - for fuzzy message
372 // msgctx string-context
373 // msgid untranslated-string
374 // -- For singular:
375 // msgstr translated-string
376 // -- For plural:
377 // msgid_plural untranslated-string-plural
378 // msgstr[0] translated-string
379 // ...
380
381 // we need line based lookahead below.
382 QStringList lines;
383 while (!in.atEnd())
384 lines.append(in.readLine().trimmed());
385 lines.append(QString());
386
387 int l = 0;
388 PoItem item;
389 for (; l != lines.size(); ++l) {
390 QString line = lines.at(l);
391 if (line.isEmpty())
392 continue;
393 if (isTranslationLine(line)) {
394 bool isObsolete = line.startsWith(QLatin1String("#~ msgstr"));
395 const QString prefix = QLatin1String(isObsolete ? "#~ " : "");
396 while (true) {
397 int idx = line.indexOf(QLatin1Char(' '), prefix.length());
398 item.msgStr.append(slurpEscapedString(lines, l, idx, prefix, cd));
399 if (l + 1 >= lines.size() || !isTranslationLine(lines.at(l + 1)))
400 break;
401 ++l;
402 line = lines.at(l);
403 }
404 if (item.msgId.isEmpty()) {
405 QRegExp rx(QLatin1String("\\bX-Language: ([^\n]*)\n"));
406 int idx = rx.indexIn(item.msgStr.first());
407 if (idx >= 0) {
408 translator.setLanguageCode(rx.cap(1));
409 item.msgStr.first().remove(idx, rx.matchedLength());
410 }
411 QRegExp rx2(QLatin1String("\\bX-Source-Language: ([^\n]*)\n"));
412 int idx2 = rx2.indexIn(item.msgStr.first());
413 if (idx2 >= 0) {
414 translator.setSourceLanguageCode(rx2.cap(1));
415 item.msgStr.first().remove(idx2, rx2.matchedLength());
416 }
417 if (item.msgStr.first().indexOf(
418 QRegExp(QLatin1String("\\bX-Virgin-Header:[^\n]*\n"))) >= 0) {
419 item = PoItem();
420 continue;
421 }
422 }
423 // build translator message
424 TranslatorMessage msg;
425 msg.setContext(item.context);
426 if (!item.references.isEmpty()) {
427 foreach (const QString &ref,
428 item.references.split(QRegExp(QLatin1String("\\s")),
429 QString::SkipEmptyParts)) {
430 int pos = ref.lastIndexOf(QLatin1Char(':'));
431 if (pos != -1)
432 msg.addReference(ref.left(pos), ref.mid(pos + 1).toInt());
433 }
434 } else if (isObsolete) {
435 msg.setFileName(QLatin1String(MAGIC_OBSOLETE_REFERENCE));
436 }
437 msg.setId(item.id);
438 msg.setSourceText(item.msgId);
439 msg.setOldSourceText(item.oldMsgId);
440 msg.setComment(item.tscomment);
441 msg.setOldComment(item.oldTscomment);
442 msg.setExtraComment(item.automaticComments);
443 msg.setTranslatorComment(item.translatorComments);
444 msg.setPlural(item.isPlural || item.msgStr.size() > 1);
445 msg.setTranslations(item.msgStr);
446 if (isObsolete)
447 msg.setType(TranslatorMessage::Obsolete);
448 else if (item.isFuzzy)
449 msg.setType(TranslatorMessage::Unfinished);
450 else
451 msg.setType(TranslatorMessage::Finished);
452 msg.setExtras(item.extra);
453
454 //qDebug() << "WRITE: " << context;
455 //qDebug() << "SOURCE: " << msg.sourceText();
456 //qDebug() << flags << msg.m_extra;
457 translator.append(msg);
458 item = PoItem();
459 } else if (line.startsWith(QLatin1Char('#'))) {
460 switch(line.size() < 2 ? 0 : line.at(1).unicode()) {
461 case ':':
462 item.references += line.mid(3);
463 item.references += newline;
464 break;
465 case ',': {
466 QStringList flags =
467 line.mid(2).split(QRegExp(QLatin1String("[, ]")),
468 QString::SkipEmptyParts);
469 if (flags.removeOne(QLatin1String("fuzzy")))
470 item.isFuzzy = true;
471 TranslatorMessage::ExtraData::const_iterator it =
472 item.extra.find(QLatin1String("po-flags"));
473 if (it != item.extra.end())
474 flags.prepend(*it);
475 if (!flags.isEmpty())
476 item.extra[QLatin1String("po-flags")] = flags.join(QLatin1String(", "));
477 break;
478 }
479 case 0:
480 item.translatorComments += newline;
481 break;
482 case ' ':
483 slurpComment(item.translatorComments, lines, l);
484 break;
485 case '.':
486 if (line.startsWith(QLatin1String("#. ts-context "))) {
487 item.context = line.mid(14);
488 } else if (line.startsWith(QLatin1String("#. ts-id "))) {
489 item.id = line.mid(9);
490 } else {
491 item.automaticComments += line.mid(3);
492 item.automaticComments += newline;
493 }
494 break;
495 case '|':
496 if (line.startsWith(QLatin1String("#| msgid "))) {
497 item.oldMsgId = slurpEscapedString(lines, l, 9, QLatin1String("#| "), cd);
498 } else if (line.startsWith(QLatin1String("#| msgid_plural "))) {
499 QString extra = slurpEscapedString(lines, l, 16, QLatin1String("#| "), cd);
500 if (extra != item.oldMsgId)
501 item.extra[QLatin1String("po-old_msgid_plural")] = extra;
502 } else if (line.startsWith(QLatin1String("#| msgctxt "))) {
503 item.oldTscomment = slurpEscapedString(lines, l, 11, QLatin1String("#| "), cd);
504 } else {
505 cd.appendError(QString(QLatin1String("PO-format parse error in line %1: '%2'\n"))
506 .arg(l + 1).arg(lines[l]));
507 error = true;
508 }
509 break;
510 case '~':
511 if (line.startsWith(QLatin1String("#~ msgid "))) {
512 item.msgId = slurpEscapedString(lines, l, 9, QLatin1String("#~ "), cd);
513 } else if (line.startsWith(QLatin1String("#~ msgid_plural "))) {
514 QString extra = slurpEscapedString(lines, l, 16, QLatin1String("#~ "), cd);
515 if (extra != item.msgId)
516 item.extra[QLatin1String("po-msgid_plural")] = extra;
517 item.isPlural = true;
518 } else if (line.startsWith(QLatin1String("#~ msgctxt "))) {
519 item.tscomment = slurpEscapedString(lines, l, 11, QLatin1String("#~ "), cd);
520 } else {
521 cd.appendError(QString(QLatin1String("PO-format parse error in line %1: '%2'\n"))
522 .arg(l + 1).arg(lines[l]));
523 error = true;
524 }
525 break;
526 default:
527 cd.appendError(QString(QLatin1String("PO-format parse error in line %1: '%2'\n"))
528 .arg(l + 1).arg(lines[l]));
529 error = true;
530 break;
531 }
532 } else if (line.startsWith(QLatin1String("msgctxt "))) {
533 item.tscomment = slurpEscapedString(lines, l, 8, QString(), cd);
534 } else if (line.startsWith(QLatin1String("msgid "))) {
535 item.msgId = slurpEscapedString(lines, l, 6, QString(), cd);
536 } else if (line.startsWith(QLatin1String("msgid_plural "))) {
537 QString extra = slurpEscapedString(lines, l, 13, QString(), cd);
538 if (extra != item.msgId)
539 item.extra[QLatin1String("po-msgid_plural")] = extra;
540 item.isPlural = true;
541 } else {
542 cd.appendError(QString(QLatin1String("PO-format error in line %1: '%2'\n"))
543 .arg(l + 1).arg(lines[l]));
544 error = true;
545 }
546 }
547 return !error && cd.errors().isEmpty();
548}
549
550bool savePO(const Translator &translator, QIODevice &dev, ConversionData &cd)
551{
552 bool ok = true;
553 QTextStream out(&dev);
554 //qDebug() << "OUT CODEC: " << out.codec()->name();
555
556 bool first = true;
557 if (translator.messages().isEmpty() || !translator.messages().first().sourceText().isEmpty()) {
558 out <<
559 "# SOME DESCRIPTIVE TITLE.\n"
560 "# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER\n"
561 "# This file is distributed under the same license as the PACKAGE package.\n"
562 "# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.\n"
563 "#\n"
564 "#, fuzzy\n"
565 "msgid \"\"\n"
566 "msgstr \"\"\n"
567 "\"X-Virgin-Header: remove this line if you change anything in the header.\\n\"\n";
568 if (!translator.languageCode().isEmpty())
569 out << "\"X-Language: " << translator.languageCode() << "\\n\"\n";
570 if (!translator.sourceLanguageCode().isEmpty())
571 out << "\"X-Source-Language: " << translator.sourceLanguageCode() << "\\n\"\n";
572 first = false;
573 }
574 foreach (const TranslatorMessage &msg, translator.messages()) {
575 if (!first)
576 out << endl;
577
578 if (!msg.translatorComment().isEmpty())
579 out << poEscapedLines(QLatin1String("#"), true, msg.translatorComment());
580
581 if (!msg.extraComment().isEmpty())
582 out << poEscapedLines(QLatin1String("#."), true, msg.extraComment());
583
584 if (!msg.context().isEmpty())
585 out << QLatin1String("#. ts-context ") << msg.context() << '\n';
586 if (!msg.id().isEmpty())
587 out << QLatin1String("#. ts-id ") << msg.id() << '\n';
588
589 if (!msg.fileName().isEmpty() && msg.fileName() != QLatin1String(MAGIC_OBSOLETE_REFERENCE)) {
590 QStringList refs;
591 foreach (const TranslatorMessage::Reference &ref, msg.allReferences())
592 refs.append(QString(QLatin1String("%2:%1"))
593 .arg(ref.lineNumber()).arg(ref.fileName()));
594 out << poWrappedEscapedLines(QLatin1String("#:"), true, refs.join(QLatin1String(" ")));
595 }
596
597 bool noWrap = false;
598 QStringList flags;
599 if (msg.type() == TranslatorMessage::Unfinished)
600 flags.append(QLatin1String("fuzzy"));
601 TranslatorMessage::ExtraData::const_iterator itr =
602 msg.extras().find(QLatin1String("po-flags"));
603 if (itr != msg.extras().end()) {
604 if (itr->split(QLatin1String(", ")).contains(QLatin1String("no-wrap")))
605 noWrap = true;
606 flags.append(*itr);
607 }
608 if (!flags.isEmpty())
609 out << "#, " << flags.join(QLatin1String(", ")) << '\n';
610
611 QString prefix = QLatin1String("#| ");
612 if (!msg.oldComment().isEmpty())
613 out << poEscapedString(prefix, QLatin1String("msgctxt"), noWrap, msg.oldComment());
614 if (!msg.oldSourceText().isEmpty())
615 out << poEscapedString(prefix, QLatin1String("msgid"), noWrap, msg.oldSourceText());
616 QString plural = msg.extra(QLatin1String("po-old_msgid_plural"));
617 if (!plural.isEmpty())
618 out << poEscapedString(prefix, QLatin1String("msgid_plural"), noWrap, plural);
619 prefix = QLatin1String((msg.type() == TranslatorMessage::Obsolete) ? "#~ " : "");
620 if (!msg.comment().isEmpty())
621 out << poEscapedString(prefix, QLatin1String("msgctxt"), noWrap, msg.comment());
622 out << poEscapedString(prefix, QLatin1String("msgid"), noWrap, msg.sourceText());
623 if (!msg.isPlural()) {
624 QString transl = msg.translation();
625 if (first) {
626 transl.remove(QRegExp(QLatin1String("\\bX-Language:[^\n]*\n")));
627 if (!translator.languageCode().isEmpty())
628 transl += QLatin1String("X-Language: ") + translator.languageCode() + QLatin1Char('\n');
629 }
630 out << poEscapedString(prefix, QLatin1String("msgstr"), noWrap, transl);
631 } else {
632 QString plural = msg.extra(QLatin1String("po-msgid_plural"));
633 if (plural.isEmpty())
634 plural = msg.sourceText();
635 out << poEscapedString(prefix, QLatin1String("msgid_plural"), noWrap, plural);
636 QStringList translations = translator.normalizedTranslations(msg, cd, &ok);
637 for (int i = 0; i != translations.size(); ++i) {
638 out << poEscapedString(prefix, QString::fromLatin1("msgstr[%1]").arg(i), noWrap,
639 translations.at(i));
640 }
641 }
642 first = false;
643 }
644 return ok;
645}
646
647int initPO()
648{
649 Translator::FileFormat format;
650 format.extension = QLatin1String("po");
651 format.description = QObject::tr("GNU Gettext localization files");
652 format.loader = &loadPO;
653 format.saver = &savePO;
654 format.fileType = Translator::FileFormat::TranslationSource;
655 format.priority = 1;
656 Translator::registerFileFormat(format);
657 return 1;
658}
659
660Q_CONSTRUCTOR_FUNCTION(initPO)
661
662QT_END_NAMESPACE
Note: See TracBrowser for help on using the repository browser.