From: Hamatoma Date: Sun, 8 Aug 2021 22:05:44 +0000 (+0200) Subject: i18n_text_parser works X-Git-Url: https://gitweb.hamatoma.de/?a=commitdiff_plain;h=093a93cb777d0702e379f5e71490d63718042778;p=exhibition.git i18n_text_parser works --- diff --git a/bin/i18n_text_parser.dart b/bin/i18n_text_parser.dart index bbf660d..c8e56c8 100644 --- a/bin/i18n_text_parser.dart +++ b/bin/i18n_text_parser.dart @@ -6,100 +6,206 @@ import "package:path/path.dart"; void main(List args) {} class I18nTextParser { + final globalModule = '!global'; + final errorModule = '!error'; final BaseLogger logger; - final modules = >{}; + final mapModules = >{}; + final mapPlurals = >{}; final foundFiles = {}; var regExpFiles = RegExp(r'.dart$'); String currentModule = ''; final moduleVariables = {}; String currentFile = ''; + String currentArguments = ''; List lines = []; int currentLineNo = 0; + int currentIxLines = 0; // ...........................1.............1.........2 final regExpModule = RegExp(r'''(\w+) = (I18N\(\)|i18N)\.module\(["'](.*?)['"]\);'''); final regExpDelimiter = RegExp(r'''["']'''); // ..........1.............1..2..............................2 - final regExpText = - RegExp(r'(i18N|I18N\(\))\.(tr|trPlural|trMulti|trWithArgs)\('); + final regExpText = RegExp(r'(i18N|I18N\(\))\.(tr|ntr|trMulti|trArgs)\('); final regExpStringConstant = RegExp(r'''^\s*(r?)(["'])(.*?)\2'''); final regExpVariable = RegExp(r'^\w+'); final regExpEmptyString = RegExp(r'^\s*$'); - I18nTextParser(this.logger); + RegExpMatch? currentMatch; - /// Gets the text (multiple lines) from the [arguments]. - /// [arguments] is the string behind the 'trMulti('. - void handleMultiText(String arguments) { - logger.error('not implemented: trMulti()'); - } + I18nTextParser(this.logger); - /// Gets the text (multiple lines) from the [arguments] . - /// [arguments] is the string behind the 'trPlural('. - void handlePluralText(String arguments) { - logger.error('not implemented: trPlural()'); + /// Handles the state if the [currentArguments] is empty: it will be updated + /// by the next line. + void handleEmptyArgument() { + if (regExpEmptyString.firstMatch(currentArguments) != null) { + currentArguments = currentIxLines >= lines.length + ? '' + : lines[currentIxLines++].trimLeft(); + } } - /// Gets the text (single line) from the [arguments] and store it. - /// [arguments] is the string behind the 'tr('. - void handleSimpleText(String arguments) { - RegExpMatch? match; - String module = '!global'; - int lineNo = currentLineNo; - if (regExpEmptyString.firstMatch(arguments) != null) { - arguments = lineNo >= lines.length ? '' : lines[lineNo++].trimLeft(); - } - String? key; - if ((match = regExpStringConstant.firstMatch(arguments)) != null) { - key = match?.group(3); - arguments = arguments.substring(match!.end).trimLeft(); - if (arguments.startsWith(',')) { - arguments = arguments.substring(1).trimLeft(); - if (arguments.isEmpty) { - arguments = lineNo >= lines.length ? '' : lines[lineNo++].trimLeft(); - } - if ((match = regExpStringConstant.firstMatch(arguments)) != null) { - module = match!.group(3)!; - } else if ((match = regExpVariable.firstMatch(arguments)) != null) { - final variable = match!.group(0); + String handleModuleArgument() { + String module = globalModule; + if (hasComma()) { + handleEmptyArgument(); + if (isStringConstant()) { + module = currentMatch!.group(3)!; + } else { + if ((currentMatch = regExpVariable.firstMatch(currentArguments)) != + null) { + final variable = currentMatch!.group(0); if (moduleVariables.containsKey(variable)) { module = moduleVariables[variable]!; } else { logger.error( '$currentFile-$currentLineNo: unknown module variable: $variable'); + module = errorModule; } + } + } + } + return module; + } + + /// Gets the text (multiple lines) from the [currentArguments]. + /// [currentArguments] is the string behind the 'trMulti('. + void handleMultiText() { + logger.error('not implemented: trMulti()'); + } + + /// Gets the text (multiple lines) from the [currentArguments] . + /// [currentArguments] is the string behind the 'trPlural('. + void handlePluralText() { + currentIxLines = currentLineNo; + handleEmptyArgument(); + if (!isStringConstant()) { + logger.error( + '$currentFile-$currentLineNo: cannot analyse parameters of ntr().' + ' Missing first string constant.'); + } else { + String key1 = currentMatch!.group(3)!; + if (!hasComma()) { + logger.error( + '$currentFile-$currentLineNo: cannot analyse parameters of ntr().' + ' Missing second string constant.'); + } else { + handleEmptyArgument(); + if (!isStringConstant()) { + logger.error( + '$currentFile-$currentLineNo: cannot analyse parameters of tr().' + ' Second argument is not a string constant.'); } else { - module = currentModule; + String key2 = currentMatch!.group(3)!; + String module = handleModuleArgument(); + putPluralKey(key1, key2, module); } } } - if (key == null) { + } + + /// Gets the text (single line) from the [currentArguments] and store it. + /// [currentArguments] is the string behind the 'tr('. + void handleSimpleText() { + currentIxLines = currentLineNo; + handleEmptyArgument(); + if (!isStringConstant()) { logger.error( - '$currentFile-$currentLineNo: cannot analyse parameters of tr()'); + '$currentFile-$currentLineNo: cannot analyse parameters of tr().' + ' Missing string constant.'); } else { - putText(key, currentFile, currentLineNo, module); + String module; + final key = currentMatch?.group(3); + module = handleModuleArgument(); + putText(key!, module); } } - /// Gets the text (with placeholders) from the [arguments] and store it. - /// [arguments] is the string behind the 'trArgs('. - void handleTextWithArgs(String? arguments) { - logger.error('not implemented: trArgs()'); + /// Gets the text (with placeholders) from the [currentArguments] and store it. + /// [currentArguments] is the string behind the 'trArgs('. + void handleTextWithArgs() { + currentIxLines = currentLineNo; + handleEmptyArgument(); + if (!isStringConstant()) { + logger.error( + '$currentFile-$currentLineNo: cannot analyse parameters of trArgs().' + ' Missing string constant.'); + } else { + String module; + final key = currentMatch?.group(3); + module = handleModuleArgument(); + putText(key!, module); + } + } + + /// Tests whether the [currentArguments] is preceded by a ','. + /// If true the comma is removed from the [currentArguments]. + bool hasComma() { + final rc = currentArguments.startsWith(','); + if (rc) { + currentArguments = currentArguments.substring(1).trimLeft(); + } + return rc; + } + + /// Tests whether the [currentArguments] is preceded by a string constant. + /// If true the [currentMatch] contains the string content and the + /// string constant is removed from the [currentArguments]. + bool isStringConstant() { + if ((currentMatch = regExpStringConstant.firstMatch(currentArguments)) != + null) { + currentArguments = + currentArguments.substring(currentMatch!.end).trimLeft(); + } + return currentMatch != null; + } + + /// Stores the data of a plural text entry. + void putPluralKey(String key1, String key2, String module) { + if (!mapPlurals.containsKey(module)) { + mapPlurals[module] = {}; + } + String reference = '#: $currentFile:$currentLineNo'; + if (module != errorModule && mapModules[module]?[key1] != null) { + logger.error('$currentFile-$currentLineNo: key already defined as normal ' + 'key. Modify key with the invisible tag <#1> or <#2> or ... at the ' + 'end to make it unique.\nFirst found in ' + + mapModules[module]![key1]!); + putText(key1, errorModule); + } else if (mapPlurals[module]!.containsKey(key1)) { + if (mapPlurals[module]![key1]!.keyPlural == key2) { + mapPlurals[module]![key1]!.references.add(reference); + } else { + logger.error('$currentFile-$currentLineNo: multiple definition of a ' + 'plural key but keyPlural is different. Modify key1 with the' + ' invisible tag <#1> or <#2> or ... at the end to make it ' + 'unique.\nFirst found in ' + + mapPlurals[module]![key1]!.references[0]); + } + } else { + mapPlurals[module]![key1] = PluralValue(key2, [reference]); + } } /// Stores the [key] in the [module] map with location info [filename] and /// [lineNo]. - void putText(String key, String filename, int lineNo, [String module = '']) { + void putText(String key, [String module = '']) { if (module.isEmpty) { module = currentModule; } - if (!modules.containsKey(module)) { - modules[module] = {}; + if (!mapModules.containsKey(module)) { + mapModules[module] = {}; } - if (modules[module]!.containsKey(key)) { - modules[module]![key] = modules[module]![key]! + '\n#: $filename:$lineNo'; + if (module != errorModule && mapPlurals[module]?[key] != null) { + logger.error('$currentFile-$currentLineNo: key already defined as plural ' + 'key. Modify key with the invisible tag <#1> or <#2> or ... at ' + 'the end to make it unique.\nFirst found in ' + + mapPlurals[module]![key]!.references[0]); + putText(key, errorModule); + } else if (mapModules[module]!.containsKey(key)) { + mapModules[module]![key] = + mapModules[module]![key]! + '\n#: $currentFile:$currentLineNo'; } else { - modules[module]![key] = '#: $filename:$lineNo'; + mapModules[module]![key] = '#: $currentFile:$currentLineNo'; } } @@ -135,6 +241,7 @@ class I18nTextParser { /// Scans the [file] for I18N texts. /// [directory]: used for logging. void scanFile(File file, String directory) { + logger.log('parsing ${file.path}', LEVEL_DETAIL); try { currentFile = join(directory, file.path); lines = file.readAsLinesSync(); @@ -148,19 +255,19 @@ class I18nTextParser { moduleVariables[match.group(1)!] = currentModule; } else { for (match in regExpText.allMatches(line)) { - final restLine = line.substring(match.end); + currentArguments = line.substring(match.end); switch (match.group(2)!) { case 'tr': - handleSimpleText(restLine); + handleSimpleText(); break; case 'trMulti': - handleMultiText(restLine); + handleMultiText(); break; - case 'trPlural': - handlePluralText(restLine); + case 'ntr': + handlePluralText(); break; case 'trArgs': - handleTextWithArgs(restLine); + handleTextWithArgs(); break; } } @@ -171,24 +278,75 @@ class I18nTextParser { } } + String convertString(String key) { + final rc = key.replaceAll('"', r'\"'); + return rc; + } + /// Writes the *.pot files to the [directory]. void writePot(String directory) { FileSync().ensureDirectory(directory); - for (var module in modules.keys) { - final map = modules[module]!; + final doneModules = {}; + for (var module in mapModules.keys) { + doneModules.add(module); + final map = mapModules[module]!; final keys = map.keys.toList(); + Map? map2; + if (mapPlurals.containsKey(module)) { + map2 = mapPlurals[module]; + keys.addAll(map2!.keys); + } keys.sort(); - final buffer = StringBuffer('# Texts of module $module, parsed by i18n_text_parser\n'); + final buffer = StringBuffer( + '# Texts of module $module, parsed by i18n_text_parser\n'); for (var key in keys) { - buffer.writeln(); - buffer.writeln(map[key]); - final key2 = key.replaceAll('"', r'\"'); + buffer.writeln(); + buffer.writeln(map.containsKey(key) + ? map[key] + : map2![key]!.references.join('\n')); + var key2 = convertString(key); buffer.writeln('msgid "$key2"'); - buffer.writeln('msgstr ""'); + if (map.containsKey(key)) { + buffer.writeln('msgstr ""'); + } else { + key2 = convertString(map2![key]!.keyPlural); + buffer.writeln('msgid_plural "$key2"'); + buffer.writeln('msgstr[0] ""'); + buffer.writeln('msgstr[1] ""'); + } } final fn = join(directory, '$module.pot'); logger.log('writing $fn', LEVEL_DETAIL); File(fn).writeAsStringSync(buffer.toString()); } + for (var module in mapPlurals.keys) { + if (!doneModules.contains(module)) { + final map = mapPlurals[module]; + final keys = map!.keys.toList(); + keys.sort(); + final buffer = StringBuffer( + '# Texts of module $module, parsed by i18n_text_parser\n'); + for (var key in keys) { + buffer.writeln(); + buffer.writeln(map[key]!.references.join('\n')); + var key2 = convertString(key); + buffer.writeln('msgid "$key2"'); + key2 = convertString(map[key]!.keyPlural); + buffer.writeln('msgid_plural "$key2"'); + buffer.writeln('msgstr[0] ""'); + buffer.writeln('msgstr[1] ""'); + } + final fn = join(directory, '$module.pot'); + logger.log('writing $fn', LEVEL_DETAIL); + File(fn).writeAsStringSync(buffer.toString()); + } + } } } + +/// Stores the data of a plural text. +class PluralValue { + final List references; + final String keyPlural; + PluralValue(this.keyPlural, this.references); +} diff --git a/lib/base/i18n.dart b/lib/base/i18n.dart index 0763460..b70b2f6 100644 --- a/lib/base/i18n.dart +++ b/lib/base/i18n.dart @@ -31,10 +31,11 @@ class I18N { } /// Translates the [key] into the local language using the namespace [module]. - /// [key] contains placeholders "{0}", "{1}"... which will be replaced by - /// the matching entry in the array [args]. - /// Returns the translation or (if not found) the key. - String trArgs(String key, List args, [String module = '!global']) { + /// [key] contains placeholders "{0}", "{1}"... that is replaced by + /// the corresponding entry in the array [args]. + /// Returns the translation or (if not found) the key with replaced + /// placeholders. + String trArgs(String key, String module, List args) { String rc = tr(key, module); for (var ix = 0; ix < args.length; ix++) { rc = rc.replaceAll('{$ix}', args[ix].toString()); @@ -47,8 +48,7 @@ class I18N { /// If [count] is 1 the translation of [keySingular] is returned. /// Otherwise the translation of [keyPlural] is returned. /// Returns the translation or (if not found) the key. - String trPlural(String keySingular, String keyPlural, int count, - [String module = '!global']) { + String ntr(String keySingular, String keyPlural, String module, int count) { String rc = tr(count == 1 ? keySingular : keyPlural, module); return rc; } diff --git a/test/i18n_text_parser_test.dart b/test/i18n_text_parser_test.dart index 3e854af..fcba747 100644 --- a/test/i18n_text_parser_test.dart +++ b/test/i18n_text_parser_test.dart @@ -14,12 +14,12 @@ void main() { final parser = I18nTextParser(logger); parser.scanDirectory(baseDir); parser.writePot(targetDir); - expect(parser.modules, isNotEmpty); - expect(parser.modules.containsKey('Example'), isTrue); - expect(parser.modules.containsKey('!global'), isTrue); - expect(parser.modules.containsKey('Sample'), isTrue); - expect(parser.modules['Example']!.containsKey('introduction'), isTrue); - expect(parser.modules['Sample']!.containsKey('info'), isTrue); + expect(parser.mapModules, isNotEmpty); + expect(parser.mapModules.containsKey('Example'), isTrue); + expect(parser.mapModules.containsKey('!global'), isTrue); + expect(parser.mapModules.containsKey('Sample'), isTrue); + expect(parser.mapModules['Example']!.containsKey('introduction'), isTrue); + expect(parser.mapModules['Sample']!.containsKey('info'), isTrue); var fn = join(targetDir, 'Example.pot'); var content = fileSync.fileAsString(fn); expect(content, '''# Texts of module Example, parsed by i18n_text_parser @@ -36,18 +36,38 @@ msgstr "" content = fileSync.fileAsString(fn); expect(content, '''# Texts of module Sample, parsed by i18n_text_parser -#: /tmp/unittest/i18n/lib/base/args.dart:5 +#: /tmp/unittest/i18n/lib/base/args.dart:8 +msgid "Name: {0} Role: {1}" +msgstr "" + +#: /tmp/unittest/i18n/lib/base/args.dart:8 msgid "info" msgstr "" + +#: /tmp/unittest/i18n/lib/base/args.dart:4 +msgid "one piece" +msgid_plural "%d pieces" +msgstr[0] "" +msgstr[1] "" '''); fn = join(targetDir, '!global.pot'); content = fileSync.fileAsString(fn); expect(content, '''# Texts of module !global, parsed by i18n_text_parser -#: /tmp/unittest/i18n/lib/base/simple.dart:10 -#: /tmp/unittest/i18n/lib/base/args.dart:9 +#: /tmp/unittest/i18n/lib/base/simple.dart:11 +#: /tmp/unittest/i18n/lib/base/args.dart:12 msgid "Status line" msgstr "" +'''); + fn = join(targetDir, 'Statistic.pot'); + content = fileSync.fileAsString(fn); + expect(content, '''# Texts of module Statistic, parsed by i18n_text_parser + +#: /tmp/unittest/i18n/lib/base/simple.dart:9 +msgid "one piece" +msgid_plural "%d pieces" +msgstr[0] "" +msgstr[1] "" '''); }); } @@ -66,6 +86,7 @@ String header(){ "description", M); } +String count() => i18N.ntr('one piece', '%d pieces', "Statistic", 3); String footer(){ return I18N().tr("Status line"); } @@ -73,8 +94,11 @@ String footer(){ fileSync.toFile(join(subDir, 'args.dart'), r'''import 'dart:io'; final i18N = I18N(); final M = i18N.module("Sample"); +String count() => i18N.ntr( + 'one piece', + '%d pieces', M, countItems('any')); String header(String user, String role){ - return i18N.trArgs('Name: {0} Role: {1}', [user, role], M) + " " + i18N.tr( + return i18N.trArgs('Name: {0} Role: {1}', M, [user, role]) + " " + i18N.tr( r'info', M); } String footer(){