From 912092646c2109fef8239bb8ea65263018eb635a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20L=20F=20S=20Bacci?= Date: Tue, 11 Jul 2023 15:14:58 -0300 Subject: [PATCH 01/50] First part of revcheck algorithm. --- README.md | 5 ++ scripts/translation/.gitignore | 2 + scripts/translation/README.md | 13 +++++ scripts/translation/TODO | 3 ++ scripts/translation/lib/RevcheckData.php | 18 +++++++ scripts/translation/lib/RevcheckFileInfo.php | 35 +++++++++++++ scripts/translation/lib/RevcheckFileList.php | 54 +++++++++++++++++++ scripts/translation/lib/RevcheckIgnore.php | 55 ++++++++++++++++++++ scripts/translation/lib/require_all.php | 6 +++ 9 files changed, 191 insertions(+) create mode 100644 scripts/translation/.gitignore create mode 100644 scripts/translation/README.md create mode 100644 scripts/translation/TODO create mode 100644 scripts/translation/lib/RevcheckData.php create mode 100644 scripts/translation/lib/RevcheckFileInfo.php create mode 100644 scripts/translation/lib/RevcheckFileList.php create mode 100644 scripts/translation/lib/RevcheckIgnore.php create mode 100644 scripts/translation/lib/require_all.php diff --git a/README.md b/README.md index e2f0d5925..a6af2b5cf 100644 --- a/README.md +++ b/README.md @@ -143,3 +143,8 @@ and find issues with it, they are located in the `scripts/qa/` directory. There might be some more just in `scripts/` but they need to be checked if they are still relevant and/or given some love. +# Translation Tools + +There are also various scripts to ensure the quality and synchrony of +documentation translations, located in the `scripts/translation/` directory. + diff --git a/scripts/translation/.gitignore b/scripts/translation/.gitignore new file mode 100644 index 000000000..08b61f3ee --- /dev/null +++ b/scripts/translation/.gitignore @@ -0,0 +1,2 @@ +# Persistent data shared between scripts +.cache/ diff --git a/scripts/translation/README.md b/scripts/translation/README.md new file mode 100644 index 000000000..ca8a4fd68 --- /dev/null +++ b/scripts/translation/README.md @@ -0,0 +1,13 @@ +Some useful scripts for maintaining translation consistency of manual + +Some of these scripts only test some file contents or XML structure +of translated files against their equivalents on `en/` directory. +Others will try modify the translations in place, changing the +translated files. Use with care. + +Not all translations are identical, or use the same conventions. +So not all scripts will be of use for all translations. The +assumptions of each script are described in each file. + +The `lib/` directory contains common code and functionality +across these scripts. diff --git a/scripts/translation/TODO b/scripts/translation/TODO new file mode 100644 index 000000000..cb72a9f73 --- /dev/null +++ b/scripts/translation/TODO @@ -0,0 +1,3 @@ + +- en/chmonly is ignored on revcheck, but it appears translatable + diff --git a/scripts/translation/lib/RevcheckData.php b/scripts/translation/lib/RevcheckData.php new file mode 100644 index 000000000..c8f161d16 --- /dev/null +++ b/scripts/translation/lib/RevcheckData.php @@ -0,0 +1,18 @@ +sourceFiles = new RevcheckFileList( $sourceDir ); + $this->targetFiles = new RevcheckFileList( $targetDir ); + } +} \ No newline at end of file diff --git a/scripts/translation/lib/RevcheckFileInfo.php b/scripts/translation/lib/RevcheckFileInfo.php new file mode 100644 index 000000000..353d0106d --- /dev/null +++ b/scripts/translation/lib/RevcheckFileInfo.php @@ -0,0 +1,35 @@ +path = $path; + $this->name = $name; + $this->size = $size; + $this->date = $date; + $this->key = ltrim( $path . '/' . $name , '/' ); + $this->syncStatus = RevcheckStatus::Untranslated; + } +} \ No newline at end of file diff --git a/scripts/translation/lib/RevcheckFileList.php b/scripts/translation/lib/RevcheckFileList.php new file mode 100644 index 000000000..75bd6a50f --- /dev/null +++ b/scripts/translation/lib/RevcheckFileList.php @@ -0,0 +1,54 @@ +loadTree( $lang ); + } + + function loadTree( $lang ) + { + $dir = new \DirectoryIterator( $lang ); + if ( $dir === false ) + die( "$lang is not a directory.\n" ); + $cwd = getcwd(); + chdir( $lang ); + $this->loadTreeRecurse( $lang , "" , $ret ); + chdir( $cwd ); + return $ret; + } + + function loadTreeRecurse( $lang , $path , & $output ) + { + $todoDirs = []; + $dir = new DirectoryIterator( $path == "" ? "." : $path ); + if ( $dir === false ) + die( "$path is not a directory.\n" ); + + foreach( $dir as $entry ) + { + $name = $entry->getFilename(); + if ( $name[0] == '.' ) + continue; + if ( $entry->isDir() ) + { + $todoDirs[] = ltrim( $path . '/' . $name , '/' ); + continue; + } + + $file = new RevcheckFileInfo( $path , $name , $entry->getSize() ); + if ( RevcheckIgnore::ignore( $file->key ) ) + continue; + $output[ $file->key ] = $file; + } + + sort( $todoDirs ); + foreach( $todoDirs as $path ) + $this->loadTreeRecurse( $lang , $path , $output ); + } +} \ No newline at end of file diff --git a/scripts/translation/lib/RevcheckIgnore.php b/scripts/translation/lib/RevcheckIgnore.php new file mode 100644 index 000000000..c0c07acda --- /dev/null +++ b/scripts/translation/lib/RevcheckIgnore.php @@ -0,0 +1,55 @@ + Date: Wed, 12 Jul 2023 21:48:48 -0300 Subject: [PATCH 02/50] Hash and timestamp details for source files. --- scripts/translation/lib/GitLogParser.php | 62 +++++++++++++++++++ scripts/translation/lib/RevcheckFileInfo.php | 23 +++---- scripts/translation/lib/RevcheckFileList.php | 23 ++++--- .../{RevcheckData.php => RevcheckInfo.php} | 6 +- scripts/translation/lib/require_all.php | 3 +- 5 files changed, 93 insertions(+), 24 deletions(-) create mode 100644 scripts/translation/lib/GitLogParser.php rename scripts/translation/lib/{RevcheckData.php => RevcheckInfo.php} (70%) diff --git a/scripts/translation/lib/GitLogParser.php b/scripts/translation/lib/GitLogParser.php new file mode 100644 index 000000000..6d9302632 --- /dev/null +++ b/scripts/translation/lib/GitLogParser.php @@ -0,0 +1,62 @@ + 0 ) + continue; + // empty lines + if ( trim( $line ) == "" ) + continue; + // commit message + if ( str_starts_with( $line , ' ' ) ) + { + // commits marked with `[skip-revcheck]` are skiped in full + if ( stristr( $line, '[skip-revcheck]' ) !== false ) + $skip = true; + continue; + } + // otherwise, a filename + $filename = trim( $line ); + + // newer file hash already parsed? + $info = $list->get( $filename ); + if ( $info == null || $info->hash != "" ) + continue; + + $info->date = $date; + $info->hash = $hash; + } + pclose( $fp ); + chdir( $cwd ); + } +} \ No newline at end of file diff --git a/scripts/translation/lib/RevcheckFileInfo.php b/scripts/translation/lib/RevcheckFileInfo.php index 353d0106d..e90f359d0 100644 --- a/scripts/translation/lib/RevcheckFileInfo.php +++ b/scripts/translation/lib/RevcheckFileInfo.php @@ -15,21 +15,18 @@ enum RevcheckStatus class RevcheckFileInfo { - public string $path; - public string $name; - public int $size; - public DateTime $date; - public string $key; + public string $file; // from fs + public int $size; // from fs + public int $date; // from vcs + public string $hash; // from vcs + public RevcheckStatus $status; - public RevcheckStatus $syncStatus; - - function __construct( string $path , string $name , int $size , $date = new DateTime('@0') ) + function __construct( string $file , int $size ) { - $this->path = $path; - $this->name = $name; + $this->file = $file; $this->size = $size; - $this->date = $date; - $this->key = ltrim( $path . '/' . $name , '/' ); - $this->syncStatus = RevcheckStatus::Untranslated; + $this->date = 0; + $this->hash = ""; + $this->status = RevcheckStatus::Untranslated; } } \ No newline at end of file diff --git a/scripts/translation/lib/RevcheckFileList.php b/scripts/translation/lib/RevcheckFileList.php index 75bd6a50f..c864c78e5 100644 --- a/scripts/translation/lib/RevcheckFileList.php +++ b/scripts/translation/lib/RevcheckFileList.php @@ -4,13 +4,18 @@ class RevcheckFileList { - private $list; + var $list = array(); function __construct( $lang ) { $this->loadTree( $lang ); } + function get( $file ): RevcheckFileInfo|null + { + return $this->list[ $file ] ?? null; + } + function loadTree( $lang ) { $dir = new \DirectoryIterator( $lang ); @@ -18,12 +23,11 @@ function loadTree( $lang ) die( "$lang is not a directory.\n" ); $cwd = getcwd(); chdir( $lang ); - $this->loadTreeRecurse( $lang , "" , $ret ); + $this->loadTreeRecurse( $lang , "" ); chdir( $cwd ); - return $ret; } - function loadTreeRecurse( $lang , $path , & $output ) + function loadTreeRecurse( $lang , $path ) { $todoDirs = []; $dir = new DirectoryIterator( $path == "" ? "." : $path ); @@ -33,22 +37,23 @@ function loadTreeRecurse( $lang , $path , & $output ) foreach( $dir as $entry ) { $name = $entry->getFilename(); + $key = ltrim( $path . '/' . $name , '/' ); if ( $name[0] == '.' ) continue; if ( $entry->isDir() ) { - $todoDirs[] = ltrim( $path . '/' . $name , '/' ); + $todoDirs[] = $key; continue; } - $file = new RevcheckFileInfo( $path , $name , $entry->getSize() ); - if ( RevcheckIgnore::ignore( $file->key ) ) + if ( RevcheckIgnore::ignore( $key ) ) continue; - $output[ $file->key ] = $file; + $file = new RevcheckFileInfo( $key , $entry->getSize() ); + $this->list[ $key ] = $file; } sort( $todoDirs ); foreach( $todoDirs as $path ) - $this->loadTreeRecurse( $lang , $path , $output ); + $this->loadTreeRecurse( $lang , $path ); } } \ No newline at end of file diff --git a/scripts/translation/lib/RevcheckData.php b/scripts/translation/lib/RevcheckInfo.php similarity index 70% rename from scripts/translation/lib/RevcheckData.php rename to scripts/translation/lib/RevcheckInfo.php index c8f161d16..914fa69e8 100644 --- a/scripts/translation/lib/RevcheckData.php +++ b/scripts/translation/lib/RevcheckInfo.php @@ -5,14 +5,18 @@ /** * Calculate translation sync/diff status from two directories */ -class RevcheckData +class RevcheckInfo { public RevcheckFileList $sourceFiles; public RevcheckFileList $targetFiles; function __construct( $sourceDir , $targetDir ) { + // Load file tree $this->sourceFiles = new RevcheckFileList( $sourceDir ); $this->targetFiles = new RevcheckFileList( $targetDir ); + + // Source translation files get hashes from VCS + GitLogParser::parseInto( $sourceDir , $this->sourceFiles ); } } \ No newline at end of file diff --git a/scripts/translation/lib/require_all.php b/scripts/translation/lib/require_all.php index 85f665070..97a8d1fd5 100644 --- a/scripts/translation/lib/require_all.php +++ b/scripts/translation/lib/require_all.php @@ -1,6 +1,7 @@ Date: Thu, 13 Jul 2023 13:47:37 -0300 Subject: [PATCH 03/50] Revtag parser. --- scripts/translation/lib/GitLogParser.php | 2 + scripts/translation/lib/RevcheckFileInfo.php | 10 ++- scripts/translation/lib/RevcheckFileList.php | 2 +- scripts/translation/lib/RevcheckInfo.php | 7 +- scripts/translation/lib/RevtagParser.php | 90 +++++++++++++++++++ scripts/translation/lib/XmlUtil.php | 34 +++++++ .../lib/{require_all.php => require.php} | 2 + 7 files changed, 140 insertions(+), 7 deletions(-) create mode 100644 scripts/translation/lib/RevtagParser.php create mode 100644 scripts/translation/lib/XmlUtil.php rename scripts/translation/lib/{require_all.php => require.php} (74%) diff --git a/scripts/translation/lib/GitLogParser.php b/scripts/translation/lib/GitLogParser.php index 6d9302632..a7120a57b 100644 --- a/scripts/translation/lib/GitLogParser.php +++ b/scripts/translation/lib/GitLogParser.php @@ -1,5 +1,7 @@ sourceFiles = new RevcheckFileList( $sourceDir ); $this->targetFiles = new RevcheckFileList( $targetDir ); - // Source translation files get hashes from VCS + // Source files get hashes from VCS GitLogParser::parseInto( $sourceDir , $this->sourceFiles ); + + // Translated files get info from revtags + RevtagParser::parseInto( $targetDir , $this->targetFiles ); } } \ No newline at end of file diff --git a/scripts/translation/lib/RevtagParser.php b/scripts/translation/lib/RevtagParser.php new file mode 100644 index 000000000..ab288bb56 --- /dev/null +++ b/scripts/translation/lib/RevtagParser.php @@ -0,0 +1,90 @@ +list as $entry ) + { + $tag = RevtagParser::parseFile( $lang . '/' . $entry->file ); + $entry->revtag = $tag; + } + } + + public static function parseFile( string $filename ): RevtagInfo|null + { + $doc = XmlUtil::loadFile( $filename ); + $ret = new RevtagInfo; + RevtagParser::parseNodeRecurse( $doc , $ret , $filename ); + return $ret; + } + + public static function parseText( string $contents ): RevtagInfo|null + { + $doc = XmlUtil::loadText( $contents ); + $ret = new RevtagInfo; + RevtagParser::parseNodeRecurse( $doc , $ret ); + return $ret; + } + + public static function parseNodeRecurse( DOMNode $node , RevtagInfo & $ret , $filename = "" ) + { + if ( $node->nodeType == XML_COMMENT_NODE ) + RevtagParser::parseComment( $node , $ret , $filename ); + + foreach( $node->childNodes as $child ) + RevtagParser::parseNodeRecurse( $child , $ret , $filename ); + } + + public static function parseComment( DOMNode $node , RevtagInfo & $ret , $filename = "" ) + { + $text = trim( $node->textContent ); + + if ( str_starts_with( $text , "EN-" ) ) + { + $match = array(); + $regex = "/EN-Revision:\s*(\S+)\s*Maintainer:\s*(\S+)\s*Status:\s*(\S+)/"; + if ( preg_match( $regex , $text , $match ) ) + { + $ret->revision = $match[1]; + $ret->maintainer = $match[2]; + $ret->status = $match[3]; + + if ( $ret->revision != "" && strlen( $ret->revision ) != 40 ) + $ret->errors .= "Wrong hash size: {$ret->revision}\n"; + if ( $ret->maintainer == "" ) + $ret->errors .= "Empty maintainer.\n"; + if ( $ret->status != "" && strlen( $ret->revision ) != 40 ) + $ret->errors .= "Empty status.\n"; + } + } + + if ( str_starts_with( $text , "CRED" ) ) + { + $match = array(); + $regex = "/CREDITS:\s*(\S+)/"; + if ( preg_match( $regex , $text , $match ) ) + { + $ret->credits = $match[1]; + + if ( $ret->credits != "" && strlen( $ret->revision ) != 40 ) + $ret->errors .= "Empty credits.\n"; + } + } + } +} \ No newline at end of file diff --git a/scripts/translation/lib/XmlUtil.php b/scripts/translation/lib/XmlUtil.php new file mode 100644 index 000000000..2280de403 --- /dev/null +++ b/scripts/translation/lib/XmlUtil.php @@ -0,0 +1,34 @@ +preserveWhiteSpace = true; + $doc->recover = true; + $doc->resolveExternals = false; + $doc->substituteEntities = false; + + $doc->loadXML( $contents ); + + libxml_clear_errors(); + libxml_use_internal_errors( $was ); + + return $doc; + } +} \ No newline at end of file diff --git a/scripts/translation/lib/require_all.php b/scripts/translation/lib/require.php similarity index 74% rename from scripts/translation/lib/require_all.php rename to scripts/translation/lib/require.php index 97a8d1fd5..c8ec443cf 100644 --- a/scripts/translation/lib/require_all.php +++ b/scripts/translation/lib/require.php @@ -5,3 +5,5 @@ require_once __DIR__ . '/RevcheckFileInfo.php'; require_once __DIR__ . '/RevcheckFileList.php'; require_once __DIR__ . '/RevcheckIgnore.php'; +require_once __DIR__ . '/RevtagParser.php'; +require_once __DIR__ . '/XmlUtil.php'; \ No newline at end of file From 7b74f59752a33d5182d668393e32fced7e971b9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20L=20F=20S=20Bacci?= Date: Fri, 14 Jul 2023 14:11:34 -0300 Subject: [PATCH 04/50] Rename for future use. --- scripts/translation/lib/{RevcheckInfo.php => RevcheckRun.php} | 2 +- scripts/translation/lib/require.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename scripts/translation/lib/{RevcheckInfo.php => RevcheckRun.php} (97%) diff --git a/scripts/translation/lib/RevcheckInfo.php b/scripts/translation/lib/RevcheckRun.php similarity index 97% rename from scripts/translation/lib/RevcheckInfo.php rename to scripts/translation/lib/RevcheckRun.php index f5aed1b13..0554cd4ca 100644 --- a/scripts/translation/lib/RevcheckInfo.php +++ b/scripts/translation/lib/RevcheckRun.php @@ -5,7 +5,7 @@ /** * Calculate translation sync/diff status from two directories */ -class RevcheckInfo +class RevcheckRun { public RevcheckFileList $sourceFiles; public RevcheckFileList $targetFiles; diff --git a/scripts/translation/lib/require.php b/scripts/translation/lib/require.php index c8ec443cf..7bb80b4ce 100644 --- a/scripts/translation/lib/require.php +++ b/scripts/translation/lib/require.php @@ -1,9 +1,9 @@ Date: Fri, 14 Jul 2023 17:39:39 -0300 Subject: [PATCH 05/50] Implementation of revcheck algorithm. --- scripts/translation/README.md | 5 +- scripts/translation/TODO | 3 - scripts/translation/lib/GitLogParser.php | 2 +- scripts/translation/lib/RevcheckFileInfo.php | 21 ++-- scripts/translation/lib/RevcheckRun.php | 100 ++++++++++++++++++- scripts/translation/lib/RevtagParser.php | 14 +-- scripts/translation/lib/XmlUtil.php | 2 +- scripts/translation/lib/require.php | 1 + 8 files changed, 122 insertions(+), 26 deletions(-) delete mode 100644 scripts/translation/TODO diff --git a/scripts/translation/README.md b/scripts/translation/README.md index ca8a4fd68..796435afa 100644 --- a/scripts/translation/README.md +++ b/scripts/translation/README.md @@ -1,4 +1,4 @@ -Some useful scripts for maintaining translation consistency of manual +# Some useful scripts for maintaining translation consistency of manual Some of these scripts only test some file contents or XML structure of translated files against their equivalents on `en/` directory. @@ -11,3 +11,6 @@ assumptions of each script are described in each file. The `lib/` directory contains common code and functionality across these scripts. + +See `MIGRATION.md` for procedures and breaking changes expected +when migrating from previous `revcheck.php`. diff --git a/scripts/translation/TODO b/scripts/translation/TODO deleted file mode 100644 index cb72a9f73..000000000 --- a/scripts/translation/TODO +++ /dev/null @@ -1,3 +0,0 @@ - -- en/chmonly is ignored on revcheck, but it appears translatable - diff --git a/scripts/translation/lib/GitLogParser.php b/scripts/translation/lib/GitLogParser.php index a7120a57b..16ae6eea6 100644 --- a/scripts/translation/lib/GitLogParser.php +++ b/scripts/translation/lib/GitLogParser.php @@ -3,7 +3,7 @@ require_once __DIR__ . '/require.php'; /** - * Parse git log to complement file state + * Parse `git log` to complement file state */ class GitLogParser diff --git a/scripts/translation/lib/RevcheckFileInfo.php b/scripts/translation/lib/RevcheckFileInfo.php index d4f55ff36..39b90e659 100644 --- a/scripts/translation/lib/RevcheckFileInfo.php +++ b/scripts/translation/lib/RevcheckFileInfo.php @@ -2,15 +2,15 @@ require_once __DIR__ . '/require.php'; -enum RevcheckStatus +enum RevcheckStatus :string { - case Untranslated ;//= 'Untranslated'; - case RevTagProblem ;//= 'RevTagProblem'; - case TranslatedWip ;//= 'TranslatedWip'; - case TranslatedOk ;//= 'TranslatedOk'; - case TranslatedOld ;//= 'TranslatedOld'; - case TranslatedCritial ;//= 'TranslatedCritial'; - case NotInEnTree ;//= 'NotInEnTree'; + case Untranslated = 'Untranslated'; + case RevTagProblem = 'RevTagProblem'; + case TranslatedWip = 'TranslatedWip'; + case TranslatedOk = 'TranslatedOk'; + case TranslatedOld = 'TranslatedOld'; + case TranslatedCritial = 'TranslatedCritial'; + case NotInEnTree = 'NotInEnTree'; } class RevcheckFileInfo @@ -20,8 +20,8 @@ class RevcheckFileInfo public int $date; // from vcs, source only public string $hash; // from vcs, source only - public RevcheckStatus $status; // target only - public RevtagInfo $revtag; // target only + public RevcheckStatus $status; // target only + public RevtagInfo|null $revtag; // target only function __construct( string $file , int $size ) { @@ -30,5 +30,6 @@ function __construct( string $file , int $size ) $this->date = 0; $this->hash = ""; $this->status = RevcheckStatus::Untranslated; + $this->revtag = null; } } \ No newline at end of file diff --git a/scripts/translation/lib/RevcheckRun.php b/scripts/translation/lib/RevcheckRun.php index 0554cd4ca..dc0e8be27 100644 --- a/scripts/translation/lib/RevcheckRun.php +++ b/scripts/translation/lib/RevcheckRun.php @@ -7,19 +7,113 @@ */ class RevcheckRun { + public string $sourceDir; + public string $targetDir; public RevcheckFileList $sourceFiles; public RevcheckFileList $targetFiles; + // Separated lists + public array $filesOk; + public array $filesOld; + public array $filesCritical; + public array $filesRevtagProblem; + public array $filesUntranslated; + public array $filesNotInEn; + public array $filesWip; + function __construct( $sourceDir , $targetDir ) { - // Load file tree + $this->sourceDir = $sourceDir; + $this->targetDir = $targetDir; + + // load respective file tree $this->sourceFiles = new RevcheckFileList( $sourceDir ); $this->targetFiles = new RevcheckFileList( $targetDir ); - // Source files get hashes from VCS + // original files get info from version control GitLogParser::parseInto( $sourceDir , $this->sourceFiles ); - // Translated files get info from revtags + // translated files get info from file contents RevtagParser::parseInto( $targetDir , $this->targetFiles ); + + $this->calculateStatus(); + } + + private function calculateStatus() + { + // All status are marked in source files, + // except notinen, that are marked on target. + + foreach( $this->sourceFiles->list as $source ) + { + $target = $this->targetFiles->get( $source->file ); + $monthAgo = strtotime( '-1 month' ); + + // Untranslated + + if ( $target == null ) + { + $source->status = RevcheckStatus::Untranslated; + $this->filesUntranslated[] = $source; + continue; + } + + // RevTagProblem + + if ( $target->revtag == null || strlen( $target->revtag->revision ) != 40 ) + { + $source->status = RevcheckStatus::RevTagProblem; + $this->filesRevtagProblem[] = $source; + continue; + } + + $target->hash = $target->revtag->revision; + + // TranslatedOk + + if ( $source->hash == $target->hash && $target->revtag->status == "ready" ) + { + $source->status = RevcheckStatus::TranslatedOk; + $this->filesOk[] = $source; + continue; + } + + GitDiffParser::parseNumstatInto( $this->sourceDir , $source ); + + // TranslatedWip + + if ( $target->revtag->status != "ready" ) + { + $source->status = RevcheckStatus::TranslatedWip; + $this->filesWip[] = $source; + continue; + } + + // TranslatedOld + // TranslatedCritial + + if ( $source->date > $monthAgo ) + { + $source->status = RevcheckStatus::TranslatedOld; + $this->filesOld[] = $source; + } + else + { + $source->status = RevcheckStatus::TranslatedCritial; + $this->filesCritical[] = $source; + } + } + + // NotInEnTree + + foreach( $this->targetFiles->list as $target ) + { + $sourcePath = $this->sourceDir . '/' . $target->file; + if( is_file( $sourcePath ) == false ) + { + $target->status = RevcheckStatus::NotInEnTree; + $this->filesNotInEn[] = $target; + } + } } } \ No newline at end of file diff --git a/scripts/translation/lib/RevtagParser.php b/scripts/translation/lib/RevtagParser.php index ab288bb56..040a9848b 100644 --- a/scripts/translation/lib/RevtagParser.php +++ b/scripts/translation/lib/RevtagParser.php @@ -20,10 +20,7 @@ class RevtagParser static function parseInto( string $lang , RevcheckFileList & $list ) { foreach( $list->list as $entry ) - { - $tag = RevtagParser::parseFile( $lang . '/' . $entry->file ); - $entry->revtag = $tag; - } + $entry->revtag = RevtagParser::parseFile( $lang . '/' . $entry->file ); } public static function parseFile( string $filename ): RevtagInfo|null @@ -57,13 +54,16 @@ public static function parseComment( DOMNode $node , RevtagInfo & $ret , $filena if ( str_starts_with( $text , "EN-" ) ) { + // /EN-Revision:\s*(\S+)\s*Maintainer:\s*(\S+)\s*Status:\s*(\S+)/ // restric maintainer with no spaces + // /EN-Revision:\s*(\S+)\s*Maintainer:\s(.*?)\sStatus:\s*(\S+)/ // accepts maintainer with spaces + $match = array(); $regex = "/EN-Revision:\s*(\S+)\s*Maintainer:\s*(\S+)\s*Status:\s*(\S+)/"; if ( preg_match( $regex , $text , $match ) ) { - $ret->revision = $match[1]; - $ret->maintainer = $match[2]; - $ret->status = $match[3]; + $ret->revision = trim( $match[1] ); + $ret->maintainer = trim( $match[2] ); + $ret->status = trim( $match[3] ); if ( $ret->revision != "" && strlen( $ret->revision ) != 40 ) $ret->errors .= "Wrong hash size: {$ret->revision}\n"; diff --git a/scripts/translation/lib/XmlUtil.php b/scripts/translation/lib/XmlUtil.php index 2280de403..80214a215 100644 --- a/scripts/translation/lib/XmlUtil.php +++ b/scripts/translation/lib/XmlUtil.php @@ -16,7 +16,7 @@ public static function loadFile( $filename ):DOMDocument public static function loadText( $contents ):DOMDocument { - $was = libxml_use_internal_errors( true ); // no entity warning + $was = libxml_use_internal_errors( true ); // do not print warnings $doc = new DOMDocument(); $doc->preserveWhiteSpace = true; diff --git a/scripts/translation/lib/require.php b/scripts/translation/lib/require.php index 7bb80b4ce..fe87bab8b 100644 --- a/scripts/translation/lib/require.php +++ b/scripts/translation/lib/require.php @@ -1,5 +1,6 @@ Date: Sun, 16 Jul 2023 18:06:50 -0300 Subject: [PATCH 06/50] Export some calculated status to easily use on other tools. --- scripts/translation/MIGRATION.md | 17 ++++++++ scripts/translation/lib/CacheUtil.php | 38 +++++++++++++++++ scripts/translation/lib/GitDiffParser.php | 13 ++++++ scripts/translation/lib/QaFileInfo.php | 44 ++++++++++++++++++++ scripts/translation/lib/RevcheckFileInfo.php | 9 ++-- scripts/translation/lib/RevcheckRun.php | 23 ++++------ scripts/translation/lib/require.php | 2 + 7 files changed, 127 insertions(+), 19 deletions(-) create mode 100644 scripts/translation/MIGRATION.md create mode 100644 scripts/translation/lib/CacheUtil.php create mode 100644 scripts/translation/lib/GitDiffParser.php create mode 100644 scripts/translation/lib/QaFileInfo.php diff --git a/scripts/translation/MIGRATION.md b/scripts/translation/MIGRATION.md new file mode 100644 index 000000000..937ce5f48 --- /dev/null +++ b/scripts/translation/MIGRATION.md @@ -0,0 +1,17 @@ + +# Migration + +## Old vs critical + +This revcheck will now emit critical status for files outdated for more than 30 days. + +## Maintainers with spaces + +The regex on `RevtagParser` was narrowed to not accept maintainer's names +with spaces. This need to be confirmed on all active translations, or +the regex modified to accept spaces again. + +## en/chmonly + +`en/chmonly` is ignored on revcheck, but it appears translatable. If it's a +`en/` only directory, this should be uncommented on RevcheckIgnore. diff --git a/scripts/translation/lib/CacheUtil.php b/scripts/translation/lib/CacheUtil.php new file mode 100644 index 000000000..7e3bc80c7 --- /dev/null +++ b/scripts/translation/lib/CacheUtil.php @@ -0,0 +1,38 @@ +sourceHash = $sourceHash; + $this->targetHash = $targetHash; + $this->file = $file; + $this->days = $days; + } + + public static function cacheLoad() :array + { + return CacheUtil::load( "" , "QaFileInfo.phps" , $itens ); + } + + public static function cacheSave( array $itens ) + { + // PHP serialize() + CacheUtil::save( "" , "QaFileInfo.phps" , $itens ); + + // CSV + $filename = CacheUtil::prepareFilename( "" , "QaFileInfo.csv" , true ); + $fp = fopen( $filename , 'w' ); + foreach( $itens as $item ) + { + $line = array( $item->sourceHash , $item->targetHash , $item->file , $item->days ); + fputcsv( $fp , $line ); + } + fclose($fp); + } +} \ No newline at end of file diff --git a/scripts/translation/lib/RevcheckFileInfo.php b/scripts/translation/lib/RevcheckFileInfo.php index 39b90e659..c61b4a00b 100644 --- a/scripts/translation/lib/RevcheckFileInfo.php +++ b/scripts/translation/lib/RevcheckFileInfo.php @@ -6,10 +6,9 @@ enum RevcheckStatus :string { case Untranslated = 'Untranslated'; case RevTagProblem = 'RevTagProblem'; - case TranslatedWip = 'TranslatedWip'; case TranslatedOk = 'TranslatedOk'; case TranslatedOld = 'TranslatedOld'; - case TranslatedCritial = 'TranslatedCritial'; + case TranslatedWip = 'TranslatedWip'; case NotInEnTree = 'NotInEnTree'; } @@ -17,8 +16,9 @@ class RevcheckFileInfo { public string $file; // from fs public int $size; // from fs - public int $date; // from vcs, source only public string $hash; // from vcs, source only + public int $date; // from vcs, source only + public int $days; // derived public RevcheckStatus $status; // target only public RevtagInfo|null $revtag; // target only @@ -27,8 +27,9 @@ function __construct( string $file , int $size ) { $this->file = $file; $this->size = $size; - $this->date = 0; $this->hash = ""; + $this->date = 0; + $this->days = 0; $this->status = RevcheckStatus::Untranslated; $this->revtag = null; } diff --git a/scripts/translation/lib/RevcheckRun.php b/scripts/translation/lib/RevcheckRun.php index dc0e8be27..2cdb57c3f 100644 --- a/scripts/translation/lib/RevcheckRun.php +++ b/scripts/translation/lib/RevcheckRun.php @@ -15,11 +15,11 @@ class RevcheckRun // Separated lists public array $filesOk; public array $filesOld; - public array $filesCritical; public array $filesRevtagProblem; public array $filesUntranslated; public array $filesNotInEn; public array $filesWip; + public array $qaList; function __construct( $sourceDir , $targetDir ) { @@ -47,7 +47,6 @@ private function calculateStatus() foreach( $this->sourceFiles->list as $source ) { $target = $this->targetFiles->get( $source->file ); - $monthAgo = strtotime( '-1 month' ); // Untranslated @@ -68,6 +67,8 @@ private function calculateStatus() } $target->hash = $target->revtag->revision; + $daysOld = ( strtotime( "now" ) - $source->date ) / 86400; + $this->qaList[] = new QaFileInfo( $source->hash , $target->hash , $source->file , $daysOld ); // TranslatedOk @@ -90,26 +91,18 @@ private function calculateStatus() } // TranslatedOld - // TranslatedCritial - if ( $source->date > $monthAgo ) - { - $source->status = RevcheckStatus::TranslatedOld; - $this->filesOld[] = $source; - } - else - { - $source->status = RevcheckStatus::TranslatedCritial; - $this->filesCritical[] = $source; - } + $source->days = $daysOld; + $source->status = RevcheckStatus::TranslatedOld; + $this->filesOld[] = $source; } // NotInEnTree foreach( $this->targetFiles->list as $target ) { - $sourcePath = $this->sourceDir . '/' . $target->file; - if( is_file( $sourcePath ) == false ) + $source = $this->sourceFiles->get( $target->file ); + if ( $source == null ) { $target->status = RevcheckStatus::NotInEnTree; $this->filesNotInEn[] = $target; diff --git a/scripts/translation/lib/require.php b/scripts/translation/lib/require.php index fe87bab8b..d0e582f11 100644 --- a/scripts/translation/lib/require.php +++ b/scripts/translation/lib/require.php @@ -1,7 +1,9 @@ Date: Mon, 17 Jul 2023 13:22:03 -0300 Subject: [PATCH 07/50] qaxml.a.check.php, tool to check tag-attribute-values. --- scripts/translation/MIGRATION.md | 17 ------- scripts/translation/README.md | 22 +++++++++ scripts/translation/lib/QaFileInfo.php | 10 ++-- scripts/translation/lib/RevcheckRun.php | 8 ++- scripts/translation/lib/RevtagParser.php | 4 +- scripts/translation/lib/XmlUtil.php | 15 ++++++ scripts/translation/qaxml.a.check.php | 62 ++++++++++++++++++++++++ scripts/translation/qaxml.e.check.php | 5 ++ scripts/translation/qaxml.p.check.php | 5 ++ scripts/translation/qaxml.t.check.php | 5 ++ 10 files changed, 129 insertions(+), 24 deletions(-) delete mode 100644 scripts/translation/MIGRATION.md create mode 100644 scripts/translation/qaxml.a.check.php create mode 100644 scripts/translation/qaxml.e.check.php create mode 100644 scripts/translation/qaxml.p.check.php create mode 100644 scripts/translation/qaxml.t.check.php diff --git a/scripts/translation/MIGRATION.md b/scripts/translation/MIGRATION.md deleted file mode 100644 index 937ce5f48..000000000 --- a/scripts/translation/MIGRATION.md +++ /dev/null @@ -1,17 +0,0 @@ - -# Migration - -## Old vs critical - -This revcheck will now emit critical status for files outdated for more than 30 days. - -## Maintainers with spaces - -The regex on `RevtagParser` was narrowed to not accept maintainer's names -with spaces. This need to be confirmed on all active translations, or -the regex modified to accept spaces again. - -## en/chmonly - -`en/chmonly` is ignored on revcheck, but it appears translatable. If it's a -`en/` only directory, this should be uncommented on RevcheckIgnore. diff --git a/scripts/translation/README.md b/scripts/translation/README.md index 796435afa..08fbfc96c 100644 --- a/scripts/translation/README.md +++ b/scripts/translation/README.md @@ -14,3 +14,25 @@ across these scripts. See `MIGRATION.md` for procedures and breaking changes expected when migrating from previous `revcheck.php`. + +# Migration + +## No critical + +This revcheck will not emit a critical status, for files outdated for more +than 30 days, as this was silently removed on + + +There is a new `days` property that tracks how old a file is in days, to make +decisions on coloring revcheck or deprecating translations. + +## Maintainers with spaces + +The regex on `RevtagParser` was narrowed to not accept maintainer's names +with spaces. This need to be confirmed on all active translations, or +the regex modified to accept spaces again. + +## en/chmonly + +`en/chmonly` is ignored on revcheck, but it appears translatable. If it's a +`en/` only directory, this should be uncommented on RevcheckIgnore. diff --git a/scripts/translation/lib/QaFileInfo.php b/scripts/translation/lib/QaFileInfo.php index 036f0ee05..5c177b8ef 100644 --- a/scripts/translation/lib/QaFileInfo.php +++ b/scripts/translation/lib/QaFileInfo.php @@ -10,20 +10,24 @@ class QaFileInfo { public string $sourceHash; public string $targetHash; + public string $sourceDir; + public string $targetDir; public string $file; public int $days; - function __construct( string $sourceHash , string $targetHash , string $file , int $days ) + function __construct( string $sourceHash , string $targetHash , string $sourceDir , string $targetDir , string $file , int $days ) { $this->sourceHash = $sourceHash; $this->targetHash = $targetHash; + $this->sourceDir = $sourceDir; + $this->targetDir = $targetDir; $this->file = $file; $this->days = $days; } public static function cacheLoad() :array { - return CacheUtil::load( "" , "QaFileInfo.phps" , $itens ); + return CacheUtil::load( "" , "QaFileInfo.phps" ); } public static function cacheSave( array $itens ) @@ -36,7 +40,7 @@ public static function cacheSave( array $itens ) $fp = fopen( $filename , 'w' ); foreach( $itens as $item ) { - $line = array( $item->sourceHash , $item->targetHash , $item->file , $item->days ); + $line = array( $item->sourceHash , $item->targetHash , $item->sourceDir , $item->targetDir , $item->file , $item->days ); fputcsv( $fp , $line ); } fclose($fp); diff --git a/scripts/translation/lib/RevcheckRun.php b/scripts/translation/lib/RevcheckRun.php index 2cdb57c3f..8bd4a0cf3 100644 --- a/scripts/translation/lib/RevcheckRun.php +++ b/scripts/translation/lib/RevcheckRun.php @@ -21,7 +21,7 @@ class RevcheckRun public array $filesWip; public array $qaList; - function __construct( $sourceDir , $targetDir ) + function __construct( string $sourceDir , string $targetDir , bool $writeResults = true ) { $this->sourceDir = $sourceDir; $this->targetDir = $targetDir; @@ -36,7 +36,11 @@ function __construct( $sourceDir , $targetDir ) // translated files get info from file contents RevtagParser::parseInto( $targetDir , $this->targetFiles ); + // match and mix $this->calculateStatus(); + + if ( $writeResults ) + QaFileInfo::cacheSave( $this->qaList ); } private function calculateStatus() @@ -68,7 +72,7 @@ private function calculateStatus() $target->hash = $target->revtag->revision; $daysOld = ( strtotime( "now" ) - $source->date ) / 86400; - $this->qaList[] = new QaFileInfo( $source->hash , $target->hash , $source->file , $daysOld ); + $this->qaList[] = new QaFileInfo( $source->hash , $target->hash , $this->sourceDir , $this->targetDir , $source->file , $daysOld ); // TranslatedOk diff --git a/scripts/translation/lib/RevtagParser.php b/scripts/translation/lib/RevtagParser.php index 040a9848b..bc3bc2644 100644 --- a/scripts/translation/lib/RevtagParser.php +++ b/scripts/translation/lib/RevtagParser.php @@ -54,7 +54,7 @@ public static function parseComment( DOMNode $node , RevtagInfo & $ret , $filena if ( str_starts_with( $text , "EN-" ) ) { - // /EN-Revision:\s*(\S+)\s*Maintainer:\s*(\S+)\s*Status:\s*(\S+)/ // restric maintainer with no spaces + // /EN-Revision:\s*(\S+)\s*Maintainer:\s*(\S+)\s*Status:\s*(\S+)/ // restric maintainer without spaces // /EN-Revision:\s*(\S+)\s*Maintainer:\s(.*?)\sStatus:\s*(\S+)/ // accepts maintainer with spaces $match = array(); @@ -66,7 +66,7 @@ public static function parseComment( DOMNode $node , RevtagInfo & $ret , $filena $ret->status = trim( $match[3] ); if ( $ret->revision != "" && strlen( $ret->revision ) != 40 ) - $ret->errors .= "Wrong hash size: {$ret->revision}\n"; + $ret->errors .= "Wrong hash format or size: {$ret->revision}\n"; if ( $ret->maintainer == "" ) $ret->errors .= "Empty maintainer.\n"; if ( $ret->status != "" && strlen( $ret->revision ) != 40 ) diff --git a/scripts/translation/lib/XmlUtil.php b/scripts/translation/lib/XmlUtil.php index 80214a215..e22144934 100644 --- a/scripts/translation/lib/XmlUtil.php +++ b/scripts/translation/lib/XmlUtil.php @@ -8,6 +8,21 @@ class XmlUtil { + public static function listNodeType( DOMNode $node , int $type ) + { + $ret = array(); + XmlUtil::listNodeTypeRecurse( $node , $type , $ret ); + return $ret; + } + + public static function listNodeTypeRecurse( DOMNode $node , int $type, array & $ret ) + { + if ( $node->nodeType == $type ) + $ret[] = $node; + foreach( $node->childNodes as $child ) + XmlUtil::listNodeTypeRecurse( $child , $type, $ret ); + } + public static function loadFile( $filename ):DOMDocument { $contents = file_get_contents( $filename ); diff --git a/scripts/translation/qaxml.a.check.php b/scripts/translation/qaxml.a.check.php new file mode 100644 index 000000000..79ffdbf39 --- /dev/null +++ b/scripts/translation/qaxml.a.check.php @@ -0,0 +1,62 @@ +sourceHash != $qafile->targetHash ) + continue; + + $source = $qafile->sourceDir . '/' . $qafile->file; + $target = $qafile->targetDir . '/' . $qafile->file; + + $s = XmlUtil::loadFile( $source ); + $t = XmlUtil::loadFile( $target ); + + $s = XmlUtil::listNodeType( $s , XML_ELEMENT_NODE ); + $t = XmlUtil::listNodeType( $t , XML_ELEMENT_NODE ); + + $s = extractTriple( $s ); + $t = extractTriple( $t ); + + if ( implode( "\n" , $s ) == implode( "\n" , $t ) ) + continue; + + $intersect = array_intersect( $s, $t ); + $onlySource = array_diff( $s , $intersect ); + $onlyTarget = array_diff( $t , $intersect ); + + print "qaxml.a.check: {$source} {$target}\n\n"; + + foreach( $onlySource as $only ) + print "+ {$only}\n"; + foreach( $onlyTarget as $only ) + print "- {$only}\n"; + + if ( count( $onlySource ) == 0 && count( $onlyTarget ) == 0 ) + { + for ( $i = 0 ; $i < count($s) ; $i++ ) + if ( $s[$i] != $t[$i] ) + { + print "+ {$s[$i]}\n"; + print "- {$t[$i]}\n"; + } + } + + print "\n"; +} + +function extractTriple( array $list ) +{ + $ret = array(); + foreach( $list as $elem ) + foreach( $elem->attributes as $attrib ) + $ret[] = "{$elem->nodeName} {$attrib->nodeName} {$attrib->nodeValue}"; + return $ret; +} \ No newline at end of file diff --git a/scripts/translation/qaxml.e.check.php b/scripts/translation/qaxml.e.check.php new file mode 100644 index 000000000..655434223 --- /dev/null +++ b/scripts/translation/qaxml.e.check.php @@ -0,0 +1,5 @@ + Date: Thu, 27 Jul 2023 14:51:10 -0300 Subject: [PATCH 08/50] Print diff in conventional order. --- scripts/translation/qaxml.a.check.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/translation/qaxml.a.check.php b/scripts/translation/qaxml.a.check.php index 79ffdbf39..23347dbb0 100644 --- a/scripts/translation/qaxml.a.check.php +++ b/scripts/translation/qaxml.a.check.php @@ -34,18 +34,18 @@ print "qaxml.a.check: {$source} {$target}\n\n"; - foreach( $onlySource as $only ) - print "+ {$only}\n"; foreach( $onlyTarget as $only ) print "- {$only}\n"; + foreach( $onlySource as $only ) + print "+ {$only}\n"; if ( count( $onlySource ) == 0 && count( $onlyTarget ) == 0 ) { for ( $i = 0 ; $i < count($s) ; $i++ ) if ( $s[$i] != $t[$i] ) { - print "+ {$s[$i]}\n"; print "- {$t[$i]}\n"; + print "+ {$s[$i]}\n"; } } From 117660a75bbcd45609e6fa8471943f7d15af6c2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20L=20F=20S=20Bacci?= Date: Wed, 2 Aug 2023 14:50:05 -0300 Subject: [PATCH 09/50] qaxml.a -- Documentation and small fixes. --- scripts/translation/README.md | 14 ++++++++++++-- scripts/translation/configure.php | 13 +++++++++++++ scripts/translation/lib/RevcheckRun.php | 4 +++- scripts/translation/lib/require.php | 4 ++++ .../translation/{qaxml.a.check.php => qaxml.a.php} | 2 +- scripts/translation/qaxml.e.check.php | 5 ----- scripts/translation/qaxml.p.check.php | 5 ----- scripts/translation/qaxml.t.check.php | 5 ----- 8 files changed, 33 insertions(+), 19 deletions(-) create mode 100644 scripts/translation/configure.php rename scripts/translation/{qaxml.a.check.php => qaxml.a.php} (96%) delete mode 100644 scripts/translation/qaxml.e.check.php delete mode 100644 scripts/translation/qaxml.p.check.php delete mode 100644 scripts/translation/qaxml.t.check.php diff --git a/scripts/translation/README.md b/scripts/translation/README.md index 08fbfc96c..838a3b661 100644 --- a/scripts/translation/README.md +++ b/scripts/translation/README.md @@ -12,8 +12,18 @@ assumptions of each script are described in each file. The `lib/` directory contains common code and functionality across these scripts. -See `MIGRATION.md` for procedures and breaking changes expected -when migrating from previous `revcheck.php`. +Before using the scritps, it need be configured: +``` +php doc-base/scripts/translation/configure.php $LANG_DIR +``` + +## qaxml.a.php + +`qaxml.a.php` checks if all updated files if the translated file have +the same tag-attribute-value triples. Tags with attributes are extensivly +utilized in manual for linking and XIncluding. Translated files with +missing os mistyped attributes may cause build failing or missing +parts not resolved by XIncludes. # Migration diff --git a/scripts/translation/configure.php b/scripts/translation/configure.php new file mode 100644 index 000000000..d64a26095 --- /dev/null +++ b/scripts/translation/configure.php @@ -0,0 +1,13 @@ +hash = $target->revtag->revision; $daysOld = ( strtotime( "now" ) - $source->date ) / 86400; - $this->qaList[] = new QaFileInfo( $source->hash , $target->hash , $this->sourceDir , $this->targetDir , $source->file , $daysOld ); + $daysOld = (int)$daysOld; + $qaInfo = new QaFileInfo( $source->hash , $target->hash , $this->sourceDir , $this->targetDir , $source->file , $daysOld ); + $this->qaList[ $source->file ] = $qaInfo; // TranslatedOk diff --git a/scripts/translation/lib/require.php b/scripts/translation/lib/require.php index d0e582f11..0caca38bf 100644 --- a/scripts/translation/lib/require.php +++ b/scripts/translation/lib/require.php @@ -1,5 +1,9 @@ Date: Thu, 3 Aug 2023 11:17:47 -0300 Subject: [PATCH 10/50] Initial support for [skip-revcheck]. --- scripts/translation/README.md | 6 ++--- scripts/translation/lib/GitLogParser.php | 23 ++++++++++++++++---- scripts/translation/lib/RevcheckFileInfo.php | 11 +++++----- 3 files changed, 28 insertions(+), 12 deletions(-) diff --git a/scripts/translation/README.md b/scripts/translation/README.md index 838a3b661..f9215e2f9 100644 --- a/scripts/translation/README.md +++ b/scripts/translation/README.md @@ -19,11 +19,11 @@ php doc-base/scripts/translation/configure.php $LANG_DIR ## qaxml.a.php -`qaxml.a.php` checks if all updated files if the translated file have -the same tag-attribute-value triples. Tags with attributes are extensivly +`qaxml.a.php` checks if all updated translated files have +the same tag-attribute-value triples. Tag's attributes are extensively utilized in manual for linking and XIncluding. Translated files with missing os mistyped attributes may cause build failing or missing -parts not resolved by XIncludes. +parts not copied by XIncludes. # Migration diff --git a/scripts/translation/lib/GitLogParser.php b/scripts/translation/lib/GitLogParser.php index 16ae6eea6..f782f514a 100644 --- a/scripts/translation/lib/GitLogParser.php +++ b/scripts/translation/lib/GitLogParser.php @@ -42,22 +42,37 @@ static function parseInto( string $lang , RevcheckFileList & $list ) // commit message if ( str_starts_with( $line , ' ' ) ) { - // commits marked with `[skip-revcheck]` are skiped in full + // commits with this mark are ignored if ( stristr( $line, '[skip-revcheck]' ) !== false ) $skip = true; continue; } // otherwise, a filename $filename = trim( $line ); - - // newer file hash already parsed? $info = $list->get( $filename ); - if ( $info == null || $info->hash != "" ) + + // untracked file (deleted, renamed) + if ( $info == null ) continue; + // do not track skiped commits + if ( $skip ) + { + if ( $info->hash == "" && $info->skip != "" ) + fwrite( STDERR , "Double [skip-revcheck] on $filename\n" ); + $info->skip = $hash; + continue; + } + + // already found a more recent hash + if ( $info->hash != "" ) + continue; + + // finally, the oldest commit $info->date = $date; $info->hash = $hash; } + pclose( $fp ); chdir( $cwd ); } diff --git a/scripts/translation/lib/RevcheckFileInfo.php b/scripts/translation/lib/RevcheckFileInfo.php index c61b4a00b..7b04d5fae 100644 --- a/scripts/translation/lib/RevcheckFileInfo.php +++ b/scripts/translation/lib/RevcheckFileInfo.php @@ -14,11 +14,12 @@ enum RevcheckStatus :string class RevcheckFileInfo { - public string $file; // from fs - public int $size; // from fs - public string $hash; // from vcs, source only - public int $date; // from vcs, source only - public int $days; // derived + public string $file = ""; // from fs + public int $size = 0 ; // from fs + public string $hash = ""; // from vcs, source only + public string $skip = ""; // from vcs, source only + public int $date = 0 ; // from vcs, source only + public int $days = 0 ; // derived public RevcheckStatus $status; // target only public RevtagInfo|null $revtag; // target only From 03b9760a1ea0d8f6566bd8dff8b067a47364b2b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20L=20F=20S=20Bacci?= Date: Thu, 3 Aug 2023 12:53:10 -0300 Subject: [PATCH 11/50] [skip-revcheck] compatibility. --- scripts/translation/lib/GitLogParser.php | 13 ++++++------- scripts/translation/lib/RevcheckRun.php | 11 +++++++++-- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/scripts/translation/lib/GitLogParser.php b/scripts/translation/lib/GitLogParser.php index f782f514a..5ba82db1d 100644 --- a/scripts/translation/lib/GitLogParser.php +++ b/scripts/translation/lib/GitLogParser.php @@ -55,22 +55,21 @@ static function parseInto( string $lang , RevcheckFileList & $list ) if ( $info == null ) continue; - // do not track skiped commits + // tracks the first skiped hash, if newer than a non skiped hash if ( $skip ) { - if ( $info->hash == "" && $info->skip != "" ) - fwrite( STDERR , "Double [skip-revcheck] on $filename\n" ); - $info->skip = $hash; + if ( $info->hash == "" && $info->skip == "" ) + $info->skip = $hash; continue; } - // already found a more recent hash + // ignore newer commits if ( $info->hash != "" ) continue; - // finally, the oldest commit - $info->date = $date; + // found oldest hash and date $info->hash = $hash; + $info->date = $date; } pclose( $fp ); diff --git a/scripts/translation/lib/RevcheckRun.php b/scripts/translation/lib/RevcheckRun.php index 5706796a1..06f9fe074 100644 --- a/scripts/translation/lib/RevcheckRun.php +++ b/scripts/translation/lib/RevcheckRun.php @@ -71,14 +71,21 @@ private function calculateStatus() } $target->hash = $target->revtag->revision; + + // translation compares ok from multiple hashs. See https://github.com/php/doc-base/commit/090ff07aa03c3e4ad7320a4ace9ffb6d5ede722f + $wobblyOkHash = $source->hash; // L372 + if ( $source->skip == $target->hash ) // R391 + $wobblyOkHash = $source->skip; // R392 + $daysOld = ( strtotime( "now" ) - $source->date ) / 86400; $daysOld = (int)$daysOld; - $qaInfo = new QaFileInfo( $source->hash , $target->hash , $this->sourceDir , $this->targetDir , $source->file , $daysOld ); + + $qaInfo = new QaFileInfo( $wobblyOkHash , $target->hash , $this->sourceDir , $this->targetDir , $source->file , $daysOld ); $this->qaList[ $source->file ] = $qaInfo; // TranslatedOk - if ( $source->hash == $target->hash && $target->revtag->status == "ready" ) + if ( $target->revtag->status == "ready" && $wobblyOkHash == $target->hash ) { $source->status = RevcheckStatus::TranslatedOk; $this->filesOk[] = $source; From ea00f5e25acb7b63e8a07d6ee351a436b4691d60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20L=20F=20S=20Bacci?= Date: Thu, 3 Aug 2023 12:53:41 -0300 Subject: [PATCH 12/50] Compact output. --- scripts/translation/qaxml.a.php | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/scripts/translation/qaxml.a.php b/scripts/translation/qaxml.a.php index 9ca7b719d..34e8a1730 100644 --- a/scripts/translation/qaxml.a.php +++ b/scripts/translation/qaxml.a.php @@ -32,24 +32,22 @@ $onlySource = array_diff( $s , $intersect ); $onlyTarget = array_diff( $t , $intersect ); - print "qaxml.a.check: {$source} {$target}\n\n"; + print "qaxml.a: {$target}\n"; foreach( $onlyTarget as $only ) - print "- {$only}\n"; + print "< {$only}\n"; foreach( $onlySource as $only ) - print "+ {$only}\n"; + print "> {$only}\n"; if ( count( $onlySource ) == 0 && count( $onlyTarget ) == 0 ) { for ( $i = 0 ; $i < count($s) ; $i++ ) if ( $s[$i] != $t[$i] ) { - print "- {$t[$i]}\n"; - print "+ {$s[$i]}\n"; + print "< {$t[$i]}\n"; + print "> {$s[$i]}\n"; } } - - print "\n"; } function extractTriple( array $list ) From 7058ea8d4979bdd0ffc6909746029a281633b7a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20L=20F=20S=20Bacci?= Date: Fri, 4 Aug 2023 11:31:07 -0300 Subject: [PATCH 13/50] Script to check entities usage in translatoins. --- scripts/translation/lib/XmlUtil.php | 24 ++++++++++++- scripts/translation/qaxml.a.php | 25 +++++++++----- scripts/translation/qaxml.e.php | 52 +++++++++++++++++++++++++++++ 3 files changed, 91 insertions(+), 10 deletions(-) create mode 100644 scripts/translation/qaxml.e.php diff --git a/scripts/translation/lib/XmlUtil.php b/scripts/translation/lib/XmlUtil.php index e22144934..dfa3d487a 100644 --- a/scripts/translation/lib/XmlUtil.php +++ b/scripts/translation/lib/XmlUtil.php @@ -8,6 +8,28 @@ class XmlUtil { + public static function extractEntities( $filename ) + { + $was = libxml_use_internal_errors( true ); + + $doc = new DOMDocument(); + $doc->recover = true; + $doc->resolveExternals = false; + $doc->load( $filename ); + + $errors = libxml_get_errors(); + libxml_clear_errors(); + libxml_use_internal_errors( $was ); + + $ret = array(); + foreach ($errors as $error) + { + if ( preg_match( "/Entity '(\S+)' not defined/" , $error->message , $matches ) ) + $ret[] = $matches[1]; + } + return $ret; + } + public static function listNodeType( DOMNode $node , int $type ) { $ret = array(); @@ -31,7 +53,7 @@ public static function loadFile( $filename ):DOMDocument public static function loadText( $contents ):DOMDocument { - $was = libxml_use_internal_errors( true ); // do not print warnings + $was = libxml_use_internal_errors( true ); $doc = new DOMDocument(); $doc->preserveWhiteSpace = true; diff --git a/scripts/translation/qaxml.a.php b/scripts/translation/qaxml.a.php index 34e8a1730..c856c585c 100644 --- a/scripts/translation/qaxml.a.php +++ b/scripts/translation/qaxml.a.php @@ -32,22 +32,29 @@ $onlySource = array_diff( $s , $intersect ); $onlyTarget = array_diff( $t , $intersect ); - print "qaxml.a: {$target}\n"; + print "qaxml.a: {$target}\n\n"; foreach( $onlyTarget as $only ) - print "< {$only}\n"; + print "- {$only}\n"; foreach( $onlySource as $only ) - print "> {$only}\n"; + print "+ {$only}\n"; if ( count( $onlySource ) == 0 && count( $onlyTarget ) == 0 ) { - for ( $i = 0 ; $i < count($s) ; $i++ ) - if ( $s[$i] != $t[$i] ) - { - print "< {$t[$i]}\n"; - print "> {$s[$i]}\n"; - } + $s = array_count_values( $s ); + $t = array_count_values( $t ); + foreach ($s as $key => $countSource ) + { + $countTarget = $t[$key]; + $countDiff = $countSource - $countTarget; + if ( $countDiff > 0 ) + print "* {$key} +{$countDiff}\n"; + if ( $countDiff < 0 ) + print "* {$key} {$countDiff}\n"; + } } + + print "\n"; } function extractTriple( array $list ) diff --git a/scripts/translation/qaxml.e.php b/scripts/translation/qaxml.e.php new file mode 100644 index 000000000..916cc64cf --- /dev/null +++ b/scripts/translation/qaxml.e.php @@ -0,0 +1,52 @@ +sourceHash != $qafile->targetHash ) + continue; + + $source = $qafile->sourceDir . '/' . $qafile->file; + $target = $qafile->targetDir . '/' . $qafile->file; + + $s = XmlUtil::extractEntities( $source ); + $t = XmlUtil::extractEntities( $target ); + + if ( implode( "\n" , $s ) == implode( "\n" , $t ) ) + continue; + + $intersect = array_intersect( $s, $t ); + $onlySource = array_diff( $s , $intersect ); + $onlyTarget = array_diff( $t , $intersect ); + + print "qaxml.e: {$target}\n\n"; + + foreach( $onlyTarget as $only ) + print "- {$only}\n"; + foreach( $onlySource as $only ) + print "+ {$only}\n"; + + if ( count( $onlySource ) == 0 && count( $onlyTarget ) == 0 ) + { + $s = array_count_values( $s ); + $t = array_count_values( $t ); + foreach ($s as $key => $countSource ) + { + $countTarget = $t[$key]; + $countDiff = $countSource - $countTarget; + if ( $countDiff > 0 ) + print "* {$key} +{$countDiff}\n"; + if ( $countDiff < 0 ) + print "* {$key} {$countDiff}\n"; + } + } + + print "\n"; +} \ No newline at end of file From 1f3c5f1f1c6040e92e2dac76b4cdfec0f2ec72d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20L=20F=20S=20Bacci?= Date: Fri, 4 Aug 2023 13:29:43 -0300 Subject: [PATCH 14/50] qaxml.p and qaxml.t tools. --- scripts/translation/qaxml.p.php | 66 ++++++++++++++ scripts/translation/qaxml.t.php | 150 ++++++++++++++++++++++++++++++++ 2 files changed, 216 insertions(+) create mode 100644 scripts/translation/qaxml.p.php create mode 100644 scripts/translation/qaxml.t.php diff --git a/scripts/translation/qaxml.p.php b/scripts/translation/qaxml.p.php new file mode 100644 index 000000000..3751f05ca --- /dev/null +++ b/scripts/translation/qaxml.p.php @@ -0,0 +1,66 @@ +sourceHash != $qafile->targetHash ) + continue; + + $source = $qafile->sourceDir . '/' . $qafile->file; + $target = $qafile->targetDir . '/' . $qafile->file; + + $s = XmlUtil::loadFile( $source ); + $t = XmlUtil::loadFile( $target ); + + $s = XmlUtil::listNodeType( $s , XML_PI_NODE ); + $t = XmlUtil::listNodeType( $t , XML_PI_NODE ); + + $s = extractPiData( $s ); + $t = extractPiData( $t ); + + if ( implode( "\n" , $s ) == implode( "\n" , $t ) ) + continue; + + $intersect = array_intersect( $s, $t ); + $onlySource = array_diff( $s , $intersect ); + $onlyTarget = array_diff( $t , $intersect ); + + print "qaxml.p: {$target}\n\n"; + + foreach( $onlyTarget as $only ) + print "- {$only}\n"; + foreach( $onlySource as $only ) + print "+ {$only}\n"; + + if ( count( $onlySource ) == 0 && count( $onlyTarget ) == 0 ) + { + $s = array_count_values( $s ); + $t = array_count_values( $t ); + foreach ($s as $key => $countSource ) + { + $countTarget = $t[$key]; + $countDiff = $countSource - $countTarget; + if ( $countDiff > 0 ) + print "* {$key} +{$countDiff}\n"; + if ( $countDiff < 0 ) + print "* {$key} {$countDiff}\n"; + } + } + + print "\n"; +} + +function extractPiData( array $list ) +{ + $ret = array(); + foreach( $list as $elem ) + $ret[] = "{$elem->target} {$elem->data}"; + return $ret; +} \ No newline at end of file diff --git a/scripts/translation/qaxml.t.php b/scripts/translation/qaxml.t.php new file mode 100644 index 000000000..77f10adfe --- /dev/null +++ b/scripts/translation/qaxml.t.php @@ -0,0 +1,150 @@ + 1 ) + $tags = explode( ',' , $argv[1] ); + +foreach ( $qalist as $qafile ) +{ + if ( $qafile->sourceHash != $qafile->targetHash ) + continue; + + $output = false; + $source = $qafile->sourceDir . '/' . $qafile->file; + $target = $qafile->targetDir . '/' . $qafile->file; + + // Tag contents + + if ( count( $tags ) > 0 ) + { + $s = XmlUtil::loadFile( $source ); + $t = XmlUtil::loadFile( $target ); + + $s = XmlUtil::listNodeType( $s , XML_ELEMENT_NODE ); + $t = XmlUtil::listNodeType( $t , XML_ELEMENT_NODE ); + + $s = extractTagContentList( $s , $tags ); + $t = extractTagContentList( $t , $tags ); + + if ( implode( "\n" , $s ) == implode( "\n" , $t ) ) + continue; + + $intersect = array_intersect( $s, $t ); + $onlySource = array_diff( $s , $intersect ); + $onlyTarget = array_diff( $t , $intersect ); + + if ( ! $output ) + { + print "qaxml.t: {$target}\n\n"; + $output = true; + } + + foreach( $onlyTarget as $only ) + print "- {$only}\n"; + foreach( $onlySource as $only ) + print "+ {$only}\n"; + + if ( count( $onlySource ) == 0 && count( $onlyTarget ) == 0 ) + { + $s = array_count_values( $s ); + $t = array_count_values( $t ); + foreach ($s as $key => $countSource ) + { + $countTarget = $t[$key]; + $countDiff = $countSource - $countTarget; + if ( $countDiff > 0 ) + print "* {$key} +{$countDiff}\n"; + if ( $countDiff < 0 ) + print "* {$key} {$countDiff}\n"; + } + } + + if ( $output ) + { + print "\n"; + continue; + } + } + + // Tag count + { + $s = XmlUtil::loadFile( $source ); + $t = XmlUtil::loadFile( $target ); + + $s = XmlUtil::listNodeType( $s , XML_ELEMENT_NODE ); + $t = XmlUtil::listNodeType( $t , XML_ELEMENT_NODE ); + + $s = countTags( $s ); + $t = countTags( $t ); + + equalizeKeys( $s , $t ); + equalizeKeys( $t , $s ); + + foreach( $s as $tag => $sourceCount ) + { + $targetCount = $t[$tag]; + + if ( $sourceCount != $targetCount ) + { + if ( ! $output ) + { + print "qaxml.t: {$target}\n\n"; + $output = true; + } + + print "* {$tag} -{$sourceCount} +{$targetCount}\n"; + } + } + + if ( $output ) + print "\n"; + } +} + +function extractTagContentList( array $nodes , array $tags ) +{ + $ret = array(); + foreach( $nodes as $node ) + { + $tag = $node->nodeName; + if ( in_array( $tag , $tags ) == false ) + continue; + $text = $node->textContent; + while( true ) + { + $was = strlen( $text ); + $text = trim( $text ); + $text = str_replace( "\n" , " " , $text ); + $text = str_replace( "\r" , " " , $text ); + $text = str_replace( " " , " " , $text ); + if ( strlen( $text ) == $was ) + break; + } + $ret[] = $tag . " " . $text; + } + return $ret; +} + +function countTags( array $list ) +{ + $ret = array(); + foreach( $list as $elem ) + $ret[] = $elem->nodeName; + $ret = array_count_values( $ret ); + return $ret; +} + +function equalizeKeys( array $list , array & $other , mixed $value = 0 ) +{ + foreach( $list as $k => $v ) + if ( ! isset( $other[$k] ) ) + $other[$k] = $value; +} \ No newline at end of file From e3a7609bb891674165c1b119e0da7c4681390fe9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20L=20F=20S=20Bacci?= Date: Sat, 5 Aug 2023 11:38:39 -0300 Subject: [PATCH 15/50] Rename compare hashes to head and diff. --- scripts/translation/lib/GitLogParser.php | 22 +++++++++++--------- scripts/translation/lib/RevcheckFileInfo.php | 11 +++++----- scripts/translation/lib/RevcheckRun.php | 6 +++--- 3 files changed, 20 insertions(+), 19 deletions(-) diff --git a/scripts/translation/lib/GitLogParser.php b/scripts/translation/lib/GitLogParser.php index 5ba82db1d..3c369fc1b 100644 --- a/scripts/translation/lib/GitLogParser.php +++ b/scripts/translation/lib/GitLogParser.php @@ -55,21 +55,23 @@ static function parseInto( string $lang , RevcheckFileList & $list ) if ( $info == null ) continue; - // tracks the first skiped hash, if newer than a non skiped hash - if ( $skip ) + // the head commit + if ( $info->head == "" ) { - if ( $info->hash == "" && $info->skip == "" ) - $info->skip = $hash; - continue; + $info->head = $hash; + $info->date = $date; } - // ignore newer commits - if ( $info->hash != "" ) + // after, only tracks non skipped commits + if ( $skip ) continue; - // found oldest hash and date - $info->hash = $hash; - $info->date = $date; + // the diff commit + if ( $info->diff == "" ) + { + $info->diff = $hash; + $info->date = $date; + } } pclose( $fp ); diff --git a/scripts/translation/lib/RevcheckFileInfo.php b/scripts/translation/lib/RevcheckFileInfo.php index 7b04d5fae..184c8be6a 100644 --- a/scripts/translation/lib/RevcheckFileInfo.php +++ b/scripts/translation/lib/RevcheckFileInfo.php @@ -16,10 +16,9 @@ class RevcheckFileInfo { public string $file = ""; // from fs public int $size = 0 ; // from fs - public string $hash = ""; // from vcs, source only - public string $skip = ""; // from vcs, source only - public int $date = 0 ; // from vcs, source only - public int $days = 0 ; // derived + public string $head = ""; // from vcs, source only, head hash, may be skipped + public string $diff = ""; // from vcs, source only, diff hash, no skips + public int $date = 0 ; // from vcs, source only, date of head or diff commit public RevcheckStatus $status; // target only public RevtagInfo|null $revtag; // target only @@ -28,9 +27,9 @@ function __construct( string $file , int $size ) { $this->file = $file; $this->size = $size; - $this->hash = ""; + $this->head = ""; + $this->diff = ""; $this->date = 0; - $this->days = 0; $this->status = RevcheckStatus::Untranslated; $this->revtag = null; } diff --git a/scripts/translation/lib/RevcheckRun.php b/scripts/translation/lib/RevcheckRun.php index 06f9fe074..c6e5995c1 100644 --- a/scripts/translation/lib/RevcheckRun.php +++ b/scripts/translation/lib/RevcheckRun.php @@ -73,9 +73,9 @@ private function calculateStatus() $target->hash = $target->revtag->revision; // translation compares ok from multiple hashs. See https://github.com/php/doc-base/commit/090ff07aa03c3e4ad7320a4ace9ffb6d5ede722f - $wobblyOkHash = $source->hash; // L372 - if ( $source->skip == $target->hash ) // R391 - $wobblyOkHash = $source->skip; // R392 + $wobblyOkHash = $source->head; // L372 + if ( $target->hash == $source->diff ) // R391 + $wobblyOkHash = $source->diff; // R392 $daysOld = ( strtotime( "now" ) - $source->date ) / 86400; $daysOld = (int)$daysOld; From f3e45aa622e1f7dca9bdc49345ba80435392f612 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20L=20F=20S=20Bacci?= Date: Sat, 5 Aug 2023 11:39:21 -0300 Subject: [PATCH 16/50] Tools documentation. --- scripts/translation/README.md | 50 ++++++++++++++++++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) diff --git a/scripts/translation/README.md b/scripts/translation/README.md index f9215e2f9..1d3973855 100644 --- a/scripts/translation/README.md +++ b/scripts/translation/README.md @@ -12,7 +12,7 @@ assumptions of each script are described in each file. The `lib/` directory contains common code and functionality across these scripts. -Before using the scritps, it need be configured: +Before using the scripts, it need be configured: ``` php doc-base/scripts/translation/configure.php $LANG_DIR ``` @@ -25,6 +25,54 @@ utilized in manual for linking and XIncluding. Translated files with missing os mistyped attributes may cause build failing or missing parts not copied by XIncludes. +## qaxml.e.php + +`qaxml.e.php` checks if all updated translated files have +the same external entities as the original files. Unbalanced entities +may indicate mistyped or wrongly traduced parts. + +## qaxml.e.php + +`qaxml.p.php` checks if all updated translated files have +the same processing instructions as the original files. Unbalanced entities +may cause compilation errors, as they are utilized on manual in the build +process. + +## qaxml.t.php + +`qaxml.t.php` checks if all updated translated files have +the same tags as the original files. Different number of tags between +source texts and target translations may cause compilation errors. + +This scripts accepts one parameter of comma separated tags to check their +contents, as some tag' contents are expected *not* be translated. + +Suggested execution: +``` +php doc-base/scripts/translation/qaxml.t.php acronym +php doc-base/scripts/translation/qaxml.t.php classname +php doc-base/scripts/translation/qaxml.t.php constant +php doc-base/scripts/translation/qaxml.t.php envar +php doc-base/scripts/translation/qaxml.t.php function +php doc-base/scripts/translation/qaxml.t.php interfacename +php doc-base/scripts/translation/qaxml.t.php parameter +php doc-base/scripts/translation/qaxml.t.php type + +php doc-base/scripts/translation/qaxml.t.php classsynopsis +php doc-base/scripts/translation/qaxml.t.php constructorsynopsis +php doc-base/scripts/translation/qaxml.t.php destructorsynopsis +php doc-base/scripts/translation/qaxml.t.php fieldsynopsis +php doc-base/scripts/translation/qaxml.t.php funcsynopsis +php doc-base/scripts/translation/qaxml.t.php methodsynopsis +``` +May contain translated parts, so manual check is necessary: +``` +php doc-base/scripts/translation/qaxml.t.php code +php doc-base/scripts/translation/qaxml.t.php filename +php doc-base/scripts/translation/qaxml.t.php literal +php doc-base/scripts/translation/qaxml.t.php varname +``` + # Migration ## No critical From 997af1f129d7fd2c202d299373b92626e8c8ed61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20L=20F=20S=20Bacci?= Date: Sat, 5 Aug 2023 11:59:07 -0300 Subject: [PATCH 17/50] Ignore order in comparation. --- scripts/translation/qaxml.t.php | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/translation/qaxml.t.php b/scripts/translation/qaxml.t.php index 77f10adfe..69bc07180 100644 --- a/scripts/translation/qaxml.t.php +++ b/scripts/translation/qaxml.t.php @@ -34,16 +34,16 @@ $s = extractTagContentList( $s , $tags ); $t = extractTagContentList( $t , $tags ); - if ( implode( "\n" , $s ) == implode( "\n" , $t ) ) - continue; - $intersect = array_intersect( $s, $t ); $onlySource = array_diff( $s , $intersect ); $onlyTarget = array_diff( $t , $intersect ); + if ( count( $s ) == count( $t ) && count( $onlySource ) == 0 && count( $onlyTarget ) == 0 ) + continue; + if ( ! $output ) { - print "qaxml.t: {$target}\n\n"; + print "qaxml.t: {$target} content \n\n"; $output = true; } @@ -96,7 +96,7 @@ { if ( ! $output ) { - print "qaxml.t: {$target}\n\n"; + print "qaxml.t: {$target} count \n\n"; $output = true; } From e07946b618d3a18e156d6768b5e7ba29b7d33c7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20L=20F=20S=20Bacci?= Date: Sat, 5 Aug 2023 12:00:48 -0300 Subject: [PATCH 18/50] Simplify output. --- scripts/translation/qaxml.t.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/translation/qaxml.t.php b/scripts/translation/qaxml.t.php index 69bc07180..bbd8a9e70 100644 --- a/scripts/translation/qaxml.t.php +++ b/scripts/translation/qaxml.t.php @@ -43,7 +43,7 @@ if ( ! $output ) { - print "qaxml.t: {$target} content \n\n"; + print "qaxml.t: {$target}\n\n"; $output = true; } @@ -96,7 +96,7 @@ { if ( ! $output ) { - print "qaxml.t: {$target} count \n\n"; + print "qaxml.t: {$target}\n\n"; $output = true; } From 6d4a3fbf3d569990dd305f660062229df98499fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20L=20F=20S=20Bacci?= Date: Mon, 7 Aug 2023 15:20:40 -0300 Subject: [PATCH 19/50] Small fixes. --- scripts/translation/README.md | 2 +- scripts/translation/qaxml.t.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/translation/README.md b/scripts/translation/README.md index 1d3973855..d15d193d6 100644 --- a/scripts/translation/README.md +++ b/scripts/translation/README.md @@ -31,7 +31,7 @@ parts not copied by XIncludes. the same external entities as the original files. Unbalanced entities may indicate mistyped or wrongly traduced parts. -## qaxml.e.php +## qaxml.p.php `qaxml.p.php` checks if all updated translated files have the same processing instructions as the original files. Unbalanced entities diff --git a/scripts/translation/qaxml.t.php b/scripts/translation/qaxml.t.php index bbd8a9e70..aa4cafa4e 100644 --- a/scripts/translation/qaxml.t.php +++ b/scripts/translation/qaxml.t.php @@ -128,7 +128,7 @@ function extractTagContentList( array $nodes , array $tags ) if ( strlen( $text ) == $was ) break; } - $ret[] = $tag . " " . $text; + $ret[] = $tag . ">" . $text; } return $ret; } From 66d963055572040b9f0046ad67cb5272cde5507b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20L=20F=20S=20Bacci?= Date: Mon, 7 Aug 2023 16:24:49 -0300 Subject: [PATCH 20/50] Simple and complex output. --- scripts/translation/qaxml.t.php | 85 +++++++++++++++++++++++++++++++-- 1 file changed, 80 insertions(+), 5 deletions(-) diff --git a/scripts/translation/qaxml.t.php b/scripts/translation/qaxml.t.php index aa4cafa4e..8a779f702 100644 --- a/scripts/translation/qaxml.t.php +++ b/scripts/translation/qaxml.t.php @@ -21,7 +21,7 @@ $source = $qafile->sourceDir . '/' . $qafile->file; $target = $qafile->targetDir . '/' . $qafile->file; - // Tag contents + // Tag contents, text if ( count( $tags ) > 0 ) { @@ -31,8 +31,61 @@ $s = XmlUtil::listNodeType( $s , XML_ELEMENT_NODE ); $t = XmlUtil::listNodeType( $t , XML_ELEMENT_NODE ); - $s = extractTagContentList( $s , $tags ); - $t = extractTagContentList( $t , $tags ); + $s = extractTagsInnerText( $s , $tags ); + $t = extractTagsInnerText( $t , $tags ); + + $intersect = array_intersect( $s, $t ); + $onlySource = array_diff( $s , $intersect ); + $onlyTarget = array_diff( $t , $intersect ); + + if ( count( $s ) == count( $t ) && count( $onlySource ) == 0 && count( $onlyTarget ) == 0 ) + continue; + + if ( ! $output ) + { + print "qaxml.t: {$target}\n\n"; + $output = true; + } + + foreach( $onlyTarget as $only ) + print "- {$only}\n"; + foreach( $onlySource as $only ) + print "+ {$only}\n"; + + if ( count( $onlySource ) == 0 && count( $onlyTarget ) == 0 ) + { + $s = array_count_values( $s ); + $t = array_count_values( $t ); + foreach ($s as $key => $countSource ) + { + $countTarget = $t[$key]; + $countDiff = $countSource - $countTarget; + if ( $countDiff > 0 ) + print "* {$key} +{$countDiff}\n"; + if ( $countDiff < 0 ) + print "* {$key} {$countDiff}\n"; + } + } + + if ( $output ) + { + print "\n"; + continue; + } + } + + // Tag contents, XML + + if ( count( $tags ) > 0 && ! $output ) + { + $s = XmlUtil::loadFile( $source ); + $t = XmlUtil::loadFile( $target ); + + $s = XmlUtil::listNodeType( $s , XML_ELEMENT_NODE ); + $t = XmlUtil::listNodeType( $t , XML_ELEMENT_NODE ); + + $s = extractTagsInnerXmls( $s , $tags ); + $t = extractTagsInnerXmls( $t , $tags ); $intersect = array_intersect( $s, $t ); $onlySource = array_diff( $s , $intersect ); @@ -109,7 +162,7 @@ } } -function extractTagContentList( array $nodes , array $tags ) +function extractTagsInnerText( array $nodes , array $tags ) { $ret = array(); foreach( $nodes as $node ) @@ -121,7 +174,6 @@ function extractTagContentList( array $nodes , array $tags ) while( true ) { $was = strlen( $text ); - $text = trim( $text ); $text = str_replace( "\n" , " " , $text ); $text = str_replace( "\r" , " " , $text ); $text = str_replace( " " , " " , $text ); @@ -133,6 +185,29 @@ function extractTagContentList( array $nodes , array $tags ) return $ret; } +function extractTagsInnerXmls( array $nodes , array $tags ) +{ + $ret = array(); + foreach( $nodes as $node ) + { + $tag = $node->nodeName; + if ( in_array( $tag , $tags ) == false ) + continue; + $text = $node->ownerDocument->saveXML( $node ); + while( true ) + { + $was = strlen( $text ); + $text = str_replace( "\n" , " " , $text ); + $text = str_replace( "\r" , " " , $text ); + $text = str_replace( " " , " " , $text ); + if ( strlen( $text ) == $was ) + break; + } + $ret[] = $text; + } + return $ret; +} + function countTags( array $list ) { $ret = array(); From 40edf104b8da2f14dc33156f7b80ae7b0cd207bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20L=20F=20S=20Bacci?= Date: Mon, 7 Aug 2023 16:32:03 -0300 Subject: [PATCH 21/50] `bookinfo.xml` is expectd to tag differ. --- scripts/translation/qaxml.t.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/translation/qaxml.t.php b/scripts/translation/qaxml.t.php index 8a779f702..1a7ffbf37 100644 --- a/scripts/translation/qaxml.t.php +++ b/scripts/translation/qaxml.t.php @@ -14,6 +14,8 @@ foreach ( $qalist as $qafile ) { + if ( $qafile->file == "bookinfo.xml" ) + continue; if ( $qafile->sourceHash != $qafile->targetHash ) continue; From 7cc43f5223a41480689030a49e2c3e8231fdf746 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20L=20F=20S=20Bacci?= Date: Tue, 8 Aug 2023 14:43:37 -0300 Subject: [PATCH 22/50] Case insentivity on ``. --- scripts/translation/qaxml.t.php | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/scripts/translation/qaxml.t.php b/scripts/translation/qaxml.t.php index 1a7ffbf37..4e46f5358 100644 --- a/scripts/translation/qaxml.t.php +++ b/scripts/translation/qaxml.t.php @@ -182,6 +182,20 @@ function extractTagsInnerText( array $nodes , array $tags ) if ( strlen( $text ) == $was ) break; } + // Types not case-sensitive: https://github.com/php/doc-en/issues/2658 + if ( $tag == "type" ) + { + switch( strtolower( $text ) ) + { + case "array": + case "string": + case "float": + case "bool": + case "null": + $text = strtolower( $text ); + break; + } + } $ret[] = $tag . ">" . $text; } return $ret; From 84818f6a148d46ec6641234771635d5e8597d730 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20L=20F=20S=20Bacci?= Date: Fri, 11 Aug 2023 14:19:53 -0300 Subject: [PATCH 23/50] =?UTF-8?q?Consistent=20output=20volunteer=C2=A0myse?= =?UTF-8?q?lf=20for=20revisions.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/translation/qaxml.t.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/translation/qaxml.t.php b/scripts/translation/qaxml.t.php index 4e46f5358..7ce5a2021 100644 --- a/scripts/translation/qaxml.t.php +++ b/scripts/translation/qaxml.t.php @@ -155,7 +155,7 @@ $output = true; } - print "* {$tag} -{$sourceCount} +{$targetCount}\n"; + print "* {$tag} -{$targetCount} +{$sourceCount}\n" } } @@ -238,4 +238,4 @@ function equalizeKeys( array $list , array & $other , mixed $value = 0 ) foreach( $list as $k => $v ) if ( ! isset( $other[$k] ) ) $other[$k] = $value; -} \ No newline at end of file +} From 817fa07bc18e416658510c678a2fdbd08452ba74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20L=20F=20S=20Bacci?= Date: Fri, 11 Aug 2023 14:20:30 -0300 Subject: [PATCH 24/50] =?UTF-8?q?Consistent=20output=20volunteer=C2=A0myse?= =?UTF-8?q?lf=20for=20revisions.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CODEOWNERS | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 CODEOWNERS diff --git a/CODEOWNERS b/CODEOWNERS new file mode 100644 index 000000000..8903550ba --- /dev/null +++ b/CODEOWNERS @@ -0,0 +1,11 @@ +# The following volunteers have self-identified as subject matter experts +# or interested parties over a particular area of this repository. +# While requesting a review from someone does not obligate that person to +# review a pull request, these reviewers might have valuable knowledge of +# the problem area and could aid in deciding whether a pull request is ready +# for merging. +# +# For more information, see the GitHub CODEOWNERS documentation: +# https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners + +/scripts/translation/ @alfsb From 0cd98f523fdb2abcc9ef9a1e0c15efce87d128da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20L=20F=20S=20Bacci?= Date: Sat, 12 Aug 2023 12:31:12 -0300 Subject: [PATCH 25/50] Missing ; --- scripts/translation/qaxml.t.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/translation/qaxml.t.php b/scripts/translation/qaxml.t.php index 7ce5a2021..0e6db0ff7 100644 --- a/scripts/translation/qaxml.t.php +++ b/scripts/translation/qaxml.t.php @@ -155,7 +155,7 @@ $output = true; } - print "* {$tag} -{$targetCount} +{$sourceCount}\n" + print "* {$tag} -{$targetCount} +{$sourceCount}\n"; } } From 918bfc630e0b97ae361ed2a90737be2a0c5799a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20L=20F=20S=20Bacci?= Date: Sun, 13 Aug 2023 20:45:21 -0300 Subject: [PATCH 26/50] Fix revtag capture and error reporting. --- scripts/translation/lib/RevtagParser.php | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/scripts/translation/lib/RevtagParser.php b/scripts/translation/lib/RevtagParser.php index bc3bc2644..8f7c3dddc 100644 --- a/scripts/translation/lib/RevtagParser.php +++ b/scripts/translation/lib/RevtagParser.php @@ -66,23 +66,23 @@ public static function parseComment( DOMNode $node , RevtagInfo & $ret , $filena $ret->status = trim( $match[3] ); if ( $ret->revision != "" && strlen( $ret->revision ) != 40 ) - $ret->errors .= "Wrong hash format or size: {$ret->revision}\n"; + $ret->errors .= "Wrong hash size: {$ret->revision}\n"; if ( $ret->maintainer == "" ) $ret->errors .= "Empty maintainer.\n"; - if ( $ret->status != "" && strlen( $ret->revision ) != 40 ) + if ( $ret->status == "" ) $ret->errors .= "Empty status.\n"; } } - if ( str_starts_with( $text , "CRED" ) ) + if ( str_starts_with( $text , "CREDITS:" ) ) { $match = array(); - $regex = "/CREDITS:\s*(\S+)/"; + $regex = "/CREDITS:(.*)/"; if ( preg_match( $regex , $text , $match ) ) { - $ret->credits = $match[1]; + $ret->credits = trim( $match[1] ); - if ( $ret->credits != "" && strlen( $ret->revision ) != 40 ) + if ( $ret->credits == "" ) $ret->errors .= "Empty credits.\n"; } } From 60c9cae3f73a2078ab604a8b93723bbe75259a96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20L=20F=20S=20Bacci?= Date: Mon, 14 Aug 2023 10:59:53 -0300 Subject: [PATCH 27/50] Organize require for all files in `lib/` --- scripts/translation/README.md | 2 +- scripts/translation/configure.php | 11 +++++++---- scripts/translation/lib/CacheUtil.php | 2 +- scripts/translation/lib/GitDiffParser.php | 2 +- scripts/translation/lib/GitLogParser.php | 2 +- scripts/translation/lib/QaFileInfo.php | 2 +- scripts/translation/lib/RevcheckFileInfo.php | 8 ++++++-- scripts/translation/lib/RevcheckFileList.php | 6 +++++- scripts/translation/lib/RevcheckIgnore.php | 6 ++++++ scripts/translation/lib/RevcheckRun.php | 3 ++- scripts/translation/lib/RevtagParser.php | 2 +- scripts/translation/lib/XmlUtil.php | 2 +- scripts/translation/lib/{require.php => all.php} | 0 scripts/translation/qaxml.a.php | 2 +- scripts/translation/qaxml.e.php | 2 +- scripts/translation/qaxml.p.php | 2 +- scripts/translation/qaxml.t.php | 2 +- 17 files changed, 37 insertions(+), 19 deletions(-) rename scripts/translation/lib/{require.php => all.php} (100%) diff --git a/scripts/translation/README.md b/scripts/translation/README.md index d15d193d6..90007326b 100644 --- a/scripts/translation/README.md +++ b/scripts/translation/README.md @@ -45,7 +45,7 @@ the same tags as the original files. Different number of tags between source texts and target translations may cause compilation errors. This scripts accepts one parameter of comma separated tags to check their -contents, as some tag' contents are expected *not* be translated. +contents, as some tag's contents are expected *not* be translated. Suggested execution: ``` diff --git a/scripts/translation/configure.php b/scripts/translation/configure.php index d64a26095..63addaf09 100644 --- a/scripts/translation/configure.php +++ b/scripts/translation/configure.php @@ -1,13 +1,16 @@ Date: Mon, 14 Aug 2023 11:42:09 -0300 Subject: [PATCH 28/50] New detail output for `qaxml.t.php`. --- scripts/translation/README.md | 7 ++++++- scripts/translation/configure.php | 16 +++++++++++++++- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/scripts/translation/README.md b/scripts/translation/README.md index 90007326b..55f1bb12e 100644 --- a/scripts/translation/README.md +++ b/scripts/translation/README.md @@ -44,9 +44,14 @@ process. the same tags as the original files. Different number of tags between source texts and target translations may cause compilation errors. -This scripts accepts one parameter of comma separated tags to check their +Usage: `php qaxml.t.php [--detail] [tag[,tag]]` + +`[tag[,tag]]` is a comma separated tag list to check their contents, as some tag's contents are expected *not* be translated. +`--detail` will also print line defintions of each mismatched tag, +to facilitate bitsecting. + Suggested execution: ``` php doc-base/scripts/translation/qaxml.t.php acronym diff --git a/scripts/translation/configure.php b/scripts/translation/configure.php index 63addaf09..bc4492929 100644 --- a/scripts/translation/configure.php +++ b/scripts/translation/configure.php @@ -13,4 +13,18 @@ return; } -new RevcheckRun( 'en' , $argv[1] , true ); +$run = new RevcheckRun( 'en' , $argv[1] , true ); + +foreach( $run->targetFiles->list as $file ) +{ + if ( $file->revtag == null ) continue; + + //print $file->file . " : " . $file->revtag->maintainer . " : " . $file->revtag->credits . "\n"; + //continue; + + if ( $file->revtag->errors != "" ) + { + //print $file->file . "\n"; + print $file->revtag->errors ; "\n"; + } +} From bd4846d2ad3e3e8b94976f29e7955ecb819f1e4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20L=20F=20S=20Bacci?= Date: Mon, 14 Aug 2023 11:42:49 -0300 Subject: [PATCH 29/50] New detail output for `qaxml.t.php`. --- scripts/translation/qaxml.t.php | 53 ++++++++++++++++++++++++++++++--- 1 file changed, 49 insertions(+), 4 deletions(-) diff --git a/scripts/translation/qaxml.t.php b/scripts/translation/qaxml.t.php index 68f99e50d..2ef35c8b9 100644 --- a/scripts/translation/qaxml.t.php +++ b/scripts/translation/qaxml.t.php @@ -6,11 +6,21 @@ require_once __DIR__ . '/lib/all.php'; -$qalist = QaFileInfo::cacheLoad(); - $tags = array(); -if ( count($argv) > 1 ) - $tags = explode( ',' , $argv[1] ); +$showDetail = false; + +array_shift( $argv ); + +if ( count( $argv ) > 0 && $argv[0] == '--detail' ) +{ + $showDetail = true; + array_shift( $argv ); +} + +if ( count( $argv ) > 0 ) + $tags = explode( ',' , $argv[0] ); + +$qalist = QaFileInfo::cacheLoad(); foreach ( $qalist as $qafile ) { @@ -156,6 +166,9 @@ } print "* {$tag} -{$targetCount} +{$sourceCount}\n"; + + if ( $showDetail ) + printTagUsageDetail( $source , $target , $tag ); } } @@ -239,3 +252,35 @@ function equalizeKeys( array $list , array & $other , mixed $value = 0 ) if ( ! isset( $other[$k] ) ) $other[$k] = $value; } + +function printTagUsageDetail( string $source , string $target , string $tag ) +{ + print "\n"; + $s = collectTagDefinitions( $source , $tag ); + $t = collectTagDefinitions( $target , $tag ); + $min = min( count( $s ) , count( $t ) ); + for( $i = 0 ; $i < $min ; $i++ ) + { + $d = $s[$i] - $t[$i]; + print "\t{$tag}\t{$s[$i]}\t{$t[$i]}\t{$d}\n"; + } + for( $i = $min ; $i < count($s) ; $i++ ) + print "\t{$tag}\t{$s[$i]}\t\t\n"; + for( $i = $min ; $i < count($t) ; $i++ ) + print "\t{$tag}\t\t{$t[$i]}\t\n"; + print "\n"; +} + +function collectTagDefinitions( string $file , string $tag ) +{ + $ret = array(); + $text = XmlUtil::loadFile( $file ); + $list = XmlUtil::listNodeType( $text , XML_ELEMENT_NODE ); + foreach( $list as $node ) + { + if ( $node->nodeName != $tag ) + continue; + $ret[] = $node->getLineNo(); + } + return $ret; +} \ No newline at end of file From efbefa743ea72020bcb85fb96ad127c64d9cf5af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20L=20F=20S=20Bacci?= Date: Tue, 15 Aug 2023 13:47:11 -0300 Subject: [PATCH 30/50] Detect and ignore expected outputs. --- scripts/translation/configure.php | 16 +--- scripts/translation/qaxml.t.php | 126 ++++++++++++++++++++---------- 2 files changed, 87 insertions(+), 55 deletions(-) diff --git a/scripts/translation/configure.php b/scripts/translation/configure.php index bc4492929..63addaf09 100644 --- a/scripts/translation/configure.php +++ b/scripts/translation/configure.php @@ -13,18 +13,4 @@ return; } -$run = new RevcheckRun( 'en' , $argv[1] , true ); - -foreach( $run->targetFiles->list as $file ) -{ - if ( $file->revtag == null ) continue; - - //print $file->file . " : " . $file->revtag->maintainer . " : " . $file->revtag->credits . "\n"; - //continue; - - if ( $file->revtag->errors != "" ) - { - //print $file->file . "\n"; - print $file->revtag->errors ; "\n"; - } -} +new RevcheckRun( 'en' , $argv[1] , true ); diff --git a/scripts/translation/qaxml.t.php b/scripts/translation/qaxml.t.php index 2ef35c8b9..42b0d15e5 100644 --- a/scripts/translation/qaxml.t.php +++ b/scripts/translation/qaxml.t.php @@ -29,10 +29,12 @@ if ( $qafile->sourceHash != $qafile->targetHash ) continue; - $output = false; $source = $qafile->sourceDir . '/' . $qafile->file; $target = $qafile->targetDir . '/' . $qafile->file; + $header = "qaxml.t: {$target}\n\n"; + $output = new TextBufferHasher(); + // Tag contents, text if ( count( $tags ) > 0 ) @@ -53,16 +55,12 @@ if ( count( $s ) == count( $t ) && count( $onlySource ) == 0 && count( $onlyTarget ) == 0 ) continue; - if ( ! $output ) - { - print "qaxml.t: {$target}\n\n"; - $output = true; - } + $output->pushFirst( $header ); foreach( $onlyTarget as $only ) - print "- {$only}\n"; + $output->push( "- {$only}\n" ); foreach( $onlySource as $only ) - print "+ {$only}\n"; + $output->push( "+ {$only}\n" ); if ( count( $onlySource ) == 0 && count( $onlyTarget ) == 0 ) { @@ -73,22 +71,18 @@ $countTarget = $t[$key]; $countDiff = $countSource - $countTarget; if ( $countDiff > 0 ) - print "* {$key} +{$countDiff}\n"; + $output->push( "* {$key} +{$countDiff}\n" ); if ( $countDiff < 0 ) - print "* {$key} {$countDiff}\n"; + $output->push( "* {$key} {$countDiff}\n" ); } } - if ( $output ) - { - print "\n"; - continue; - } + $output->pushExtra( "\n" ); } - // Tag contents, XML + // Tag contents, XML, if other checks passed - if ( count( $tags ) > 0 && ! $output ) + if ( count( $tags ) > 0 && count( $output->texts ) == 0 ) { $s = XmlUtil::loadFile( $source ); $t = XmlUtil::loadFile( $target ); @@ -106,16 +100,12 @@ if ( count( $s ) == count( $t ) && count( $onlySource ) == 0 && count( $onlyTarget ) == 0 ) continue; - if ( ! $output ) - { - print "qaxml.t: {$target}\n\n"; - $output = true; - } + $output->pushFirst( $header ); foreach( $onlyTarget as $only ) - print "- {$only}\n"; + $output->push( "- {$only}\n" ); foreach( $onlySource as $only ) - print "+ {$only}\n"; + $output->push( "+ {$only}\n" ); if ( count( $onlySource ) == 0 && count( $onlyTarget ) == 0 ) { @@ -126,17 +116,13 @@ $countTarget = $t[$key]; $countDiff = $countSource - $countTarget; if ( $countDiff > 0 ) - print "* {$key} +{$countDiff}\n"; + $output->push( "* {$key} +{$countDiff}\n" ); if ( $countDiff < 0 ) - print "* {$key} {$countDiff}\n"; + $output->push( "* {$key} {$countDiff}\n" ); } } - if ( $output ) - { - print "\n"; - continue; - } + $output->pushExtra( "\n" ); } // Tag count @@ -159,21 +145,46 @@ if ( $sourceCount != $targetCount ) { - if ( ! $output ) - { - print "qaxml.t: {$target}\n\n"; - $output = true; - } + $output->pushFirst( $header ); - print "* {$tag} -{$targetCount} +{$sourceCount}\n"; + $output->push( "* {$tag} -{$targetCount} +{$sourceCount}\n" ); if ( $showDetail ) printTagUsageDetail( $source , $target , $tag ); + + $output->pushExtra( "\n" ); } } + } + + // Output + + $marks = XmlUtil::listNodeType( XmlUtil::loadFile( $target ) , XML_COMMENT_NODE ); + foreach( $marks as $k => $v ) + $marks[$k] = trim( $v->nodeValue ); + + if ( count( $output->texts ) > 0 ) + { + $hash = $output->hash(); + $mark = "qaxml.t.php ignore $hash"; + + $key = array_search( $mark , $marks ); + if ( $key !== false ) + { + unset( $marks[$key] ); + } + else + { + $output->print(); + print "# Annotate with '' to ignore.\n\n"; + } - if ( $output ) - print "\n"; + foreach ( $marks as $item ) + if ( str_starts_with( $item , "qaxml.t.php ignore" ) ) + { + print $header; + print "# Unconsumed annotation: $item\n\n"; + } } } @@ -283,4 +294,39 @@ function collectTagDefinitions( string $file , string $tag ) $ret[] = $node->getLineNo(); } return $ret; -} \ No newline at end of file +} + +class TextBufferHasher +{ + public array $texts = array(); + + function hash() : string + { + if ( count( $this->texts) == 0 ) + return ""; + return md5( implode( "" , $this->texts ) ); + } + + function push( string $text ) + { + $this->texts[] = $text; + } + + function pushFirst( string $text ) + { + if ( count( $this->texts ) == 0 ) + $this->push( $text ); + } + + function pushExtra( string $text ) + { + if ( count( $this->texts ) > 0 ) + $this->push( $text ); + } + + function print() + { + foreach( $this->texts as $text ) + print $text; + } +} From 258bc50be330ed8f01744d7d3c58ed641addc064 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20L=20F=20S=20Bacci?= Date: Thu, 17 Aug 2023 15:16:36 -0300 Subject: [PATCH 31/50] New `qarvt.php` to validate revtag expected format. --- scripts/translation/README.md | 39 +++++++++++----- scripts/translation/lib/RevtagParser.php | 12 +++-- scripts/translation/qarvt.php | 59 ++++++++++++++++++++++++ scripts/translation/qaxml.t.php | 2 +- 4 files changed, 94 insertions(+), 18 deletions(-) create mode 100644 scripts/translation/qarvt.php diff --git a/scripts/translation/README.md b/scripts/translation/README.md index 55f1bb12e..7224cec78 100644 --- a/scripts/translation/README.md +++ b/scripts/translation/README.md @@ -17,6 +17,11 @@ Before using the scripts, it need be configured: php doc-base/scripts/translation/configure.php $LANG_DIR ``` +## qarvt.php + +`qarvt.a.php` checks if all translated files have revtags in the +expected format. + ## qaxml.a.php `qaxml.a.php` checks if all updated translated files have @@ -52,7 +57,21 @@ contents, as some tag's contents are expected *not* be translated. `--detail` will also print line defintions of each mismatched tag, to facilitate bitsecting. -Suggested execution: +## Suggested execution + +Structural checks: + +``` +php doc-base/scripts/translation/configure.php $LANG_DIR + +php doc-base/scripts/translation/qarvt.php + +php doc-base/scripts/translation/qaxml.a.php +php doc-base/scripts/translation/qaxml.e.php +php doc-base/scripts/translation/qaxml.p.php +php doc-base/scripts/translation/qaxml.t.php +``` +Tags where is expected no translations: ``` php doc-base/scripts/translation/qaxml.t.php acronym php doc-base/scripts/translation/qaxml.t.php classname @@ -62,15 +81,20 @@ php doc-base/scripts/translation/qaxml.t.php function php doc-base/scripts/translation/qaxml.t.php interfacename php doc-base/scripts/translation/qaxml.t.php parameter php doc-base/scripts/translation/qaxml.t.php type - php doc-base/scripts/translation/qaxml.t.php classsynopsis php doc-base/scripts/translation/qaxml.t.php constructorsynopsis php doc-base/scripts/translation/qaxml.t.php destructorsynopsis php doc-base/scripts/translation/qaxml.t.php fieldsynopsis php doc-base/scripts/translation/qaxml.t.php funcsynopsis php doc-base/scripts/translation/qaxml.t.php methodsynopsis + +php doc-base/scripts/translation/qaxml.t.php code +php doc-base/scripts/translation/qaxml.t.php computeroutput +php doc-base/scripts/translation/qaxml.t.php filename +php doc-base/scripts/translation/qaxml.t.php literal +php doc-base/scripts/translation/qaxml.t.php varname ``` -May contain translated parts, so manual check is necessary: +Tags where is expected few translations: ``` php doc-base/scripts/translation/qaxml.t.php code php doc-base/scripts/translation/qaxml.t.php filename @@ -80,15 +104,6 @@ php doc-base/scripts/translation/qaxml.t.php varname # Migration -## No critical - -This revcheck will not emit a critical status, for files outdated for more -than 30 days, as this was silently removed on - - -There is a new `days` property that tracks how old a file is in days, to make -decisions on coloring revcheck or deprecating translations. - ## Maintainers with spaces The regex on `RevtagParser` was narrowed to not accept maintainer's names diff --git a/scripts/translation/lib/RevtagParser.php b/scripts/translation/lib/RevtagParser.php index e88d74a6b..c9e28d6f2 100644 --- a/scripts/translation/lib/RevtagParser.php +++ b/scripts/translation/lib/RevtagParser.php @@ -12,7 +12,7 @@ class RevtagInfo public string $maintainer = ""; public string $status = ""; public string $credits = ""; - public string $errors = ""; + public array $errors = []; } class RevtagParser @@ -66,12 +66,14 @@ public static function parseComment( DOMNode $node , RevtagInfo & $ret , $filena $ret->status = trim( $match[3] ); if ( $ret->revision != "" && strlen( $ret->revision ) != 40 ) - $ret->errors .= "Wrong hash size: {$ret->revision}\n"; + $ret->errors[] = "Wrong hash size: {$ret->revision}"; if ( $ret->maintainer == "" ) - $ret->errors .= "Empty maintainer.\n"; + $ret->errors[] = "Empty maintainer."; if ( $ret->status == "" ) - $ret->errors .= "Empty status.\n"; + $ret->errors[] = "Empty status."; } + else + $ret->errors[] = "No revtag."; } if ( str_starts_with( $text , "CREDITS:" ) ) @@ -83,7 +85,7 @@ public static function parseComment( DOMNode $node , RevtagInfo & $ret , $filena $ret->credits = trim( $match[1] ); if ( $ret->credits == "" ) - $ret->errors .= "Empty credits.\n"; + $ret->errors[] = "Empty credits."; } } } diff --git a/scripts/translation/qarvt.php b/scripts/translation/qarvt.php new file mode 100644 index 000000000..be44cf8c1 --- /dev/null +++ b/scripts/translation/qarvt.php @@ -0,0 +1,59 @@ + 0 ) + { + foreach( $qalist as $qa ) + { + $langDir = $qa->targetDir; + break; + } + } + else + print_usage_exit(); +} + +$list = new RevcheckFileList( $langDir ); + +foreach( $list->list as $item ) +{ + $file = $langDir . '/' . $item->file; + $revt = RevtagParser::parseFile( $file ); + + if ( count( $revt->errors ) == 0 ) + continue; + + print "qarvt: $file\n"; + foreach( $revt->errors as $error ) + print " $error\n"; + print "\n"; +} + +function print_usage_exit() +{ + fwrite( STDERR , " Wrong paramater count. Usage:\n" ); + fwrite( STDERR , " {$argv[0]} [lang_dir]:\n" ); + exit; +} diff --git a/scripts/translation/qaxml.t.php b/scripts/translation/qaxml.t.php index 42b0d15e5..0c7908cc9 100644 --- a/scripts/translation/qaxml.t.php +++ b/scripts/translation/qaxml.t.php @@ -176,7 +176,7 @@ else { $output->print(); - print "# Annotate with '' to ignore.\n\n"; + print "# To ignore, annotate with: \n\n"; } foreach ( $marks as $item ) From 34f2e3d65d75ca3da6531702f7a5ceba1afc8b35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20L=20F=20S=20Bacci?= Date: Fri, 18 Aug 2023 10:48:17 -0300 Subject: [PATCH 32/50] Offer ignore comment only if checking tag's contents. --- scripts/translation/qaxml.t.php | 80 ++++++++++++++++++--------------- 1 file changed, 43 insertions(+), 37 deletions(-) diff --git a/scripts/translation/qaxml.t.php b/scripts/translation/qaxml.t.php index 0c7908cc9..e6ffa81f3 100644 --- a/scripts/translation/qaxml.t.php +++ b/scripts/translation/qaxml.t.php @@ -8,6 +8,7 @@ $tags = array(); $showDetail = false; +$showIgnore = false; array_shift( $argv ); @@ -32,10 +33,9 @@ $source = $qafile->sourceDir . '/' . $qafile->file; $target = $qafile->targetDir . '/' . $qafile->file; - $header = "qaxml.t: {$target}\n\n"; - $output = new TextBufferHasher(); + $output = new TextBufferHasher( "qaxml.t: {$target}\n\n" ); - // Tag contents, text + // Check tag contents, inner text if ( count( $tags ) > 0 ) { @@ -55,7 +55,7 @@ if ( count( $s ) == count( $t ) && count( $onlySource ) == 0 && count( $onlyTarget ) == 0 ) continue; - $output->pushFirst( $header ); + $showIgnore = true; foreach( $onlyTarget as $only ) $output->push( "- {$only}\n" ); @@ -80,9 +80,9 @@ $output->pushExtra( "\n" ); } - // Tag contents, XML, if other checks passed + // Check tag contents, inner XML - if ( count( $tags ) > 0 && count( $output->texts ) == 0 ) + if ( count( $tags ) > 0 && $output->isEmpty() ) { $s = XmlUtil::loadFile( $source ); $t = XmlUtil::loadFile( $target ); @@ -100,7 +100,7 @@ if ( count( $s ) == count( $t ) && count( $onlySource ) == 0 && count( $onlyTarget ) == 0 ) continue; - $output->pushFirst( $header ); + $showIgnore = true; foreach( $onlyTarget as $only ) $output->push( "- {$only}\n" ); @@ -125,7 +125,9 @@ $output->pushExtra( "\n" ); } - // Tag count + // Check tag count + + if ( $output->isEmpty() ) { $s = XmlUtil::loadFile( $source ); $t = XmlUtil::loadFile( $target ); @@ -145,47 +147,40 @@ if ( $sourceCount != $targetCount ) { - $output->pushFirst( $header ); - + $showIgnore = false; $output->push( "* {$tag} -{$targetCount} +{$sourceCount}\n" ); if ( $showDetail ) printTagUsageDetail( $source , $target , $tag ); - - $output->pushExtra( "\n" ); } } + $output->pushExtra( "\n" ); } - // Output - - $marks = XmlUtil::listNodeType( XmlUtil::loadFile( $target ) , XML_COMMENT_NODE ); - foreach( $marks as $k => $v ) - $marks[$k] = trim( $v->nodeValue ); - - if ( count( $output->texts ) > 0 ) + if ( $showIgnore ) { $hash = $output->hash(); $mark = "qaxml.t.php ignore $hash"; + $marks = XmlUtil::listNodeType( XmlUtil::loadFile( $target ) , XML_COMMENT_NODE ); + foreach( $marks as $k => $v ) + $marks[$k] = trim( $v->nodeValue ); $key = array_search( $mark , $marks ); - if ( $key !== false ) - { - unset( $marks[$key] ); - } + + if ( $key === false ) + $output->push( " To ignore, annotate with: \n" ); else - { - $output->print(); - print "# To ignore, annotate with: \n\n"; - } + unset( $marks[$key] ); foreach ( $marks as $item ) if ( str_starts_with( $item , "qaxml.t.php ignore" ) ) - { - print $header; - print "# Unconsumed annotation: $item\n\n"; - } + $output->push( " Unconsumed annotation: $item\n" ); + + $output->pushExtra( "\n" ); } + + if ( $output->isEmpty() == false ) + $output->print(); } function extractTagsInnerText( array $nodes , array $tags ) @@ -298,24 +293,34 @@ function collectTagDefinitions( string $file , string $tag ) class TextBufferHasher { + public string $header = ""; public array $texts = array(); + function __construct( string $header = "" ) + { + $this->header = $header; + } + function hash() : string { if ( count( $this->texts) == 0 ) return ""; - return md5( implode( "" , $this->texts ) ); + $text = $this->header . implode( "" , $this->texts ); + $text = str_replace( " " , "" , $text ); + $text = str_replace( "\n" , "" , $text ); + $text = str_replace( "\r" , "" , $text ); + $text = str_replace( "\t" , "" , $text ); + return md5( $text ); } - function push( string $text ) + function isEmpty() : bool { - $this->texts[] = $text; + return count( $this->texts ) == 0; } - function pushFirst( string $text ) + function push( string $text ) { - if ( count( $this->texts ) == 0 ) - $this->push( $text ); + $this->texts[] = $text; } function pushExtra( string $text ) @@ -326,6 +331,7 @@ function pushExtra( string $text ) function print() { + print $this->header; foreach( $this->texts as $text ) print $text; } From aa9b422c6c127ec5b36bbc4417f579b4e202b446 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20L=20F=20S=20Bacci?= Date: Fri, 18 Aug 2023 12:28:17 -0300 Subject: [PATCH 33/50] Save ignore locally, outside of global XML. --- scripts/translation/lib/CacheFile.php | 43 +++++++++++ scripts/translation/lib/all.php | 1 + scripts/translation/qaxml.t.php | 102 +++++++++++++++++--------- 3 files changed, 113 insertions(+), 33 deletions(-) create mode 100644 scripts/translation/lib/CacheFile.php diff --git a/scripts/translation/lib/CacheFile.php b/scripts/translation/lib/CacheFile.php new file mode 100644 index 000000000..f0ec2fb3d --- /dev/null +++ b/scripts/translation/lib/CacheFile.php @@ -0,0 +1,43 @@ +filename = CacheFile::prepareFilename( "" , $file , true ); + } + + public function load() + { + if ( file_exists( $this->filename ) == false ) + return null; + $data = file_get_contents( $this->filename ); + return unserialize( gzdecode( $data ) ); + } + + public function save( $data ) + { + $contents = gzencode( serialize( $data ) ); + file_put_contents( $this->filename , $contents ); + } + + public static function prepareFilename( string $path , string $file , bool $createDirs = false ) + { + $baseDir = CacheUtil::CACHE_DIR; + $outPath = rtrim( $baseDir , '/' ) . '/' . $path; + $outFile = rtrim( $outPath , '/' ) . '/' . $file; + if ( $createDirs && file_exists( $outPath ) == false ) + mkdir( $outPath , 0777 , true ); + return $outFile; + } +} \ No newline at end of file diff --git a/scripts/translation/lib/all.php b/scripts/translation/lib/all.php index 0caca38bf..24ae76df1 100644 --- a/scripts/translation/lib/all.php +++ b/scripts/translation/lib/all.php @@ -4,6 +4,7 @@ ini_set( 'display_startup_errors' , 1 ); error_reporting( E_ALL ); +require_once __DIR__ . '/CacheFile.php'; require_once __DIR__ . '/CacheUtil.php'; require_once __DIR__ . '/GitDiffParser.php'; require_once __DIR__ . '/GitLogParser.php'; diff --git a/scripts/translation/qaxml.t.php b/scripts/translation/qaxml.t.php index e6ffa81f3..85bd3933a 100644 --- a/scripts/translation/qaxml.t.php +++ b/scripts/translation/qaxml.t.php @@ -10,16 +10,46 @@ $showDetail = false; $showIgnore = false; -array_shift( $argv ); +$igfile = new CacheFile( "qaxml.t.ignore" ); +$ignore = $igfile->load(); +if ( $ignore == null ) + $ignore = array(); -if ( count( $argv ) > 0 && $argv[0] == '--detail' ) +$cmd0 = array_shift( $argv ); + +while ( count( $argv ) > 0 ) { - $showDetail = true; - array_shift( $argv ); -} + $arg = array_shift( $argv ); + + if ( $arg == "--detail" ) + { + $showDetail = true; + continue; + } + + if ( str_starts_with( $arg , "--add-ignore=" ) ) + { + $add = substr( $arg , 13 ); + $ignore[] = $add; + $igfile->save( $ignore ); + exit; + } + + if ( str_starts_with( $arg , "--del-ignore=" ) ) + { + $del = substr( $arg , 13 ); + $key = array_search( $del , $ignore ); + + if ( $key === false ) + print "Ignore mark not found.\n"; + else + unset( $ignore[$key] ); + $igfile->save( $ignore ); + exit; + } -if ( count( $argv ) > 0 ) - $tags = explode( ',' , $argv[0] ); + $tags = explode( ',' , $arg ); +} $qalist = QaFileInfo::cacheLoad(); @@ -33,7 +63,7 @@ $source = $qafile->sourceDir . '/' . $qafile->file; $target = $qafile->targetDir . '/' . $qafile->file; - $output = new TextBufferHasher( "qaxml.t: {$target}\n\n" ); + $output = new OutputBufferHasher( "qaxml.t: {$target}\n\n" ); // Check tag contents, inner text @@ -151,36 +181,38 @@ $output->push( "* {$tag} -{$targetCount} +{$sourceCount}\n" ); if ( $showDetail ) - printTagUsageDetail( $source , $target , $tag ); + printTagUsageDetail( $source , $target , $tag , $output ); } } $output->pushExtra( "\n" ); } + // Output && Ignore + if ( $showIgnore ) { $hash = $output->hash(); - $mark = "qaxml.t.php ignore $hash"; + $mark = "{$hash},{$qafile->file}"; - $marks = XmlUtil::listNodeType( XmlUtil::loadFile( $target ) , XML_COMMENT_NODE ); - foreach( $marks as $k => $v ) - $marks[$k] = trim( $v->nodeValue ); - $key = array_search( $mark , $marks ); + $key = array_search( $mark , $ignore ); if ( $key === false ) - $output->push( " To ignore, annotate with: \n" ); + { + $output->push( " To ignore, run:\n php $cmd0 --add-ignore=$mark\n" ); + } else - unset( $marks[$key] ); - - foreach ( $marks as $item ) - if ( str_starts_with( $item , "qaxml.t.php ignore" ) ) - $output->push( " Unconsumed annotation: $item\n" ); + { + unset( $ignore[$key] ); + $output->clear(); + } - $output->pushExtra( "\n" ); + foreach ( $ignore as $item ) + if ( str_ends_with( $item , ",$qafile->file" ) ) + $output->push( " Unused ignore. To drop, run:\n php $cmd0 --del-ignore=$mark\n" ); } - if ( $output->isEmpty() == false ) - $output->print(); + $output->pushExtra( "\n" ); + $output->print(); } function extractTagsInnerText( array $nodes , array $tags ) @@ -259,22 +291,19 @@ function equalizeKeys( array $list , array & $other , mixed $value = 0 ) $other[$k] = $value; } -function printTagUsageDetail( string $source , string $target , string $tag ) +function printTagUsageDetail( string $source , string $target , string $tag , OutputBufferHasher $output ) { - print "\n"; + $output->push( "\n" ); $s = collectTagDefinitions( $source , $tag ); $t = collectTagDefinitions( $target , $tag ); $min = min( count( $s ) , count( $t ) ); for( $i = 0 ; $i < $min ; $i++ ) - { - $d = $s[$i] - $t[$i]; - print "\t{$tag}\t{$s[$i]}\t{$t[$i]}\t{$d}\n"; - } + $output->push( "\t{$tag}\t{$s[$i]}\t{$t[$i]}\n" ); for( $i = $min ; $i < count($s) ; $i++ ) - print "\t{$tag}\t{$s[$i]}\t\t\n"; + $output->push( "\t{$tag}\t{$s[$i]}\t\t\n" ); for( $i = $min ; $i < count($t) ; $i++ ) - print "\t{$tag}\t\t{$t[$i]}\t\n"; - print "\n"; + $output->push( "\t{$tag}\t\t{$t[$i]}\t\n" ); + $output->push( "\n" ); } function collectTagDefinitions( string $file , string $tag ) @@ -291,7 +320,7 @@ function collectTagDefinitions( string $file , string $tag ) return $ret; } -class TextBufferHasher +class OutputBufferHasher { public string $header = ""; public array $texts = array(); @@ -301,6 +330,11 @@ function __construct( string $header = "" ) $this->header = $header; } + function clear() + { + $this->texts = array(); + } + function hash() : string { if ( count( $this->texts) == 0 ) @@ -331,6 +365,8 @@ function pushExtra( string $text ) function print() { + if ( $this->isEmpty() ) + return; print $this->header; foreach( $this->texts as $text ) print $text; From 2ab0f2f1c04ae5c999d36e11ffa01f5c3e440b14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20L=20F=20S=20Bacci?= Date: Fri, 18 Aug 2023 13:16:38 -0300 Subject: [PATCH 34/50] Enhance ignore marking to consider arguments and intenal state. --- scripts/translation/lib/CacheFile.php | 4 ++-- scripts/translation/qaxml.t.php | 30 +++++++++++++-------------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/scripts/translation/lib/CacheFile.php b/scripts/translation/lib/CacheFile.php index f0ec2fb3d..b24939aa8 100644 --- a/scripts/translation/lib/CacheFile.php +++ b/scripts/translation/lib/CacheFile.php @@ -17,10 +17,10 @@ function __construct( string $file ) $this->filename = CacheFile::prepareFilename( "" , $file , true ); } - public function load() + public function load( mixed $init = null ) { if ( file_exists( $this->filename ) == false ) - return null; + return $init; $data = file_get_contents( $this->filename ); return unserialize( gzdecode( $data ) ); } diff --git a/scripts/translation/qaxml.t.php b/scripts/translation/qaxml.t.php index 85bd3933a..911215799 100644 --- a/scripts/translation/qaxml.t.php +++ b/scripts/translation/qaxml.t.php @@ -11,9 +11,6 @@ $showIgnore = false; $igfile = new CacheFile( "qaxml.t.ignore" ); -$ignore = $igfile->load(); -if ( $ignore == null ) - $ignore = array(); $cmd0 = array_shift( $argv ); @@ -29,6 +26,7 @@ if ( str_starts_with( $arg , "--add-ignore=" ) ) { + $ignore = $igfile->load( array() ); $add = substr( $arg , 13 ); $ignore[] = $add; $igfile->save( $ignore ); @@ -37,6 +35,7 @@ if ( str_starts_with( $arg , "--del-ignore=" ) ) { + $ignore = $igfile->load( array() ); $del = substr( $arg , 13 ); $key = array_search( $del , $ignore ); @@ -64,6 +63,7 @@ $target = $qafile->targetDir . '/' . $qafile->file; $output = new OutputBufferHasher( "qaxml.t: {$target}\n\n" ); + $ignore = $igfile->load( array() ); // Check tag contents, inner text @@ -191,27 +191,27 @@ if ( $showIgnore ) { - $hash = $output->hash(); - $mark = "{$hash},{$qafile->file}"; + $prefix = $output->hash( $tags ); + $suffix = md5( implode( "" , $tags ) ) . ',' . $qafile->file; + $mark = "{$prefix},{$suffix}"; - $key = array_search( $mark , $ignore ); - - if ( $key === false ) - { - $output->push( " To ignore, run:\n php $cmd0 --add-ignore=$mark\n" ); - } + if ( in_array( $mark , $ignore ) ) + $output->clear(); else + $output->push( " To ignore, run:\n php $cmd0 --add-ignore=$mark\n" ); + + while ( in_array( $mark , $ignore ) ) { + $key = array_search( $mark , $ignore ); unset( $ignore[$key] ); - $output->clear(); } - foreach ( $ignore as $item ) - if ( str_ends_with( $item , ",$qafile->file" ) ) + if ( str_ends_with( $item , $suffix ) ) $output->push( " Unused ignore. To drop, run:\n php $cmd0 --del-ignore=$mark\n" ); + + $output->pushExtra( "\n" ); } - $output->pushExtra( "\n" ); $output->print(); } From b1d1205957325f7a7fe6b1d24425b121e3a5c2e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20L=20F=20S=20Bacci?= Date: Fri, 18 Aug 2023 14:16:21 -0300 Subject: [PATCH 35/50] Ignore usage in all cases. --- scripts/translation/lib/CacheFile.php | 11 ++++++----- scripts/translation/qaxml.a.php | 2 ++ scripts/translation/qaxml.e.php | 2 ++ scripts/translation/qaxml.p.php | 2 ++ scripts/translation/qaxml.t.php | 16 +++++++--------- 5 files changed, 19 insertions(+), 14 deletions(-) diff --git a/scripts/translation/lib/CacheFile.php b/scripts/translation/lib/CacheFile.php index b24939aa8..749083ff7 100644 --- a/scripts/translation/lib/CacheFile.php +++ b/scripts/translation/lib/CacheFile.php @@ -14,7 +14,7 @@ class CacheFile function __construct( string $file ) { - $this->filename = CacheFile::prepareFilename( "" , $file , true ); + $this->filename = CacheFile::prepareFilename( $file , true ); } public function load( mixed $init = null ) @@ -31,12 +31,13 @@ public function save( $data ) file_put_contents( $this->filename , $contents ); } - public static function prepareFilename( string $path , string $file , bool $createDirs = false ) + public static function prepareFilename( string $file , bool $createCacheDirs = false ) { - $baseDir = CacheUtil::CACHE_DIR; - $outPath = rtrim( $baseDir , '/' ) . '/' . $path; + if ( str_starts_with( $file , '/' ) ) + return $file; + $outPath = CacheUtil::CACHE_DIR . '/' . dirname( $file ); $outFile = rtrim( $outPath , '/' ) . '/' . $file; - if ( $createDirs && file_exists( $outPath ) == false ) + if ( $createCacheDirs && file_exists( $outPath ) == false ) mkdir( $outPath , 0777 , true ); return $outFile; } diff --git a/scripts/translation/qaxml.a.php b/scripts/translation/qaxml.a.php index 42e84be0b..c351ad69f 100644 --- a/scripts/translation/qaxml.a.php +++ b/scripts/translation/qaxml.a.php @@ -10,6 +10,8 @@ foreach ( $qalist as $qafile ) { + if ( $qafile->file == "bookinfo.xml" ) + continue; if ( $qafile->sourceHash != $qafile->targetHash ) continue; diff --git a/scripts/translation/qaxml.e.php b/scripts/translation/qaxml.e.php index 4372d7923..2a6753462 100644 --- a/scripts/translation/qaxml.e.php +++ b/scripts/translation/qaxml.e.php @@ -10,6 +10,8 @@ foreach ( $qalist as $qafile ) { + if ( $qafile->file == "bookinfo.xml" ) + continue; if ( $qafile->sourceHash != $qafile->targetHash ) continue; diff --git a/scripts/translation/qaxml.p.php b/scripts/translation/qaxml.p.php index cf3564561..6efebcb07 100644 --- a/scripts/translation/qaxml.p.php +++ b/scripts/translation/qaxml.p.php @@ -10,6 +10,8 @@ foreach ( $qalist as $qafile ) { + if ( $qafile->file == "bookinfo.xml" ) + continue; if ( $qafile->sourceHash != $qafile->targetHash ) continue; diff --git a/scripts/translation/qaxml.t.php b/scripts/translation/qaxml.t.php index 911215799..c268adb9a 100644 --- a/scripts/translation/qaxml.t.php +++ b/scripts/translation/qaxml.t.php @@ -8,9 +8,8 @@ $tags = array(); $showDetail = false; -$showIgnore = false; -$igfile = new CacheFile( "qaxml.t.ignore" ); +$igfile = new CacheFile( getcwd() . "/.qaxml.t.ignore" ); $cmd0 = array_shift( $argv ); @@ -27,6 +26,8 @@ if ( str_starts_with( $arg , "--add-ignore=" ) ) { $ignore = $igfile->load( array() ); + if ( count( $ignore ) == 0 ) + print "Creating file ignore file on current working directory.\n"; $add = substr( $arg , 13 ); $ignore[] = $add; $igfile->save( $ignore ); @@ -85,8 +86,6 @@ if ( count( $s ) == count( $t ) && count( $onlySource ) == 0 && count( $onlyTarget ) == 0 ) continue; - $showIgnore = true; - foreach( $onlyTarget as $only ) $output->push( "- {$only}\n" ); foreach( $onlySource as $only ) @@ -130,8 +129,6 @@ if ( count( $s ) == count( $t ) && count( $onlySource ) == 0 && count( $onlyTarget ) == 0 ) continue; - $showIgnore = true; - foreach( $onlyTarget as $only ) $output->push( "- {$only}\n" ); foreach( $onlySource as $only ) @@ -177,7 +174,6 @@ if ( $sourceCount != $targetCount ) { - $showIgnore = false; $output->push( "* {$tag} -{$targetCount} +{$sourceCount}\n" ); if ( $showDetail ) @@ -187,9 +183,9 @@ $output->pushExtra( "\n" ); } - // Output && Ignore + // Ignore - if ( $showIgnore ) + if ( $output->isEmpty() == false ) { $prefix = $output->hash( $tags ); $suffix = md5( implode( "" , $tags ) ) . ',' . $qafile->file; @@ -212,6 +208,8 @@ $output->pushExtra( "\n" ); } + // Output + $output->print(); } From 1292b1f47d4c2d71dae277de34b9f9b69720d0ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20L=20F=20S=20Bacci?= Date: Mon, 21 Aug 2023 23:27:48 -0300 Subject: [PATCH 36/50] Generic algorithm to detect count and contents mismatch. --- scripts/translation/lib/RevcheckRun.php | 14 +-- scripts/translation/qaxml.a.php | 40 ++++---- scripts/translation/qaxml.e.php | 40 ++++---- scripts/translation/qaxml.p.php | 40 ++++---- scripts/translation/qaxml.t.php | 122 ++++++++++-------------- 5 files changed, 121 insertions(+), 135 deletions(-) diff --git a/scripts/translation/lib/RevcheckRun.php b/scripts/translation/lib/RevcheckRun.php index 518b6e2a1..ee71607a6 100644 --- a/scripts/translation/lib/RevcheckRun.php +++ b/scripts/translation/lib/RevcheckRun.php @@ -14,13 +14,13 @@ class RevcheckRun public RevcheckFileList $targetFiles; // Separated lists - public array $filesOk; - public array $filesOld; - public array $filesRevtagProblem; - public array $filesUntranslated; - public array $filesNotInEn; - public array $filesWip; - public array $qaList; + public array $filesOk = []; + public array $filesOld = []; + public array $filesRevtagProblem = []; + public array $filesUntranslated = []; + public array $filesNotInEn = []; + public array $filesWip = []; + public array $qaList = []; function __construct( string $sourceDir , string $targetDir , bool $writeResults = true ) { diff --git a/scripts/translation/qaxml.a.php b/scripts/translation/qaxml.a.php index c351ad69f..1c0f50740 100644 --- a/scripts/translation/qaxml.a.php +++ b/scripts/translation/qaxml.a.php @@ -30,33 +30,35 @@ if ( implode( "\n" , $s ) == implode( "\n" , $t ) ) continue; - $intersect = array_intersect( $s, $t ); - $onlySource = array_diff( $s , $intersect ); - $onlyTarget = array_diff( $t , $intersect ); + $header = true; + $match = array(); - print "qaxml.a: {$target}\n\n"; + foreach( $s as $v ) + $match[$v] = array( 0 , 0 ); + foreach( $t as $v ) + $match[$v] = array( 0 , 0 ); - foreach( $onlyTarget as $only ) - print "- {$only}\n"; - foreach( $onlySource as $only ) - print "+ {$only}\n"; + foreach( $s as $v ) + $match[$v][0] += 1; + foreach( $t as $v ) + $match[$v][1] += 1; - if ( count( $onlySource ) == 0 && count( $onlyTarget ) == 0 ) + foreach( $match as $k => $v ) { - $s = array_count_values( $s ); - $t = array_count_values( $t ); - foreach ($s as $key => $countSource ) + if ( $v[0] == $v[1] ) + continue; + + if ( $header ) { - $countTarget = $t[$key]; - $countDiff = $countSource - $countTarget; - if ( $countDiff > 0 ) - print "* {$key} +{$countDiff}\n"; - if ( $countDiff < 0 ) - print "* {$key} {$countDiff}\n"; + print "qaxml.a: {$target}\n\n"; + $header = false; } + + print "* {$k} -{$v[1]} +{$v[0]}\n"; } - print "\n"; + if ( ! $header ) + print "\n"; } function extractTriple( array $list ) diff --git a/scripts/translation/qaxml.e.php b/scripts/translation/qaxml.e.php index 2a6753462..88fcacae1 100644 --- a/scripts/translation/qaxml.e.php +++ b/scripts/translation/qaxml.e.php @@ -24,31 +24,33 @@ if ( implode( "\n" , $s ) == implode( "\n" , $t ) ) continue; - $intersect = array_intersect( $s, $t ); - $onlySource = array_diff( $s , $intersect ); - $onlyTarget = array_diff( $t , $intersect ); + $header = true; + $match = array(); - print "qaxml.e: {$target}\n\n"; + foreach( $s as $v ) + $match[$v] = array( 0 , 0 ); + foreach( $t as $v ) + $match[$v] = array( 0 , 0 ); - foreach( $onlyTarget as $only ) - print "- {$only}\n"; - foreach( $onlySource as $only ) - print "+ {$only}\n"; + foreach( $s as $v ) + $match[$v][0] += 1; + foreach( $t as $v ) + $match[$v][1] += 1; - if ( count( $onlySource ) == 0 && count( $onlyTarget ) == 0 ) + foreach( $match as $k => $v ) { - $s = array_count_values( $s ); - $t = array_count_values( $t ); - foreach ($s as $key => $countSource ) + if ( $v[0] == $v[1] ) + continue; + + if ( $header ) { - $countTarget = $t[$key]; - $countDiff = $countSource - $countTarget; - if ( $countDiff > 0 ) - print "* {$key} +{$countDiff}\n"; - if ( $countDiff < 0 ) - print "* {$key} {$countDiff}\n"; + print "qaxml.e: {$target}\n\n"; + $header = false; } + + print "* &{$k}; -{$v[1]} +{$v[0]}\n"; } - print "\n"; + if ( ! $header ) + print "\n"; } \ No newline at end of file diff --git a/scripts/translation/qaxml.p.php b/scripts/translation/qaxml.p.php index 6efebcb07..dd30a75e0 100644 --- a/scripts/translation/qaxml.p.php +++ b/scripts/translation/qaxml.p.php @@ -30,33 +30,35 @@ if ( implode( "\n" , $s ) == implode( "\n" , $t ) ) continue; - $intersect = array_intersect( $s, $t ); - $onlySource = array_diff( $s , $intersect ); - $onlyTarget = array_diff( $t , $intersect ); + $header = true; + $match = array(); - print "qaxml.p: {$target}\n\n"; + foreach( $s as $v ) + $match[$v] = array( 0 , 0 ); + foreach( $t as $v ) + $match[$v] = array( 0 , 0 ); - foreach( $onlyTarget as $only ) - print "- {$only}\n"; - foreach( $onlySource as $only ) - print "+ {$only}\n"; + foreach( $s as $v ) + $match[$v][0] += 1; + foreach( $t as $v ) + $match[$v][1] += 1; - if ( count( $onlySource ) == 0 && count( $onlyTarget ) == 0 ) + foreach( $match as $k => $v ) { - $s = array_count_values( $s ); - $t = array_count_values( $t ); - foreach ($s as $key => $countSource ) + if ( $v[0] == $v[1] ) + continue; + + if ( $header ) { - $countTarget = $t[$key]; - $countDiff = $countSource - $countTarget; - if ( $countDiff > 0 ) - print "* {$key} +{$countDiff}\n"; - if ( $countDiff < 0 ) - print "* {$key} {$countDiff}\n"; + print "qaxml.p: {$target}\n\n"; + $header = false; } + + print "* {$k} -{$v[1]} +{$v[0]}\n"; } - print "\n"; + if ( ! $header ) + print "\n"; } function extractPiData( array $list ) diff --git a/scripts/translation/qaxml.t.php b/scripts/translation/qaxml.t.php index c268adb9a..c71cbc176 100644 --- a/scripts/translation/qaxml.t.php +++ b/scripts/translation/qaxml.t.php @@ -79,31 +79,23 @@ $s = extractTagsInnerText( $s , $tags ); $t = extractTagsInnerText( $t , $tags ); - $intersect = array_intersect( $s, $t ); - $onlySource = array_diff( $s , $intersect ); - $onlyTarget = array_diff( $t , $intersect ); + $match = array(); - if ( count( $s ) == count( $t ) && count( $onlySource ) == 0 && count( $onlyTarget ) == 0 ) - continue; + foreach( $s as $v ) + $match[$v] = array( 0 , 0 ); + foreach( $t as $v ) + $match[$v] = array( 0 , 0 ); - foreach( $onlyTarget as $only ) - $output->push( "- {$only}\n" ); - foreach( $onlySource as $only ) - $output->push( "+ {$only}\n" ); + foreach( $s as $v ) + $match[$v][0] += 1; + foreach( $t as $v ) + $match[$v][1] += 1; - if ( count( $onlySource ) == 0 && count( $onlyTarget ) == 0 ) + foreach( $match as $k => $v ) { - $s = array_count_values( $s ); - $t = array_count_values( $t ); - foreach ($s as $key => $countSource ) - { - $countTarget = $t[$key]; - $countDiff = $countSource - $countTarget; - if ( $countDiff > 0 ) - $output->push( "* {$key} +{$countDiff}\n" ); - if ( $countDiff < 0 ) - $output->push( "* {$key} {$countDiff}\n" ); - } + if ( $v[0] == $v[1] ) + continue; + $output->push( "* {$k} -{$v[1]} +{$v[0]}\n" ); } $output->pushExtra( "\n" ); @@ -122,31 +114,23 @@ $s = extractTagsInnerXmls( $s , $tags ); $t = extractTagsInnerXmls( $t , $tags ); - $intersect = array_intersect( $s, $t ); - $onlySource = array_diff( $s , $intersect ); - $onlyTarget = array_diff( $t , $intersect ); + $match = array(); - if ( count( $s ) == count( $t ) && count( $onlySource ) == 0 && count( $onlyTarget ) == 0 ) - continue; + foreach( $s as $v ) + $match[$v] = array( 0 , 0 ); + foreach( $t as $v ) + $match[$v] = array( 0 , 0 ); - foreach( $onlyTarget as $only ) - $output->push( "- {$only}\n" ); - foreach( $onlySource as $only ) - $output->push( "+ {$only}\n" ); + foreach( $s as $v ) + $match[$v][0] += 1; + foreach( $t as $v ) + $match[$v][1] += 1; - if ( count( $onlySource ) == 0 && count( $onlyTarget ) == 0 ) + foreach( $match as $k => $v ) { - $s = array_count_values( $s ); - $t = array_count_values( $t ); - foreach ($s as $key => $countSource ) - { - $countTarget = $t[$key]; - $countDiff = $countSource - $countTarget; - if ( $countDiff > 0 ) - $output->push( "* {$key} +{$countDiff}\n" ); - if ( $countDiff < 0 ) - $output->push( "* {$key} {$countDiff}\n" ); - } + if ( $v[0] == $v[1] ) + continue; + $output->push( "* {$k} -{$v[1]} +{$v[0]}\n" ); } $output->pushExtra( "\n" ); @@ -162,24 +146,28 @@ $s = XmlUtil::listNodeType( $s , XML_ELEMENT_NODE ); $t = XmlUtil::listNodeType( $t , XML_ELEMENT_NODE ); - $s = countTags( $s ); - $t = countTags( $t ); + $s = extractNodeName( $s ); + $t = extractNodeName( $t ); - equalizeKeys( $s , $t ); - equalizeKeys( $t , $s ); + $match = array(); - foreach( $s as $tag => $sourceCount ) - { - $targetCount = $t[$tag]; + foreach( $s as $v ) + $match[$v] = array( 0 , 0 ); + foreach( $t as $v ) + $match[$v] = array( 0 , 0 ); - if ( $sourceCount != $targetCount ) - { - $output->push( "* {$tag} -{$targetCount} +{$sourceCount}\n" ); + foreach( $s as $v ) + $match[$v][0] += 1; + foreach( $t as $v ) + $match[$v][1] += 1; - if ( $showDetail ) - printTagUsageDetail( $source , $target , $tag , $output ); - } + foreach( $match as $k => $v ) + { + if ( $v[0] == $v[1] ) + continue; + $output->push( "* {$k} -{$v[1]} +{$v[0]}\n" ); } + $output->pushExtra( "\n" ); } @@ -213,6 +201,14 @@ $output->print(); } +function extractNodeName( array $list ) +{ + $ret = array(); + foreach( $list as $elem ) + $ret[] = $elem->nodeName; + return $ret; +} + function extractTagsInnerText( array $nodes , array $tags ) { $ret = array(); @@ -273,22 +269,6 @@ function extractTagsInnerXmls( array $nodes , array $tags ) return $ret; } -function countTags( array $list ) -{ - $ret = array(); - foreach( $list as $elem ) - $ret[] = $elem->nodeName; - $ret = array_count_values( $ret ); - return $ret; -} - -function equalizeKeys( array $list , array & $other , mixed $value = 0 ) -{ - foreach( $list as $k => $v ) - if ( ! isset( $other[$k] ) ) - $other[$k] = $value; -} - function printTagUsageDetail( string $source , string $target , string $tag , OutputBufferHasher $output ) { $output->push( "\n" ); From cda81296d48573699e0dd5d1a876a2623c9780f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20L=20F=20S=20Bacci?= Date: Mon, 21 Aug 2023 23:58:11 -0300 Subject: [PATCH 37/50] Count filtered. --- scripts/translation/qaxml.t.php | 48 +++++++++++++++++---------------- 1 file changed, 25 insertions(+), 23 deletions(-) diff --git a/scripts/translation/qaxml.t.php b/scripts/translation/qaxml.t.php index c71cbc176..84af9f434 100644 --- a/scripts/translation/qaxml.t.php +++ b/scripts/translation/qaxml.t.php @@ -146,8 +146,8 @@ $s = XmlUtil::listNodeType( $s , XML_ELEMENT_NODE ); $t = XmlUtil::listNodeType( $t , XML_ELEMENT_NODE ); - $s = extractNodeName( $s ); - $t = extractNodeName( $t ); + $s = extractNodeName( $s , $tags ); + $t = extractNodeName( $t , $tags ); $match = array(); @@ -173,39 +173,41 @@ // Ignore - if ( $output->isEmpty() == false ) - { - $prefix = $output->hash( $tags ); - $suffix = md5( implode( "" , $tags ) ) . ',' . $qafile->file; - $mark = "{$prefix},{$suffix}"; + if ( false ) + if ( $output->isEmpty() == false ) + { + $prefix = $output->hash( $tags ); + $suffix = md5( implode( "" , $tags ) ) . ',' . $qafile->file; + $mark = "{$prefix},{$suffix}"; - if ( in_array( $mark , $ignore ) ) - $output->clear(); - else - $output->push( " To ignore, run:\n php $cmd0 --add-ignore=$mark\n" ); + if ( in_array( $mark , $ignore ) ) + $output->clear(); + else + $output->push( " To ignore, run:\n php $cmd0 --add-ignore=$mark\n" ); - while ( in_array( $mark , $ignore ) ) - { - $key = array_search( $mark , $ignore ); - unset( $ignore[$key] ); - } - foreach ( $ignore as $item ) - if ( str_ends_with( $item , $suffix ) ) - $output->push( " Unused ignore. To drop, run:\n php $cmd0 --del-ignore=$mark\n" ); + while ( in_array( $mark , $ignore ) ) + { + $key = array_search( $mark , $ignore ); + unset( $ignore[$key] ); + } + foreach ( $ignore as $item ) + if ( str_ends_with( $item , $suffix ) ) + $output->push( " Unused ignore. To drop, run:\n php $cmd0 --del-ignore=$mark\n" ); - $output->pushExtra( "\n" ); - } + $output->pushExtra( "\n" ); + } // Output $output->print(); } -function extractNodeName( array $list ) +function extractNodeName( array $list , array $tags ) { $ret = array(); foreach( $list as $elem ) - $ret[] = $elem->nodeName; + if ( in_array( $elem->nodeName , $tags) || count( $tags ) == 0 ) + $ret[] = $elem->nodeName; return $ret; } From e8dcd15b1c859ef3f7a45480c15f1fe864ed7eb7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20L=20F=20S=20Bacci?= Date: Tue, 22 Aug 2023 08:29:27 -0300 Subject: [PATCH 38/50] Generalize case insensitivity on basic types --- scripts/translation/qaxml.t.php | 88 ++++++++++++++++++++------------- 1 file changed, 54 insertions(+), 34 deletions(-) diff --git a/scripts/translation/qaxml.t.php b/scripts/translation/qaxml.t.php index 84af9f434..e5dfabceb 100644 --- a/scripts/translation/qaxml.t.php +++ b/scripts/translation/qaxml.t.php @@ -76,6 +76,9 @@ $s = XmlUtil::listNodeType( $s , XML_ELEMENT_NODE ); $t = XmlUtil::listNodeType( $t , XML_ELEMENT_NODE ); + typesNotCaseSensitive( $s ); + typesNotCaseSensitive( $t ); + $s = extractTagsInnerText( $s , $tags ); $t = extractTagsInnerText( $t , $tags ); @@ -111,6 +114,9 @@ $s = XmlUtil::listNodeType( $s , XML_ELEMENT_NODE ); $t = XmlUtil::listNodeType( $t , XML_ELEMENT_NODE ); + typesNotCaseSensitive( $s ); + typesNotCaseSensitive( $t ); + $s = extractTagsInnerXmls( $s , $tags ); $t = extractTagsInnerXmls( $t , $tags ); @@ -146,6 +152,9 @@ $s = XmlUtil::listNodeType( $s , XML_ELEMENT_NODE ); $t = XmlUtil::listNodeType( $t , XML_ELEMENT_NODE ); + typesNotCaseSensitive( $s ); + typesNotCaseSensitive( $t ); + $s = extractNodeName( $s , $tags ); $t = extractNodeName( $t , $tags ); @@ -173,29 +182,28 @@ // Ignore - if ( false ) - if ( $output->isEmpty() == false ) - { - $prefix = $output->hash( $tags ); - $suffix = md5( implode( "" , $tags ) ) . ',' . $qafile->file; - $mark = "{$prefix},{$suffix}"; - - if ( in_array( $mark , $ignore ) ) - $output->clear(); - else - $output->push( " To ignore, run:\n php $cmd0 --add-ignore=$mark\n" ); + if ( $output->isEmpty() == false ) + { + $prefix = $output->hash( $tags ); + $suffix = md5( implode( "" , $tags ) ) . ',' . $qafile->file; + $mark = "{$prefix},{$suffix}"; - while ( in_array( $mark , $ignore ) ) - { - $key = array_search( $mark , $ignore ); - unset( $ignore[$key] ); - } - foreach ( $ignore as $item ) - if ( str_ends_with( $item , $suffix ) ) - $output->push( " Unused ignore. To drop, run:\n php $cmd0 --del-ignore=$mark\n" ); + if ( in_array( $mark , $ignore ) ) + $output->clear(); + else + $output->push( " To ignore, run:\n php $cmd0 --add-ignore=$mark\n" ); - $output->pushExtra( "\n" ); + while ( in_array( $mark , $ignore ) ) + { + $key = array_search( $mark , $ignore ); + unset( $ignore[$key] ); } + foreach ( $ignore as $item ) + if ( str_ends_with( $item , $suffix ) ) + $output->push( " Unused ignore. To drop, run:\n php $cmd0 --del-ignore=$mark\n" ); + + $output->pushExtra( "\n" ); + } // Output @@ -211,6 +219,32 @@ function extractNodeName( array $list , array $tags ) return $ret; } +function typesNotCaseSensitive( array & $nodes ) +{ + // Types not case-sensitive: https://github.com/php/doc-en/issues/2658 + + if ( $nodes == null ) + return; + + foreach( $nodes as $node ) + { + if ( $node->nodeName == "type" ) + { + $text = trim( strtolower( $node->nodeValue ) ); + switch( $text ) + { + case "array": + case "string": + case "float": + case "bool": + case "null": + $node->nodeValue = $text; + break; + } + } + } +} + function extractTagsInnerText( array $nodes , array $tags ) { $ret = array(); @@ -229,20 +263,6 @@ function extractTagsInnerText( array $nodes , array $tags ) if ( strlen( $text ) == $was ) break; } - // Types not case-sensitive: https://github.com/php/doc-en/issues/2658 - if ( $tag == "type" ) - { - switch( strtolower( $text ) ) - { - case "array": - case "string": - case "float": - case "bool": - case "null": - $text = strtolower( $text ); - break; - } - } $ret[] = $tag . ">" . $text; } return $ret; From ae34a2e70d6c8b458a533a8310dc494a339ce319 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20L=20F=20S=20Bacci?= Date: Tue, 22 Aug 2023 08:36:58 -0300 Subject: [PATCH 39/50] Configurable ignore output. --- scripts/translation/qaxml.t.php | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/scripts/translation/qaxml.t.php b/scripts/translation/qaxml.t.php index e5dfabceb..68122afbd 100644 --- a/scripts/translation/qaxml.t.php +++ b/scripts/translation/qaxml.t.php @@ -8,6 +8,7 @@ $tags = array(); $showDetail = false; +$showIgnore = true; $igfile = new CacheFile( getcwd() . "/.qaxml.t.ignore" ); @@ -48,6 +49,12 @@ exit; } + if ( str_starts_with( $arg , "--disable-ignore" ) ) + { + $showIgnore = false; + continue; + } + $tags = explode( ',' , $arg ); } @@ -182,7 +189,7 @@ // Ignore - if ( $output->isEmpty() == false ) + if ( $showIgnore && $output->isEmpty() == false ) { $prefix = $output->hash( $tags ); $suffix = md5( implode( "" , $tags ) ) . ',' . $qafile->file; From 99d9e306af4e0f86a0194177e38fa0d4f02ffb5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20L=20F=20S=20Bacci?= Date: Tue, 22 Aug 2023 09:22:26 -0300 Subject: [PATCH 40/50] Fix wrong hash on unused message. --- scripts/translation/qaxml.t.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/translation/qaxml.t.php b/scripts/translation/qaxml.t.php index 68122afbd..fc9cc986c 100644 --- a/scripts/translation/qaxml.t.php +++ b/scripts/translation/qaxml.t.php @@ -205,8 +205,8 @@ $key = array_search( $mark , $ignore ); unset( $ignore[$key] ); } - foreach ( $ignore as $item ) - if ( str_ends_with( $item , $suffix ) ) + foreach ( $ignore as $mark ) + if ( str_ends_with( $mark , $suffix ) ) $output->push( " Unused ignore. To drop, run:\n php $cmd0 --del-ignore=$mark\n" ); $output->pushExtra( "\n" ); From 1c1c59b15d909f52005d7eaf2750fec969381d55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20L=20F=20S=20Bacci?= Date: Wed, 23 Aug 2023 14:07:25 -0300 Subject: [PATCH 41/50] Simplify output. --- scripts/translation/qarvt.php | 8 ++-- scripts/translation/qaxml.t.php | 83 ++++++++++++++++++--------------- 2 files changed, 49 insertions(+), 42 deletions(-) diff --git a/scripts/translation/qarvt.php b/scripts/translation/qarvt.php index be44cf8c1..17d9e94cf 100644 --- a/scripts/translation/qarvt.php +++ b/scripts/translation/qarvt.php @@ -16,7 +16,7 @@ $langDir = $argv[1]; break; default: - print_usage_exit(); + print_usage_exit($argv[0]); return; } @@ -32,7 +32,7 @@ } } else - print_usage_exit(); + print_usage_exit($argv[0]); } $list = new RevcheckFileList( $langDir ); @@ -51,9 +51,9 @@ print "\n"; } -function print_usage_exit() +function print_usage_exit($cmd) { fwrite( STDERR , " Wrong paramater count. Usage:\n" ); - fwrite( STDERR , " {$argv[0]} [lang_dir]:\n" ); + fwrite( STDERR , " {$cmd} [lang_dir]:\n" ); exit; } diff --git a/scripts/translation/qaxml.t.php b/scripts/translation/qaxml.t.php index fc9cc986c..41a95ad1e 100644 --- a/scripts/translation/qaxml.t.php +++ b/scripts/translation/qaxml.t.php @@ -91,10 +91,10 @@ $match = array(); - foreach( $s as $v ) - $match[$v] = array( 0 , 0 ); foreach( $t as $v ) $match[$v] = array( 0 , 0 ); + foreach( $s as $v ) + $match[$v] = array( 0 , 0 ); foreach( $s as $v ) $match[$v][0] += 1; @@ -102,13 +102,9 @@ $match[$v][1] += 1; foreach( $match as $k => $v ) - { - if ( $v[0] == $v[1] ) - continue; - $output->push( "* {$k} -{$v[1]} +{$v[0]}\n" ); - } + $output->addDiff( $k , $v[0] , $v[1] ); - $output->pushExtra( "\n" ); + $output->addFooter( "\n" ); } // Check tag contents, inner XML @@ -129,10 +125,10 @@ $match = array(); - foreach( $s as $v ) - $match[$v] = array( 0 , 0 ); foreach( $t as $v ) $match[$v] = array( 0 , 0 ); + foreach( $s as $v ) + $match[$v] = array( 0 , 0 ); foreach( $s as $v ) $match[$v][0] += 1; @@ -140,13 +136,9 @@ $match[$v][1] += 1; foreach( $match as $k => $v ) - { - if ( $v[0] == $v[1] ) - continue; - $output->push( "* {$k} -{$v[1]} +{$v[0]}\n" ); - } + $output->addDiff( $k , $v[0] , $v[1] ); - $output->pushExtra( "\n" ); + $output->addFooter( "\n" ); } // Check tag count @@ -167,10 +159,10 @@ $match = array(); - foreach( $s as $v ) - $match[$v] = array( 0 , 0 ); foreach( $t as $v ) $match[$v] = array( 0 , 0 ); + foreach( $s as $v ) + $match[$v] = array( 0 , 0 ); foreach( $s as $v ) $match[$v][0] += 1; @@ -178,13 +170,9 @@ $match[$v][1] += 1; foreach( $match as $k => $v ) - { - if ( $v[0] == $v[1] ) - continue; - $output->push( "* {$k} -{$v[1]} +{$v[0]}\n" ); - } + $output->addDiff( $k , $v[0] , $v[1] ); - $output->pushExtra( "\n" ); + $output->addFooter( "\n" ); } // Ignore @@ -198,7 +186,7 @@ if ( in_array( $mark , $ignore ) ) $output->clear(); else - $output->push( " To ignore, run:\n php $cmd0 --add-ignore=$mark\n" ); + $output->add( " php $cmd0 --add-ignore=$mark\n" ); while ( in_array( $mark , $ignore ) ) { @@ -207,9 +195,9 @@ } foreach ( $ignore as $mark ) if ( str_ends_with( $mark , $suffix ) ) - $output->push( " Unused ignore. To drop, run:\n php $cmd0 --del-ignore=$mark\n" ); + $output->add( " php $cmd0 --del-ignore=$mark\n" ); - $output->pushExtra( "\n" ); + $output->addFooter( "\n" ); } // Output @@ -300,17 +288,17 @@ function extractTagsInnerXmls( array $nodes , array $tags ) function printTagUsageDetail( string $source , string $target , string $tag , OutputBufferHasher $output ) { - $output->push( "\n" ); + $output->add( "\n" ); $s = collectTagDefinitions( $source , $tag ); $t = collectTagDefinitions( $target , $tag ); $min = min( count( $s ) , count( $t ) ); for( $i = 0 ; $i < $min ; $i++ ) - $output->push( "\t{$tag}\t{$s[$i]}\t{$t[$i]}\n" ); + $output->add( "\t{$tag}\t{$s[$i]}\t{$t[$i]}\n" ); for( $i = $min ; $i < count($s) ; $i++ ) - $output->push( "\t{$tag}\t{$s[$i]}\t\t\n" ); + $output->add( "\t{$tag}\t{$s[$i]}\t\t\n" ); for( $i = $min ; $i < count($t) ; $i++ ) - $output->push( "\t{$tag}\t\t{$t[$i]}\t\n" ); - $output->push( "\n" ); + $output->add( "\t{$tag}\t\t{$t[$i]}\t\n" ); + $output->add( "\n" ); } function collectTagDefinitions( string $file , string $tag ) @@ -354,20 +342,39 @@ function hash() : string return md5( $text ); } - function isEmpty() : bool + function add( string $text ) { - return count( $this->texts ) == 0; + $this->texts[] = $text; } - function push( string $text ) + function addDiff( string $text , int $sourceCount , int $targetCount ) { - $this->texts[] = $text; + if ( $sourceCount == $targetCount ) + return; + $prefix = "* "; + $suffix = " -{$targetCount} +{$sourceCount}"; + if ( $sourceCount == 0 ) + { + $prefix = "- "; + $suffix = $targetCount == 1 ? "" : " -{$targetCount}"; + } + if ( $targetCount == 0 ) + { + $prefix = "+ "; + $suffix = $sourceCount == 1 ? "" : " +{$sourceCount}"; + } + $this->add( "{$prefix}{$text}{$suffix}\n" ); } - function pushExtra( string $text ) + function addFooter( string $text ) { if ( count( $this->texts ) > 0 ) - $this->push( $text ); + $this->add( $text ); + } + + function isEmpty() : bool + { + return count( $this->texts ) == 0; } function print() From 43624a2ce752d8d940a82af9c83db442770243a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20L=20F=20S=20Bacci?= Date: Thu, 24 Aug 2023 12:59:56 -0300 Subject: [PATCH 42/50] Add ignore capacity on qaxml.e.php. --- scripts/translation/lib/OutputIgnoreArgv.php | 79 ++++++++++++++ .../translation/lib/OutputIgnoreBuffer.php | 103 ++++++++++++++++++ scripts/translation/lib/all.php | 2 + scripts/translation/qaxml.e.php | 15 +-- 4 files changed, 190 insertions(+), 9 deletions(-) create mode 100644 scripts/translation/lib/OutputIgnoreArgv.php create mode 100644 scripts/translation/lib/OutputIgnoreBuffer.php diff --git a/scripts/translation/lib/OutputIgnoreArgv.php b/scripts/translation/lib/OutputIgnoreArgv.php new file mode 100644 index 000000000..bafd38562 --- /dev/null +++ b/scripts/translation/lib/OutputIgnoreArgv.php @@ -0,0 +1,79 @@ +command = $argv[0]; + + foreach( $argv as $key => $arg ) + { + if ( str_starts_with( $arg , "--add-ignore=" ) ) + { + $file = OutputIgnoreArgv::cacheFile(); + $list = $file->load( array() ); + $line = substr( $arg , 13 ); + if ( ! in_array( $line , $list ) ) + { + $list[] = $line; + $file->save( $list ); + } + exit; + } + + if ( str_starts_with( $arg , "--del-ignore=" ) ) + { + $file = OutputIgnoreArgv::cacheFile(); + $list = $file->load( array() ); + $line = substr( $arg , 13 ); + $dels = 0; + while ( in_array( $line , $list ) ) + { + $key = array_search( $line , $list ); + unset( $list[$key] ); + $dels++; + } + if ( $dels == 0 ) + print "Ignore mark not found.\n"; + else + $file->save( $list ); + exit; + } + + if ( $arg == "--disable-ignore" ) + { + $this->showIgnore = false; + unset( $argv[$key] ); + } + } + + $copy = $argv; + array_shift( $copy ); + $this->options = implode( " " , $copy ); + } + + public static function cacheFile() + { + return new CacheFile( getcwd() . "/.qa.ignore" ); + } + + function pushAddIgnore( OutputIgnoreBuffer $output, string $mark ) + { + $output->add( " php {$this->command} --add-ignore=$mark\n" ); + } + + function pushDelIgnore( OutputIgnoreBuffer $output, string $mark ) + { + $output->add( " php {$this->command} --del-ignore=$mark\n" ); + } +} \ No newline at end of file diff --git a/scripts/translation/lib/OutputIgnoreBuffer.php b/scripts/translation/lib/OutputIgnoreBuffer.php new file mode 100644 index 000000000..730d8fec1 --- /dev/null +++ b/scripts/translation/lib/OutputIgnoreBuffer.php @@ -0,0 +1,103 @@ +args = $args; + $this->header = $header; + $this->filename = $filename; + } + + function add( string $text ) + { + $this->texts[] = $text; + } + + function addDiff( string $text , int $sourceCount , int $targetCount ) + { + if ( $sourceCount == $targetCount ) + return; + $prefix = "* "; + $suffix = " -{$targetCount} +{$sourceCount}"; + if ( $sourceCount == 0 ) + { + $prefix = "- "; + $suffix = $targetCount == 1 ? "" : " -{$targetCount}"; + } + if ( $targetCount == 0 ) + { + $prefix = "+ "; + $suffix = $sourceCount == 1 ? "" : " +{$sourceCount}"; + } + $this->add( "{$prefix}{$text}{$suffix}\n" ); + } + + function addLine() + { + if ( count( $this->texts ) > 0 ) + $this->add( "\n" ); + } + + function print( bool $showIgnore = true ) + { + if ( count( $this->texts ) == 0 ) + return; + + $head = $this->filename . ':' . $this->hash( false ) . ':'; + $mark = $head . $this->hash( true ); + $marks = OutputIgnoreArgv::cacheFile()->load( array() ); + + if ( $showIgnore ) + { + if ( in_array( $mark , $marks ) ) + $this->texts = array(); + else + $this->args->pushAddIgnore( $this , $mark ); + + // old marks + while ( in_array( $mark , $marks ) ) + { + $key = array_search( $mark , $marks ); + unset( $marks[$key] ); + } + foreach ( $marks as $mark ) + if ( $mark != null ) + if ( str_starts_with( $mark , $head ) ) + $this->args->pushDelIgnore( $this , $mark ); + + $this->addLine( "\n" ); + } + + if ( count( $this->texts) == 0 ) + return; + + print $this->header; + foreach( $this->texts as $text ) + print $text; + } + + private function hash( bool $withContents ) : string + { + $text = $this->header . $this->args->options; + if ( $withContents ) + $text .= implode( "" , $this->texts ); + $text = str_replace( " " , "" , $text ); + $text = str_replace( "\n" , "" , $text ); + $text = str_replace( "\r" , "" , $text ); + $text = str_replace( "\t" , "" , $text ); + return hash( "crc32b" , $text ); + } +} \ No newline at end of file diff --git a/scripts/translation/lib/all.php b/scripts/translation/lib/all.php index 24ae76df1..900809b55 100644 --- a/scripts/translation/lib/all.php +++ b/scripts/translation/lib/all.php @@ -8,6 +8,8 @@ require_once __DIR__ . '/CacheUtil.php'; require_once __DIR__ . '/GitDiffParser.php'; require_once __DIR__ . '/GitLogParser.php'; +require_once __DIR__ . '/OutputIgnoreArgv.php'; +require_once __DIR__ . '/OutputIgnoreBuffer.php'; require_once __DIR__ . '/QaFileInfo.php'; require_once __DIR__ . '/RevcheckFileInfo.php'; require_once __DIR__ . '/RevcheckFileList.php'; diff --git a/scripts/translation/qaxml.e.php b/scripts/translation/qaxml.e.php index 88fcacae1..70dffe40e 100644 --- a/scripts/translation/qaxml.e.php +++ b/scripts/translation/qaxml.e.php @@ -7,6 +7,7 @@ require_once __DIR__ . '/lib/all.php'; $qalist = QaFileInfo::cacheLoad(); +$outarg = new OutputIgnoreArgv( $argv ); foreach ( $qalist as $qafile ) { @@ -24,7 +25,8 @@ if ( implode( "\n" , $s ) == implode( "\n" , $t ) ) continue; - $header = true; + $output = new OutputIgnoreBuffer( $outarg , "qaxml.e: {$target}\n\n" , $target ); + $match = array(); foreach( $s as $v ) @@ -42,15 +44,10 @@ if ( $v[0] == $v[1] ) continue; - if ( $header ) - { - print "qaxml.e: {$target}\n\n"; - $header = false; - } + $output->add( "* &{$k}; -{$v[1]} +{$v[0]}\n" ); - print "* &{$k}; -{$v[1]} +{$v[0]}\n"; } - if ( ! $header ) - print "\n"; + $output->addLine(); + $output->print(); } \ No newline at end of file From 2e3585af7808296f8a5c54369454ae4bbb309323 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20L=20F=20S=20Bacci?= Date: Thu, 24 Aug 2023 20:42:01 -0300 Subject: [PATCH 43/50] Convert qaxml.t.php to use OutputIgnoreBuffer. --- .../translation/lib/OutputIgnoreBuffer.php | 19 +- scripts/translation/qaxml.e.php | 1 - scripts/translation/qaxml.t.php | 264 ++++++------------ 3 files changed, 99 insertions(+), 185 deletions(-) diff --git a/scripts/translation/lib/OutputIgnoreBuffer.php b/scripts/translation/lib/OutputIgnoreBuffer.php index 730d8fec1..735f147b6 100644 --- a/scripts/translation/lib/OutputIgnoreBuffer.php +++ b/scripts/translation/lib/OutputIgnoreBuffer.php @@ -47,20 +47,22 @@ function addDiff( string $text , int $sourceCount , int $targetCount ) function addLine() { - if ( count( $this->texts ) > 0 ) + if ( count( $this->texts ) > 0 && end( $this->texts ) != "\n" ) $this->add( "\n" ); } - function print( bool $showIgnore = true ) + function print() : bool { if ( count( $this->texts ) == 0 ) - return; + return false; + + $this->addLine( "\n" ); $head = $this->filename . ':' . $this->hash( false ) . ':'; $mark = $head . $this->hash( true ); $marks = OutputIgnoreArgv::cacheFile()->load( array() ); - if ( $showIgnore ) + if ( $this->args->showIgnore ) { if ( in_array( $mark , $marks ) ) $this->texts = array(); @@ -78,15 +80,18 @@ function print( bool $showIgnore = true ) if ( str_starts_with( $mark , $head ) ) $this->args->pushDelIgnore( $this , $mark ); - $this->addLine( "\n" ); } - if ( count( $this->texts) == 0 ) - return; + $this->addLine( "\n" ); + + if ( count( $this->texts ) == 0 ) + return false; print $this->header; foreach( $this->texts as $text ) print $text; + + return true; } private function hash( bool $withContents ) : string diff --git a/scripts/translation/qaxml.e.php b/scripts/translation/qaxml.e.php index 70dffe40e..58130279f 100644 --- a/scripts/translation/qaxml.e.php +++ b/scripts/translation/qaxml.e.php @@ -45,7 +45,6 @@ continue; $output->add( "* &{$k}; -{$v[1]} +{$v[0]}\n" ); - } $output->addLine(); diff --git a/scripts/translation/qaxml.t.php b/scripts/translation/qaxml.t.php index 41a95ad1e..914beddf7 100644 --- a/scripts/translation/qaxml.t.php +++ b/scripts/translation/qaxml.t.php @@ -8,12 +8,11 @@ $tags = array(); $showDetail = false; -$showIgnore = true; -$igfile = new CacheFile( getcwd() . "/.qaxml.t.ignore" ); - -$cmd0 = array_shift( $argv ); +$qalist = QaFileInfo::cacheLoad(); +$outarg = new OutputIgnoreArgv( $argv ); +array_shift( $argv ); while ( count( $argv ) > 0 ) { $arg = array_shift( $argv ); @@ -21,45 +20,13 @@ if ( $arg == "--detail" ) { $showDetail = true; - continue; - } - - if ( str_starts_with( $arg , "--add-ignore=" ) ) - { - $ignore = $igfile->load( array() ); - if ( count( $ignore ) == 0 ) - print "Creating file ignore file on current working directory.\n"; - $add = substr( $arg , 13 ); - $ignore[] = $add; - $igfile->save( $ignore ); - exit; - } - - if ( str_starts_with( $arg , "--del-ignore=" ) ) - { - $ignore = $igfile->load( array() ); - $del = substr( $arg , 13 ); - $key = array_search( $del , $ignore ); - - if ( $key === false ) - print "Ignore mark not found.\n"; - else - unset( $ignore[$key] ); - $igfile->save( $ignore ); - exit; - } - - if ( str_starts_with( $arg , "--disable-ignore" ) ) - { - $showIgnore = false; + $outarg->showIgnore = false; continue; } $tags = explode( ',' , $arg ); } -$qalist = QaFileInfo::cacheLoad(); - foreach ( $qalist as $qafile ) { if ( $qafile->file == "bookinfo.xml" ) @@ -70,8 +37,7 @@ $source = $qafile->sourceDir . '/' . $qafile->file; $target = $qafile->targetDir . '/' . $qafile->file; - $output = new OutputBufferHasher( "qaxml.t: {$target}\n\n" ); - $ignore = $igfile->load( array() ); + $output = new OutputIgnoreBuffer( $outarg , "qaxml.t: {$target}\n\n" , $target ); // Check tag contents, inner text @@ -104,12 +70,17 @@ foreach( $match as $k => $v ) $output->addDiff( $k , $v[0] , $v[1] ); - $output->addFooter( "\n" ); + if ( $showDetail ) + foreach( $match as $tag => $v ) + printTagUsageDetail( $source , $target , $tag , $output ); + + if ( $output->print() ) + continue; } // Check tag contents, inner XML - if ( count( $tags ) > 0 && $output->isEmpty() ) + if ( count( $tags ) > 0 ) { $s = XmlUtil::loadFile( $source ); $t = XmlUtil::loadFile( $target ); @@ -138,69 +109,46 @@ foreach( $match as $k => $v ) $output->addDiff( $k , $v[0] , $v[1] ); - $output->addFooter( "\n" ); + if ( $showDetail ) + foreach( $match as $tag => $v ) + printTagUsageDetail( $source , $target , $tag , $output ); + + if ( $output->print() ) + continue; } // Check tag count - if ( $output->isEmpty() ) - { - $s = XmlUtil::loadFile( $source ); - $t = XmlUtil::loadFile( $target ); - - $s = XmlUtil::listNodeType( $s , XML_ELEMENT_NODE ); - $t = XmlUtil::listNodeType( $t , XML_ELEMENT_NODE ); - - typesNotCaseSensitive( $s ); - typesNotCaseSensitive( $t ); - - $s = extractNodeName( $s , $tags ); - $t = extractNodeName( $t , $tags ); - - $match = array(); - - foreach( $t as $v ) - $match[$v] = array( 0 , 0 ); - foreach( $s as $v ) - $match[$v] = array( 0 , 0 ); - - foreach( $s as $v ) - $match[$v][0] += 1; - foreach( $t as $v ) - $match[$v][1] += 1; + $s = XmlUtil::loadFile( $source ); + $t = XmlUtil::loadFile( $target ); - foreach( $match as $k => $v ) - $output->addDiff( $k , $v[0] , $v[1] ); + $s = XmlUtil::listNodeType( $s , XML_ELEMENT_NODE ); + $t = XmlUtil::listNodeType( $t , XML_ELEMENT_NODE ); - $output->addFooter( "\n" ); - } + typesNotCaseSensitive( $s ); + typesNotCaseSensitive( $t ); - // Ignore + $s = extractNodeName( $s , $tags ); + $t = extractNodeName( $t , $tags ); - if ( $showIgnore && $output->isEmpty() == false ) - { - $prefix = $output->hash( $tags ); - $suffix = md5( implode( "" , $tags ) ) . ',' . $qafile->file; - $mark = "{$prefix},{$suffix}"; + $match = array(); - if ( in_array( $mark , $ignore ) ) - $output->clear(); - else - $output->add( " php $cmd0 --add-ignore=$mark\n" ); + foreach( $t as $v ) + $match[$v] = array( 0 , 0 ); + foreach( $s as $v ) + $match[$v] = array( 0 , 0 ); - while ( in_array( $mark , $ignore ) ) - { - $key = array_search( $mark , $ignore ); - unset( $ignore[$key] ); - } - foreach ( $ignore as $mark ) - if ( str_ends_with( $mark , $suffix ) ) - $output->add( " php $cmd0 --del-ignore=$mark\n" ); + foreach( $s as $v ) + $match[$v][0] += 1; + foreach( $t as $v ) + $match[$v][1] += 1; - $output->addFooter( "\n" ); - } + foreach( $match as $k => $v ) + $output->addDiff( $k , $v[0] , $v[1] ); - // Output + if ( $showDetail ) + foreach( $match as $tag => $v ) + printTagUsageDetail( $source , $target , $tag , $output ); $output->print(); } @@ -286,19 +234,53 @@ function extractTagsInnerXmls( array $nodes , array $tags ) return $ret; } -function printTagUsageDetail( string $source , string $target , string $tag , OutputBufferHasher $output ) +function printTagUsageDetail( string $source , string $target , string $tag , OutputIgnoreBuffer $output ) { - $output->add( "\n" ); - $s = collectTagDefinitions( $source , $tag ); - $t = collectTagDefinitions( $target , $tag ); - $min = min( count( $s ) , count( $t ) ); - for( $i = 0 ; $i < $min ; $i++ ) - $output->add( "\t{$tag}\t{$s[$i]}\t{$t[$i]}\n" ); - for( $i = $min ; $i < count($s) ; $i++ ) - $output->add( "\t{$tag}\t{$s[$i]}\t\t\n" ); - for( $i = $min ; $i < count($t) ; $i++ ) - $output->add( "\t{$tag}\t\t{$t[$i]}\t\n" ); - $output->add( "\n" ); + $source = collectTagDefinitions( $source , $tag ); + $target = collectTagDefinitions( $target , $tag ); + if ( count( $source ) == count($target) ) + return; + $output->addLine(); + $s = null; + $t = null; + while ( count( $source ) > 0 || count( $target ) > 0 ) + { + if ( $s == null ) + $s = array_shift( $source ); + if ( $t == null ) + $t = array_shift( $target ); + if ( $s != null && $t != null ) + { + if ( abs( $s - $t ) < 1 ) + { + $output->add( "\t{$tag}\t{$s}\t{$t}\n" ); + $s = null; + $t = null; + continue; + } + if ( $s < $t ) + { + array_unshift( $target , $t ); + $t = null; + } + else + { + array_unshift( $source , $s ); + $s = null; + } + } + if ( $s != null ) + { + $output->add( "\t{$tag}\t{$s}\t-\n" ); + $s = null; + } + if ( $t != null ) + { + $output->add( "\t{$tag}\t-\t{$t}\n" ); + $t = null; + } + } + $output->addLine(); } function collectTagDefinitions( string $file , string $tag ) @@ -313,76 +295,4 @@ function collectTagDefinitions( string $file , string $tag ) $ret[] = $node->getLineNo(); } return $ret; -} - -class OutputBufferHasher -{ - public string $header = ""; - public array $texts = array(); - - function __construct( string $header = "" ) - { - $this->header = $header; - } - - function clear() - { - $this->texts = array(); - } - - function hash() : string - { - if ( count( $this->texts) == 0 ) - return ""; - $text = $this->header . implode( "" , $this->texts ); - $text = str_replace( " " , "" , $text ); - $text = str_replace( "\n" , "" , $text ); - $text = str_replace( "\r" , "" , $text ); - $text = str_replace( "\t" , "" , $text ); - return md5( $text ); - } - - function add( string $text ) - { - $this->texts[] = $text; - } - - function addDiff( string $text , int $sourceCount , int $targetCount ) - { - if ( $sourceCount == $targetCount ) - return; - $prefix = "* "; - $suffix = " -{$targetCount} +{$sourceCount}"; - if ( $sourceCount == 0 ) - { - $prefix = "- "; - $suffix = $targetCount == 1 ? "" : " -{$targetCount}"; - } - if ( $targetCount == 0 ) - { - $prefix = "+ "; - $suffix = $sourceCount == 1 ? "" : " +{$sourceCount}"; - } - $this->add( "{$prefix}{$text}{$suffix}\n" ); - } - - function addFooter( string $text ) - { - if ( count( $this->texts ) > 0 ) - $this->add( $text ); - } - - function isEmpty() : bool - { - return count( $this->texts ) == 0; - } - - function print() - { - if ( $this->isEmpty() ) - return; - print $this->header; - foreach( $this->texts as $text ) - print $text; - } -} +} \ No newline at end of file From beec19b17b2540b041bd06fb3a9219175cfa8f75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20L=20F=20S=20Bacci?= Date: Fri, 25 Aug 2023 10:34:24 -0300 Subject: [PATCH 44/50] Fix instable output on qaxml.t.php. --- scripts/translation/lib/OutputIgnoreArgv.php | 2 +- .../translation/lib/OutputIgnoreBuffer.php | 12 ++-- scripts/translation/qaxml.t.php | 65 ++++++++++--------- 3 files changed, 41 insertions(+), 38 deletions(-) diff --git a/scripts/translation/lib/OutputIgnoreArgv.php b/scripts/translation/lib/OutputIgnoreArgv.php index bafd38562..40a3a83bf 100644 --- a/scripts/translation/lib/OutputIgnoreArgv.php +++ b/scripts/translation/lib/OutputIgnoreArgv.php @@ -64,7 +64,7 @@ function __construct( array & $argv ) public static function cacheFile() { - return new CacheFile( getcwd() . "/.qa.ignore" ); + return new CacheFile( getcwd() . "/.qaxml.ignore" ); } function pushAddIgnore( OutputIgnoreBuffer $output, string $mark ) diff --git a/scripts/translation/lib/OutputIgnoreBuffer.php b/scripts/translation/lib/OutputIgnoreBuffer.php index 735f147b6..8aeaacb3b 100644 --- a/scripts/translation/lib/OutputIgnoreBuffer.php +++ b/scripts/translation/lib/OutputIgnoreBuffer.php @@ -8,6 +8,8 @@ class OutputIgnoreBuffer { + public int $printCount = 0; + private string $header = ""; private string $filename = ""; private array $texts = array(); @@ -51,12 +53,14 @@ function addLine() $this->add( "\n" ); } - function print() : bool + function print() { if ( count( $this->texts ) == 0 ) - return false; + return; $this->addLine( "\n" ); + if ( count ( $this->texts ) > 0 ) + $this->printCount++; $head = $this->filename . ':' . $this->hash( false ) . ':'; $mark = $head . $this->hash( true ); @@ -85,13 +89,11 @@ function print() : bool $this->addLine( "\n" ); if ( count( $this->texts ) == 0 ) - return false; + return; print $this->header; foreach( $this->texts as $text ) print $text; - - return true; } private function hash( bool $withContents ) : string diff --git a/scripts/translation/qaxml.t.php b/scripts/translation/qaxml.t.php index 914beddf7..b845c6e34 100644 --- a/scripts/translation/qaxml.t.php +++ b/scripts/translation/qaxml.t.php @@ -39,9 +39,9 @@ $output = new OutputIgnoreBuffer( $outarg , "qaxml.t: {$target}\n\n" , $target ); - // Check tag contents, inner text + // First check, by tag contents, inner text - if ( count( $tags ) > 0 ) + if ( count( $tags ) > 0 && $output->printCount == 0 ) { $s = XmlUtil::loadFile( $source ); $t = XmlUtil::loadFile( $target ); @@ -74,13 +74,12 @@ foreach( $match as $tag => $v ) printTagUsageDetail( $source , $target , $tag , $output ); - if ( $output->print() ) - continue; + $output->print(); } - // Check tag contents, inner XML + // Second check, by tag contents, inner XML - if ( count( $tags ) > 0 ) + if ( count( $tags ) > 0 && $output->printCount == 0 ) { $s = XmlUtil::loadFile( $source ); $t = XmlUtil::loadFile( $target ); @@ -113,44 +112,46 @@ foreach( $match as $tag => $v ) printTagUsageDetail( $source , $target , $tag , $output ); - if ( $output->print() ) - continue; + $output->print(); } - // Check tag count + // Last check, simple tag count - $s = XmlUtil::loadFile( $source ); - $t = XmlUtil::loadFile( $target ); + if ( $output->printCount == 0 ) + { + $s = XmlUtil::loadFile( $source ); + $t = XmlUtil::loadFile( $target ); - $s = XmlUtil::listNodeType( $s , XML_ELEMENT_NODE ); - $t = XmlUtil::listNodeType( $t , XML_ELEMENT_NODE ); + $s = XmlUtil::listNodeType( $s , XML_ELEMENT_NODE ); + $t = XmlUtil::listNodeType( $t , XML_ELEMENT_NODE ); - typesNotCaseSensitive( $s ); - typesNotCaseSensitive( $t ); + typesNotCaseSensitive( $s ); + typesNotCaseSensitive( $t ); - $s = extractNodeName( $s , $tags ); - $t = extractNodeName( $t , $tags ); + $s = extractNodeName( $s , $tags ); + $t = extractNodeName( $t , $tags ); - $match = array(); + $match = array(); - foreach( $t as $v ) - $match[$v] = array( 0 , 0 ); - foreach( $s as $v ) - $match[$v] = array( 0 , 0 ); + foreach( $t as $v ) + $match[$v] = array( 0 , 0 ); + foreach( $s as $v ) + $match[$v] = array( 0 , 0 ); - foreach( $s as $v ) - $match[$v][0] += 1; - foreach( $t as $v ) - $match[$v][1] += 1; + foreach( $s as $v ) + $match[$v][0] += 1; + foreach( $t as $v ) + $match[$v][1] += 1; - foreach( $match as $k => $v ) - $output->addDiff( $k , $v[0] , $v[1] ); + foreach( $match as $k => $v ) + $output->addDiff( $k , $v[0] , $v[1] ); - if ( $showDetail ) - foreach( $match as $tag => $v ) - printTagUsageDetail( $source , $target , $tag , $output ); + if ( $showDetail ) + foreach( $match as $tag => $v ) + printTagUsageDetail( $source , $target , $tag , $output ); - $output->print(); + $output->print(); + } } function extractNodeName( array $list , array $tags ) From ed29a72a5a2d7e95ac3d56378c3ab4fc755e1e78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20L=20F=20S=20Bacci?= Date: Thu, 14 Dec 2023 10:48:39 -0300 Subject: [PATCH 45/50] License code to PHP License --- scripts/translation/configure.php | 17 +++++++++++-- scripts/translation/lib/CacheFile.php | 23 ++++++++++++++---- scripts/translation/lib/CacheUtil.php | 23 ++++++++++++++---- scripts/translation/lib/GitDiffParser.php | 23 ++++++++++++++---- scripts/translation/lib/GitLogParser.php | 23 ++++++++++++++---- scripts/translation/lib/OutputIgnoreArgv.php | 23 ++++++++++++++---- .../translation/lib/OutputIgnoreBuffer.php | 23 ++++++++++++++---- scripts/translation/lib/QaFileInfo.php | 23 ++++++++++++++---- scripts/translation/lib/RevcheckFileInfo.php | 23 ++++++++++++++---- scripts/translation/lib/RevcheckFileList.php | 23 ++++++++++++++---- scripts/translation/lib/RevcheckIgnore.php | 23 ++++++++++++++---- scripts/translation/lib/RevcheckRun.php | 24 +++++++++++++++---- scripts/translation/lib/RevtagParser.php | 23 ++++++++++++++---- scripts/translation/lib/XmlUtil.php | 23 ++++++++++++++---- scripts/translation/lib/all.php | 19 ++++++++++++++- scripts/translation/qarvt.php | 17 +++++++++++-- scripts/translation/qaxml.a.php | 19 ++++++++++++--- scripts/translation/qaxml.e.php | 19 ++++++++++++--- scripts/translation/qaxml.p.php | 19 ++++++++++++--- scripts/translation/qaxml.t.php | 19 ++++++++++++--- 20 files changed, 347 insertions(+), 82 deletions(-) diff --git a/scripts/translation/configure.php b/scripts/translation/configure.php index 63addaf09..5df8324c0 100644 --- a/scripts/translation/configure.php +++ b/scripts/translation/configure.php @@ -1,7 +1,20 @@ | + * +----------------------------------------------------------------------+ + * | Description: Generate cached data for revcheck and QA tools. | + * +----------------------------------------------------------------------+ */ require_once __DIR__ . '/lib/all.php'; diff --git a/scripts/translation/lib/CacheFile.php b/scripts/translation/lib/CacheFile.php index 749083ff7..50e27348a 100644 --- a/scripts/translation/lib/CacheFile.php +++ b/scripts/translation/lib/CacheFile.php @@ -1,11 +1,24 @@ | + * +----------------------------------------------------------------------+ + * | Description: Class to handle data persistence. | + * +----------------------------------------------------------------------+ */ +require_once __DIR__ . '/all.php'; + class CacheFile { const CACHE_DIR = __DIR__ . '/../.cache'; @@ -41,4 +54,4 @@ public static function prepareFilename( string $file , bool $createCacheDirs = f mkdir( $outPath , 0777 , true ); return $outFile; } -} \ No newline at end of file +} diff --git a/scripts/translation/lib/CacheUtil.php b/scripts/translation/lib/CacheUtil.php index 6c6300be0..bc71e98fe 100644 --- a/scripts/translation/lib/CacheUtil.php +++ b/scripts/translation/lib/CacheUtil.php @@ -1,11 +1,24 @@ | + * +----------------------------------------------------------------------+ + * | Description: Common functions do load and save to cache files. | + * +----------------------------------------------------------------------+ */ +require_once __DIR__ . '/all.php'; + class CacheUtil { const CACHE_DIR = __DIR__ . '/../.cache'; @@ -35,4 +48,4 @@ public static function prepareFilename( string $path , string $file , bool $crea mkdir( $outPath , 0777 , true ); return $outFile; } -} \ No newline at end of file +} diff --git a/scripts/translation/lib/GitDiffParser.php b/scripts/translation/lib/GitDiffParser.php index 71b2afe77..9df98ac07 100644 --- a/scripts/translation/lib/GitDiffParser.php +++ b/scripts/translation/lib/GitDiffParser.php @@ -1,13 +1,26 @@ | + * +----------------------------------------------------------------------+ + * | Description: Parse `git diff` to complement file state. | + * +----------------------------------------------------------------------+ */ +require_once __DIR__ . '/all.php'; + class GitDiffParser { public static function parseNumstatInto( string $dir , RevcheckFileInfo $file ) {} -} \ No newline at end of file +} diff --git a/scripts/translation/lib/GitLogParser.php b/scripts/translation/lib/GitLogParser.php index ed80fc095..151d18456 100644 --- a/scripts/translation/lib/GitLogParser.php +++ b/scripts/translation/lib/GitLogParser.php @@ -1,11 +1,24 @@ | + * +----------------------------------------------------------------------+ + * | Description: Parse `git log` to complement file state. | + * +----------------------------------------------------------------------+ */ +require_once __DIR__ . '/all.php'; + class GitLogParser { static function parseInto( string $lang , RevcheckFileList & $list ) @@ -77,4 +90,4 @@ static function parseInto( string $lang , RevcheckFileList & $list ) pclose( $fp ); chdir( $cwd ); } -} \ No newline at end of file +} diff --git a/scripts/translation/lib/OutputIgnoreArgv.php b/scripts/translation/lib/OutputIgnoreArgv.php index 40a3a83bf..1b75c8ddf 100644 --- a/scripts/translation/lib/OutputIgnoreArgv.php +++ b/scripts/translation/lib/OutputIgnoreArgv.php @@ -1,11 +1,24 @@ | + * +----------------------------------------------------------------------+ + * | Description: Intercept and process $argv parameters. | + * +----------------------------------------------------------------------+ */ +require_once __DIR__ . '/all.php'; + class OutputIgnoreArgv { public string $command = ""; @@ -76,4 +89,4 @@ function pushDelIgnore( OutputIgnoreBuffer $output, string $mark ) { $output->add( " php {$this->command} --del-ignore=$mark\n" ); } -} \ No newline at end of file +} diff --git a/scripts/translation/lib/OutputIgnoreBuffer.php b/scripts/translation/lib/OutputIgnoreBuffer.php index 8aeaacb3b..3db7573d1 100644 --- a/scripts/translation/lib/OutputIgnoreBuffer.php +++ b/scripts/translation/lib/OutputIgnoreBuffer.php @@ -1,11 +1,24 @@ | + * +----------------------------------------------------------------------+ + * | Description: Cache the output and shows if if not marked to ignore. | + * +----------------------------------------------------------------------+ */ +require_once __DIR__ . '/all.php'; + class OutputIgnoreBuffer { public int $printCount = 0; @@ -107,4 +120,4 @@ private function hash( bool $withContents ) : string $text = str_replace( "\t" , "" , $text ); return hash( "crc32b" , $text ); } -} \ No newline at end of file +} diff --git a/scripts/translation/lib/QaFileInfo.php b/scripts/translation/lib/QaFileInfo.php index 7b243a449..2b8eb3ded 100644 --- a/scripts/translation/lib/QaFileInfo.php +++ b/scripts/translation/lib/QaFileInfo.php @@ -1,11 +1,24 @@ | + * +----------------------------------------------------------------------+ + * | Description: Common data for revcheck and QA tools. | + * +----------------------------------------------------------------------+ */ +require_once __DIR__ . '/all.php'; + class QaFileInfo { public string $sourceHash; @@ -45,4 +58,4 @@ public static function cacheSave( array $itens ) } fclose($fp); } -} \ No newline at end of file +} diff --git a/scripts/translation/lib/RevcheckFileInfo.php b/scripts/translation/lib/RevcheckFileInfo.php index 7448b5835..027c93fd8 100644 --- a/scripts/translation/lib/RevcheckFileInfo.php +++ b/scripts/translation/lib/RevcheckFileInfo.php @@ -1,11 +1,24 @@ | + * +----------------------------------------------------------------------+ + * | Description: General data of a file in a documentation tree. | + * +----------------------------------------------------------------------+ */ +require_once __DIR__ . '/all.php'; + enum RevcheckStatus :string { case Untranslated = 'Untranslated'; @@ -37,4 +50,4 @@ function __construct( string $file , int $size ) $this->status = RevcheckStatus::Untranslated; $this->revtag = null; } -} \ No newline at end of file +} diff --git a/scripts/translation/lib/RevcheckFileList.php b/scripts/translation/lib/RevcheckFileList.php index cc46b6e55..4c2eae255 100644 --- a/scripts/translation/lib/RevcheckFileList.php +++ b/scripts/translation/lib/RevcheckFileList.php @@ -1,11 +1,24 @@ | + * +----------------------------------------------------------------------+ + * | Description: General file transversal, ordered file listing. | + * +----------------------------------------------------------------------+ */ +require_once __DIR__ . '/all.php'; + class RevcheckFileList { var $list = array(); @@ -60,4 +73,4 @@ function loadTreeRecurse( $lang , $path ) foreach( $todoDirs as $path ) $this->loadTreeRecurse( $lang , $path ); } -} \ No newline at end of file +} diff --git a/scripts/translation/lib/RevcheckIgnore.php b/scripts/translation/lib/RevcheckIgnore.php index 9f0d0036e..a2a36d026 100644 --- a/scripts/translation/lib/RevcheckIgnore.php +++ b/scripts/translation/lib/RevcheckIgnore.php @@ -1,11 +1,24 @@ | + * +----------------------------------------------------------------------+ + * | Description: Files ignored on translation tree. | + * +----------------------------------------------------------------------+ */ +require_once __DIR__ . '/all.php'; + class RevcheckIgnore { public static function ignore( $filename ) : bool @@ -58,4 +71,4 @@ public static function ignore( $filename ) : bool // At least, do not ignore return false; } -} \ No newline at end of file +} diff --git a/scripts/translation/lib/RevcheckRun.php b/scripts/translation/lib/RevcheckRun.php index ee71607a6..ef9a01fb1 100644 --- a/scripts/translation/lib/RevcheckRun.php +++ b/scripts/translation/lib/RevcheckRun.php @@ -1,11 +1,25 @@ | + * +----------------------------------------------------------------------+ + * | Description: Calculate translation sync/diff status from two | + * | directories. | + * +----------------------------------------------------------------------+ */ +require_once __DIR__ . '/all.php'; + class RevcheckRun { public string $sourceDir; @@ -123,4 +137,4 @@ private function calculateStatus() } } } -} \ No newline at end of file +} diff --git a/scripts/translation/lib/RevtagParser.php b/scripts/translation/lib/RevtagParser.php index c9e28d6f2..82a291c6f 100644 --- a/scripts/translation/lib/RevtagParser.php +++ b/scripts/translation/lib/RevtagParser.php @@ -1,11 +1,24 @@ | + * +----------------------------------------------------------------------+ + * | Description: Parse revision and credits from XML comments. | + * +----------------------------------------------------------------------+ */ +require_once __DIR__ . '/all.php'; + class RevtagInfo { public string $revision = ""; @@ -89,4 +102,4 @@ public static function parseComment( DOMNode $node , RevtagInfo & $ret , $filena } } } -} \ No newline at end of file +} diff --git a/scripts/translation/lib/XmlUtil.php b/scripts/translation/lib/XmlUtil.php index 1e665bac4..292cffee2 100644 --- a/scripts/translation/lib/XmlUtil.php +++ b/scripts/translation/lib/XmlUtil.php @@ -1,11 +1,24 @@ | + * +----------------------------------------------------------------------+ + * | Description: Misc funcionality dealing with raw XML. | + * +----------------------------------------------------------------------+ */ +require_once __DIR__ . '/all.php'; + class XmlUtil { public static function extractEntities( $filename ) @@ -68,4 +81,4 @@ public static function loadText( $contents ):DOMDocument return $doc; } -} \ No newline at end of file +} diff --git a/scripts/translation/lib/all.php b/scripts/translation/lib/all.php index 900809b55..aa9ca28eb 100644 --- a/scripts/translation/lib/all.php +++ b/scripts/translation/lib/all.php @@ -1,4 +1,21 @@ | + * +----------------------------------------------------------------------+ + * | Description: Old style, require all file. | + * +----------------------------------------------------------------------+ + */ ini_set( 'display_errors' , 1 ); ini_set( 'display_startup_errors' , 1 ); @@ -16,4 +33,4 @@ require_once __DIR__ . '/RevcheckIgnore.php'; require_once __DIR__ . '/RevcheckRun.php'; require_once __DIR__ . '/RevtagParser.php'; -require_once __DIR__ . '/XmlUtil.php'; \ No newline at end of file +require_once __DIR__ . '/XmlUtil.php'; diff --git a/scripts/translation/qarvt.php b/scripts/translation/qarvt.php index 17d9e94cf..e76196729 100644 --- a/scripts/translation/qarvt.php +++ b/scripts/translation/qarvt.php @@ -1,7 +1,20 @@ | + * +----------------------------------------------------------------------+ + * | Description: Check format for revtags and credits on XML comments. | + * +----------------------------------------------------------------------+ */ require_once __DIR__ . '/lib/all.php'; diff --git a/scripts/translation/qaxml.a.php b/scripts/translation/qaxml.a.php index 1c0f50740..6662b580d 100644 --- a/scripts/translation/qaxml.a.php +++ b/scripts/translation/qaxml.a.php @@ -1,7 +1,20 @@ | + * +----------------------------------------------------------------------+ + * | Description: Compare attributes between XMLs. | + * +----------------------------------------------------------------------+ */ require_once __DIR__ . '/lib/all.php'; @@ -68,4 +81,4 @@ function extractTriple( array $list ) foreach( $elem->attributes as $attrib ) $ret[] = "{$elem->nodeName} {$attrib->nodeName} {$attrib->nodeValue}"; return $ret; -} \ No newline at end of file +} diff --git a/scripts/translation/qaxml.e.php b/scripts/translation/qaxml.e.php index 58130279f..6d09cb1b1 100644 --- a/scripts/translation/qaxml.e.php +++ b/scripts/translation/qaxml.e.php @@ -1,7 +1,20 @@ | + * +----------------------------------------------------------------------+ + * | Description: Compare entities usage between XMLs. | + * +----------------------------------------------------------------------+ */ require_once __DIR__ . '/lib/all.php'; @@ -49,4 +62,4 @@ $output->addLine(); $output->print(); -} \ No newline at end of file +} diff --git a/scripts/translation/qaxml.p.php b/scripts/translation/qaxml.p.php index dd30a75e0..95cca01d2 100644 --- a/scripts/translation/qaxml.p.php +++ b/scripts/translation/qaxml.p.php @@ -1,7 +1,20 @@ | + * +----------------------------------------------------------------------+ + * | Description: Compare PIs usage between XMLs. | + * +----------------------------------------------------------------------+ */ require_once __DIR__ . '/lib/all.php'; @@ -67,4 +80,4 @@ function extractPiData( array $list ) foreach( $list as $elem ) $ret[] = "{$elem->target} {$elem->data}"; return $ret; -} \ No newline at end of file +} diff --git a/scripts/translation/qaxml.t.php b/scripts/translation/qaxml.t.php index b845c6e34..3917302c9 100644 --- a/scripts/translation/qaxml.t.php +++ b/scripts/translation/qaxml.t.php @@ -1,7 +1,20 @@ | + * +----------------------------------------------------------------------+ + * | Description: Compare tag count and contents between XMLs. | + * +----------------------------------------------------------------------+ */ require_once __DIR__ . '/lib/all.php'; @@ -296,4 +309,4 @@ function collectTagDefinitions( string $file , string $tag ) $ret[] = $node->getLineNo(); } return $ret; -} \ No newline at end of file +} From 0861659359ca03d07b9b83ceda676488630f7c45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20L=20F=20S=20Bacci?= Date: Fri, 15 Dec 2023 11:43:55 -0300 Subject: [PATCH 46/50] Fix dynamic property creation into temporary --- scripts/translation/lib/RevcheckRun.php | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/scripts/translation/lib/RevcheckRun.php b/scripts/translation/lib/RevcheckRun.php index ef9a01fb1..8a14dfbf3 100644 --- a/scripts/translation/lib/RevcheckRun.php +++ b/scripts/translation/lib/RevcheckRun.php @@ -85,22 +85,25 @@ private function calculateStatus() continue; } - $target->hash = $target->revtag->revision; + // Translation compares ok from multiple hashs. The head hash or the last non-skiped hash. + // See https://github.com/php/doc-base/blob/090ff07aa03c3e4ad7320a4ace9ffb6d5ede722f/scripts/revcheck.php#L374 + // and https://github.com/php/doc-base/blob/090ff07aa03c3e4ad7320a4ace9ffb6d5ede722f/scripts/revcheck.php#L392 . - // translation compares ok from multiple hashs. See https://github.com/php/doc-base/commit/090ff07aa03c3e4ad7320a4ace9ffb6d5ede722f - $wobblyOkHash = $source->head; // L372 - if ( $target->hash == $source->diff ) // R391 - $wobblyOkHash = $source->diff; // R392 + $sourceHash = $source->head; + $targetHash = $target->revtag->revision; + + if ( $targetHash == $source->diff ) + $sourceHash = $source->diff; $daysOld = ( strtotime( "now" ) - $source->date ) / 86400; $daysOld = (int)$daysOld; - $qaInfo = new QaFileInfo( $wobblyOkHash , $target->hash , $this->sourceDir , $this->targetDir , $source->file , $daysOld ); + $qaInfo = new QaFileInfo( $sourceHash , $targetHash , $this->sourceDir , $this->targetDir , $source->file , $daysOld ); $this->qaList[ $source->file ] = $qaInfo; // TranslatedOk - if ( $target->revtag->status == "ready" && $wobblyOkHash == $target->hash ) + if ( $target->revtag->status == "ready" && $sourceHash == $targetHash ) { $source->status = RevcheckStatus::TranslatedOk; $this->filesOk[] = $source; From ed0a911db35abf889060b45a8bc817eba9695407 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20L=20F=20S=20Bacci?= Date: Fri, 15 Dec 2023 11:52:55 -0300 Subject: [PATCH 47/50] Usage and help info --- scripts/translation/configure.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/translation/configure.php b/scripts/translation/configure.php index 5df8324c0..88d5ff40c 100644 --- a/scripts/translation/configure.php +++ b/scripts/translation/configure.php @@ -19,10 +19,10 @@ require_once __DIR__ . '/lib/all.php'; -if ( count( $argv ) < 2 ) +if ( count( $argv ) < 2 || in_array( '--help' , $argv ) || in_array( '-h' , $argv ) ) { - fwrite( STDERR , " Missing paramater. Usage:\n" ); - fwrite( STDERR , " {$argv[0]} [lang_dir]:\n" ); + fwrite( STDERR , "Usage: {$argv[0]} [lang_dir]\n\n" ); + fwrite( STDERR , "See https://github.com/php/doc-base/tree/master/scripts/translation#readme for more info.\n" ); return; } From 07eee250c59c6ea39a2d326cd12a2020f0e00500 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20L=20F=20S=20Bacci?= Date: Fri, 15 Dec 2023 12:49:40 -0300 Subject: [PATCH 48/50] Documentation fixes --- scripts/translation/README.md | 7 +------ scripts/translation/lib/RevtagParser.php | 2 +- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/scripts/translation/README.md b/scripts/translation/README.md index 7224cec78..7890be067 100644 --- a/scripts/translation/README.md +++ b/scripts/translation/README.md @@ -87,16 +87,11 @@ php doc-base/scripts/translation/qaxml.t.php destructorsynopsis php doc-base/scripts/translation/qaxml.t.php fieldsynopsis php doc-base/scripts/translation/qaxml.t.php funcsynopsis php doc-base/scripts/translation/qaxml.t.php methodsynopsis - -php doc-base/scripts/translation/qaxml.t.php code -php doc-base/scripts/translation/qaxml.t.php computeroutput -php doc-base/scripts/translation/qaxml.t.php filename -php doc-base/scripts/translation/qaxml.t.php literal -php doc-base/scripts/translation/qaxml.t.php varname ``` Tags where is expected few translations: ``` php doc-base/scripts/translation/qaxml.t.php code +php doc-base/scripts/translation/qaxml.t.php computeroutput php doc-base/scripts/translation/qaxml.t.php filename php doc-base/scripts/translation/qaxml.t.php literal php doc-base/scripts/translation/qaxml.t.php varname diff --git a/scripts/translation/lib/RevtagParser.php b/scripts/translation/lib/RevtagParser.php index 82a291c6f..88fb8c23d 100644 --- a/scripts/translation/lib/RevtagParser.php +++ b/scripts/translation/lib/RevtagParser.php @@ -67,7 +67,7 @@ public static function parseComment( DOMNode $node , RevtagInfo & $ret , $filena if ( str_starts_with( $text , "EN-" ) ) { - // /EN-Revision:\s*(\S+)\s*Maintainer:\s*(\S+)\s*Status:\s*(\S+)/ // restric maintainer without spaces + // /EN-Revision:\s*(\S+)\s*Maintainer:\s*(\S+)\s*Status:\s*(\S+)/ // restrict maintainer without spaces // /EN-Revision:\s*(\S+)\s*Maintainer:\s(.*?)\sStatus:\s*(\S+)/ // accepts maintainer with spaces $match = array(); From 821023f0e9ae33c1d1d6b71c42260203cfba3d59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20L=20F=20S=20Bacci?= Date: Fri, 15 Dec 2023 13:18:41 -0300 Subject: [PATCH 49/50] Pass by reference is not necessary with classes --- scripts/translation/lib/RevtagParser.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/translation/lib/RevtagParser.php b/scripts/translation/lib/RevtagParser.php index 88fb8c23d..aac675cc8 100644 --- a/scripts/translation/lib/RevtagParser.php +++ b/scripts/translation/lib/RevtagParser.php @@ -52,7 +52,7 @@ public static function parseText( string $contents ): RevtagInfo|null return $ret; } - public static function parseNodeRecurse( DOMNode $node , RevtagInfo & $ret , $filename = "" ) + public static function parseNodeRecurse( DOMNode $node , RevtagInfo $ret , $filename = "" ) { if ( $node->nodeType == XML_COMMENT_NODE ) RevtagParser::parseComment( $node , $ret , $filename ); @@ -61,7 +61,7 @@ public static function parseNodeRecurse( DOMNode $node , RevtagInfo & $ret , $fi RevtagParser::parseNodeRecurse( $child , $ret , $filename ); } - public static function parseComment( DOMNode $node , RevtagInfo & $ret , $filename = "" ) + public static function parseComment( DOMNode $node , RevtagInfo $ret , $filename = "" ) { $text = trim( $node->textContent ); From bfa2f5283fd448afb8fa8e3ba553caa020704d2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20L=20F=20S=20Bacci?= Date: Fri, 15 Dec 2023 13:45:45 -0300 Subject: [PATCH 50/50] Encapsulate private property --- scripts/translation/lib/RevcheckFileList.php | 7 ++++++- scripts/translation/lib/RevcheckRun.php | 4 ++-- scripts/translation/lib/RevtagParser.php | 2 +- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/scripts/translation/lib/RevcheckFileList.php b/scripts/translation/lib/RevcheckFileList.php index 4c2eae255..a227912b7 100644 --- a/scripts/translation/lib/RevcheckFileList.php +++ b/scripts/translation/lib/RevcheckFileList.php @@ -21,7 +21,7 @@ class RevcheckFileList { - var $list = array(); + private $list = array(); function __construct( $lang ) { @@ -33,6 +33,11 @@ function get( $file ): RevcheckFileInfo|null return $this->list[ $file ] ?? null; } + function iterator(): Iterator + { + return new ArrayIterator( $this->list ); + } + function loadTree( $lang ) { $dir = new \DirectoryIterator( $lang ); diff --git a/scripts/translation/lib/RevcheckRun.php b/scripts/translation/lib/RevcheckRun.php index 8a14dfbf3..8debc0408 100644 --- a/scripts/translation/lib/RevcheckRun.php +++ b/scripts/translation/lib/RevcheckRun.php @@ -63,7 +63,7 @@ private function calculateStatus() // All status are marked in source files, // except notinen, that are marked on target. - foreach( $this->sourceFiles->list as $source ) + foreach( $this->sourceFiles->iterator() as $source ) { $target = $this->targetFiles->get( $source->file ); @@ -130,7 +130,7 @@ private function calculateStatus() // NotInEnTree - foreach( $this->targetFiles->list as $target ) + foreach( $this->targetFiles->iterator() as $target ) { $source = $this->sourceFiles->get( $target->file ); if ( $source == null ) diff --git a/scripts/translation/lib/RevtagParser.php b/scripts/translation/lib/RevtagParser.php index aac675cc8..a04db2fd1 100644 --- a/scripts/translation/lib/RevtagParser.php +++ b/scripts/translation/lib/RevtagParser.php @@ -32,7 +32,7 @@ class RevtagParser { static function parseInto( string $lang , RevcheckFileList & $list ) { - foreach( $list->list as $entry ) + foreach( $list->iterator() as $entry ) $entry->revtag = RevtagParser::parseFile( $lang . '/' . $entry->file ); }