From 64c7ef79a3fda9e41ae69142a5abfccebbd68359 Mon Sep 17 00:00:00 2001 From: Joas Schilling Date: Mon, 24 Feb 2025 22:25:33 +0100 Subject: [PATCH 1/6] fix(translation): Notify on broken German translations Signed-off-by: Joas Schilling --- translations-app/handleAppTranslations.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/translations-app/handleAppTranslations.sh b/translations-app/handleAppTranslations.sh index e3dd5d8c..f906315d 100755 --- a/translations-app/handleAppTranslations.sh +++ b/translations-app/handleAppTranslations.sh @@ -141,3 +141,11 @@ do echo "done with $version" done + +# End of verbose mode +set +x + +if [ $(jq '.translations[]' l10n/de.json | grep 'Benötigt keine Übersetzung. Hier wird nur die formelle Übersetzung verwendet (de_DE).' | wc -l) -ne 0 ]; then + echo "German language file contains the 'Benötigt keine Übersetzung. Hier wird nur die formelle Übersetzung verwendet (de_DE).' hint." 1>&2 + exit 3 +fi From e8ed337ae9c0947dbd4db6741c823e08c92ab283 Mon Sep 17 00:00:00 2001 From: Joas Schilling Date: Tue, 25 Feb 2025 15:51:33 +0100 Subject: [PATCH 2/6] fix(translation): Check other translation restrictions as well Signed-off-by: Joas Schilling --- translations-app/handleAppTranslations.sh | 30 ++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/translations-app/handleAppTranslations.sh b/translations-app/handleAppTranslations.sh index f906315d..acf33e99 100755 --- a/translations-app/handleAppTranslations.sh +++ b/translations-app/handleAppTranslations.sh @@ -145,7 +145,35 @@ done # End of verbose mode set +x +EXIT_CODE=0 +# Confirm German translation does not have the false "don't translate" warning if [ $(jq '.translations[]' l10n/de.json | grep 'Benötigt keine Übersetzung. Hier wird nur die formelle Übersetzung verwendet (de_DE).' | wc -l) -ne 0 ]; then echo "German language file contains the 'Benötigt keine Übersetzung. Hier wird nur die formelle Übersetzung verwendet (de_DE).' hint." 1>&2 - exit 3 + EXIT=3 fi + +# Confirm English source does not contain the pipe character which breaks the Symfony Translation library +if [ $(jq '.translations | keys[]' l10n/en_GB.json | grep '|' | wc -l) -ne 0 ]; then + echo "English source contains the pipe character" 1>&2 + EXIT=4 +fi + +for file in $(ls l10n/*.json) +do + # Make sure only RTL languages contain such characters + if [ "$file" != "l10n/ar.json" -a "$file" != "l10n/fa.json" -a "$file" != "l10n/he.json" -a "$file" != "l10n/ps.json" -a "$file" != "l10n/ug.json" -a "$file" != "l10n/ur_PK.json" ]; then + if [ $(jq '.translations[]' $file | grep -E '(\x{061C}|\x{0623}|\x{200E}|\x{200F}|\x{202A}|\x{202B}|\x{202C}|\x{202D}|\x{202E}|\x{2066}|\x{2067}|\x{2068}|\x{2069}|\x{206C}|\x{206D})' | wc -l) -ne 0 ]; then + echo "$file contains a RTL limited characters in the translations" 1>&2 + EXIT=5 + fi + fi + + # Confirm translations do not contain the pipe character which breaks the Symfony Translation library + if [ $(jq '.translations[]' $file | grep '|' | wc -l) -ne 0 ]; then + echo "$file contains the pipe character" 1>&2 + EXIT=6 + fi +done + + +exit $EXIT_CODE From 77f2851540ee369b7f5a4d34b5ccd6963e54efdf Mon Sep 17 00:00:00 2001 From: Joas Schilling Date: Tue, 25 Feb 2025 15:58:23 +0100 Subject: [PATCH 3/6] fix(translations): Also check unicode characters for now Signed-off-by: Joas Schilling --- translations-app/handleAppTranslations.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/translations-app/handleAppTranslations.sh b/translations-app/handleAppTranslations.sh index acf33e99..174e33dc 100755 --- a/translations-app/handleAppTranslations.sh +++ b/translations-app/handleAppTranslations.sh @@ -158,6 +158,12 @@ if [ $(jq '.translations | keys[]' l10n/en_GB.json | grep '|' | wc -l) -ne 0 ]; EXIT=4 fi +# Confirm English source does not contain the unicode single quote character +if [ $(jq '.translations | keys[]' l10n/en_GB.json | grep -E '(´|’)' | wc -l) -ne 0 ]; then + echo "English source contains unicode single quote character, that should be replaced by normal single quotes" 1>&2 + EXIT=4 +fi + for file in $(ls l10n/*.json) do # Make sure only RTL languages contain such characters From 1a5412062af6a3ff69386f20a4ebc38097f07e05 Mon Sep 17 00:00:00 2001 From: Joas Schilling Date: Tue, 25 Feb 2025 16:19:12 +0100 Subject: [PATCH 4/6] fix(translation): Check against triple dots Signed-off-by: Joas Schilling --- translations-app/handleAppTranslations.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/translations-app/handleAppTranslations.sh b/translations-app/handleAppTranslations.sh index 174e33dc..1537c4b6 100755 --- a/translations-app/handleAppTranslations.sh +++ b/translations-app/handleAppTranslations.sh @@ -164,6 +164,12 @@ if [ $(jq '.translations | keys[]' l10n/en_GB.json | grep -E '(´|’)' | wc -l) EXIT=4 fi +# Confirm English source does not use triple dots +if [ $(jq '.translations | keys[]' l10n/en_GB.json | grep '\.\.\.' | wc -l) -ne 0 ]; then + echo "English source contains three consecutive dots. Unicode … should be used instead" 1>&2 + EXIT=4 +fi + for file in $(ls l10n/*.json) do # Make sure only RTL languages contain such characters From a061d6c0eb3bc03cc53608d0cd795a391662eb31 Mon Sep 17 00:00:00 2001 From: Joas Schilling Date: Tue, 25 Feb 2025 16:41:08 +0100 Subject: [PATCH 5/6] fix(translations): Check for leading or trailing whitespace in source Signed-off-by: Joas Schilling --- translations-app/handleAppTranslations.sh | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/translations-app/handleAppTranslations.sh b/translations-app/handleAppTranslations.sh index 1537c4b6..d674c99d 100755 --- a/translations-app/handleAppTranslations.sh +++ b/translations-app/handleAppTranslations.sh @@ -154,19 +154,37 @@ fi # Confirm English source does not contain the pipe character which breaks the Symfony Translation library if [ $(jq '.translations | keys[]' l10n/en_GB.json | grep '|' | wc -l) -ne 0 ]; then + echo "" 1>&2 echo "English source contains the pipe character" 1>&2 + echo "---" 1>&2 + jq '.translations | keys[]' l10n/en_GB.json | grep '|' 1>&2 EXIT=4 fi # Confirm English source does not contain the unicode single quote character if [ $(jq '.translations | keys[]' l10n/en_GB.json | grep -E '(´|’)' | wc -l) -ne 0 ]; then + echo "" 1>&2 echo "English source contains unicode single quote character, that should be replaced by normal single quotes" 1>&2 + echo "---" 1>&2 + jq '.translations | keys[]' l10n/en_GB.json | grep -E '(´|’)' 1>&2 EXIT=4 fi # Confirm English source does not use triple dots if [ $(jq '.translations | keys[]' l10n/en_GB.json | grep '\.\.\.' | wc -l) -ne 0 ]; then + echo "" 1>&2 echo "English source contains three consecutive dots. Unicode … should be used instead" 1>&2 + echo "---" 1>&2 + jq '.translations | keys[]' l10n/en_GB.json | grep '\.\.\.' 1>&2 + EXIT=4 +fi + +# Check for leading or trailing spaces +if [ $(jq '.translations | keys[]' l10n/en_GB.json | grep -E '(^\"(\s|\\t|\\n)|(\s|\\t|\\n)\"$)' | wc -l) -ne 0 ]; then + echo "" 1>&2 + echo "English source contains leading or trailing white spaces, tabs or new lines" 1>&2 + echo "---" 1>&2 + jq '.translations | keys[]' l10n/en_GB.json | grep -E '(^\"(\s|\\t|\\n)|(\s|\\t|\\n)\"$)' 1>&2 EXIT=4 fi From e6c61330ed3db08749dd7538cc0c318b6bac84d9 Mon Sep 17 00:00:00 2001 From: Joas Schilling Date: Tue, 25 Feb 2025 17:34:59 +0100 Subject: [PATCH 6/6] fix(translation): Move validation to a file, so users can check it locally Signed-off-by: Joas Schilling --- translations-app/handleAppTranslations.sh | 66 +--------------------- translations/Dockerfile | 1 + translations/handleAppsTranslations.sh | 11 ++++ translations/handleTranslations.sh | 15 +++++ translations/validateTranslationFiles.sh | 67 +++++++++++++++++++++++ 5 files changed, 97 insertions(+), 63 deletions(-) create mode 100755 translations/validateTranslationFiles.sh diff --git a/translations-app/handleAppTranslations.sh b/translations-app/handleAppTranslations.sh index d674c99d..86b62c39 100755 --- a/translations-app/handleAppTranslations.sh +++ b/translations-app/handleAppTranslations.sh @@ -143,67 +143,7 @@ do done # End of verbose mode -set +x +set +xe -EXIT_CODE=0 -# Confirm German translation does not have the false "don't translate" warning -if [ $(jq '.translations[]' l10n/de.json | grep 'Benötigt keine Übersetzung. Hier wird nur die formelle Übersetzung verwendet (de_DE).' | wc -l) -ne 0 ]; then - echo "German language file contains the 'Benötigt keine Übersetzung. Hier wird nur die formelle Übersetzung verwendet (de_DE).' hint." 1>&2 - EXIT=3 -fi - -# Confirm English source does not contain the pipe character which breaks the Symfony Translation library -if [ $(jq '.translations | keys[]' l10n/en_GB.json | grep '|' | wc -l) -ne 0 ]; then - echo "" 1>&2 - echo "English source contains the pipe character" 1>&2 - echo "---" 1>&2 - jq '.translations | keys[]' l10n/en_GB.json | grep '|' 1>&2 - EXIT=4 -fi - -# Confirm English source does not contain the unicode single quote character -if [ $(jq '.translations | keys[]' l10n/en_GB.json | grep -E '(´|’)' | wc -l) -ne 0 ]; then - echo "" 1>&2 - echo "English source contains unicode single quote character, that should be replaced by normal single quotes" 1>&2 - echo "---" 1>&2 - jq '.translations | keys[]' l10n/en_GB.json | grep -E '(´|’)' 1>&2 - EXIT=4 -fi - -# Confirm English source does not use triple dots -if [ $(jq '.translations | keys[]' l10n/en_GB.json | grep '\.\.\.' | wc -l) -ne 0 ]; then - echo "" 1>&2 - echo "English source contains three consecutive dots. Unicode … should be used instead" 1>&2 - echo "---" 1>&2 - jq '.translations | keys[]' l10n/en_GB.json | grep '\.\.\.' 1>&2 - EXIT=4 -fi - -# Check for leading or trailing spaces -if [ $(jq '.translations | keys[]' l10n/en_GB.json | grep -E '(^\"(\s|\\t|\\n)|(\s|\\t|\\n)\"$)' | wc -l) -ne 0 ]; then - echo "" 1>&2 - echo "English source contains leading or trailing white spaces, tabs or new lines" 1>&2 - echo "---" 1>&2 - jq '.translations | keys[]' l10n/en_GB.json | grep -E '(^\"(\s|\\t|\\n)|(\s|\\t|\\n)\"$)' 1>&2 - EXIT=4 -fi - -for file in $(ls l10n/*.json) -do - # Make sure only RTL languages contain such characters - if [ "$file" != "l10n/ar.json" -a "$file" != "l10n/fa.json" -a "$file" != "l10n/he.json" -a "$file" != "l10n/ps.json" -a "$file" != "l10n/ug.json" -a "$file" != "l10n/ur_PK.json" ]; then - if [ $(jq '.translations[]' $file | grep -E '(\x{061C}|\x{0623}|\x{200E}|\x{200F}|\x{202A}|\x{202B}|\x{202C}|\x{202D}|\x{202E}|\x{2066}|\x{2067}|\x{2068}|\x{2069}|\x{206C}|\x{206D})' | wc -l) -ne 0 ]; then - echo "$file contains a RTL limited characters in the translations" 1>&2 - EXIT=5 - fi - fi - - # Confirm translations do not contain the pipe character which breaks the Symfony Translation library - if [ $(jq '.translations[]' $file | grep '|' | wc -l) -ne 0 ]; then - echo "$file contains the pipe character" 1>&2 - EXIT=6 - fi -done - - -exit $EXIT_CODE +/validateTranslationFiles.sh +exit $? diff --git a/translations/Dockerfile b/translations/Dockerfile index 870f439f..ed884b59 100644 --- a/translations/Dockerfile +++ b/translations/Dockerfile @@ -32,6 +32,7 @@ RUN mkdir -p /app ADD gitconfig /root/.gitconfig ADD known_hosts /root/.ssh/known_hosts ADD handleTranslations.sh /handleTranslations.sh +ADD validateTranslationFiles.sh /validateTranslationFiles.sh ADD translationtool/translationtool.phar /translationtool.phar WORKDIR /app diff --git a/translations/handleAppsTranslations.sh b/translations/handleAppsTranslations.sh index 663af9ec..1dda924d 100755 --- a/translations/handleAppsTranslations.sh +++ b/translations/handleAppsTranslations.sh @@ -62,3 +62,14 @@ done git commit -m "Fix(l10n): Update translations from Transifex" -s || true git push origin master echo "done" + +set +xe + +EXIT_CODE=0 +for app in $(ls .) +do + if [ -d "$app/l10n" ]; then + /validateTranslationFiles.sh $app + EXIT_CODE=$(($?+$EXIT_CODE)) + fi +done; diff --git a/translations/handleTranslations.sh b/translations/handleTranslations.sh index 70b3347f..43c2cff5 100755 --- a/translations/handleTranslations.sh +++ b/translations/handleTranslations.sh @@ -103,3 +103,18 @@ do echo "done with $version" done + +set +xe +EXIT_CODE=0 +/validateTranslationFiles.sh core +EXIT_CODE=$(($?+$EXIT_CODE)) +/validateTranslationFiles.sh lib +EXIT_CODE=$(($?+$EXIT_CODE)) + +for app in $(ls apps) +do + if [ -d "apps/$app/l10n" ]; then + /validateTranslationFiles.sh apps/$app + EXIT_CODE=$(($?+$EXIT_CODE)) + fi +done; diff --git a/translations/validateTranslationFiles.sh b/translations/validateTranslationFiles.sh new file mode 100755 index 00000000..25e20c74 --- /dev/null +++ b/translations/validateTranslationFiles.sh @@ -0,0 +1,67 @@ +#!/bin/sh + +cd $1 + +EXIT_CODE=0 +# Confirm German translation does not have the false "don't translate" warning +if [ $(jq '.translations[]' l10n/de.json | grep 'Benötigt keine Übersetzung. Hier wird nur die formelle Übersetzung verwendet (de_DE).' | wc -l) -ne 0 ]; then + echo "German language file $1/l10n/de.json contains the 'Benötigt keine Übersetzung. Hier wird nur die formelle Übersetzung verwendet (de_DE).' hint." 1>&2 + EXIT_CODE=3 +fi + +# Confirm English source does not contain the pipe character which breaks the Symfony Translation library +if [ $(jq '.translations | keys[]' l10n/en_GB.json | grep '|' | wc -l) -ne 0 ]; then + echo "" 1>&2 + echo "English source $1/l10n/en_GB.json contains the pipe character" 1>&2 + echo "---" 1>&2 + jq '.translations | keys[]' l10n/en_GB.json | grep '|' 1>&2 + EXIT_CODE=4 +fi + +# Confirm English source does not contain the unicode single quote character +if [ $(jq '.translations | keys[]' l10n/en_GB.json | grep -E '(´|’)' | wc -l) -ne 0 ]; then + echo "" 1>&2 + echo "English source $1/l10n/en_GB.json contains unicode single quote character, that should be replaced by normal single quotes" 1>&2 + echo "---" 1>&2 + jq '.translations | keys[]' l10n/en_GB.json | grep -E '(´|’)' 1>&2 + EXIT_CODE=4 +fi + +# Confirm English source does not use triple dots +if [ $(jq '.translations | keys[]' l10n/en_GB.json | grep '\.\.\.' | wc -l) -ne 0 ]; then + echo "" 1>&2 + echo "English source $1/l10n/en_GB.json contains three consecutive dots. Unicode … should be used instead" 1>&2 + echo "---" 1>&2 + jq '.translations | keys[]' l10n/en_GB.json | grep '\.\.\.' 1>&2 + EXIT_CODE=4 +fi + +# Check for leading or trailing spaces +if [ $(jq '.translations | keys[]' l10n/en_GB.json | grep -E '(^\"(\s|\\t|\\n)|(\s|\\t|\\n)\"$)' | wc -l) -ne 0 ]; then + echo "" 1>&2 + echo "English source $1/l10n/en_GB.json contains leading or trailing white spaces, tabs or new lines" 1>&2 + echo "---" 1>&2 + jq '.translations | keys[]' l10n/en_GB.json | grep -E '(^\"(\s|\\t|\\n)|(\s|\\t|\\n)\"$)' 1>&2 + EXIT_CODE=4 +fi + +for file in $(ls l10n/*.json) +do + # Make sure only RTL languages contain such characters + if [ "$file" != "l10n/ar.json" -a "$file" != "l10n/fa.json" -a "$file" != "l10n/he.json" -a "$file" != "l10n/ps.json" -a "$file" != "l10n/ug.json" -a "$file" != "l10n/ur_PK.json" ]; then + if [ $(jq '.translations[]' $file | grep -E '(\x{061C}|\x{0623}|\x{200E}|\x{200F}|\x{202A}|\x{202B}|\x{202C}|\x{202D}|\x{202E}|\x{2066}|\x{2067}|\x{2068}|\x{2069}|\x{206C}|\x{206D})' | wc -l) -ne 0 ]; then + echo "$1/$file contains a RTL limited characters in the translations" 1>&2 + EXIT_CODE=5 + fi + fi + + # Confirm translations do not contain the pipe character which breaks the Symfony Translation library + if [ $(jq '.translations[]' $file | grep '|' | wc -l) -ne 0 ]; then + echo "$1/$file contains the pipe character" 1>&2 + EXIT_CODE=6 + fi +done + +cd - + +exit $EXIT_CODE