Convert locale files from ini to json format

.github/workflows/files-changed.yml

@@ -19,6 +19,8 @@ on:
         value: ${{ jobs.detect.outputs.swagger }}
       yaml:
         value: ${{ jobs.detect.outputs.yaml }}
+      locale:
+        value: ${{ jobs.detect.outputs.locale }}
 
 jobs:
   detect:
@@ -33,6 +35,7 @@ jobs:
       docker: ${{ steps.changes.outputs.docker }}
       swagger: ${{ steps.changes.outputs.swagger }}
       yaml: ${{ steps.changes.outputs.yaml }}
+      locale: ${{ steps.changes.outputs.locale }}
     steps:
       - uses: actions/checkout@v4
       - uses: dorny/paths-filter@v3
@@ -48,7 +51,6 @@ jobs:
               - "Makefile"
               - ".golangci.yml"
               - ".editorconfig"
-              - "options/locale/locale_en-US.ini"
 
             frontend:
               - "*.js"
@@ -63,6 +65,9 @@ jobs:
               - ".eslintrc.cjs"
               - ".npmrc"
 
+            locale:
+              - "options/locale/*.json"
+
             docs:
               - "**/*.md"
               - ".markdownlint.yaml"

.github/workflows/pull-compliance.yml

@@ -110,6 +110,14 @@ jobs:
         env:
           TAGS: bindata gogit sqlite sqlite_unlock_notify
 
+  checks-locale:
+    if: needs.files-changed.outputs.locale == 'true'
+    needs: files-changed
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - run: make translation-check
+
   checks-backend:
     if: needs.files-changed.outputs.backend == 'true' || needs.files-changed.outputs.actions == 'true'
     needs: files-changed

Makefile

@@ -168,8 +168,8 @@ WEB_DIRS := web_src/js web_src/css
 
 ESLINT_FILES := web_src/js tools *.ts tests/e2e
 STYLELINT_FILES := web_src/css web_src/js/components/*.vue
-SPELLCHECK_FILES := $(GO_DIRS) $(WEB_DIRS) templates options/locale/locale_en-US.ini .github $(filter-out CHANGELOG.md, $(wildcard *.go *.md *.yml *.yaml *.toml)) $(filter-out tools/misspellings.csv, $(wildcard tools/*))
-EDITORCONFIG_FILES := templates .github/workflows options/locale/locale_en-US.ini
+SPELLCHECK_FILES := $(GO_DIRS) $(WEB_DIRS) templates options/locale/locale_en-US.json .github $(filter-out CHANGELOG.md, $(wildcard *.go *.md *.yml *.yaml *.toml)) $(filter-out tools/misspellings.csv, $(wildcard tools/*))
+EDITORCONFIG_FILES := templates .github/workflows options/locale/locale_en-US.json
 
 GO_SOURCES := $(wildcard *.go)
 GO_SOURCES += $(shell find $(GO_DIRS) -type f -name "*.go")
@@ -918,15 +918,21 @@ lockfile-check:
 		exit 1; \
 	fi
 
-.PHONY: update-translations
-update-translations:
-	mkdir -p ./translations
-	cd ./translations && curl -L https://crowdin.com/download/project/gitea.zip > gitea.zip && unzip gitea.zip
-	rm ./translations/gitea.zip
-	$(SED_INPLACE) -e 's/="/=/g' -e 's/"$$//g' ./translations/*.ini
-	$(SED_INPLACE) -e 's/\\"/"/g' ./translations/*.ini
-	mv ./translations/*.ini ./options/locale/
-	rmdir ./translations
+LOCALE_DIR := options/locale
+LOCALE_FILES := $(wildcard $(LOCALE_DIR)/*.json)
+
+.PHONY: translation-check
+translation-check:
+	@for f in $(LOCALE_FILES); do \
+		if ! $(NODE_VARS) pnpm exec jsonlint -q $$f > /dev/null 2>&1; then \
+			echo "Invalid JSON syntax: $$f"; \
+			exit 1; \
+		fi; \
+		if ! $(NODE_VARS) pnpm exec find-duplicated-property-keys -s $$f > /dev/null 2>&1; then \
+			echo "Duplicate key found in: $$f"; \
+			exit 1; \
+		fi; \
+	done
 
 .PHONY: generate-gitignore
 generate-gitignore: ## update gitignore files

build/update-locales.sh

@@ -1,52 +1,22 @@
 #!/bin/sh
 
 # this script runs in alpine image which only has `sh` shell
-
-set +e
-if sed --version 2>/dev/null | grep -q GNU; then
-  SED_INPLACE="sed -i"
-else
-  SED_INPLACE="sed -i ''"
-fi
-set -e
-
-if [ ! -f ./options/locale/locale_en-US.ini ]; then
+if [ ! -f ./options/locale/locale_en-US.json ]; then
   echo "please run this script in the root directory of the project"
   exit 1
 fi
 
-mv ./options/locale/locale_en-US.ini ./options/
-
-# the "ini" library for locale has many quirks, its behavior is different from Crowdin.
-# see i18n_test.go for more details
-
-# this script helps to unquote the Crowdin outputs for the quirky ini library
-# * find all `key="...\"..."` lines
-# * remove the leading quote
-# * remove the trailing quote
-# * unescape the quotes
-# * eg: key="...\"..." => key=..."...
-$SED_INPLACE -r -e '/^[-.A-Za-z0-9_]+[ ]*=[ ]*".*"$/ {
-	s/^([-.A-Za-z0-9_]+)[ ]*=[ ]*"/\1=/
-	s/"$//
-	s/\\"/"/g
-	}' ./options/locale/*.ini
-
-# * if the escaped line is incomplete like `key="...` or `key=..."`, quote it with backticks
-# * eg: key="... => key=`"...`
-# * eg: key=..." => key=`..."`
-$SED_INPLACE -r -e 's/^([-.A-Za-z0-9_]+)[ ]*=[ ]*(".*[^"])$/\1=`\2`/' ./options/locale/*.ini
-$SED_INPLACE -r -e 's/^([-.A-Za-z0-9_]+)[ ]*=[ ]*([^"].*")$/\1=`\2`/' ./options/locale/*.ini
+mv ./options/locale/locale_en-US.json ./options/
 
 # Remove translation under 25% of en_us
-baselines=$(wc -l "./options/locale_en-US.ini" | cut -d" " -f1)
+baselines=$(wc -l "./options/locale_en-US.json" | cut -d" " -f1)
 baselines=$((baselines / 4))
-for filename in ./options/locale/*.ini; do
+for filename in ./options/locale/*.json; do
   lines=$(wc -l "$filename" | cut -d" " -f1)
   if [ $lines -lt $baselines ]; then
     echo "Removing $filename: $lines/$baselines"
     rm "$filename"
   fi
 done
 
-mv ./options/locale_en-US.ini ./options/locale/
+mv ./options/locale_en-US.json ./options/locale/

crowdin.yml

@@ -4,9 +4,9 @@ base_path: "."
 base_url: "https://api.crowdin.com"
 preserve_hierarchy: true
 files:
-  - source: "/options/locale/locale_en-US.ini"
-    translation: "/options/locale/locale_%locale%.ini"
-    type: "ini"
+  - source: "/options/locale/locale_en-US.json"
+    translation: "/options/locale/locale_%locale%.json"
+    type: "json"
     skip_untranslated_strings: true
     export_only_approved: true
     update_option: "update_as_unapproved"

modules/translation/i18n/i18n.go

@@ -32,7 +32,7 @@ type LocaleStore interface {
 	// HasLang returns whether a given language is present in the store
 	HasLang(langName string) bool
 	// AddLocaleByIni adds a new language to the store
-	AddLocaleByIni(langName, langDesc string, source, moreSource []byte) error
+	AddLocaleByJSON(langName, langDesc string, source, moreSource []byte) error
 }
 
 // ResetDefaultLocales resets the current default locales

modules/translation/i18n/i18n_test.go

@@ -13,24 +13,23 @@ import (
 
 func TestLocaleStore(t *testing.T) {
 	testData1 := []byte(`
-.dot.name = Dot Name
-fmt = %[1]s %[2]s
+{
+".dot.name": "Dot Name",
+"fmt": "%[1]s %[2]s",
 
-[section]
-sub = Sub String
-mixed = test value; <span style="color: red\; background: none;">%s</span>
-`)
+"section.sub": "Sub String",
+"section.mixed": "test value; <span style=\"color: red; background: none;\">%s</span>"
+}`)
 
 	testData2 := []byte(`
-fmt = %[2]s %[1]s
-
-[section]
-sub = Changed Sub String
-`)
+{
+	"fmt": "%[2]s %[1]s",
+	"section.sub": "Changed Sub String"
+}`)
 
 	ls := NewLocaleStore()
-	assert.NoError(t, ls.AddLocaleByIni("lang1", "Lang1", testData1, nil))
-	assert.NoError(t, ls.AddLocaleByIni("lang2", "Lang2", testData2, nil))
+	assert.NoError(t, ls.AddLocaleByJSON("lang1", "Lang1", testData1, nil))
+	assert.NoError(t, ls.AddLocaleByJSON("lang2", "Lang2", testData2, nil))
 	ls.SetDefaultLang("lang1")
 
 	lang1, _ := ls.Locale("lang1")
@@ -69,17 +68,21 @@ sub = Changed Sub String
 
 func TestLocaleStoreMoreSource(t *testing.T) {
 	testData1 := []byte(`
-a=11
-b=12
+{
+	"a": "11",
+	"b": "12"
+}
 `)
 
 	testData2 := []byte(`
-b=21
-c=22
+{
+	"b": "21",
+	"c": "22"
+}
 `)
 
 	ls := NewLocaleStore()
-	assert.NoError(t, ls.AddLocaleByIni("lang1", "Lang1", testData1, testData2))
+	assert.NoError(t, ls.AddLocaleByJSON("lang1", "Lang1", testData1, testData2))
 	lang1, _ := ls.Locale("lang1")
 	assert.Equal(t, "11", lang1.TrString("a"))
 	assert.Equal(t, "21", lang1.TrString("b"))
@@ -120,7 +123,7 @@ func (e *errorPointerReceiver) Error() string {
 
 func TestLocaleWithTemplate(t *testing.T) {
 	ls := NewLocaleStore()
-	assert.NoError(t, ls.AddLocaleByIni("lang1", "Lang1", []byte(`key=<a>%s</a>`), nil))
+	assert.NoError(t, ls.AddLocaleByJSON("lang1", "Lang1", []byte(`{"key":"<a>%s</a>"}`), nil))
 	lang1, _ := ls.Locale("lang1")
 
 	tmpl := template.New("test").Funcs(template.FuncMap{"tr": lang1.TrHTML})
@@ -150,57 +153,3 @@ func TestLocaleWithTemplate(t *testing.T) {
 		assert.Equal(t, c.want, buf.String())
 	}
 }
-
-func TestLocaleStoreQuirks(t *testing.T) {
-	const nl = "\n"
-	q := func(q1, s string, q2 ...string) string {
-		return q1 + s + strings.Join(q2, "")
-	}
-	testDataList := []struct {
-		in   string
-		out  string
-		hint string
-	}{
-		{` xx`, `xx`, "simple, no quote"},
-		{`" xx"`, ` xx`, "simple, double-quote"},
-		{`' xx'`, ` xx`, "simple, single-quote"},
-		{"` xx`", ` xx`, "simple, back-quote"},
-
-		{`x\"y`, `x\"y`, "no unescape, simple"},
-		{q(`"`, `x\"y`, `"`), `"x\"y"`, "unescape, double-quote"},
-		{q(`'`, `x\"y`, `'`), `x\"y`, "no unescape, single-quote"},
-		{q("`", `x\"y`, "`"), `x\"y`, "no unescape, back-quote"},
-
-		{q(`"`, `x\"y`) + nl + "b=", `"x\"y`, "half open, double-quote"},
-		{q(`'`, `x\"y`) + nl + "b=", `'x\"y`, "half open, single-quote"},
-		{q("`", `x\"y`) + nl + "b=`", `x\"y` + nl + "b=", "half open, back-quote, multi-line"},
-
-		{`x ; y`, `x ; y`, "inline comment (;)"},
-		{`x # y`, `x # y`, "inline comment (#)"},
-		{`x \; y`, `x ; y`, `inline comment (\;)`},
-		{`x \# y`, `x # y`, `inline comment (\#)`},
-	}
-
-	for _, testData := range testDataList {
-		ls := NewLocaleStore()
-		err := ls.AddLocaleByIni("lang1", "Lang1", []byte("a="+testData.in), nil)
-		lang1, _ := ls.Locale("lang1")
-		assert.NoError(t, err, testData.hint)
-		assert.Equal(t, testData.out, lang1.TrString("a"), testData.hint)
-		assert.NoError(t, ls.Close())
-	}
-
-	// TODO: Crowdin needs the strings to be quoted correctly and doesn't like incomplete quotes
-	//       and Crowdin always outputs quoted strings if there are quotes in the strings.
-	//       So, Gitea's `key="quoted" unquoted` content shouldn't be used on Crowdin directly,
-	//       it should be converted to `key="\"quoted\" unquoted"` first.
-	// TODO: We can not use UnescapeValueDoubleQuotes=true, because there are a lot of back-quotes in en-US.ini,
-	//       then Crowdin will output:
-	//       > key = "`x \" y`"
-	//       Then Gitea will read a string with back-quotes, which is incorrect.
-	// TODO: Crowdin might generate multi-line strings, quoted by double-quote, it's not supported by LocaleStore
-	//       LocaleStore uses back-quote for multi-line strings, it's not supported by Crowdin.
-	// TODO: Crowdin doesn't support back-quote as string quoter, it mainly uses double-quote
-	//       so, the following line will be parsed as: value="`first", comment="second`" on Crowdin
-	//       > a = `first; second`
-}

modules/translation/i18n/localestore.go

@@ -10,8 +10,8 @@ import (
 	"html/template"
 	"slices"
 
+	"code.gitea.io/gitea/modules/json"
 	"code.gitea.io/gitea/modules/log"
-	"code.gitea.io/gitea/modules/setting"
 )
 
 // This file implements the static LocaleStore that will not watch for changes
@@ -40,8 +40,8 @@ func NewLocaleStore() LocaleStore {
 	return &localeStore{localeMap: make(map[string]*locale), trKeyToIdxMap: make(map[string]int)}
 }
 
-// AddLocaleByIni adds locale by ini into the store
-func (store *localeStore) AddLocaleByIni(langName, langDesc string, source, moreSource []byte) error {
+// AddLocaleByJSON adds locale by JSON into the store
+func (store *localeStore) AddLocaleByJSON(langName, langDesc string, source, moreSource []byte) error {
 	if _, ok := store.localeMap[langName]; ok {
 		return errors.New("lang has already been added")
 	}
@@ -52,28 +52,46 @@ func (store *localeStore) AddLocaleByIni(langName, langDesc string, source, more
 	l := &locale{store: store, langName: langName, idxToMsgMap: make(map[int]string)}
 	store.localeMap[l.langName] = l
 
-	iniFile, err := setting.NewConfigProviderForLocale(source, moreSource)
-	if err != nil {
-		return fmt.Errorf("unable to load ini: %w", err)
-	}
+	addFunc := func(source []byte) error {
+		if len(source) == 0 {
+			return nil
+		}
 
-	for _, section := range iniFile.Sections() {
-		for _, key := range section.Keys() {
-			var trKey string
-			if section.Name() == "" || section.Name() == "DEFAULT" {
-				trKey = key.Name()
-			} else {
-				trKey = section.Name() + "." + key.Name()
-			}
-			idx, ok := store.trKeyToIdxMap[trKey]
-			if !ok {
-				idx = len(store.trKeyToIdxMap)
-				store.trKeyToIdxMap[trKey] = idx
+		values := make(map[string]any)
+		if err := json.Unmarshal(source, &values); err != nil {
+			return fmt.Errorf("unable to load json: %w", err)
+		}
+		for trKey, value := range values {
+			switch v := value.(type) {
+			case string:
+				idx, ok := store.trKeyToIdxMap[trKey]
+				if !ok {
+					idx = len(store.trKeyToIdxMap)
+					store.trKeyToIdxMap[trKey] = idx
+				}
+				l.idxToMsgMap[idx] = v
+			case map[string]any:
+				for key, val := range v {
+					idx, ok := store.trKeyToIdxMap[trKey+"."+key]
+					if !ok {
+						idx = len(store.trKeyToIdxMap)
+						store.trKeyToIdxMap[trKey+"."+key] = idx
+					}
+					l.idxToMsgMap[idx] = val.(string)
+				}
+			default:
+				return fmt.Errorf("unsupported value type %T for key %q", v, trKey)
 			}
-			l.idxToMsgMap[idx] = key.Value()
 		}
+		return nil
 	}
 
+	if err := addFunc(source); err != nil {
+		return fmt.Errorf("unable to load json: %w", err)
+	}
+	if err := addFunc(moreSource); err != nil {
+		return fmt.Errorf("unable to load json: %w", err)
+	}
 	return nil
 }