1
0
Fork 0
mirror of https://codeberg.org/forgejo/forgejo.git synced 2025-08-06 18:35:23 +02:00

Initial support for localization and pluralization with go-i18n-JSON-v2 format

This commit is contained in:
Benedikt Straub 2024-12-30 20:03:37 +01:00
parent 376a2e19ea
commit a2787bb09e
No known key found for this signature in database
GPG key ID: E5C73DDA51986AEC
61 changed files with 1317 additions and 51 deletions

View file

@ -22,20 +22,7 @@ func (k *KeyLocale) HasKey(trKey string) bool {
// TrHTML implements Locale.
func (k *KeyLocale) TrHTML(trKey string, trArgs ...any) template.HTML {
args := slices.Clone(trArgs)
for i, v := range args {
switch v := v.(type) {
case nil, bool, int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64, float32, float64, template.HTML:
// for most basic types (including template.HTML which is safe), just do nothing and use it
case string:
args[i] = template.HTMLEscapeString(v)
case fmt.Stringer:
args[i] = template.HTMLEscapeString(v.String())
default:
args[i] = template.HTMLEscapeString(fmt.Sprint(v))
}
}
return template.HTML(k.TrString(trKey, args...))
return template.HTML(k.TrString(trKey, PrepareArgsForHTML(trArgs...)...))
}
// TrString implements Locale.
@ -43,6 +30,11 @@ func (k *KeyLocale) TrString(trKey string, trArgs ...any) string {
return FormatDummy(trKey, trArgs...)
}
// TrPluralString implements Locale.
func (k *KeyLocale) TrPluralString(count any, trKey string, trArgs ...any) template.HTML {
return template.HTML(FormatDummy(trKey, PrepareArgsForHTML(trArgs...)...))
}
func FormatDummy(trKey string, args ...any) string {
if len(args) == 0 {
return fmt.Sprintf("(%s)", trKey)

View file

@ -8,6 +8,8 @@ import (
)
var (
ErrLocaleAlreadyExist = util.SilentWrap{Message: "lang already exists", Err: util.ErrAlreadyExist}
ErrUncertainArguments = util.SilentWrap{Message: "arguments to i18n should not contain uncertain slices", Err: util.ErrInvalidArgument}
ErrLocaleAlreadyExist = util.SilentWrap{Message: "lang already exists", Err: util.ErrAlreadyExist}
ErrLocaleDoesNotExist = util.SilentWrap{Message: "lang does not exist", Err: util.ErrNotExist}
ErrTranslationDoesNotExist = util.SilentWrap{Message: "translation does not exist", Err: util.ErrNotExist}
ErrUncertainArguments = util.SilentWrap{Message: "arguments to i18n should not contain uncertain slices", Err: util.ErrInvalidArgument}
)

View file

@ -8,11 +8,28 @@ import (
"io"
)
type (
PluralFormIndex uint8
PluralFormRule func(int64) PluralFormIndex
)
const (
PluralFormZero PluralFormIndex = iota
PluralFormOne
PluralFormTwo
PluralFormFew
PluralFormMany
PluralFormOther
)
var DefaultLocales = NewLocaleStore()
type Locale interface {
// TrString translates a given key and arguments for a language
TrString(trKey string, trArgs ...any) string
// TrPluralString translates a given pluralized key and arguments for a language.
// This function returns an error if new-style support for the given key is not available.
TrPluralString(count any, trKey string, trArgs ...any) template.HTML
// TrHTML translates a given key and arguments for a language, string arguments are escaped to HTML
TrHTML(trKey string, trArgs ...any) template.HTML
// HasKey reports if a locale has a translation for a given key
@ -31,8 +48,10 @@ type LocaleStore interface {
Locale(langName string) (Locale, bool)
// HasLang returns whether a given language is present in the store
HasLang(langName string) bool
// AddLocaleByIni adds a new language to the store
AddLocaleByIni(langName, langDesc string, source, moreSource []byte) error
// AddLocaleByIni adds a new old-style language to the store
AddLocaleByIni(langName, langDesc string, pluralRule PluralFormRule, source, moreSource []byte) error
// AddLocaleByJSON adds new-style content to an existing language to the store
AddToLocaleFromJSON(langName string, source []byte) error
}
// ResetDefaultLocales resets the current default locales

View file

@ -12,6 +12,26 @@ import (
"github.com/stretchr/testify/require"
)
var MockPluralRule PluralFormRule = func(n int64) PluralFormIndex {
if n == 0 {
return PluralFormZero
}
if n == 1 {
return PluralFormOne
}
if n >= 2 && n <= 4 {
return PluralFormFew
}
return PluralFormOther
}
var MockPluralRuleEnglish PluralFormRule = func(n int64) PluralFormIndex {
if n == 1 {
return PluralFormOne
}
return PluralFormOther
}
func TestLocaleStore(t *testing.T) {
testData1 := []byte(`
.dot.name = Dot Name
@ -27,11 +47,48 @@ fmt = %[2]s %[1]s
[section]
sub = Changed Sub String
commits = fallback value for commits
`)
testDataJSON2 := []byte(`
{
"section.json": "the JSON is %s",
"section.commits": {
"one": "one %d commit",
"few": "some %d commits",
"other": "lots of %d commits"
},
"section.incomplete": {
"few": "some %d objects (translated)"
},
"nested": {
"outer": {
"inner": {
"json": "Hello World",
"issue": {
"one": "one %d issue",
"few": "some %d issues",
"other": "lots of %d issues"
}
}
}
}
}
`)
testDataJSON1 := []byte(`
{
"section.incomplete": {
"one": "[untranslated] some %d object",
"other": "[untranslated] some %d objects"
}
}
`)
ls := NewLocaleStore()
require.NoError(t, ls.AddLocaleByIni("lang1", "Lang1", testData1, nil))
require.NoError(t, ls.AddLocaleByIni("lang2", "Lang2", testData2, nil))
require.NoError(t, ls.AddLocaleByIni("lang1", "Lang1", MockPluralRuleEnglish, testData1, nil))
require.NoError(t, ls.AddLocaleByIni("lang2", "Lang2", MockPluralRule, testData2, nil))
require.NoError(t, ls.AddToLocaleFromJSON("lang1", testDataJSON1))
require.NoError(t, ls.AddToLocaleFromJSON("lang2", testDataJSON2))
ls.SetDefaultLang("lang1")
lang1, _ := ls.Locale("lang1")
@ -56,6 +113,45 @@ sub = Changed Sub String
result2 := lang2.TrHTML("section.mixed", "a&b")
assert.EqualValues(t, `test value; <span style="color: red; background: none;">a&amp;b</span>`, result2)
result = lang2.TrString("section.json", "valid")
assert.Equal(t, "the JSON is valid", result)
result = lang2.TrString("nested.outer.inner.json")
assert.Equal(t, "Hello World", result)
result = lang2.TrString("section.commits")
assert.Equal(t, "lots of %d commits", result)
result2 = lang2.TrPluralString(1, "section.commits", 1)
assert.EqualValues(t, "one 1 commit", result2)
result2 = lang2.TrPluralString(3, "section.commits", 3)
assert.EqualValues(t, "some 3 commits", result2)
result2 = lang2.TrPluralString(8, "section.commits", 8)
assert.EqualValues(t, "lots of 8 commits", result2)
result2 = lang2.TrPluralString(0, "section.commits")
assert.EqualValues(t, "section.commits", result2)
result2 = lang2.TrPluralString(1, "nested.outer.inner.issue", 1)
assert.EqualValues(t, "one 1 issue", result2)
result2 = lang2.TrPluralString(3, "nested.outer.inner.issue", 3)
assert.EqualValues(t, "some 3 issues", result2)
result2 = lang2.TrPluralString(9, "nested.outer.inner.issue", 9)
assert.EqualValues(t, "lots of 9 issues", result2)
result2 = lang2.TrPluralString(3, "section.incomplete", 3)
assert.EqualValues(t, "some 3 objects (translated)", result2)
result2 = lang2.TrPluralString(1, "section.incomplete", 1)
assert.EqualValues(t, "[untranslated] some 1 object", result2)
result2 = lang2.TrPluralString(7, "section.incomplete", 7)
assert.EqualValues(t, "[untranslated] some 7 objects", result2)
langs, descs := ls.ListLangNameDesc()
assert.ElementsMatch(t, []string{"lang1", "lang2"}, langs)
assert.ElementsMatch(t, []string{"Lang1", "Lang2"}, descs)
@ -77,7 +173,7 @@ c=22
`)
ls := NewLocaleStore()
require.NoError(t, ls.AddLocaleByIni("lang1", "Lang1", testData1, testData2))
require.NoError(t, ls.AddLocaleByIni("lang1", "Lang1", MockPluralRule, testData1, testData2))
lang1, _ := ls.Locale("lang1")
assert.Equal(t, "11", lang1.TrString("a"))
assert.Equal(t, "21", lang1.TrString("b"))
@ -118,7 +214,7 @@ func (e *errorPointerReceiver) Error() string {
func TestLocaleWithTemplate(t *testing.T) {
ls := NewLocaleStore()
require.NoError(t, ls.AddLocaleByIni("lang1", "Lang1", []byte(`key=<a>%s</a>`), nil))
require.NoError(t, ls.AddLocaleByIni("lang1", "Lang1", MockPluralRule, []byte(`key=<a>%s</a>`), nil))
lang1, _ := ls.Locale("lang1")
tmpl := template.New("test").Funcs(template.FuncMap{"tr": lang1.TrHTML})
@ -181,7 +277,7 @@ func TestLocaleStoreQuirks(t *testing.T) {
for _, testData := range testDataList {
ls := NewLocaleStore()
err := ls.AddLocaleByIni("lang1", "Lang1", []byte("a="+testData.in), nil)
err := ls.AddLocaleByIni("lang1", "Lang1", nil, []byte("a="+testData.in), nil)
lang1, _ := ls.Locale("lang1")
require.NoError(t, err, testData.hint)
assert.Equal(t, testData.out, lang1.TrString("a"), testData.hint)

View file

@ -8,8 +8,10 @@ import (
"html/template"
"slices"
"code.gitea.io/gitea/modules/json"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/util"
)
// This file implements the static LocaleStore that will not watch for changes
@ -18,6 +20,9 @@ type locale struct {
store *localeStore
langName string
idxToMsgMap map[int]string // the map idx is generated by store's trKeyToIdxMap
newStyleMessages map[string]string
pluralRule PluralFormRule
}
var _ Locale = (*locale)(nil)
@ -38,8 +43,19 @@ func NewLocaleStore() LocaleStore {
return &localeStore{localeMap: make(map[string]*locale), trKeyToIdxMap: make(map[string]int)}
}
const (
PluralFormSeparator string = "\036"
)
// A note about pluralization rules.
// go-i18n supports plural rules in theory.
// In practice, it relies on another library that hardcodes a list of common languages
// and their plural rules, and does not support languages not hardcoded there.
// So we pretend that all languages are English and use our own function to extract
// the correct plural form for a given count and language.
// AddLocaleByIni adds locale by ini into the store
func (store *localeStore) AddLocaleByIni(langName, langDesc string, source, moreSource []byte) error {
func (store *localeStore) AddLocaleByIni(langName, langDesc string, pluralRule PluralFormRule, source, moreSource []byte) error {
if _, ok := store.localeMap[langName]; ok {
return ErrLocaleAlreadyExist
}
@ -47,7 +63,7 @@ func (store *localeStore) AddLocaleByIni(langName, langDesc string, source, more
store.langNames = append(store.langNames, langName)
store.langDescs = append(store.langDescs, langDesc)
l := &locale{store: store, langName: langName, idxToMsgMap: make(map[int]string)}
l := &locale{store: store, langName: langName, idxToMsgMap: make(map[int]string), pluralRule: pluralRule, newStyleMessages: make(map[string]string)}
store.localeMap[l.langName] = l
iniFile, err := setting.NewConfigProviderForLocale(source, moreSource)
@ -78,6 +94,98 @@ func (store *localeStore) AddLocaleByIni(langName, langDesc string, source, more
return nil
}
func RecursivelyAddTranslationsFromJSON(locale *locale, object map[string]any, prefix string) error {
for key, value := range object {
var fullkey string
if prefix != "" {
fullkey = prefix + "." + key
} else {
fullkey = key
}
switch v := value.(type) {
case string:
// Check whether we are adding a plural form to the parent object, or a new nested JSON object.
if key == "zero" || key == "one" || key == "two" || key == "few" || key == "many" {
locale.newStyleMessages[prefix+PluralFormSeparator+key] = v
} else if key == "other" {
locale.newStyleMessages[prefix] = v
} else {
locale.newStyleMessages[fullkey] = v
}
case map[string]any:
err := RecursivelyAddTranslationsFromJSON(locale, v, fullkey)
if err != nil {
return err
}
case nil:
default:
return fmt.Errorf("Unrecognized JSON value '%s'", value)
}
}
return nil
}
func (store *localeStore) AddToLocaleFromJSON(langName string, source []byte) error {
locale, ok := store.localeMap[langName]
if !ok {
return ErrLocaleDoesNotExist
}
var result map[string]any
if err := json.Unmarshal(source, &result); err != nil {
return err
}
return RecursivelyAddTranslationsFromJSON(locale, result, "")
}
func (l *locale) LookupNewStyleMessage(trKey string) string {
if msg, ok := l.newStyleMessages[trKey]; ok {
return msg
}
return ""
}
func (l *locale) LookupPlural(trKey string, count any) string {
n, err := util.ToInt64(count)
if err != nil {
log.Error("Invalid plural count '%s'", count)
return ""
}
pluralForm := l.pluralRule(n)
suffix := ""
switch pluralForm {
case PluralFormZero:
suffix = PluralFormSeparator + "zero"
case PluralFormOne:
suffix = PluralFormSeparator + "one"
case PluralFormTwo:
suffix = PluralFormSeparator + "two"
case PluralFormFew:
suffix = PluralFormSeparator + "few"
case PluralFormMany:
suffix = PluralFormSeparator + "many"
case PluralFormOther:
// No suffix for the "other" string.
default:
log.Error("Invalid plural form index %d for count %d", pluralForm, count)
return ""
}
if result, ok := l.newStyleMessages[trKey+suffix]; ok {
return result
}
log.Error("Missing translation for plural form index %d for count %d", pluralForm, count)
return ""
}
func (store *localeStore) HasLang(langName string) bool {
_, ok := store.localeMap[langName]
return ok
@ -113,22 +221,37 @@ func (store *localeStore) Close() error {
func (l *locale) TrString(trKey string, trArgs ...any) string {
format := trKey
idx, ok := l.store.trKeyToIdxMap[trKey]
found := false
if ok {
if msg, ok := l.idxToMsgMap[idx]; ok {
format = msg // use the found translation
found = true
} else if def, ok := l.store.localeMap[l.store.defaultLang]; ok {
// try to use default locale's translation
if msg, ok := def.idxToMsgMap[idx]; ok {
format = msg
if msg := l.LookupNewStyleMessage(trKey); msg != "" {
format = msg
} else {
// First fallback: old-style translation
idx, ok := l.store.trKeyToIdxMap[trKey]
found := false
if ok {
if msg, ok := l.idxToMsgMap[idx]; ok {
format = msg // use the found translation
found = true
}
}
}
if !found {
log.Error("Missing translation %q", trKey)
if !found {
// Second fallback: new-style default language
if defaultLang, ok := l.store.localeMap[l.store.defaultLang]; ok {
if msg := defaultLang.LookupNewStyleMessage(trKey); msg != "" {
format = msg
} else {
// Third fallback: old-style default language
if msg, ok := defaultLang.idxToMsgMap[idx]; ok {
format = msg
found = true
}
}
}
if !found {
log.Error("Missing translation %q", trKey)
}
}
}
msg, err := Format(format, trArgs...)
@ -138,7 +261,7 @@ func (l *locale) TrString(trKey string, trArgs ...any) string {
return msg
}
func (l *locale) TrHTML(trKey string, trArgs ...any) template.HTML {
func PrepareArgsForHTML(trArgs ...any) []any {
args := slices.Clone(trArgs)
for i, v := range args {
switch v := v.(type) {
@ -152,7 +275,30 @@ func (l *locale) TrHTML(trKey string, trArgs ...any) template.HTML {
args[i] = template.HTMLEscapeString(fmt.Sprint(v))
}
}
return template.HTML(l.TrString(trKey, args...))
return args
}
func (l *locale) TrHTML(trKey string, trArgs ...any) template.HTML {
return template.HTML(l.TrString(trKey, PrepareArgsForHTML(trArgs...)...))
}
func (l *locale) TrPluralString(count any, trKey string, trArgs ...any) template.HTML {
message := l.LookupPlural(trKey, count)
if message == "" {
if defaultLang, ok := l.store.localeMap[l.store.defaultLang]; ok {
message = defaultLang.LookupPlural(trKey, count)
}
if message == "" {
message = trKey
}
}
message, err := Format(message, PrepareArgsForHTML(trArgs...)...)
if err != nil {
log.Error("Error whilst formatting %q in %s: %v", trKey, l.langName, err)
}
return template.HTML(message)
}
// HasKey returns whether a key is present in this locale or not