mirror of
https://codeberg.org/forgejo/forgejo.git
synced 2025-08-06 18:35:23 +02:00
Initial support for localization and pluralization with go-i18n-JSON-v2 format
This commit is contained in:
parent
376a2e19ea
commit
a2787bb09e
61 changed files with 1317 additions and 51 deletions
|
@ -22,20 +22,7 @@ func (k *KeyLocale) HasKey(trKey string) bool {
|
|||
|
||||
// TrHTML implements Locale.
|
||||
func (k *KeyLocale) TrHTML(trKey string, trArgs ...any) template.HTML {
|
||||
args := slices.Clone(trArgs)
|
||||
for i, v := range args {
|
||||
switch v := v.(type) {
|
||||
case nil, bool, int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64, float32, float64, template.HTML:
|
||||
// for most basic types (including template.HTML which is safe), just do nothing and use it
|
||||
case string:
|
||||
args[i] = template.HTMLEscapeString(v)
|
||||
case fmt.Stringer:
|
||||
args[i] = template.HTMLEscapeString(v.String())
|
||||
default:
|
||||
args[i] = template.HTMLEscapeString(fmt.Sprint(v))
|
||||
}
|
||||
}
|
||||
return template.HTML(k.TrString(trKey, args...))
|
||||
return template.HTML(k.TrString(trKey, PrepareArgsForHTML(trArgs...)...))
|
||||
}
|
||||
|
||||
// TrString implements Locale.
|
||||
|
@ -43,6 +30,11 @@ func (k *KeyLocale) TrString(trKey string, trArgs ...any) string {
|
|||
return FormatDummy(trKey, trArgs...)
|
||||
}
|
||||
|
||||
// TrPluralString implements Locale.
|
||||
func (k *KeyLocale) TrPluralString(count any, trKey string, trArgs ...any) template.HTML {
|
||||
return template.HTML(FormatDummy(trKey, PrepareArgsForHTML(trArgs...)...))
|
||||
}
|
||||
|
||||
func FormatDummy(trKey string, args ...any) string {
|
||||
if len(args) == 0 {
|
||||
return fmt.Sprintf("(%s)", trKey)
|
||||
|
|
|
@ -8,6 +8,8 @@ import (
|
|||
)
|
||||
|
||||
var (
|
||||
ErrLocaleAlreadyExist = util.SilentWrap{Message: "lang already exists", Err: util.ErrAlreadyExist}
|
||||
ErrUncertainArguments = util.SilentWrap{Message: "arguments to i18n should not contain uncertain slices", Err: util.ErrInvalidArgument}
|
||||
ErrLocaleAlreadyExist = util.SilentWrap{Message: "lang already exists", Err: util.ErrAlreadyExist}
|
||||
ErrLocaleDoesNotExist = util.SilentWrap{Message: "lang does not exist", Err: util.ErrNotExist}
|
||||
ErrTranslationDoesNotExist = util.SilentWrap{Message: "translation does not exist", Err: util.ErrNotExist}
|
||||
ErrUncertainArguments = util.SilentWrap{Message: "arguments to i18n should not contain uncertain slices", Err: util.ErrInvalidArgument}
|
||||
)
|
||||
|
|
|
@ -8,11 +8,28 @@ import (
|
|||
"io"
|
||||
)
|
||||
|
||||
type (
|
||||
PluralFormIndex uint8
|
||||
PluralFormRule func(int64) PluralFormIndex
|
||||
)
|
||||
|
||||
const (
|
||||
PluralFormZero PluralFormIndex = iota
|
||||
PluralFormOne
|
||||
PluralFormTwo
|
||||
PluralFormFew
|
||||
PluralFormMany
|
||||
PluralFormOther
|
||||
)
|
||||
|
||||
var DefaultLocales = NewLocaleStore()
|
||||
|
||||
type Locale interface {
|
||||
// TrString translates a given key and arguments for a language
|
||||
TrString(trKey string, trArgs ...any) string
|
||||
// TrPluralString translates a given pluralized key and arguments for a language.
|
||||
// This function returns an error if new-style support for the given key is not available.
|
||||
TrPluralString(count any, trKey string, trArgs ...any) template.HTML
|
||||
// TrHTML translates a given key and arguments for a language, string arguments are escaped to HTML
|
||||
TrHTML(trKey string, trArgs ...any) template.HTML
|
||||
// HasKey reports if a locale has a translation for a given key
|
||||
|
@ -31,8 +48,10 @@ type LocaleStore interface {
|
|||
Locale(langName string) (Locale, bool)
|
||||
// HasLang returns whether a given language is present in the store
|
||||
HasLang(langName string) bool
|
||||
// AddLocaleByIni adds a new language to the store
|
||||
AddLocaleByIni(langName, langDesc string, source, moreSource []byte) error
|
||||
// AddLocaleByIni adds a new old-style language to the store
|
||||
AddLocaleByIni(langName, langDesc string, pluralRule PluralFormRule, source, moreSource []byte) error
|
||||
// AddLocaleByJSON adds new-style content to an existing language to the store
|
||||
AddToLocaleFromJSON(langName string, source []byte) error
|
||||
}
|
||||
|
||||
// ResetDefaultLocales resets the current default locales
|
||||
|
|
|
@ -12,6 +12,26 @@ import (
|
|||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
var MockPluralRule PluralFormRule = func(n int64) PluralFormIndex {
|
||||
if n == 0 {
|
||||
return PluralFormZero
|
||||
}
|
||||
if n == 1 {
|
||||
return PluralFormOne
|
||||
}
|
||||
if n >= 2 && n <= 4 {
|
||||
return PluralFormFew
|
||||
}
|
||||
return PluralFormOther
|
||||
}
|
||||
|
||||
var MockPluralRuleEnglish PluralFormRule = func(n int64) PluralFormIndex {
|
||||
if n == 1 {
|
||||
return PluralFormOne
|
||||
}
|
||||
return PluralFormOther
|
||||
}
|
||||
|
||||
func TestLocaleStore(t *testing.T) {
|
||||
testData1 := []byte(`
|
||||
.dot.name = Dot Name
|
||||
|
@ -27,11 +47,48 @@ fmt = %[2]s %[1]s
|
|||
|
||||
[section]
|
||||
sub = Changed Sub String
|
||||
commits = fallback value for commits
|
||||
`)
|
||||
|
||||
testDataJSON2 := []byte(`
|
||||
{
|
||||
"section.json": "the JSON is %s",
|
||||
"section.commits": {
|
||||
"one": "one %d commit",
|
||||
"few": "some %d commits",
|
||||
"other": "lots of %d commits"
|
||||
},
|
||||
"section.incomplete": {
|
||||
"few": "some %d objects (translated)"
|
||||
},
|
||||
"nested": {
|
||||
"outer": {
|
||||
"inner": {
|
||||
"json": "Hello World",
|
||||
"issue": {
|
||||
"one": "one %d issue",
|
||||
"few": "some %d issues",
|
||||
"other": "lots of %d issues"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
`)
|
||||
testDataJSON1 := []byte(`
|
||||
{
|
||||
"section.incomplete": {
|
||||
"one": "[untranslated] some %d object",
|
||||
"other": "[untranslated] some %d objects"
|
||||
}
|
||||
}
|
||||
`)
|
||||
|
||||
ls := NewLocaleStore()
|
||||
require.NoError(t, ls.AddLocaleByIni("lang1", "Lang1", testData1, nil))
|
||||
require.NoError(t, ls.AddLocaleByIni("lang2", "Lang2", testData2, nil))
|
||||
require.NoError(t, ls.AddLocaleByIni("lang1", "Lang1", MockPluralRuleEnglish, testData1, nil))
|
||||
require.NoError(t, ls.AddLocaleByIni("lang2", "Lang2", MockPluralRule, testData2, nil))
|
||||
require.NoError(t, ls.AddToLocaleFromJSON("lang1", testDataJSON1))
|
||||
require.NoError(t, ls.AddToLocaleFromJSON("lang2", testDataJSON2))
|
||||
ls.SetDefaultLang("lang1")
|
||||
|
||||
lang1, _ := ls.Locale("lang1")
|
||||
|
@ -56,6 +113,45 @@ sub = Changed Sub String
|
|||
result2 := lang2.TrHTML("section.mixed", "a&b")
|
||||
assert.EqualValues(t, `test value; <span style="color: red; background: none;">a&b</span>`, result2)
|
||||
|
||||
result = lang2.TrString("section.json", "valid")
|
||||
assert.Equal(t, "the JSON is valid", result)
|
||||
|
||||
result = lang2.TrString("nested.outer.inner.json")
|
||||
assert.Equal(t, "Hello World", result)
|
||||
|
||||
result = lang2.TrString("section.commits")
|
||||
assert.Equal(t, "lots of %d commits", result)
|
||||
|
||||
result2 = lang2.TrPluralString(1, "section.commits", 1)
|
||||
assert.EqualValues(t, "one 1 commit", result2)
|
||||
|
||||
result2 = lang2.TrPluralString(3, "section.commits", 3)
|
||||
assert.EqualValues(t, "some 3 commits", result2)
|
||||
|
||||
result2 = lang2.TrPluralString(8, "section.commits", 8)
|
||||
assert.EqualValues(t, "lots of 8 commits", result2)
|
||||
|
||||
result2 = lang2.TrPluralString(0, "section.commits")
|
||||
assert.EqualValues(t, "section.commits", result2)
|
||||
|
||||
result2 = lang2.TrPluralString(1, "nested.outer.inner.issue", 1)
|
||||
assert.EqualValues(t, "one 1 issue", result2)
|
||||
|
||||
result2 = lang2.TrPluralString(3, "nested.outer.inner.issue", 3)
|
||||
assert.EqualValues(t, "some 3 issues", result2)
|
||||
|
||||
result2 = lang2.TrPluralString(9, "nested.outer.inner.issue", 9)
|
||||
assert.EqualValues(t, "lots of 9 issues", result2)
|
||||
|
||||
result2 = lang2.TrPluralString(3, "section.incomplete", 3)
|
||||
assert.EqualValues(t, "some 3 objects (translated)", result2)
|
||||
|
||||
result2 = lang2.TrPluralString(1, "section.incomplete", 1)
|
||||
assert.EqualValues(t, "[untranslated] some 1 object", result2)
|
||||
|
||||
result2 = lang2.TrPluralString(7, "section.incomplete", 7)
|
||||
assert.EqualValues(t, "[untranslated] some 7 objects", result2)
|
||||
|
||||
langs, descs := ls.ListLangNameDesc()
|
||||
assert.ElementsMatch(t, []string{"lang1", "lang2"}, langs)
|
||||
assert.ElementsMatch(t, []string{"Lang1", "Lang2"}, descs)
|
||||
|
@ -77,7 +173,7 @@ c=22
|
|||
`)
|
||||
|
||||
ls := NewLocaleStore()
|
||||
require.NoError(t, ls.AddLocaleByIni("lang1", "Lang1", testData1, testData2))
|
||||
require.NoError(t, ls.AddLocaleByIni("lang1", "Lang1", MockPluralRule, testData1, testData2))
|
||||
lang1, _ := ls.Locale("lang1")
|
||||
assert.Equal(t, "11", lang1.TrString("a"))
|
||||
assert.Equal(t, "21", lang1.TrString("b"))
|
||||
|
@ -118,7 +214,7 @@ func (e *errorPointerReceiver) Error() string {
|
|||
|
||||
func TestLocaleWithTemplate(t *testing.T) {
|
||||
ls := NewLocaleStore()
|
||||
require.NoError(t, ls.AddLocaleByIni("lang1", "Lang1", []byte(`key=<a>%s</a>`), nil))
|
||||
require.NoError(t, ls.AddLocaleByIni("lang1", "Lang1", MockPluralRule, []byte(`key=<a>%s</a>`), nil))
|
||||
lang1, _ := ls.Locale("lang1")
|
||||
|
||||
tmpl := template.New("test").Funcs(template.FuncMap{"tr": lang1.TrHTML})
|
||||
|
@ -181,7 +277,7 @@ func TestLocaleStoreQuirks(t *testing.T) {
|
|||
|
||||
for _, testData := range testDataList {
|
||||
ls := NewLocaleStore()
|
||||
err := ls.AddLocaleByIni("lang1", "Lang1", []byte("a="+testData.in), nil)
|
||||
err := ls.AddLocaleByIni("lang1", "Lang1", nil, []byte("a="+testData.in), nil)
|
||||
lang1, _ := ls.Locale("lang1")
|
||||
require.NoError(t, err, testData.hint)
|
||||
assert.Equal(t, testData.out, lang1.TrString("a"), testData.hint)
|
||||
|
|
|
@ -8,8 +8,10 @@ import (
|
|||
"html/template"
|
||||
"slices"
|
||||
|
||||
"code.gitea.io/gitea/modules/json"
|
||||
"code.gitea.io/gitea/modules/log"
|
||||
"code.gitea.io/gitea/modules/setting"
|
||||
"code.gitea.io/gitea/modules/util"
|
||||
)
|
||||
|
||||
// This file implements the static LocaleStore that will not watch for changes
|
||||
|
@ -18,6 +20,9 @@ type locale struct {
|
|||
store *localeStore
|
||||
langName string
|
||||
idxToMsgMap map[int]string // the map idx is generated by store's trKeyToIdxMap
|
||||
|
||||
newStyleMessages map[string]string
|
||||
pluralRule PluralFormRule
|
||||
}
|
||||
|
||||
var _ Locale = (*locale)(nil)
|
||||
|
@ -38,8 +43,19 @@ func NewLocaleStore() LocaleStore {
|
|||
return &localeStore{localeMap: make(map[string]*locale), trKeyToIdxMap: make(map[string]int)}
|
||||
}
|
||||
|
||||
const (
|
||||
PluralFormSeparator string = "\036"
|
||||
)
|
||||
|
||||
// A note about pluralization rules.
|
||||
// go-i18n supports plural rules in theory.
|
||||
// In practice, it relies on another library that hardcodes a list of common languages
|
||||
// and their plural rules, and does not support languages not hardcoded there.
|
||||
// So we pretend that all languages are English and use our own function to extract
|
||||
// the correct plural form for a given count and language.
|
||||
|
||||
// AddLocaleByIni adds locale by ini into the store
|
||||
func (store *localeStore) AddLocaleByIni(langName, langDesc string, source, moreSource []byte) error {
|
||||
func (store *localeStore) AddLocaleByIni(langName, langDesc string, pluralRule PluralFormRule, source, moreSource []byte) error {
|
||||
if _, ok := store.localeMap[langName]; ok {
|
||||
return ErrLocaleAlreadyExist
|
||||
}
|
||||
|
@ -47,7 +63,7 @@ func (store *localeStore) AddLocaleByIni(langName, langDesc string, source, more
|
|||
store.langNames = append(store.langNames, langName)
|
||||
store.langDescs = append(store.langDescs, langDesc)
|
||||
|
||||
l := &locale{store: store, langName: langName, idxToMsgMap: make(map[int]string)}
|
||||
l := &locale{store: store, langName: langName, idxToMsgMap: make(map[int]string), pluralRule: pluralRule, newStyleMessages: make(map[string]string)}
|
||||
store.localeMap[l.langName] = l
|
||||
|
||||
iniFile, err := setting.NewConfigProviderForLocale(source, moreSource)
|
||||
|
@ -78,6 +94,98 @@ func (store *localeStore) AddLocaleByIni(langName, langDesc string, source, more
|
|||
return nil
|
||||
}
|
||||
|
||||
func RecursivelyAddTranslationsFromJSON(locale *locale, object map[string]any, prefix string) error {
|
||||
for key, value := range object {
|
||||
var fullkey string
|
||||
if prefix != "" {
|
||||
fullkey = prefix + "." + key
|
||||
} else {
|
||||
fullkey = key
|
||||
}
|
||||
|
||||
switch v := value.(type) {
|
||||
case string:
|
||||
// Check whether we are adding a plural form to the parent object, or a new nested JSON object.
|
||||
|
||||
if key == "zero" || key == "one" || key == "two" || key == "few" || key == "many" {
|
||||
locale.newStyleMessages[prefix+PluralFormSeparator+key] = v
|
||||
} else if key == "other" {
|
||||
locale.newStyleMessages[prefix] = v
|
||||
} else {
|
||||
locale.newStyleMessages[fullkey] = v
|
||||
}
|
||||
|
||||
case map[string]any:
|
||||
err := RecursivelyAddTranslationsFromJSON(locale, v, fullkey)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
case nil:
|
||||
default:
|
||||
return fmt.Errorf("Unrecognized JSON value '%s'", value)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (store *localeStore) AddToLocaleFromJSON(langName string, source []byte) error {
|
||||
locale, ok := store.localeMap[langName]
|
||||
if !ok {
|
||||
return ErrLocaleDoesNotExist
|
||||
}
|
||||
|
||||
var result map[string]any
|
||||
if err := json.Unmarshal(source, &result); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return RecursivelyAddTranslationsFromJSON(locale, result, "")
|
||||
}
|
||||
|
||||
func (l *locale) LookupNewStyleMessage(trKey string) string {
|
||||
if msg, ok := l.newStyleMessages[trKey]; ok {
|
||||
return msg
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (l *locale) LookupPlural(trKey string, count any) string {
|
||||
n, err := util.ToInt64(count)
|
||||
if err != nil {
|
||||
log.Error("Invalid plural count '%s'", count)
|
||||
return ""
|
||||
}
|
||||
|
||||
pluralForm := l.pluralRule(n)
|
||||
suffix := ""
|
||||
switch pluralForm {
|
||||
case PluralFormZero:
|
||||
suffix = PluralFormSeparator + "zero"
|
||||
case PluralFormOne:
|
||||
suffix = PluralFormSeparator + "one"
|
||||
case PluralFormTwo:
|
||||
suffix = PluralFormSeparator + "two"
|
||||
case PluralFormFew:
|
||||
suffix = PluralFormSeparator + "few"
|
||||
case PluralFormMany:
|
||||
suffix = PluralFormSeparator + "many"
|
||||
case PluralFormOther:
|
||||
// No suffix for the "other" string.
|
||||
default:
|
||||
log.Error("Invalid plural form index %d for count %d", pluralForm, count)
|
||||
return ""
|
||||
}
|
||||
|
||||
if result, ok := l.newStyleMessages[trKey+suffix]; ok {
|
||||
return result
|
||||
}
|
||||
|
||||
log.Error("Missing translation for plural form index %d for count %d", pluralForm, count)
|
||||
return ""
|
||||
}
|
||||
|
||||
func (store *localeStore) HasLang(langName string) bool {
|
||||
_, ok := store.localeMap[langName]
|
||||
return ok
|
||||
|
@ -113,22 +221,37 @@ func (store *localeStore) Close() error {
|
|||
func (l *locale) TrString(trKey string, trArgs ...any) string {
|
||||
format := trKey
|
||||
|
||||
idx, ok := l.store.trKeyToIdxMap[trKey]
|
||||
found := false
|
||||
if ok {
|
||||
if msg, ok := l.idxToMsgMap[idx]; ok {
|
||||
format = msg // use the found translation
|
||||
found = true
|
||||
} else if def, ok := l.store.localeMap[l.store.defaultLang]; ok {
|
||||
// try to use default locale's translation
|
||||
if msg, ok := def.idxToMsgMap[idx]; ok {
|
||||
format = msg
|
||||
if msg := l.LookupNewStyleMessage(trKey); msg != "" {
|
||||
format = msg
|
||||
} else {
|
||||
// First fallback: old-style translation
|
||||
idx, ok := l.store.trKeyToIdxMap[trKey]
|
||||
found := false
|
||||
if ok {
|
||||
if msg, ok := l.idxToMsgMap[idx]; ok {
|
||||
format = msg // use the found translation
|
||||
found = true
|
||||
}
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
log.Error("Missing translation %q", trKey)
|
||||
|
||||
if !found {
|
||||
// Second fallback: new-style default language
|
||||
if defaultLang, ok := l.store.localeMap[l.store.defaultLang]; ok {
|
||||
if msg := defaultLang.LookupNewStyleMessage(trKey); msg != "" {
|
||||
format = msg
|
||||
} else {
|
||||
// Third fallback: old-style default language
|
||||
if msg, ok := defaultLang.idxToMsgMap[idx]; ok {
|
||||
format = msg
|
||||
found = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !found {
|
||||
log.Error("Missing translation %q", trKey)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
msg, err := Format(format, trArgs...)
|
||||
|
@ -138,7 +261,7 @@ func (l *locale) TrString(trKey string, trArgs ...any) string {
|
|||
return msg
|
||||
}
|
||||
|
||||
func (l *locale) TrHTML(trKey string, trArgs ...any) template.HTML {
|
||||
func PrepareArgsForHTML(trArgs ...any) []any {
|
||||
args := slices.Clone(trArgs)
|
||||
for i, v := range args {
|
||||
switch v := v.(type) {
|
||||
|
@ -152,7 +275,30 @@ func (l *locale) TrHTML(trKey string, trArgs ...any) template.HTML {
|
|||
args[i] = template.HTMLEscapeString(fmt.Sprint(v))
|
||||
}
|
||||
}
|
||||
return template.HTML(l.TrString(trKey, args...))
|
||||
return args
|
||||
}
|
||||
|
||||
func (l *locale) TrHTML(trKey string, trArgs ...any) template.HTML {
|
||||
return template.HTML(l.TrString(trKey, PrepareArgsForHTML(trArgs...)...))
|
||||
}
|
||||
|
||||
func (l *locale) TrPluralString(count any, trKey string, trArgs ...any) template.HTML {
|
||||
message := l.LookupPlural(trKey, count)
|
||||
|
||||
if message == "" {
|
||||
if defaultLang, ok := l.store.localeMap[l.store.defaultLang]; ok {
|
||||
message = defaultLang.LookupPlural(trKey, count)
|
||||
}
|
||||
if message == "" {
|
||||
message = trKey
|
||||
}
|
||||
}
|
||||
|
||||
message, err := Format(message, PrepareArgsForHTML(trArgs...)...)
|
||||
if err != nil {
|
||||
log.Error("Error whilst formatting %q in %s: %v", trKey, l.langName, err)
|
||||
}
|
||||
return template.HTML(message)
|
||||
}
|
||||
|
||||
// HasKey returns whether a key is present in this locale or not
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue