From 88ca5e10a4cb2dbfdbf667c5c29620db13a796cd Mon Sep 17 00:00:00 2001 From: JYYYeung Date: Fri, 28 Feb 2025 08:23:23 +0000 Subject: [PATCH] feat: gets translations for country, city, region, etc. names for i18n --- Makefile | 39 ++++++++++++++++--- backend/Dockerfile | 2 +- backend/entrypoint.sh | 11 ++++++ backend/server/.env.example | 2 + backend/server/main/settings.py | 12 +++++- backend/server/requirements.txt | 3 +- .../management/commands/get-translations.py | 17 ++++++++ backend/server/worldtravel/models.py | 19 ++++++++- documentation/docs/configuration/updating.md | 10 +++++ 9 files changed, 106 insertions(+), 9 deletions(-) create mode 100644 backend/server/worldtravel/management/commands/get-translations.py diff --git a/Makefile b/Makefile index 48a5be7..6c653fe 100644 --- a/Makefile +++ b/Makefile @@ -11,11 +11,40 @@ all: dev ## Build all services (alias for dev) download-countries: ## Download countries @cd backend/server && mkdir -p media @cd backend/server && python manage.py download-countries --force + @echo "Countries downloaded" + +download-cities: ## Download cities + @cd backend/server && mkdir -p media/cities_data + @cd backend/server && python manage.py cities --import=country + # @cd backend/server && python manage.py cities --import=region + # @cd backend/server && python manage.py cities --import=city + @cd backend/server && python manage.py cities --import=alt_name + @echo "Cities downloaded" + +import-translations: ## Import translations + @cd backend/server && python manage.py get-translations + @echo "Translations Imported" dev-db: dev ## Start development database - # If adventurelog-development container not exists, create it - @docker ps -q -f name=adventurelog-development || docker run --name adventurelog-development -e POSTGRES_USER=admin -e POSTGRES_PASSWORD=admin -e POSTGRES_DB=adventurelog -p 5432:5432 -d postgis/postgis:15-3.3 - @echo "Please update the backend/.env file with the correct database credentials (uncomment the lines starting with # PGHOST, PGDATABASE, PGUSER, PGPASSWORD)" + @if [ ! "$$(docker ps -q -f name=adventurelog-development)" ]; then \ + if [ "$$(docker ps -aq -f status=exited -f name=adventurelog-development)" ]; then \ + docker rm adventurelog-development; \ + fi; \ + docker run --name adventurelog-development \ + -e POSTGRES_USER=admin \ + -e POSTGRES_PASSWORD=admin \ + -e POSTGRES_DB=adventurelog \ + -p 5432:5432 \ + -d postgis/postgis:15-3.3; \ + echo "Development database started. Please wait a few seconds for it to initialize."; \ + echo "Please update the backend/.env file with these credentials:"; \ + echo "PGHOST=localhost"; \ + echo "PGDATABASE=adventurelog"; \ + echo "PGUSER=admin"; \ + echo "PGPASSWORD=admin"; \ + else \ + echo "Development database is already running."; \ + fi web: dev ## Start web service @cd frontend && pnpm dev @@ -24,12 +53,12 @@ django: dev ## Start Django server @cd backend/server && python manage.py migrate @cd frontend && pnpm django -dev: download-countries ## Setup Development Environment +dev: ## Setup Development Environment @[ -f backend/server/.env ] || cp backend/server/.env.example backend/server/.env @[ -f frontend/.env ] || cp frontend/.env.example frontend/.env @cd frontend && pnpm install @[ -d .venv ] || python -m venv .venv - @source .venv/bin/activate + . .venv/bin/activate @pip install -r backend/server/requirements.txt diff --git a/backend/Dockerfile b/backend/Dockerfile index aa0f9f4..fdcfdb5 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -22,7 +22,7 @@ RUN pip install --upgrade pip \ && pip install -r requirements.txt # Create necessary directories -RUN mkdir -p /code/static /code/media +RUN mkdir -p /code/static /code/media /code/media/cities_data # RUN mkdir -p /code/staticfiles /code/media # Copy the Django project code into the Docker image diff --git a/backend/entrypoint.sh b/backend/entrypoint.sh index 84de7b6..9f42038 100644 --- a/backend/entrypoint.sh +++ b/backend/entrypoint.sh @@ -58,6 +58,17 @@ if [ $? -eq 137 ]; then exit 1 fi +# Sync the translations if $COUNTRY_TRANSLATIONS is true +if [ -n "$COUNTRY_TRANSLATIONS" -a "$COUNTRY_TRANSLATIONS" = "true" ]; then + echo "Syncing translations for countries..." + # Get the translations for all countries + python manage.py cities --import=country + # Get the translations for all alt names + python manage.py cities --import=alt_name + # Get the translations for all countries + python manage.py get-translations +fi + cat /code/adventurelog.txt # Start gunicorn diff --git a/backend/server/.env.example b/backend/server/.env.example index 598aeb7..81894e0 100644 --- a/backend/server/.env.example +++ b/backend/server/.env.example @@ -13,6 +13,8 @@ FRONTEND_URL='http://localhost:3000' EMAIL_BACKEND='console' +COUNTRY_TRANSLATIONS=True + # EMAIL_BACKEND='email' # EMAIL_HOST='smtp.gmail.com' # EMAIL_USE_TLS=False diff --git a/backend/server/main/settings.py b/backend/server/main/settings.py index dd099a1..4c70431 100644 --- a/backend/server/main/settings.py +++ b/backend/server/main/settings.py @@ -61,6 +61,7 @@ INSTALLED_APPS = ( 'users', 'integrations', 'django.contrib.gis', + 'cities', # 'achievements', # Not done yet, will be added later in a future update # 'widget_tweaks', # 'slippers', @@ -167,6 +168,7 @@ STATIC_URL = '/static/' MEDIA_URL = '/media/' MEDIA_ROOT = BASE_DIR / 'media' # This path must match the NGINX root STATICFILES_DIRS = [BASE_DIR / 'static'] +CITIES_DATA_DIR = MEDIA_ROOT / 'cities_data' STORAGES = { "staticfiles": { @@ -304,4 +306,12 @@ LOGGING = { # ADVENTURELOG_CDN_URL = getenv('ADVENTURELOG_CDN_URL', 'https://cdn.adventurelog.app') # https://github.com/dr5hn/countries-states-cities-database/tags -COUNTRY_REGION_JSON_VERSION = 'v2.5' \ No newline at end of file +COUNTRY_REGION_JSON_VERSION = 'v2.5' + +# English, Spanish, French, German, Italian, Chinese, Dutch, Swedish +CITIES_LOCALES = ['en', 'es', 'fr', 'de', 'it', 'zh', 'nl', 'sv', 'LANGUAGES'] +CITIES_POSTAL_CODES = [] +CITIES_PLUGINS = [ + # Reduce memory usage when importing large datasets (e.g. "allCountries.zip") + 'cities.plugin.reset_queries.Plugin', +] \ No newline at end of file diff --git a/backend/server/requirements.txt b/backend/server/requirements.txt index dcd0125..fe906f4 100644 --- a/backend/server/requirements.txt +++ b/backend/server/requirements.txt @@ -21,4 +21,5 @@ icalendar==6.1.0 ijson==3.3.0 tqdm==4.67.1 overpy==0.7 -publicsuffix2==2.20191221 \ No newline at end of file +publicsuffix2==2.20191221 +django-cities==0.6.2 \ No newline at end of file diff --git a/backend/server/worldtravel/management/commands/get-translations.py b/backend/server/worldtravel/management/commands/get-translations.py new file mode 100644 index 0000000..1f8d2b5 --- /dev/null +++ b/backend/server/worldtravel/management/commands/get-translations.py @@ -0,0 +1,17 @@ +from main.settings import CITIES_LOCALES +from worldtravel.models import Country +from django.core.management.base import BaseCommand + +class Command(BaseCommand): + help = 'Get translations for all countries' + + def handle(self, *args, **options): + countries = Country.objects.all() + countries_to_update = [] + for country in countries: + updated = country.get_translations(CITIES_LOCALES) + if updated: + countries_to_update.append(country) + # Bulk update the translations + Country.objects.bulk_update(countries_to_update, ['translations']) + print(f"Updated translations for {len(countries_to_update)} countries") \ No newline at end of file diff --git a/backend/server/worldtravel/models.py b/backend/server/worldtravel/models.py index a6ad58c..f2f116d 100644 --- a/backend/server/worldtravel/models.py +++ b/backend/server/worldtravel/models.py @@ -2,7 +2,7 @@ from django.db import models from django.contrib.auth import get_user_model from django.core.exceptions import ValidationError from django.contrib.gis.db import models as gis_models - +from cities.models import Country as CityCountry, City as CityCity, Region as CityRegion User = get_user_model() @@ -26,6 +26,23 @@ class Country(models.Model): def __str__(self): return self.name + + def get_translations(self, languages: list[str])->bool: + # get the translations for the country + translations = self.translations + try: + # get the preferred alt names for the country + alt_names = CityCountry.objects.get(code=self.country_code).alt_names.filter(language_code__in=languages, is_preferred=True) + for alt_name in alt_names: + translations[alt_name.language_code] = alt_name.name + + if self.translations != translations: + self.translations = translations + return True + return False + except CityCountry.DoesNotExist: + print(f"Country {self.name} ({self.country_code}) not found in cities.models.Country") + return False class Region(models.Model): id = models.CharField(primary_key=True) diff --git a/documentation/docs/configuration/updating.md b/documentation/docs/configuration/updating.md index 85fec59..026d7d9 100644 --- a/documentation/docs/configuration/updating.md +++ b/documentation/docs/configuration/updating.md @@ -22,3 +22,13 @@ Once you are in the container run the following command to resync the region dat ```bash python manage.py download-countries --force ``` + +## Updating the Country Translations + +If you would like to get translations for country names, you can run the following command. This will get the translations for all countries and save them to the database. + +```bash +python manage.py cities --import=country +python manage.py cities --import=alt_name # This takes a while to run, around 20 - 30 minutes, but only needs to be done once +python manage.py get-translations +``` \ No newline at end of file