Docker/AdventureLog
Docker/AdventureLog
1
0
Fork 0
mirror of https://github.com/seanmorley15/AdventureLog.git synced 2025-07-28 09:19:37 +02:00
Code Issues Releases Activity

fix: switch from json to ijson for efficient country data processing

Browse source
This commit is contained in:
Sean Morley 2025-01-13 11:23:57 -05:00
parent 6252f3cf14
commit 7f2c3c33a5
1 changed files with 6 additions and 6 deletions
Download patch file Download diff file Expand all files Collapse all files

10
backend/server/worldtravel/management/commands/download-countries.py
View file

@ -3,7 +3,7 @@ from django.core.management.base import BaseCommand
import requests import requests
from worldtravel.models import Country, Region, City from worldtravel.models import Country, Region, City
from django.db import transaction from django.db import transaction
import json import ijson
from django.conf import settings from django.conf import settings
@ -64,10 +64,10 @@ class Command(BaseCommand):
self.stdout.write(self.style.SUCCESS('Latest country, region, and state data already downloaded.')) self.stdout.write(self.style.SUCCESS('Latest country, region, and state data already downloaded.'))
return return
with open(countries_json_path, 'r') as f:
data = json.load(f)
with transaction.atomic(): with transaction.atomic():
# Process data in chunks using ijson
f = open(countries_json_path, 'rb')
parser = ijson.items(f, 'item')
existing_countries = {country.country_code: country for country in Country.objects.all()} existing_countries = {country.country_code: country for country in Country.objects.all()}
existing_regions = {region.id: region for region in Region.objects.all()} existing_regions = {region.id: region for region in Region.objects.all()}
existing_cities = {city.id: city for city in City.objects.all()} existing_cities = {city.id: city for city in City.objects.all()}
@ -83,7 +83,7 @@ class Command(BaseCommand):
processed_region_ids = set() processed_region_ids = set()
processed_city_ids = set() processed_city_ids = set()
for country in data: for country in parser:
country_code = country['iso2'] country_code = country['iso2']
country_name = country['name'] country_name = country['name']
country_subregion = country['subregion'] country_subregion = country['subregion']