From a19b9c3ed63fd9b157affb2a4b3079a1f2076ee4 Mon Sep 17 00:00:00 2001 From: Eric Holscher Date: Mon, 6 Oct 2025 16:15:37 +0300 Subject: [PATCH 1/5] Add CSV importer This is mostly for the OSS pledge to upload ads in bulk --- adserver/importers/csv_importer.py | 118 +++++++++++++++++++++++++++++ 1 file changed, 118 insertions(+) create mode 100644 adserver/importers/csv_importer.py diff --git a/adserver/importers/csv_importer.py b/adserver/importers/csv_importer.py new file mode 100644 index 00000000..3b0931ac --- /dev/null +++ b/adserver/importers/csv_importer.py @@ -0,0 +1,118 @@ +import csv +import logging +import os + +from django.core.files import File +from django.utils.text import slugify + +from adserver.models import AdType +from adserver.models import Advertisement +from adserver.models import Flight + + +log = logging.getLogger(__name__) + + +def run_csv_import(csv_path, image_dir, advertiser_slug, flight_slug, sync=False): + """ + Import advertisements from a CSV file and a directory of images. + + :param csv_path: Path to the CSV file containing ad data. + :param image_dir: Path to the directory containing ad images. + :param advertiser_slug: Slug of the advertiser. + :param flight_slug: Slug of the flight. + :param sync: Whether to write data to the database. + + Example usage: + from adserver.importers.csv_importer import run_csv_import + + run_csv_import( + csv_path="ads.csv", + image_dir="images", + advertiser_slug="ethicalads-community", + flight_slug="ethicalads-community-open-source-pledge", + sync=True + ) + """ + # Resolve relative paths to absolute paths + csv_path = os.path.abspath(csv_path) + image_dir = os.path.abspath(image_dir) + + if not os.path.exists(csv_path): + log.error(f"CSV file not found: {csv_path}") + return + + if not os.path.exists(image_dir): + log.error(f"Image directory not found: {image_dir}") + return + + if not sync: + log.warning("DRY RUN: Specify --sync to actually write data") + + # State + valid_ads = set() + # Ad types + default_ad_type = AdType.objects.get(slug="default") + + # Fetch the advertiser and flight for the given slugs + try: + flight = Flight.objects.get( + slug=flight_slug, campaign__advertiser__slug=advertiser_slug + ) + except Flight.DoesNotExist: + log.error(f"Flight not found: {advertiser_slug}/{flight_slug}") + return + + with open(csv_path, newline="", encoding="utf-8") as csvfile: + reader = csv.DictReader(csvfile) + for row in reader: + try: + name = row["name"] + image_filename = row["image"] + text = row["text"] + + image_path = os.path.join(image_dir, image_filename) + if not os.path.exists(image_path): + log.warning(f"Image not found for ad: {name} ({image_filename})") + continue + + with open(image_path, "rb") as image_file: + image = File(image_file, name=image_filename) + + slug = slugify(name) + if sync: + ad, created = Advertisement.objects.get_or_create( + slug=slug, flight=flight + ) + if created: + log.info(f"NEW AD: Created new ad {name}") + + # Update fields + ad.name = name + ad.image.save(image_filename, image, save=False) + ad.text = text + ad.live = True + ad.ad_types.add(default_ad_type) + try: + ad.save() + except Exception: + log.exception(f"Failed to save ad: {name}") + else: + log.info(f"DRY RUN: Would process ad {name}") + + valid_ads.add(slug) + + except KeyError as e: + log.error(f"Missing required column in CSV: {e}") + except Exception: + log.exception(f"Error processing row: {row}") + + # Disable ads no longer in the CSV + for ad in Advertisement.objects.filter(live=True): + if ad.slug not in valid_ads: + if sync: + log.info(f"Deactivating ad: {ad.name}") + ad.live = False + ad.save() + else: + log.info(f"DRY RUN: Would deactivate ad: {ad.name}") From 7c507403ad27cb3363d764e0d2a00def29617d96 Mon Sep 17 00:00:00 2001 From: Eric Holscher Date: Mon, 6 Oct 2025 16:45:48 +0300 Subject: [PATCH 2/5] Remove disabling --- adserver/importers/csv_importer.py | 48 ++++++++++++------------------ 1 file changed, 19 insertions(+), 29 deletions(-) diff --git a/adserver/importers/csv_importer.py b/adserver/importers/csv_importer.py index 3b0931ac..949c1ed1 100644 --- a/adserver/importers/csv_importer.py +++ b/adserver/importers/csv_importer.py @@ -15,24 +15,24 @@ def run_csv_import(csv_path, image_dir, advertiser_slug, flight_slug, sync=False): """ - Import advertisements from a CSV file and a directory of images. - - :param csv_path: Path to the CSV file containing ad data. - :param image_dir: Path to the directory containing ad images. - :param advertiser_slug: Slug of the advertiser. - :param flight_slug: Slug of the flight. - :param sync: Whether to write data to the database. - - Example usage: - from adserver.importers.csv_importer import run_csv_import - - run_csv_import( - csv_path="ads.csv", - image_dir="images", - advertiser_slug="ethicalads-community", - flight_slug="ethicalads-community-open-source-pledge", - sync=True - ) + Import advertisements from a CSV file and a directory of images. + + :param csv_path: Path to the CSV file containing ad data. + :param image_dir: Path to the directory containing ad images. + :param advertiser_slug: Slug of the advertiser. + :param flight_slug: Slug of the flight. + :param sync: Whether to write data to the database. + + Example usage: + from adserver.importers.csv_importer import run_csv_import + + run_csv_import( + csv_path="ad-data.txt", + image_dir="photos", + advertiser_slug="ethicalads-community", + flight_slug="ethicalads-community-open-source-pledge", + sync=False + ) """ # Resolve relative paths to absolute paths csv_path = os.path.abspath(csv_path) @@ -52,7 +52,7 @@ def run_csv_import(csv_path, image_dir, advertiser_slug, flight_slug, sync=False # State valid_ads = set() # Ad types - default_ad_type = AdType.objects.get(slug="default") + default_ad_type = AdType.objects.get(slug="image-v1") # Fetch the advertiser and flight for the given slugs try: @@ -106,13 +106,3 @@ def run_csv_import(csv_path, image_dir, advertiser_slug, flight_slug, sync=False log.error(f"Missing required column in CSV: {e}") except Exception: log.exception(f"Error processing row: {row}") - - # Disable ads no longer in the CSV - for ad in Advertisement.objects.filter(live=True): - if ad.slug not in valid_ads: - if sync: - log.info(f"Deactivating ad: {ad.name}") - ad.live = False - ad.save() - else: - log.info(f"DRY RUN: Would deactivate ad: {ad.name}") From 9ac88421c46bbf494829880fb6ab4d980c71afc9 Mon Sep 17 00:00:00 2001 From: Eric Holscher Date: Mon, 6 Oct 2025 17:06:20 +0300 Subject: [PATCH 3/5] Add link --- adserver/importers/csv_importer.py | 67 ++++++++++++++++-------------- 1 file changed, 35 insertions(+), 32 deletions(-) diff --git a/adserver/importers/csv_importer.py b/adserver/importers/csv_importer.py index 949c1ed1..0f333eca 100644 --- a/adserver/importers/csv_importer.py +++ b/adserver/importers/csv_importer.py @@ -1,6 +1,7 @@ import csv import logging import os +from io import BytesIO from django.core.files import File from django.utils.text import slugify @@ -13,25 +14,26 @@ log = logging.getLogger(__name__) -def run_csv_import(csv_path, image_dir, advertiser_slug, flight_slug, sync=False): +def run_csv_import(csv_path, image_dir, link, advertiser_slug, flight_slug, sync=False): """ - Import advertisements from a CSV file and a directory of images. + Import advertisements from a CSV file and a directory of images. - :param csv_path: Path to the CSV file containing ad data. - :param image_dir: Path to the directory containing ad images. - :param advertiser_slug: Slug of the advertiser. - :param flight_slug: Slug of the flight. - :param sync: Whether to write data to the database. + :param csv_path: Path to the CSV file containing ad data. + :param image_dir: Path to the directory containing ad images. + :param advertiser_slug: Slug of the advertiser. + :param flight_slug: Slug of the flight. + :param sync: Whether to write data to the database. - Example usage: - from adserver.importers.csv_importer import run_csv_import + Example usage: + from adserver.importers.csv_importer import run_csv_import run_csv_import( csv_path="ad-data.txt", image_dir="photos", + link="https://opensourcepledge.com/?ref=ethicalads-community", advertiser_slug="ethicalads-community", flight_slug="ethicalads-community-open-source-pledge", - sync=False + sync=True ) """ # Resolve relative paths to absolute paths @@ -76,29 +78,30 @@ def run_csv_import(csv_path, image_dir, advertiser_slug, flight_slug, sync=False log.warning(f"Image not found for ad: {name} ({image_filename})") continue + # Open the image file using BytesIO for consistency with open(image_path, "rb") as image_file: - image = File(image_file, name=image_filename) - - slug = slugify(name) - if sync: - ad, created = Advertisement.objects.get_or_create( - slug=slug, flight=flight - ) - if created: - log.info(f"NEW AD: Created new ad {name}") - - # Update fields - ad.name = name - ad.image.save(image_filename, image, save=False) - ad.text = text - ad.live = True - ad.ad_types.add(default_ad_type) - try: - ad.save() - except Exception: - log.exception(f"Failed to save ad: {name}") - else: - log.info(f"DRY RUN: Would process ad {name}") + image = File(BytesIO(image_file.read()), name=image_filename) + slug = slugify(name) + if sync: + ad, created = Advertisement.objects.get_or_create( + slug=slug, flight=flight + ) + if created: + log.info(f"NEW AD: Created new ad {name}") + + # Update fields + ad.name = name + ad.image.save(image_filename, image, save=False) + ad.link = link + ad.text = text + ad.live = True + ad.ad_types.add(default_ad_type) + try: + ad.save() + except Exception: + log.exception(f"Failed to save ad: {name}") + else: + log.info(f"DRY RUN: Would process ad {name}") valid_ads.add(slug) From b189f21e4f543d4254e473b32a774ea9aa321032 Mon Sep 17 00:00:00 2001 From: Eric Holscher Date: Tue, 14 Oct 2025 09:22:58 +0300 Subject: [PATCH 4/5] Update CSV to use headline/cta --- adserver/importers/csv_importer.py | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/adserver/importers/csv_importer.py b/adserver/importers/csv_importer.py index 0f333eca..6cf8becd 100644 --- a/adserver/importers/csv_importer.py +++ b/adserver/importers/csv_importer.py @@ -16,16 +16,16 @@ def run_csv_import(csv_path, image_dir, link, advertiser_slug, flight_slug, sync=False): """ - Import advertisements from a CSV file and a directory of images. + Import advertisements from a CSV file and a directory of images. - :param csv_path: Path to the CSV file containing ad data. - :param image_dir: Path to the directory containing ad images. - :param advertiser_slug: Slug of the advertiser. - :param flight_slug: Slug of the flight. - :param sync: Whether to write data to the database. + :param csv_path: Path to the CSV file containing ad data. + :param image_dir: Path to the directory containing ad images. + :param advertiser_slug: Slug of the advertiser. + :param flight_slug: Slug of the flight. + :param sync: Whether to write data to the database. - Example usage: - from adserver.importers.csv_importer import run_csv_import + Example usage: + from adserver.importers.csv_importer import run_csv_import run_csv_import( csv_path="ad-data.txt", @@ -71,7 +71,9 @@ def run_csv_import(csv_path, image_dir, link, advertiser_slug, flight_slug, sync try: name = row["name"] image_filename = row["image"] + headline = row.get("headline", "") # Extract headline, default to empty text = row["text"] + cta = row.get("cta", "") # Extract call-to-action, default to empty image_path = os.path.join(image_dir, image_filename) if not os.path.exists(image_path): @@ -84,7 +86,8 @@ def run_csv_import(csv_path, image_dir, link, advertiser_slug, flight_slug, sync slug = slugify(name) if sync: ad, created = Advertisement.objects.get_or_create( - slug=slug, flight=flight + slug=slug, + flight=flight, # Use ad ID for uniqueness ) if created: log.info(f"NEW AD: Created new ad {name}") @@ -92,8 +95,12 @@ def run_csv_import(csv_path, image_dir, link, advertiser_slug, flight_slug, sync # Update fields ad.name = name ad.image.save(image_filename, image, save=False) - ad.link = link - ad.text = text + if headline or cta: + ad.headline = headline + ad.content = text + ad.cta = cta + else: + ad.text = text ad.live = True ad.ad_types.add(default_ad_type) try: From 1d42af3502ffebe3bf191750f0a2c0ffb605e457 Mon Sep 17 00:00:00 2001 From: Eric Holscher Date: Tue, 14 Oct 2025 09:26:45 +0300 Subject: [PATCH 5/5] Clean both --- adserver/importers/csv_importer.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/adserver/importers/csv_importer.py b/adserver/importers/csv_importer.py index 6cf8becd..d33b4d37 100644 --- a/adserver/importers/csv_importer.py +++ b/adserver/importers/csv_importer.py @@ -16,15 +16,15 @@ def run_csv_import(csv_path, image_dir, link, advertiser_slug, flight_slug, sync=False): """ - Import advertisements from a CSV file and a directory of images. + Import advertisements from a CSV file and a directory of images. - :param csv_path: Path to the CSV file containing ad data. - :param image_dir: Path to the directory containing ad images. - :param advertiser_slug: Slug of the advertiser. - :param flight_slug: Slug of the flight. - :param sync: Whether to write data to the database. + :param csv_path: Path to the CSV file containing ad data. + :param image_dir: Path to the directory containing ad images. + :param advertiser_slug: Slug of the advertiser. + :param flight_slug: Slug of the flight. + :param sync: Whether to write data to the database. - Example usage: + Example usage: from adserver.importers.csv_importer import run_csv_import run_csv_import( @@ -86,8 +86,7 @@ def run_csv_import(csv_path, image_dir, link, advertiser_slug, flight_slug, sync slug = slugify(name) if sync: ad, created = Advertisement.objects.get_or_create( - slug=slug, - flight=flight, # Use ad ID for uniqueness + slug=slug, flight=flight ) if created: log.info(f"NEW AD: Created new ad {name}") @@ -99,8 +98,12 @@ def run_csv_import(csv_path, image_dir, link, advertiser_slug, flight_slug, sync ad.headline = headline ad.content = text ad.cta = cta + ad.text = "" else: ad.text = text + ad.headline = "" + ad.cta = "" + ad.content = "" ad.live = True ad.ad_types.add(default_ad_type) try: