From a81b816032264b54ac084a6009785ebfc27039b9 Mon Sep 17 00:00:00 2001
From: Melissa DeLucchi <113376043+delucchi-cmu@users.noreply.github.com>
Date: Thu, 14 Nov 2024 11:50:54 -0500
Subject: [PATCH] Test with non-utf8 csv encoding. (#48)

---
 tests/hats/io/file_io/test_file_io_cloud.py | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/tests/hats/io/file_io/test_file_io_cloud.py b/tests/hats/io/file_io/test_file_io_cloud.py
index 254f833..8fa2d94 100644
--- a/tests/hats/io/file_io/test_file_io_cloud.py
+++ b/tests/hats/io/file_io/test_file_io_cloud.py
@@ -3,6 +3,7 @@
 from hats.io import paths
 from hats.io.file_io import (
     load_csv_to_pandas,
+    load_csv_to_pandas_generator,
     load_text_file,
     read_fits_image,
     read_parquet_file_to_pandas,
@@ -39,6 +40,17 @@ def test_write_df_to_csv(tmp_cloud_path):
     pd.testing.assert_frame_equal(loaded_df, random_df)
 
 
+def test_load_csv_to_pandas_generator_encoding(tmp_cloud_path):
+    path = tmp_cloud_path / "koi8-r.csv"
+    with path.open(encoding="koi8-r", mode="w") as fh:
+        fh.write("col1,col2\nыыы,яяя\n")
+    num_reads = 0
+    for frame in load_csv_to_pandas_generator(path, chunksize=7, encoding="koi8-r"):
+        assert len(frame) == 1
+        num_reads += 1
+    assert num_reads == 1
+
+
 def test_write_point_map_roundtrip(small_sky_dir_cloud, tmp_cloud_path):
     """Test the reading/writing of a catalog point map"""
     expected_counts_skymap = read_fits_image(paths.get_point_map_file_pointer(small_sky_dir_cloud))