1use arrow_array::*;
164use arrow_cast::display::*;
165use arrow_schema::*;
166use csv::ByteRecord;
167use std::io::Write;
168
169use crate::map_csv_error;
170const DEFAULT_NULL_VALUE: &str = "";
171
172pub use csv::QuoteStyle;
187
188#[derive(Debug)]
192pub struct Writer<W: Write> {
193 writer: csv::Writer<W>,
195 has_headers: bool,
197 date_format: Option<String>,
199 datetime_format: Option<String>,
201 timestamp_format: Option<String>,
203 timestamp_tz_format: Option<String>,
205 time_format: Option<String>,
207 beginning: bool,
209 null_value: Option<String>,
211 ignore_leading_whitespace: bool,
213 ignore_trailing_whitespace: bool,
215}
216
217impl<W: Write> Writer<W> {
218 pub fn new(writer: W) -> Self {
223 let delimiter = b',';
224 WriterBuilder::new().with_delimiter(delimiter).build(writer)
225 }
226
227 pub fn write(&mut self, batch: &RecordBatch) -> Result<(), ArrowError> {
229 let num_columns = batch.num_columns();
230 if self.beginning {
231 if self.has_headers {
232 let mut headers: Vec<String> = Vec::with_capacity(num_columns);
233 batch
234 .schema()
235 .fields()
236 .iter()
237 .for_each(|field| headers.push(field.name().clone()));
238 self.writer
239 .write_record(&headers[..])
240 .map_err(map_csv_error)?;
241 }
242 self.beginning = false;
243 }
244
245 let options = FormatOptions::default()
246 .with_null(self.null_value.as_deref().unwrap_or(DEFAULT_NULL_VALUE))
247 .with_date_format(self.date_format.as_deref())
248 .with_datetime_format(self.datetime_format.as_deref())
249 .with_timestamp_format(self.timestamp_format.as_deref())
250 .with_timestamp_tz_format(self.timestamp_tz_format.as_deref())
251 .with_time_format(self.time_format.as_deref());
252
253 let converters = batch
254 .columns()
255 .iter()
256 .map(|a| {
257 if a.data_type().is_nested() {
258 Err(ArrowError::CsvError(format!(
259 "Nested type {} is not supported in CSV",
260 a.data_type()
261 )))
262 } else {
263 ArrayFormatter::try_new(a.as_ref(), &options)
264 }
265 })
266 .collect::<Result<Vec<_>, ArrowError>>()?;
267
268 let mut buffer = String::with_capacity(1024);
269 let mut byte_record = ByteRecord::with_capacity(1024, converters.len());
270
271 for row_idx in 0..batch.num_rows() {
272 byte_record.clear();
273 for (col_idx, converter) in converters.iter().enumerate() {
274 buffer.clear();
275 converter.value(row_idx).write(&mut buffer).map_err(|e| {
276 ArrowError::CsvError(format!(
277 "Error processing row {}, col {}: {e}",
278 row_idx + 1,
279 col_idx + 1
280 ))
281 })?;
282
283 let field_bytes =
284 self.get_trimmed_field_bytes(&buffer, batch.column(col_idx).data_type());
285 byte_record.push_field(field_bytes);
286 }
287
288 self.writer
289 .write_byte_record(&byte_record)
290 .map_err(map_csv_error)?;
291 }
292 self.writer.flush()?;
293
294 Ok(())
295 }
296
297 fn get_trimmed_field_bytes<'a>(&self, buffer: &'a str, data_type: &DataType) -> &'a [u8] {
299 let should_trim = (self.ignore_leading_whitespace || self.ignore_trailing_whitespace)
301 && matches!(
302 data_type,
303 DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View
304 );
305
306 if !should_trim {
307 return buffer.as_bytes();
308 }
309
310 let mut trimmed = buffer;
311 if self.ignore_leading_whitespace {
312 trimmed = trimmed.trim_start();
313 }
314 if self.ignore_trailing_whitespace {
315 trimmed = trimmed.trim_end();
316 }
317 trimmed.as_bytes()
318 }
319
320 pub fn into_inner(self) -> W {
322 self.writer.into_inner().unwrap()
324 }
325}
326
327impl<W: Write> RecordBatchWriter for Writer<W> {
328 fn write(&mut self, batch: &RecordBatch) -> Result<(), ArrowError> {
329 self.write(batch)
330 }
331
332 fn close(self) -> Result<(), ArrowError> {
333 Ok(())
334 }
335}
336
337#[derive(Clone, Debug)]
339pub struct WriterBuilder {
340 delimiter: u8,
342 has_header: bool,
344 quote: u8,
346 escape: u8,
348 terminator: Terminator,
350 double_quote: bool,
352 date_format: Option<String>,
354 datetime_format: Option<String>,
356 timestamp_format: Option<String>,
358 timestamp_tz_format: Option<String>,
360 time_format: Option<String>,
362 null_value: Option<String>,
364 ignore_leading_whitespace: bool,
366 ignore_trailing_whitespace: bool,
368 quote_style: QuoteStyle,
370}
371
372#[derive(Clone, Debug)]
374pub enum Terminator {
375 CRLF,
377 Any(u8),
379}
380
381impl Default for WriterBuilder {
382 fn default() -> Self {
383 WriterBuilder {
384 delimiter: b',',
385 has_header: true,
386 quote: b'"',
387 escape: b'\\',
388 terminator: Terminator::Any(b'\n'),
389 double_quote: true,
390 date_format: None,
391 datetime_format: None,
392 timestamp_format: None,
393 timestamp_tz_format: None,
394 time_format: None,
395 null_value: None,
396 ignore_leading_whitespace: false,
397 ignore_trailing_whitespace: false,
398 quote_style: QuoteStyle::default(),
399 }
400 }
401}
402
403impl WriterBuilder {
404 pub fn new() -> Self {
426 Self::default()
427 }
428
429 pub fn with_header(mut self, header: bool) -> Self {
431 self.has_header = header;
432 self
433 }
434
435 pub fn header(&self) -> bool {
437 self.has_header
438 }
439
440 pub fn with_delimiter(mut self, delimiter: u8) -> Self {
442 self.delimiter = delimiter;
443 self
444 }
445
446 pub fn delimiter(&self) -> u8 {
448 self.delimiter
449 }
450
451 pub fn with_quote(mut self, quote: u8) -> Self {
453 self.quote = quote;
454 self
455 }
456
457 pub fn quote(&self) -> u8 {
459 self.quote
460 }
461
462 pub fn with_escape(mut self, escape: u8) -> Self {
470 self.escape = escape;
471 self
472 }
473
474 pub fn escape(&self) -> u8 {
476 self.escape
477 }
478
479 pub fn with_double_quote(mut self, double_quote: bool) -> Self {
487 self.double_quote = double_quote;
488 self
489 }
490
491 pub fn double_quote(&self) -> bool {
493 self.double_quote
494 }
495
496 pub fn with_date_format(mut self, format: String) -> Self {
498 self.date_format = Some(format);
499 self
500 }
501
502 pub fn date_format(&self) -> Option<&str> {
504 self.date_format.as_deref()
505 }
506
507 pub fn with_datetime_format(mut self, format: String) -> Self {
509 self.datetime_format = Some(format);
510 self
511 }
512
513 pub fn datetime_format(&self) -> Option<&str> {
515 self.datetime_format.as_deref()
516 }
517
518 pub fn with_time_format(mut self, format: String) -> Self {
520 self.time_format = Some(format);
521 self
522 }
523
524 pub fn time_format(&self) -> Option<&str> {
526 self.time_format.as_deref()
527 }
528
529 pub fn with_timestamp_format(mut self, format: String) -> Self {
531 self.timestamp_format = Some(format);
532 self
533 }
534
535 pub fn timestamp_format(&self) -> Option<&str> {
537 self.timestamp_format.as_deref()
538 }
539
540 pub fn with_timestamp_tz_format(mut self, tz_format: String) -> Self {
542 self.timestamp_tz_format = Some(tz_format);
543 self
544 }
545
546 pub fn timestamp_tz_format(&self) -> Option<&str> {
548 self.timestamp_tz_format.as_deref()
549 }
550
551 pub fn with_null(mut self, null_value: String) -> Self {
553 self.null_value = Some(null_value);
554 self
555 }
556
557 pub fn null(&self) -> &str {
559 self.null_value.as_deref().unwrap_or(DEFAULT_NULL_VALUE)
560 }
561
562 pub fn with_ignore_leading_whitespace(mut self, ignore: bool) -> Self {
565 self.ignore_leading_whitespace = ignore;
566 self
567 }
568
569 pub fn ignore_leading_whitespace(&self) -> bool {
571 self.ignore_leading_whitespace
572 }
573
574 pub fn with_ignore_trailing_whitespace(mut self, ignore: bool) -> Self {
577 self.ignore_trailing_whitespace = ignore;
578 self
579 }
580
581 pub fn ignore_trailing_whitespace(&self) -> bool {
583 self.ignore_trailing_whitespace
584 }
585
586 pub fn with_quote_style(mut self, quote_style: QuoteStyle) -> Self {
602 self.quote_style = quote_style;
603 self
604 }
605
606 pub fn quote_style(&self) -> QuoteStyle {
608 self.quote_style
609 }
610
611 pub fn with_line_terminator(mut self, terminator: Terminator) -> Self {
613 self.terminator = terminator;
614 self
615 }
616
617 pub fn line_terminator(&self) -> &Terminator {
619 &self.terminator
620 }
621
622 pub fn build<W: Write>(self, writer: W) -> Writer<W> {
624 let mut builder = csv::WriterBuilder::new();
625
626 let terminator = match self.terminator {
627 Terminator::CRLF => csv::Terminator::CRLF,
628 Terminator::Any(byte) => csv::Terminator::Any(byte),
629 };
630
631 let writer = builder
632 .delimiter(self.delimiter)
633 .quote(self.quote)
634 .quote_style(self.quote_style)
635 .double_quote(self.double_quote)
636 .escape(self.escape)
637 .terminator(terminator)
638 .from_writer(writer);
639 Writer {
640 writer,
641 beginning: true,
642 has_headers: self.has_header,
643 date_format: self.date_format,
644 datetime_format: self.datetime_format,
645 time_format: self.time_format,
646 timestamp_format: self.timestamp_format,
647 timestamp_tz_format: self.timestamp_tz_format,
648 null_value: self.null_value,
649 ignore_leading_whitespace: self.ignore_leading_whitespace,
650 ignore_trailing_whitespace: self.ignore_trailing_whitespace,
651 }
652 }
653}
654
655#[cfg(test)]
656#[expect(
657 clippy::verbose_file_reads,
658 reason = "`tempfile::tempfile()` has no path, so the contents can only be read back through the handle"
659)]
660mod tests {
661 use super::*;
662
663 use crate::ReaderBuilder;
664 use arrow_array::builder::{
665 BinaryBuilder, Decimal32Builder, Decimal64Builder, Decimal128Builder, Decimal256Builder,
666 FixedSizeBinaryBuilder, LargeBinaryBuilder,
667 };
668 use arrow_array::types::*;
669 use arrow_buffer::i256;
670 use core::str;
671 use std::io::{Cursor, Read, Seek};
672 use std::sync::Arc;
673
674 #[test]
675 fn test_write_csv() {
676 let schema = Schema::new(vec![
677 Field::new("c1", DataType::Utf8, false),
678 Field::new("c2", DataType::Float64, true),
679 Field::new("c3", DataType::UInt32, false),
680 Field::new("c4", DataType::Boolean, true),
681 Field::new("c5", DataType::Timestamp(TimeUnit::Millisecond, None), true),
682 Field::new("c6", DataType::Time32(TimeUnit::Second), false),
683 Field::new_dictionary("c7", DataType::Int32, DataType::Utf8, false),
684 ]);
685
686 let c1 = StringArray::from(vec![
687 "Lorem ipsum dolor sit amet",
688 "consectetur adipiscing elit",
689 "sed do eiusmod tempor",
690 ]);
691 let c2 =
692 PrimitiveArray::<Float64Type>::from(vec![Some(123.564532), None, Some(-556132.25)]);
693 let c3 = PrimitiveArray::<UInt32Type>::from(vec![3, 2, 1]);
694 let c4 = BooleanArray::from(vec![Some(true), Some(false), None]);
695 let c5 =
696 TimestampMillisecondArray::from(vec![None, Some(1555584887378), Some(1555555555555)]);
697 let c6 = Time32SecondArray::from(vec![1234, 24680, 85563]);
698 let c7: DictionaryArray<Int32Type> =
699 vec!["cupcakes", "cupcakes", "foo"].into_iter().collect();
700
701 let batch = RecordBatch::try_new(
702 Arc::new(schema),
703 vec![
704 Arc::new(c1),
705 Arc::new(c2),
706 Arc::new(c3),
707 Arc::new(c4),
708 Arc::new(c5),
709 Arc::new(c6),
710 Arc::new(c7),
711 ],
712 )
713 .unwrap();
714
715 let mut file = tempfile::tempfile().unwrap();
716
717 let mut writer = Writer::new(&mut file);
718 let batches = vec![&batch, &batch];
719 for batch in batches {
720 writer.write(batch).unwrap();
721 }
722 drop(writer);
723
724 file.rewind().unwrap();
726 let mut buffer: Vec<u8> = vec![];
727 file.read_to_end(&mut buffer).unwrap();
728
729 let expected = r"c1,c2,c3,c4,c5,c6,c7
730Lorem ipsum dolor sit amet,123.564532,3,true,,00:20:34,cupcakes
731consectetur adipiscing elit,,2,false,2019-04-18T10:54:47.378,06:51:20,cupcakes
732sed do eiusmod tempor,-556132.25,1,,2019-04-18T02:45:55.555,23:46:03,foo
733Lorem ipsum dolor sit amet,123.564532,3,true,,00:20:34,cupcakes
734consectetur adipiscing elit,,2,false,2019-04-18T10:54:47.378,06:51:20,cupcakes
735sed do eiusmod tempor,-556132.25,1,,2019-04-18T02:45:55.555,23:46:03,foo
736";
737 assert_eq!(expected, str::from_utf8(&buffer).unwrap());
738 }
739
740 #[test]
741 fn test_write_csv_decimal() {
742 let schema = Schema::new(vec![
743 Field::new("c1", DataType::Decimal32(9, 6), true),
744 Field::new("c2", DataType::Decimal64(17, 6), true),
745 Field::new("c3", DataType::Decimal128(38, 6), true),
746 Field::new("c4", DataType::Decimal256(76, 6), true),
747 ]);
748
749 let mut c1_builder = Decimal32Builder::new().with_data_type(DataType::Decimal32(9, 6));
750 c1_builder.extend(vec![Some(-3335724), Some(2179404), None, Some(290472)]);
751 let c1 = c1_builder.finish();
752
753 let mut c2_builder = Decimal64Builder::new().with_data_type(DataType::Decimal64(17, 6));
754 c2_builder.extend(vec![Some(-3335724), Some(2179404), None, Some(290472)]);
755 let c2 = c2_builder.finish();
756
757 let mut c3_builder = Decimal128Builder::new().with_data_type(DataType::Decimal128(38, 6));
758 c3_builder.extend(vec![Some(-3335724), Some(2179404), None, Some(290472)]);
759 let c3 = c3_builder.finish();
760
761 let mut c4_builder = Decimal256Builder::new().with_data_type(DataType::Decimal256(76, 6));
762 c4_builder.extend(vec![
763 Some(i256::from_i128(-3335724)),
764 Some(i256::from_i128(2179404)),
765 None,
766 Some(i256::from_i128(290472)),
767 ]);
768 let c4 = c4_builder.finish();
769
770 let batch = RecordBatch::try_new(
771 Arc::new(schema),
772 vec![Arc::new(c1), Arc::new(c2), Arc::new(c3), Arc::new(c4)],
773 )
774 .unwrap();
775
776 let mut file = tempfile::tempfile().unwrap();
777
778 let mut writer = Writer::new(&mut file);
779 let batches = vec![&batch, &batch];
780 for batch in batches {
781 writer.write(batch).unwrap();
782 }
783 drop(writer);
784
785 file.rewind().unwrap();
787 let mut buffer: Vec<u8> = vec![];
788 file.read_to_end(&mut buffer).unwrap();
789
790 let expected = r"c1,c2,c3,c4
791-3.335724,-3.335724,-3.335724,-3.335724
7922.179404,2.179404,2.179404,2.179404
793,,,
7940.290472,0.290472,0.290472,0.290472
795-3.335724,-3.335724,-3.335724,-3.335724
7962.179404,2.179404,2.179404,2.179404
797,,,
7980.290472,0.290472,0.290472,0.290472
799";
800 assert_eq!(expected, str::from_utf8(&buffer).unwrap());
801 }
802
803 #[test]
804 fn test_write_csv_custom_options() {
805 let schema = Schema::new(vec![
806 Field::new("c1", DataType::Utf8, false),
807 Field::new("c2", DataType::Float64, true),
808 Field::new("c3", DataType::UInt32, false),
809 Field::new("c4", DataType::Boolean, true),
810 Field::new("c6", DataType::Time32(TimeUnit::Second), false),
811 ]);
812
813 let c1 = StringArray::from(vec![
814 "Lorem ipsum \ndolor sit amet",
815 "consectetur \"adipiscing\" elit",
816 "sed do eiusmod tempor",
817 ]);
818 let c2 =
819 PrimitiveArray::<Float64Type>::from(vec![Some(123.564532), None, Some(-556132.25)]);
820 let c3 = PrimitiveArray::<UInt32Type>::from(vec![3, 2, 1]);
821 let c4 = BooleanArray::from(vec![Some(true), Some(false), None]);
822 let c6 = Time32SecondArray::from(vec![1234, 24680, 85563]);
823
824 let batch = RecordBatch::try_new(
825 Arc::new(schema),
826 vec![
827 Arc::new(c1),
828 Arc::new(c2),
829 Arc::new(c3),
830 Arc::new(c4),
831 Arc::new(c6),
832 ],
833 )
834 .unwrap();
835
836 let mut file = tempfile::tempfile().unwrap();
837
838 let builder = WriterBuilder::new()
839 .with_header(false)
840 .with_delimiter(b'|')
841 .with_quote(b'\'')
842 .with_null("NULL".to_string())
843 .with_time_format("%r".to_string());
844 let mut writer = builder.build(&mut file);
845 let batches = vec![&batch];
846 for batch in batches {
847 writer.write(batch).unwrap();
848 }
849 drop(writer);
850
851 file.rewind().unwrap();
853 let mut buffer: Vec<u8> = vec![];
854 file.read_to_end(&mut buffer).unwrap();
855
856 assert_eq!(
857 "'Lorem ipsum \ndolor sit amet'|123.564532|3|true|12:20:34 AM\nconsectetur \"adipiscing\" elit|NULL|2|false|06:51:20 AM\nsed do eiusmod tempor|-556132.25|1|NULL|11:46:03 PM\n"
858 .to_string(),
859 String::from_utf8(buffer).unwrap()
860 );
861
862 let mut file = tempfile::tempfile().unwrap();
863
864 let builder = WriterBuilder::new()
865 .with_header(true)
866 .with_double_quote(false)
867 .with_escape(b'$');
868 let mut writer = builder.build(&mut file);
869 let batches = vec![&batch];
870 for batch in batches {
871 writer.write(batch).unwrap();
872 }
873 drop(writer);
874
875 file.rewind().unwrap();
876 let mut buffer: Vec<u8> = vec![];
877 file.read_to_end(&mut buffer).unwrap();
878
879 assert_eq!(
880 "c1,c2,c3,c4,c6\n\"Lorem ipsum \ndolor sit amet\",123.564532,3,true,00:20:34\n\"consectetur $\"adipiscing$\" elit\",,2,false,06:51:20\nsed do eiusmod tempor,-556132.25,1,,23:46:03\n"
881 .to_string(),
882 String::from_utf8(buffer).unwrap()
883 );
884 }
885
886 #[test]
887 fn test_conversion_consistency() {
888 let schema = Schema::new(vec![
891 Field::new("c1", DataType::Date32, false),
892 Field::new("c2", DataType::Date64, false),
893 Field::new("c3", DataType::Timestamp(TimeUnit::Nanosecond, None), false),
894 ]);
895
896 let nanoseconds = vec![
897 1599566300000000000,
898 1599566200000000000,
899 1599566100000000000,
900 ];
901 let c1 = Date32Array::from(vec![3, 2, 1]);
902 let c2 = Date64Array::from(vec![3, 2, 1]);
903 let c3 = TimestampNanosecondArray::from(nanoseconds.clone());
904
905 let batch = RecordBatch::try_new(
906 Arc::new(schema.clone()),
907 vec![Arc::new(c1), Arc::new(c2), Arc::new(c3)],
908 )
909 .unwrap();
910
911 let builder = WriterBuilder::new().with_header(false);
912
913 let mut buf: Cursor<Vec<u8>> = Default::default();
914 {
916 let mut writer = builder.build(&mut buf);
917 writer.write(&batch).unwrap();
918 }
919 buf.set_position(0);
920
921 let mut reader = ReaderBuilder::new(Arc::new(schema))
922 .with_batch_size(3)
923 .build_buffered(buf)
924 .unwrap();
925
926 let rb = reader.next().unwrap().unwrap();
927 let c1 = rb.column(0).as_any().downcast_ref::<Date32Array>().unwrap();
928 let c2 = rb.column(1).as_any().downcast_ref::<Date64Array>().unwrap();
929 let c3 = rb
930 .column(2)
931 .as_any()
932 .downcast_ref::<TimestampNanosecondArray>()
933 .unwrap();
934
935 let actual = c1.into_iter().collect::<Vec<_>>();
936 let expected = vec![Some(3), Some(2), Some(1)];
937 assert_eq!(actual, expected);
938 let actual = c2.into_iter().collect::<Vec<_>>();
939 let expected = vec![Some(3), Some(2), Some(1)];
940 assert_eq!(actual, expected);
941 let actual = c3.into_iter().collect::<Vec<_>>();
942 let expected = nanoseconds.into_iter().map(Some).collect::<Vec<_>>();
943 assert_eq!(actual, expected);
944 }
945
946 #[test]
947 fn test_write_csv_invalid_cast() {
948 let schema = Schema::new(vec![
949 Field::new("c0", DataType::UInt32, false),
950 Field::new("c1", DataType::Date64, false),
951 ]);
952
953 let c0 = UInt32Array::from(vec![Some(123), Some(234)]);
954 let c1 = Date64Array::from(vec![Some(1926632005177), Some(1926632005177685347)]);
955 let batch =
956 RecordBatch::try_new(Arc::new(schema), vec![Arc::new(c0), Arc::new(c1)]).unwrap();
957
958 let mut file = tempfile::tempfile().unwrap();
959 let mut writer = Writer::new(&mut file);
960 let batches = vec![&batch, &batch];
961
962 for batch in batches {
963 let err = writer.write(batch).unwrap_err().to_string();
964 assert_eq!(
965 err,
966 "Csv error: Error processing row 2, col 2: Cast error: Failed to convert 1926632005177685347 to temporal for Date64"
967 )
968 }
969 drop(writer);
970 }
971
972 #[test]
973 fn test_write_csv_using_rfc3339() {
974 let schema = Schema::new(vec![
975 Field::new(
976 "c1",
977 DataType::Timestamp(TimeUnit::Millisecond, Some("+00:00".into())),
978 true,
979 ),
980 Field::new("c2", DataType::Timestamp(TimeUnit::Millisecond, None), true),
981 Field::new("c3", DataType::Date32, false),
982 Field::new("c4", DataType::Time32(TimeUnit::Second), false),
983 ]);
984
985 let c1 = TimestampMillisecondArray::from(vec![Some(1555584887378), Some(1635577147000)])
986 .with_timezone("+00:00".to_string());
987 let c2 = TimestampMillisecondArray::from(vec![Some(1555584887378), Some(1635577147000)]);
988 let c3 = Date32Array::from(vec![3, 2]);
989 let c4 = Time32SecondArray::from(vec![1234, 24680]);
990
991 let batch = RecordBatch::try_new(
992 Arc::new(schema),
993 vec![Arc::new(c1), Arc::new(c2), Arc::new(c3), Arc::new(c4)],
994 )
995 .unwrap();
996
997 let mut file = tempfile::tempfile().unwrap();
998
999 let builder = WriterBuilder::new();
1000 let mut writer = builder.build(&mut file);
1001 let batches = vec![&batch];
1002 for batch in batches {
1003 writer.write(batch).unwrap();
1004 }
1005 drop(writer);
1006
1007 file.rewind().unwrap();
1008 let mut buffer: Vec<u8> = vec![];
1009 file.read_to_end(&mut buffer).unwrap();
1010
1011 assert_eq!(
1012 "c1,c2,c3,c4
10132019-04-18T10:54:47.378Z,2019-04-18T10:54:47.378,1970-01-04,00:20:34
10142021-10-30T06:59:07Z,2021-10-30T06:59:07,1970-01-03,06:51:20\n",
1015 String::from_utf8(buffer).unwrap()
1016 );
1017 }
1018
1019 #[test]
1020 fn test_write_csv_tz_format() {
1021 let schema = Schema::new(vec![
1022 Field::new(
1023 "c1",
1024 DataType::Timestamp(TimeUnit::Millisecond, Some("+02:00".into())),
1025 true,
1026 ),
1027 Field::new(
1028 "c2",
1029 DataType::Timestamp(TimeUnit::Second, Some("+04:00".into())),
1030 true,
1031 ),
1032 ]);
1033 let c1 = TimestampMillisecondArray::from(vec![Some(1_000), Some(2_000)])
1034 .with_timezone("+02:00".to_string());
1035 let c2 = TimestampSecondArray::from(vec![Some(1_000_000), None])
1036 .with_timezone("+04:00".to_string());
1037 let batch =
1038 RecordBatch::try_new(Arc::new(schema), vec![Arc::new(c1), Arc::new(c2)]).unwrap();
1039
1040 let mut file = tempfile::tempfile().unwrap();
1041 let mut writer = WriterBuilder::new()
1042 .with_timestamp_tz_format("%M:%H".to_string())
1043 .build(&mut file);
1044 writer.write(&batch).unwrap();
1045
1046 drop(writer);
1047 file.rewind().unwrap();
1048 let mut buffer: Vec<u8> = vec![];
1049 file.read_to_end(&mut buffer).unwrap();
1050
1051 let output = String::from_utf8(buffer).unwrap();
1052 assert_eq!(output, "c1,c2\n00:02,46:17\n00:02,\n");
1053 }
1054
1055 #[test]
1056 fn test_write_csv_with_lf_terminator() {
1057 let output = write_batch_with_terminator(Terminator::Any(b'\n'));
1058 assert_eq!(output, "c1,c2\nhello,1\nworld,2\n");
1059 }
1060
1061 #[test]
1062 fn test_write_csv_with_crlf_terminator() {
1063 let output = write_batch_with_terminator(Terminator::CRLF);
1064 assert_eq!(output, "c1,c2\r\nhello,1\r\nworld,2\r\n");
1065 }
1066
1067 #[test]
1068 fn test_write_csv_with_any_terminator() {
1069 let output = write_batch_with_terminator(Terminator::Any(b'|'));
1070 assert_eq!(output, "c1,c2|hello,1|world,2|");
1071 }
1072
1073 fn write_batch_with_terminator(terminator: Terminator) -> String {
1074 let schema = Schema::new(vec![
1075 Field::new("c1", DataType::Utf8, false),
1076 Field::new("c2", DataType::UInt32, false),
1077 ]);
1078
1079 let c1 = StringArray::from(vec!["hello", "world"]);
1080 let c2 = PrimitiveArray::<UInt32Type>::from(vec![1, 2]);
1081
1082 let batch =
1083 RecordBatch::try_new(Arc::new(schema), vec![Arc::new(c1), Arc::new(c2)]).unwrap();
1084
1085 let mut buf = Vec::new();
1086 let mut writer = WriterBuilder::new()
1087 .with_line_terminator(terminator)
1088 .build(&mut buf);
1089 writer.write(&batch).unwrap();
1090 drop(writer);
1091
1092 String::from_utf8(buf).unwrap()
1093 }
1094
1095 #[test]
1096 fn test_write_csv_binary() {
1097 let fixed_size = 8;
1098 let schema = SchemaRef::new(Schema::new(vec![
1099 Field::new("c1", DataType::Binary, true),
1100 Field::new("c2", DataType::FixedSizeBinary(fixed_size), true),
1101 Field::new("c3", DataType::LargeBinary, true),
1102 ]));
1103 let mut c1_builder = BinaryBuilder::new();
1104 c1_builder.append_value(b"Homer");
1105 c1_builder.append_value(b"Bart");
1106 c1_builder.append_null();
1107 c1_builder.append_value(b"Bob");
1108 let mut c2_builder = FixedSizeBinaryBuilder::new(fixed_size);
1109 c2_builder.append_value(b"Simpson ").unwrap();
1110 c2_builder.append_value(b"Simpson ").unwrap();
1111 c2_builder.append_null();
1112 c2_builder.append_value(b"Flanders").unwrap();
1113 let mut c3_builder = LargeBinaryBuilder::new();
1114 c3_builder.append_null();
1115 c3_builder.append_null();
1116 c3_builder.append_value(b"Comic Book Guy");
1117 c3_builder.append_null();
1118
1119 let batch = RecordBatch::try_new(
1120 schema,
1121 vec![
1122 Arc::new(c1_builder.finish()) as ArrayRef,
1123 Arc::new(c2_builder.finish()) as ArrayRef,
1124 Arc::new(c3_builder.finish()) as ArrayRef,
1125 ],
1126 )
1127 .unwrap();
1128
1129 let mut buf = Vec::new();
1130 let builder = WriterBuilder::new();
1131 let mut writer = builder.build(&mut buf);
1132 writer.write(&batch).unwrap();
1133 drop(writer);
1134 assert_eq!(
1135 "\
1136 c1,c2,c3\n\
1137 486f6d6572,53696d70736f6e20,\n\
1138 42617274,53696d70736f6e20,\n\
1139 ,,436f6d696320426f6f6b20477579\n\
1140 426f62,466c616e64657273,\n\
1141 ",
1142 String::from_utf8(buf).unwrap()
1143 );
1144 }
1145
1146 #[test]
1147 fn test_write_csv_whitespace_handling() {
1148 let schema = Schema::new(vec![
1149 Field::new("c1", DataType::Utf8, false),
1150 Field::new("c2", DataType::Float64, true),
1151 Field::new("c3", DataType::Utf8, true),
1152 ]);
1153
1154 let c1 = StringArray::from(vec![
1155 " leading space",
1156 "trailing space ",
1157 " both spaces ",
1158 "no spaces",
1159 ]);
1160 let c2 = PrimitiveArray::<Float64Type>::from(vec![
1161 Some(123.45),
1162 Some(678.90),
1163 None,
1164 Some(111.22),
1165 ]);
1166 let c3 = StringArray::from(vec![
1167 Some(" test "),
1168 Some("value "),
1169 None,
1170 Some(" another"),
1171 ]);
1172
1173 let batch = RecordBatch::try_new(
1174 Arc::new(schema),
1175 vec![Arc::new(c1), Arc::new(c2), Arc::new(c3)],
1176 )
1177 .unwrap();
1178
1179 let mut buf = Vec::new();
1181 let builder = WriterBuilder::new();
1182 let mut writer = builder.build(&mut buf);
1183 writer.write(&batch).unwrap();
1184 drop(writer);
1185 assert_eq!(
1186 "c1,c2,c3\n leading space,123.45, test \ntrailing space ,678.9,value \n both spaces ,,\nno spaces,111.22, another\n",
1187 String::from_utf8(buf).unwrap()
1188 );
1189
1190 let mut buf = Vec::new();
1192 let builder = WriterBuilder::new().with_ignore_leading_whitespace(true);
1193 let mut writer = builder.build(&mut buf);
1194 writer.write(&batch).unwrap();
1195 drop(writer);
1196 assert_eq!(
1197 "c1,c2,c3\nleading space,123.45,test \ntrailing space ,678.9,value \nboth spaces ,,\nno spaces,111.22,another\n",
1198 String::from_utf8(buf).unwrap()
1199 );
1200
1201 let mut buf = Vec::new();
1203 let builder = WriterBuilder::new().with_ignore_trailing_whitespace(true);
1204 let mut writer = builder.build(&mut buf);
1205 writer.write(&batch).unwrap();
1206 drop(writer);
1207 assert_eq!(
1208 "c1,c2,c3\n leading space,123.45, test\ntrailing space,678.9,value\n both spaces,,\nno spaces,111.22, another\n",
1209 String::from_utf8(buf).unwrap()
1210 );
1211
1212 let mut buf = Vec::new();
1214 let builder = WriterBuilder::new()
1215 .with_ignore_leading_whitespace(true)
1216 .with_ignore_trailing_whitespace(true);
1217 let mut writer = builder.build(&mut buf);
1218 writer.write(&batch).unwrap();
1219 drop(writer);
1220 assert_eq!(
1221 "c1,c2,c3\nleading space,123.45,test\ntrailing space,678.9,value\nboth spaces,,\nno spaces,111.22,another\n",
1222 String::from_utf8(buf).unwrap()
1223 );
1224 }
1225
1226 #[test]
1227 fn test_write_csv_whitespace_with_special_chars() {
1228 let schema = Schema::new(vec![Field::new("c1", DataType::Utf8, false)]);
1229
1230 let c1 = StringArray::from(vec![
1231 " quoted \"value\" ",
1232 " new\nline ",
1233 " comma,value ",
1234 "\ttab\tvalue\t",
1235 ]);
1236
1237 let batch = RecordBatch::try_new(Arc::new(schema), vec![Arc::new(c1)]).unwrap();
1238
1239 let mut buf = Vec::new();
1241 let builder = WriterBuilder::new()
1242 .with_ignore_leading_whitespace(true)
1243 .with_ignore_trailing_whitespace(true);
1244 let mut writer = builder.build(&mut buf);
1245 writer.write(&batch).unwrap();
1246 drop(writer);
1247
1248 assert_eq!(
1250 "c1\n\"quoted \"\"value\"\"\"\n\"new\nline\"\n\"comma,value\"\ntab\tvalue\n",
1251 String::from_utf8(buf).unwrap()
1252 );
1253 }
1254
1255 #[test]
1256 fn test_write_csv_whitespace_all_string_types() {
1257 use arrow_array::{LargeStringArray, StringViewArray};
1258
1259 let schema = Schema::new(vec![
1260 Field::new("utf8", DataType::Utf8, false),
1261 Field::new("large_utf8", DataType::LargeUtf8, false),
1262 Field::new("utf8_view", DataType::Utf8View, false),
1263 ]);
1264
1265 let utf8 = StringArray::from(vec![" leading", "trailing ", " both ", "no_spaces"]);
1266
1267 let large_utf8 =
1268 LargeStringArray::from(vec![" leading", "trailing ", " both ", "no_spaces"]);
1269
1270 let utf8_view =
1271 StringViewArray::from(vec![" leading", "trailing ", " both ", "no_spaces"]);
1272
1273 let batch = RecordBatch::try_new(
1274 Arc::new(schema),
1275 vec![Arc::new(utf8), Arc::new(large_utf8), Arc::new(utf8_view)],
1276 )
1277 .unwrap();
1278
1279 let mut buf = Vec::new();
1281 let builder = WriterBuilder::new();
1282 let mut writer = builder.build(&mut buf);
1283 writer.write(&batch).unwrap();
1284 drop(writer);
1285 assert_eq!(
1286 "utf8,large_utf8,utf8_view\n leading, leading, leading\ntrailing ,trailing ,trailing \n both , both , both \nno_spaces,no_spaces,no_spaces\n",
1287 String::from_utf8(buf).unwrap()
1288 );
1289
1290 let mut buf = Vec::new();
1292 let builder = WriterBuilder::new()
1293 .with_ignore_leading_whitespace(true)
1294 .with_ignore_trailing_whitespace(true);
1295 let mut writer = builder.build(&mut buf);
1296 writer.write(&batch).unwrap();
1297 drop(writer);
1298 assert_eq!(
1299 "utf8,large_utf8,utf8_view\nleading,leading,leading\ntrailing,trailing,trailing\nboth,both,both\nno_spaces,no_spaces,no_spaces\n",
1300 String::from_utf8(buf).unwrap()
1301 );
1302
1303 let mut buf = Vec::new();
1305 let builder = WriterBuilder::new().with_ignore_leading_whitespace(true);
1306 let mut writer = builder.build(&mut buf);
1307 writer.write(&batch).unwrap();
1308 drop(writer);
1309 assert_eq!(
1310 "utf8,large_utf8,utf8_view\nleading,leading,leading\ntrailing ,trailing ,trailing \nboth ,both ,both \nno_spaces,no_spaces,no_spaces\n",
1311 String::from_utf8(buf).unwrap()
1312 );
1313
1314 let mut buf = Vec::new();
1316 let builder = WriterBuilder::new().with_ignore_trailing_whitespace(true);
1317 let mut writer = builder.build(&mut buf);
1318 writer.write(&batch).unwrap();
1319 drop(writer);
1320 assert_eq!(
1321 "utf8,large_utf8,utf8_view\n leading, leading, leading\ntrailing,trailing,trailing\n both, both, both\nno_spaces,no_spaces,no_spaces\n",
1322 String::from_utf8(buf).unwrap()
1323 );
1324 }
1325
1326 fn write_quote_style(batch: &RecordBatch, quote_style: QuoteStyle) -> String {
1327 let mut buf = Vec::new();
1328 let mut writer = WriterBuilder::new()
1329 .with_quote_style(quote_style)
1330 .build(&mut buf);
1331 writer.write(batch).unwrap();
1332 drop(writer);
1333 String::from_utf8(buf).unwrap()
1334 }
1335
1336 fn write_quote_style_with_null(
1337 batch: &RecordBatch,
1338 quote_style: QuoteStyle,
1339 null_value: &str,
1340 ) -> String {
1341 let mut buf = Vec::new();
1342 let mut writer = WriterBuilder::new()
1343 .with_quote_style(quote_style)
1344 .with_null(null_value.to_string())
1345 .build(&mut buf);
1346 writer.write(batch).unwrap();
1347 drop(writer);
1348 String::from_utf8(buf).unwrap()
1349 }
1350
1351 #[test]
1352 fn test_write_csv_quote_style() {
1353 let schema = Schema::new(vec![
1354 Field::new("text", DataType::Utf8, false),
1355 Field::new("number", DataType::Int32, false),
1356 Field::new("float", DataType::Float64, false),
1357 ]);
1358
1359 let text = StringArray::from(vec!["hello", "world", "comma,value", "quote\"test"]);
1360 let number = Int32Array::from(vec![1, 2, 3, 4]);
1361 let float = Float64Array::from(vec![1.1, 2.2, 3.3, 4.4]);
1362
1363 let batch = RecordBatch::try_new(
1364 Arc::new(schema),
1365 vec![Arc::new(text), Arc::new(number), Arc::new(float)],
1366 )
1367 .unwrap();
1368
1369 assert_eq!(
1371 "text,number,float\nhello,1,1.1\nworld,2,2.2\n\"comma,value\",3,3.3\n\"quote\"\"test\",4,4.4\n",
1372 write_quote_style(&batch, QuoteStyle::Necessary)
1373 );
1374
1375 assert_eq!(
1377 "\"text\",\"number\",\"float\"\n\"hello\",\"1\",\"1.1\"\n\"world\",\"2\",\"2.2\"\n\"comma,value\",\"3\",\"3.3\"\n\"quote\"\"test\",\"4\",\"4.4\"\n",
1378 write_quote_style(&batch, QuoteStyle::Always)
1379 );
1380
1381 assert_eq!(
1383 "\"text\",\"number\",\"float\"\n\"hello\",1,1.1\n\"world\",2,2.2\n\"comma,value\",3,3.3\n\"quote\"\"test\",4,4.4\n",
1384 write_quote_style(&batch, QuoteStyle::NonNumeric)
1385 );
1386
1387 assert_eq!(
1390 "text,number,float\nhello,1,1.1\nworld,2,2.2\ncomma,value,3,3.3\nquote\"test,4,4.4\n",
1391 write_quote_style(&batch, QuoteStyle::Never)
1392 );
1393 }
1394
1395 #[test]
1396 fn test_write_csv_quote_style_with_nulls() {
1397 let schema = Schema::new(vec![
1398 Field::new("text", DataType::Utf8, true),
1399 Field::new("number", DataType::Int32, true),
1400 ]);
1401
1402 let text = StringArray::from(vec![Some("hello"), None, Some("world")]);
1403 let number = Int32Array::from(vec![Some(1), Some(2), None]);
1404
1405 let batch =
1406 RecordBatch::try_new(Arc::new(schema), vec![Arc::new(text), Arc::new(number)]).unwrap();
1407
1408 assert_eq!(
1410 "\"text\",\"number\"\n\"hello\",\"1\"\n\"\",\"2\"\n\"world\",\"\"\n",
1411 write_quote_style(&batch, QuoteStyle::Always)
1412 );
1413
1414 assert_eq!(
1416 "\"text\",\"number\"\n\"hello\",\"1\"\n\"NULL\",\"2\"\n\"world\",\"NULL\"\n",
1417 write_quote_style_with_null(&batch, QuoteStyle::Always, "NULL")
1418 );
1419 }
1420}