Skip to main content

arrow_csv/
writer.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4
5//! CSV Writing: [`Writer`] and [`WriterBuilder`]
6//!
7//! This CSV writer allows Arrow data (in record batches) to be written as CSV files.
8//! The writer does not support writing `ListArray` and `StructArray`.
9//!
10//! # Example
11//! ```
12//! # use arrow_array::*;
13//! # use arrow_array::types::*;
14//! # use arrow_csv::Writer;
15//! # use arrow_schema::*;
16//! # use std::sync::Arc;
17//!
18//! let schema = Schema::new(vec![
19//!     Field::new("c1", DataType::Utf8, false),
20//!     Field::new("c2", DataType::Float64, true),
21//!     Field::new("c3", DataType::UInt32, false),
22//!     Field::new("c4", DataType::Boolean, true),
23//! ]);
24//! let c1 = StringArray::from(vec![
25//!     "Lorem ipsum dolor sit amet",
26//!     "consectetur adipiscing elit",
27//!     "sed do eiusmod tempor",
28//! ]);
29//! let c2 = PrimitiveArray::<Float64Type>::from(vec![
30//!     Some(123.564532),
31//!     None,
32//!     Some(-556132.25),
33//! ]);
34//! let c3 = PrimitiveArray::<UInt32Type>::from(vec![3, 2, 1]);
35//! let c4 = BooleanArray::from(vec![Some(true), Some(false), None]);
36//!
37//! let batch = RecordBatch::try_new(
38//!     Arc::new(schema),
39//!     vec![Arc::new(c1), Arc::new(c2), Arc::new(c3), Arc::new(c4)],
40//! )
41//! .unwrap();
42//!
43//! let mut output = Vec::with_capacity(1024);
44//!
45//! let mut writer = Writer::new(&mut output);
46//! let batches = vec![&batch, &batch];
47//! for batch in batches {
48//!     writer.write(batch).unwrap();
49//! }
50//! ```
51//!
52//! # Whitespace Handling
53//!
54//! The writer supports trimming leading and trailing whitespace from string values,
55//! compatible with Apache Spark's CSV options `ignoreLeadingWhiteSpace` and
56//! `ignoreTrailingWhiteSpace`. This is useful when working with data that may have
57//! unwanted padding.
58//!
59//! Whitespace trimming is applied to all string data types:
60//! - `DataType::Utf8`
61//! - `DataType::LargeUtf8`
62//! - `DataType::Utf8View`
63//!
64//! ## Example: Use [`WriterBuilder`] to control whitespace handling
65//!
66//! ```
67//! # use arrow_array::*;
68//! # use arrow_csv::WriterBuilder;
69//! # use arrow_schema::*;
70//! # use std::sync::Arc;
71//! let schema = Schema::new(vec![
72//!     Field::new("name", DataType::Utf8, false),
73//!     Field::new("comment", DataType::Utf8, false),
74//! ]);
75//!
76//! let name = StringArray::from(vec![
77//!     "  Alice  ",   // Leading and trailing spaces
78//!     "Bob",         // No spaces
79//!     "  Charlie",   // Leading spaces only
80//! ]);
81//! let comment = StringArray::from(vec![
82//!     "  Great job!  ",
83//!     "Well done",
84//!     "Excellent  ",
85//! ]);
86//!
87//! let batch = RecordBatch::try_new(
88//!     Arc::new(schema),
89//!     vec![Arc::new(name), Arc::new(comment)],
90//! )
91//! .unwrap();
92//!
93//! // Trim both leading and trailing whitespace
94//! let mut output = Vec::new();
95//! WriterBuilder::new()
96//!     .with_ignore_leading_whitespace(true)
97//!     .with_ignore_trailing_whitespace(true)
98//!     .build(&mut output)
99//!     .write(&batch)
100//!     .unwrap();
101//! assert_eq!(
102//!     String::from_utf8(output).unwrap(),
103//!     "\
104//! name,comment\n\
105//! Alice,Great job!\n\
106//! Bob,Well done\n\
107//! Charlie,Excellent\n"
108//! );
109//! ```
110//!
111//! # Quoting Styles
112//!
113//! The writer supports different quoting styles for fields, compatible with Apache Spark's
114//! CSV options like `quoteAll`. You can control when fields are quoted using the
115//! [`QuoteStyle`] enum.
116//!
117//! ## Example
118//!
119//! ```
120//! # use arrow_array::*;
121//! # use arrow_csv::{WriterBuilder, QuoteStyle};
122//! # use arrow_schema::*;
123//! # use std::sync::Arc;
124//!
125//! let schema = Schema::new(vec![
126//!     Field::new("product", DataType::Utf8, false),
127//!     Field::new("price", DataType::Float64, false),
128//! ]);
129//!
130//! let product = StringArray::from(vec!["apple", "banana,organic", "cherry"]);
131//! let price = Float64Array::from(vec![1.50, 2.25, 3.00]);
132//!
133//! let batch = RecordBatch::try_new(
134//!     Arc::new(schema),
135//!     vec![Arc::new(product), Arc::new(price)],
136//! )
137//! .unwrap();
138//!
139//! // Default behavior (QuoteStyle::Necessary)
140//! let mut output = Vec::new();
141//! WriterBuilder::new()
142//!     .build(&mut output)
143//!     .write(&batch)
144//!     .unwrap();
145//! assert_eq!(
146//!     String::from_utf8(output).unwrap(),
147//!     "product,price\napple,1.5\n\"banana,organic\",2.25\ncherry,3.0\n"
148//! );
149//!
150//! // Quote all fields (Spark's quoteAll=true)
151//! let mut output = Vec::new();
152//! WriterBuilder::new()
153//!     .with_quote_style(QuoteStyle::Always)
154//!     .build(&mut output)
155//!     .write(&batch)
156//!     .unwrap();
157//! assert_eq!(
158//!     String::from_utf8(output).unwrap(),
159//!     "\"product\",\"price\"\n\"apple\",\"1.5\"\n\"banana,organic\",\"2.25\"\n\"cherry\",\"3.0\"\n"
160//! );
161//! ```
162
163use arrow_array::*;
164use arrow_cast::display::*;
165use arrow_schema::*;
166use csv::ByteRecord;
167use std::io::Write;
168
169use crate::map_csv_error;
170const DEFAULT_NULL_VALUE: &str = "";
171
172/// The quoting style to use when writing CSV files.
173///
174/// This type is re-exported from the `csv` crate and supports different
175/// strategies for quoting fields. It is compatible with Apache Spark's
176/// CSV options like `quoteAll`.
177///
178/// # Example
179///
180/// ```
181/// use arrow_csv::{WriterBuilder, QuoteStyle};
182///
183/// let builder = WriterBuilder::new()
184///     .with_quote_style(QuoteStyle::Always); // Equivalent to Spark's quoteAll=true
185/// ```
186pub use csv::QuoteStyle;
187
188/// A CSV writer
189///
190/// See the [module documentation](crate::writer) for examples.
191#[derive(Debug)]
192pub struct Writer<W: Write> {
193    /// The object to write to
194    writer: csv::Writer<W>,
195    /// Whether file should be written with headers, defaults to `true`
196    has_headers: bool,
197    /// The date format for date arrays, defaults to RFC3339
198    date_format: Option<String>,
199    /// The datetime format for datetime arrays, defaults to RFC3339
200    datetime_format: Option<String>,
201    /// The timestamp format for timestamp arrays, defaults to RFC3339
202    timestamp_format: Option<String>,
203    /// The timestamp format for timestamp (with timezone) arrays, defaults to RFC3339
204    timestamp_tz_format: Option<String>,
205    /// The time format for time arrays, defaults to RFC3339
206    time_format: Option<String>,
207    /// Is the beginning-of-writer
208    beginning: bool,
209    /// The value to represent null entries, defaults to [`DEFAULT_NULL_VALUE`]
210    null_value: Option<String>,
211    /// Whether to ignore leading whitespace in string values
212    ignore_leading_whitespace: bool,
213    /// Whether to ignore trailing whitespace in string values
214    ignore_trailing_whitespace: bool,
215}
216
217impl<W: Write> Writer<W> {
218    /// Create a new CsvWriter from a writable object, with default options
219    ///
220    /// See [`WriterBuilder`] for configure options, and the [module
221    /// documentation](crate::writer) for examples.
222    pub fn new(writer: W) -> Self {
223        let delimiter = b',';
224        WriterBuilder::new().with_delimiter(delimiter).build(writer)
225    }
226
227    /// Write a RecordBatch to the underlying writer
228    pub fn write(&mut self, batch: &RecordBatch) -> Result<(), ArrowError> {
229        let num_columns = batch.num_columns();
230        if self.beginning {
231            if self.has_headers {
232                let mut headers: Vec<String> = Vec::with_capacity(num_columns);
233                batch
234                    .schema()
235                    .fields()
236                    .iter()
237                    .for_each(|field| headers.push(field.name().clone()));
238                self.writer
239                    .write_record(&headers[..])
240                    .map_err(map_csv_error)?;
241            }
242            self.beginning = false;
243        }
244
245        let options = FormatOptions::default()
246            .with_null(self.null_value.as_deref().unwrap_or(DEFAULT_NULL_VALUE))
247            .with_date_format(self.date_format.as_deref())
248            .with_datetime_format(self.datetime_format.as_deref())
249            .with_timestamp_format(self.timestamp_format.as_deref())
250            .with_timestamp_tz_format(self.timestamp_tz_format.as_deref())
251            .with_time_format(self.time_format.as_deref());
252
253        let converters = batch
254            .columns()
255            .iter()
256            .map(|a| {
257                if a.data_type().is_nested() {
258                    Err(ArrowError::CsvError(format!(
259                        "Nested type {} is not supported in CSV",
260                        a.data_type()
261                    )))
262                } else {
263                    ArrayFormatter::try_new(a.as_ref(), &options)
264                }
265            })
266            .collect::<Result<Vec<_>, ArrowError>>()?;
267
268        let mut buffer = String::with_capacity(1024);
269        let mut byte_record = ByteRecord::with_capacity(1024, converters.len());
270
271        for row_idx in 0..batch.num_rows() {
272            byte_record.clear();
273            for (col_idx, converter) in converters.iter().enumerate() {
274                buffer.clear();
275                converter.value(row_idx).write(&mut buffer).map_err(|e| {
276                    ArrowError::CsvError(format!(
277                        "Error processing row {}, col {}: {e}",
278                        row_idx + 1,
279                        col_idx + 1
280                    ))
281                })?;
282
283                let field_bytes =
284                    self.get_trimmed_field_bytes(&buffer, batch.column(col_idx).data_type());
285                byte_record.push_field(field_bytes);
286            }
287
288            self.writer
289                .write_byte_record(&byte_record)
290                .map_err(map_csv_error)?;
291        }
292        self.writer.flush()?;
293
294        Ok(())
295    }
296
297    /// Returns the bytes for a field, applying whitespace trimming if configured and applicable
298    fn get_trimmed_field_bytes<'a>(&self, buffer: &'a str, data_type: &DataType) -> &'a [u8] {
299        // Only trim string types when trimming is enabled
300        let should_trim = (self.ignore_leading_whitespace || self.ignore_trailing_whitespace)
301            && matches!(
302                data_type,
303                DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View
304            );
305
306        if !should_trim {
307            return buffer.as_bytes();
308        }
309
310        let mut trimmed = buffer;
311        if self.ignore_leading_whitespace {
312            trimmed = trimmed.trim_start();
313        }
314        if self.ignore_trailing_whitespace {
315            trimmed = trimmed.trim_end();
316        }
317        trimmed.as_bytes()
318    }
319
320    /// Unwraps this `Writer<W>`, returning the underlying writer.
321    pub fn into_inner(self) -> W {
322        // Safe to call `unwrap` since `write` always flushes the writer.
323        self.writer.into_inner().unwrap()
324    }
325}
326
327impl<W: Write> RecordBatchWriter for Writer<W> {
328    fn write(&mut self, batch: &RecordBatch) -> Result<(), ArrowError> {
329        self.write(batch)
330    }
331
332    fn close(self) -> Result<(), ArrowError> {
333        Ok(())
334    }
335}
336
337/// A CSV writer builder
338#[derive(Clone, Debug)]
339pub struct WriterBuilder {
340    /// Optional column delimiter. Defaults to `b','`
341    delimiter: u8,
342    /// Whether to write column names as file headers. Defaults to `true`
343    has_header: bool,
344    /// Optional quote character. Defaults to `b'"'`
345    quote: u8,
346    /// Optional escape character. Defaults to `b'\\'`
347    escape: u8,
348    /// Optional line terminator. Defaults to `LF` (`\n`)
349    terminator: Terminator,
350    /// Enable double quote escapes. Defaults to `true`
351    double_quote: bool,
352    /// Optional date format for date arrays
353    date_format: Option<String>,
354    /// Optional datetime format for datetime arrays
355    datetime_format: Option<String>,
356    /// Optional timestamp format for timestamp arrays
357    timestamp_format: Option<String>,
358    /// Optional timestamp format for timestamp with timezone arrays
359    timestamp_tz_format: Option<String>,
360    /// Optional time format for time arrays
361    time_format: Option<String>,
362    /// Optional value to represent null
363    null_value: Option<String>,
364    /// Whether to ignore leading whitespace in string values. Defaults to `false`
365    ignore_leading_whitespace: bool,
366    /// Whether to ignore trailing whitespace in string values. Defaults to `false`
367    ignore_trailing_whitespace: bool,
368    /// The quoting style to use. Defaults to `QuoteStyle::Necessary`
369    quote_style: QuoteStyle,
370}
371
372/// The line terminator to use when writing CSV files.
373#[derive(Clone, Debug)]
374pub enum Terminator {
375    /// Use CRLF (`\r\n`) as the line terminator
376    CRLF,
377    /// Use the specified byte character as the line terminator
378    Any(u8),
379}
380
381impl Default for WriterBuilder {
382    fn default() -> Self {
383        WriterBuilder {
384            delimiter: b',',
385            has_header: true,
386            quote: b'"',
387            escape: b'\\',
388            terminator: Terminator::Any(b'\n'),
389            double_quote: true,
390            date_format: None,
391            datetime_format: None,
392            timestamp_format: None,
393            timestamp_tz_format: None,
394            time_format: None,
395            null_value: None,
396            ignore_leading_whitespace: false,
397            ignore_trailing_whitespace: false,
398            quote_style: QuoteStyle::default(),
399        }
400    }
401}
402
403impl WriterBuilder {
404    /// Create a new builder for configuring CSV [`Writer`] options.
405    ///
406    /// To convert a builder into a writer, call [`WriterBuilder::build`]. See
407    /// the [module documentation](crate::writer) for more examples.
408    ///
409    /// # Example
410    ///
411    /// ```
412    /// # use arrow_csv::{Writer, WriterBuilder};
413    /// # use std::fs::File;
414    ///
415    /// fn example() -> Writer<File> {
416    ///     let file = File::create("target/out.csv").unwrap();
417    ///
418    ///     // create a builder that doesn't write headers
419    ///     let builder = WriterBuilder::new().with_header(false);
420    ///     let writer = builder.build(file);
421    ///
422    ///     writer
423    /// }
424    /// ```
425    pub fn new() -> Self {
426        Self::default()
427    }
428
429    /// Set whether to write the CSV file with a header
430    pub fn with_header(mut self, header: bool) -> Self {
431        self.has_header = header;
432        self
433    }
434
435    /// Returns `true` if this writer is configured to write a header
436    pub fn header(&self) -> bool {
437        self.has_header
438    }
439
440    /// Set the CSV file's column delimiter as a byte character
441    pub fn with_delimiter(mut self, delimiter: u8) -> Self {
442        self.delimiter = delimiter;
443        self
444    }
445
446    /// Get the CSV file's column delimiter as a byte character
447    pub fn delimiter(&self) -> u8 {
448        self.delimiter
449    }
450
451    /// Set the CSV file's quote character as a byte character
452    pub fn with_quote(mut self, quote: u8) -> Self {
453        self.quote = quote;
454        self
455    }
456
457    /// Get the CSV file's quote character as a byte character
458    pub fn quote(&self) -> u8 {
459        self.quote
460    }
461
462    /// Set the CSV file's escape character as a byte character
463    ///
464    /// In some variants of CSV, quotes are escaped using a special escape
465    /// character like `\` (instead of escaping quotes by doubling them).
466    ///
467    /// By default, writing these idiosyncratic escapes is disabled, and is
468    /// only used when `double_quote` is disabled.
469    pub fn with_escape(mut self, escape: u8) -> Self {
470        self.escape = escape;
471        self
472    }
473
474    /// Get the CSV file's escape character as a byte character
475    pub fn escape(&self) -> u8 {
476        self.escape
477    }
478
479    /// Set whether to enable double quote escapes
480    ///
481    /// When enabled (which is the default), quotes are escaped by doubling
482    /// them. e.g., `"` escapes to `""`.
483    ///
484    /// When disabled, quotes are escaped with the escape character (which
485    /// is `\\` by default).
486    pub fn with_double_quote(mut self, double_quote: bool) -> Self {
487        self.double_quote = double_quote;
488        self
489    }
490
491    /// Get whether double quote escapes are enabled
492    pub fn double_quote(&self) -> bool {
493        self.double_quote
494    }
495
496    /// Set the CSV file's date format
497    pub fn with_date_format(mut self, format: String) -> Self {
498        self.date_format = Some(format);
499        self
500    }
501
502    /// Get the CSV file's date format if set, defaults to RFC3339
503    pub fn date_format(&self) -> Option<&str> {
504        self.date_format.as_deref()
505    }
506
507    /// Set the CSV file's datetime format
508    pub fn with_datetime_format(mut self, format: String) -> Self {
509        self.datetime_format = Some(format);
510        self
511    }
512
513    /// Get the CSV file's datetime format if set, defaults to RFC3339
514    pub fn datetime_format(&self) -> Option<&str> {
515        self.datetime_format.as_deref()
516    }
517
518    /// Set the CSV file's time format
519    pub fn with_time_format(mut self, format: String) -> Self {
520        self.time_format = Some(format);
521        self
522    }
523
524    /// Get the CSV file's datetime time if set, defaults to RFC3339
525    pub fn time_format(&self) -> Option<&str> {
526        self.time_format.as_deref()
527    }
528
529    /// Set the CSV file's timestamp format
530    pub fn with_timestamp_format(mut self, format: String) -> Self {
531        self.timestamp_format = Some(format);
532        self
533    }
534
535    /// Get the CSV file's timestamp format if set, defaults to RFC3339
536    pub fn timestamp_format(&self) -> Option<&str> {
537        self.timestamp_format.as_deref()
538    }
539
540    /// Set the CSV file's timestamp tz format
541    pub fn with_timestamp_tz_format(mut self, tz_format: String) -> Self {
542        self.timestamp_tz_format = Some(tz_format);
543        self
544    }
545
546    /// Get the CSV file's timestamp tz format if set, defaults to RFC3339
547    pub fn timestamp_tz_format(&self) -> Option<&str> {
548        self.timestamp_tz_format.as_deref()
549    }
550
551    /// Set the value to represent null in output
552    pub fn with_null(mut self, null_value: String) -> Self {
553        self.null_value = Some(null_value);
554        self
555    }
556
557    /// Get the value to represent null in output
558    pub fn null(&self) -> &str {
559        self.null_value.as_deref().unwrap_or(DEFAULT_NULL_VALUE)
560    }
561
562    /// Set whether to ignore leading whitespace in string values
563    /// For example, a string value such as "   foo" will be written as "foo"
564    pub fn with_ignore_leading_whitespace(mut self, ignore: bool) -> Self {
565        self.ignore_leading_whitespace = ignore;
566        self
567    }
568
569    /// Get whether to ignore leading whitespace in string values
570    pub fn ignore_leading_whitespace(&self) -> bool {
571        self.ignore_leading_whitespace
572    }
573
574    /// Set whether to ignore trailing whitespace in string values
575    /// For example, a string value such as "foo    " will be written as "foo"
576    pub fn with_ignore_trailing_whitespace(mut self, ignore: bool) -> Self {
577        self.ignore_trailing_whitespace = ignore;
578        self
579    }
580
581    /// Get whether to ignore trailing whitespace in string values
582    pub fn ignore_trailing_whitespace(&self) -> bool {
583        self.ignore_trailing_whitespace
584    }
585
586    /// Set the quoting style for writing CSV files
587    ///
588    /// # Example
589    ///
590    /// ```
591    /// use arrow_csv::{WriterBuilder, QuoteStyle};
592    ///
593    /// // Quote all fields (equivalent to Spark's quoteAll=true)
594    /// let builder = WriterBuilder::new()
595    ///     .with_quote_style(QuoteStyle::Always);
596    ///
597    /// // Only quote when necessary (default)
598    /// let builder = WriterBuilder::new()
599    ///     .with_quote_style(QuoteStyle::Necessary);
600    /// ```
601    pub fn with_quote_style(mut self, quote_style: QuoteStyle) -> Self {
602        self.quote_style = quote_style;
603        self
604    }
605
606    /// Get the configured quoting style
607    pub fn quote_style(&self) -> QuoteStyle {
608        self.quote_style
609    }
610
611    /// Set the CSV file's line terminator
612    pub fn with_line_terminator(mut self, terminator: Terminator) -> Self {
613        self.terminator = terminator;
614        self
615    }
616
617    /// Get the CSV file's line terminator, defaults to `LF` (`\n`)
618    pub fn line_terminator(&self) -> &Terminator {
619        &self.terminator
620    }
621
622    /// Create a new `Writer`
623    pub fn build<W: Write>(self, writer: W) -> Writer<W> {
624        let mut builder = csv::WriterBuilder::new();
625
626        let terminator = match self.terminator {
627            Terminator::CRLF => csv::Terminator::CRLF,
628            Terminator::Any(byte) => csv::Terminator::Any(byte),
629        };
630
631        let writer = builder
632            .delimiter(self.delimiter)
633            .quote(self.quote)
634            .quote_style(self.quote_style)
635            .double_quote(self.double_quote)
636            .escape(self.escape)
637            .terminator(terminator)
638            .from_writer(writer);
639        Writer {
640            writer,
641            beginning: true,
642            has_headers: self.has_header,
643            date_format: self.date_format,
644            datetime_format: self.datetime_format,
645            time_format: self.time_format,
646            timestamp_format: self.timestamp_format,
647            timestamp_tz_format: self.timestamp_tz_format,
648            null_value: self.null_value,
649            ignore_leading_whitespace: self.ignore_leading_whitespace,
650            ignore_trailing_whitespace: self.ignore_trailing_whitespace,
651        }
652    }
653}
654
655#[cfg(test)]
656#[expect(
657    clippy::verbose_file_reads,
658    reason = "`tempfile::tempfile()` has no path, so the contents can only be read back through the handle"
659)]
660mod tests {
661    use super::*;
662
663    use crate::ReaderBuilder;
664    use arrow_array::builder::{
665        BinaryBuilder, Decimal32Builder, Decimal64Builder, Decimal128Builder, Decimal256Builder,
666        FixedSizeBinaryBuilder, LargeBinaryBuilder,
667    };
668    use arrow_array::types::*;
669    use arrow_buffer::i256;
670    use core::str;
671    use std::io::{Cursor, Read, Seek};
672    use std::sync::Arc;
673
674    #[test]
675    fn test_write_csv() {
676        let schema = Schema::new(vec![
677            Field::new("c1", DataType::Utf8, false),
678            Field::new("c2", DataType::Float64, true),
679            Field::new("c3", DataType::UInt32, false),
680            Field::new("c4", DataType::Boolean, true),
681            Field::new("c5", DataType::Timestamp(TimeUnit::Millisecond, None), true),
682            Field::new("c6", DataType::Time32(TimeUnit::Second), false),
683            Field::new_dictionary("c7", DataType::Int32, DataType::Utf8, false),
684        ]);
685
686        let c1 = StringArray::from(vec![
687            "Lorem ipsum dolor sit amet",
688            "consectetur adipiscing elit",
689            "sed do eiusmod tempor",
690        ]);
691        let c2 =
692            PrimitiveArray::<Float64Type>::from(vec![Some(123.564532), None, Some(-556132.25)]);
693        let c3 = PrimitiveArray::<UInt32Type>::from(vec![3, 2, 1]);
694        let c4 = BooleanArray::from(vec![Some(true), Some(false), None]);
695        let c5 =
696            TimestampMillisecondArray::from(vec![None, Some(1555584887378), Some(1555555555555)]);
697        let c6 = Time32SecondArray::from(vec![1234, 24680, 85563]);
698        let c7: DictionaryArray<Int32Type> =
699            vec!["cupcakes", "cupcakes", "foo"].into_iter().collect();
700
701        let batch = RecordBatch::try_new(
702            Arc::new(schema),
703            vec![
704                Arc::new(c1),
705                Arc::new(c2),
706                Arc::new(c3),
707                Arc::new(c4),
708                Arc::new(c5),
709                Arc::new(c6),
710                Arc::new(c7),
711            ],
712        )
713        .unwrap();
714
715        let mut file = tempfile::tempfile().unwrap();
716
717        let mut writer = Writer::new(&mut file);
718        let batches = vec![&batch, &batch];
719        for batch in batches {
720            writer.write(batch).unwrap();
721        }
722        drop(writer);
723
724        // check that file was written successfully
725        file.rewind().unwrap();
726        let mut buffer: Vec<u8> = vec![];
727        file.read_to_end(&mut buffer).unwrap();
728
729        let expected = r"c1,c2,c3,c4,c5,c6,c7
730Lorem ipsum dolor sit amet,123.564532,3,true,,00:20:34,cupcakes
731consectetur adipiscing elit,,2,false,2019-04-18T10:54:47.378,06:51:20,cupcakes
732sed do eiusmod tempor,-556132.25,1,,2019-04-18T02:45:55.555,23:46:03,foo
733Lorem ipsum dolor sit amet,123.564532,3,true,,00:20:34,cupcakes
734consectetur adipiscing elit,,2,false,2019-04-18T10:54:47.378,06:51:20,cupcakes
735sed do eiusmod tempor,-556132.25,1,,2019-04-18T02:45:55.555,23:46:03,foo
736";
737        assert_eq!(expected, str::from_utf8(&buffer).unwrap());
738    }
739
740    #[test]
741    fn test_write_csv_decimal() {
742        let schema = Schema::new(vec![
743            Field::new("c1", DataType::Decimal32(9, 6), true),
744            Field::new("c2", DataType::Decimal64(17, 6), true),
745            Field::new("c3", DataType::Decimal128(38, 6), true),
746            Field::new("c4", DataType::Decimal256(76, 6), true),
747        ]);
748
749        let mut c1_builder = Decimal32Builder::new().with_data_type(DataType::Decimal32(9, 6));
750        c1_builder.extend(vec![Some(-3335724), Some(2179404), None, Some(290472)]);
751        let c1 = c1_builder.finish();
752
753        let mut c2_builder = Decimal64Builder::new().with_data_type(DataType::Decimal64(17, 6));
754        c2_builder.extend(vec![Some(-3335724), Some(2179404), None, Some(290472)]);
755        let c2 = c2_builder.finish();
756
757        let mut c3_builder = Decimal128Builder::new().with_data_type(DataType::Decimal128(38, 6));
758        c3_builder.extend(vec![Some(-3335724), Some(2179404), None, Some(290472)]);
759        let c3 = c3_builder.finish();
760
761        let mut c4_builder = Decimal256Builder::new().with_data_type(DataType::Decimal256(76, 6));
762        c4_builder.extend(vec![
763            Some(i256::from_i128(-3335724)),
764            Some(i256::from_i128(2179404)),
765            None,
766            Some(i256::from_i128(290472)),
767        ]);
768        let c4 = c4_builder.finish();
769
770        let batch = RecordBatch::try_new(
771            Arc::new(schema),
772            vec![Arc::new(c1), Arc::new(c2), Arc::new(c3), Arc::new(c4)],
773        )
774        .unwrap();
775
776        let mut file = tempfile::tempfile().unwrap();
777
778        let mut writer = Writer::new(&mut file);
779        let batches = vec![&batch, &batch];
780        for batch in batches {
781            writer.write(batch).unwrap();
782        }
783        drop(writer);
784
785        // check that file was written successfully
786        file.rewind().unwrap();
787        let mut buffer: Vec<u8> = vec![];
788        file.read_to_end(&mut buffer).unwrap();
789
790        let expected = r"c1,c2,c3,c4
791-3.335724,-3.335724,-3.335724,-3.335724
7922.179404,2.179404,2.179404,2.179404
793,,,
7940.290472,0.290472,0.290472,0.290472
795-3.335724,-3.335724,-3.335724,-3.335724
7962.179404,2.179404,2.179404,2.179404
797,,,
7980.290472,0.290472,0.290472,0.290472
799";
800        assert_eq!(expected, str::from_utf8(&buffer).unwrap());
801    }
802
803    #[test]
804    fn test_write_csv_custom_options() {
805        let schema = Schema::new(vec![
806            Field::new("c1", DataType::Utf8, false),
807            Field::new("c2", DataType::Float64, true),
808            Field::new("c3", DataType::UInt32, false),
809            Field::new("c4", DataType::Boolean, true),
810            Field::new("c6", DataType::Time32(TimeUnit::Second), false),
811        ]);
812
813        let c1 = StringArray::from(vec![
814            "Lorem ipsum \ndolor sit amet",
815            "consectetur \"adipiscing\" elit",
816            "sed do eiusmod tempor",
817        ]);
818        let c2 =
819            PrimitiveArray::<Float64Type>::from(vec![Some(123.564532), None, Some(-556132.25)]);
820        let c3 = PrimitiveArray::<UInt32Type>::from(vec![3, 2, 1]);
821        let c4 = BooleanArray::from(vec![Some(true), Some(false), None]);
822        let c6 = Time32SecondArray::from(vec![1234, 24680, 85563]);
823
824        let batch = RecordBatch::try_new(
825            Arc::new(schema),
826            vec![
827                Arc::new(c1),
828                Arc::new(c2),
829                Arc::new(c3),
830                Arc::new(c4),
831                Arc::new(c6),
832            ],
833        )
834        .unwrap();
835
836        let mut file = tempfile::tempfile().unwrap();
837
838        let builder = WriterBuilder::new()
839            .with_header(false)
840            .with_delimiter(b'|')
841            .with_quote(b'\'')
842            .with_null("NULL".to_string())
843            .with_time_format("%r".to_string());
844        let mut writer = builder.build(&mut file);
845        let batches = vec![&batch];
846        for batch in batches {
847            writer.write(batch).unwrap();
848        }
849        drop(writer);
850
851        // check that file was written successfully
852        file.rewind().unwrap();
853        let mut buffer: Vec<u8> = vec![];
854        file.read_to_end(&mut buffer).unwrap();
855
856        assert_eq!(
857            "'Lorem ipsum \ndolor sit amet'|123.564532|3|true|12:20:34 AM\nconsectetur \"adipiscing\" elit|NULL|2|false|06:51:20 AM\nsed do eiusmod tempor|-556132.25|1|NULL|11:46:03 PM\n"
858            .to_string(),
859            String::from_utf8(buffer).unwrap()
860        );
861
862        let mut file = tempfile::tempfile().unwrap();
863
864        let builder = WriterBuilder::new()
865            .with_header(true)
866            .with_double_quote(false)
867            .with_escape(b'$');
868        let mut writer = builder.build(&mut file);
869        let batches = vec![&batch];
870        for batch in batches {
871            writer.write(batch).unwrap();
872        }
873        drop(writer);
874
875        file.rewind().unwrap();
876        let mut buffer: Vec<u8> = vec![];
877        file.read_to_end(&mut buffer).unwrap();
878
879        assert_eq!(
880            "c1,c2,c3,c4,c6\n\"Lorem ipsum \ndolor sit amet\",123.564532,3,true,00:20:34\n\"consectetur $\"adipiscing$\" elit\",,2,false,06:51:20\nsed do eiusmod tempor,-556132.25,1,,23:46:03\n"
881            .to_string(),
882            String::from_utf8(buffer).unwrap()
883        );
884    }
885
886    #[test]
887    fn test_conversion_consistency() {
888        // test if we can serialize and deserialize whilst retaining the same type information/ precision
889
890        let schema = Schema::new(vec![
891            Field::new("c1", DataType::Date32, false),
892            Field::new("c2", DataType::Date64, false),
893            Field::new("c3", DataType::Timestamp(TimeUnit::Nanosecond, None), false),
894        ]);
895
896        let nanoseconds = vec![
897            1599566300000000000,
898            1599566200000000000,
899            1599566100000000000,
900        ];
901        let c1 = Date32Array::from(vec![3, 2, 1]);
902        let c2 = Date64Array::from(vec![3, 2, 1]);
903        let c3 = TimestampNanosecondArray::from(nanoseconds.clone());
904
905        let batch = RecordBatch::try_new(
906            Arc::new(schema.clone()),
907            vec![Arc::new(c1), Arc::new(c2), Arc::new(c3)],
908        )
909        .unwrap();
910
911        let builder = WriterBuilder::new().with_header(false);
912
913        let mut buf: Cursor<Vec<u8>> = Default::default();
914        // drop the writer early to release the borrow.
915        {
916            let mut writer = builder.build(&mut buf);
917            writer.write(&batch).unwrap();
918        }
919        buf.set_position(0);
920
921        let mut reader = ReaderBuilder::new(Arc::new(schema))
922            .with_batch_size(3)
923            .build_buffered(buf)
924            .unwrap();
925
926        let rb = reader.next().unwrap().unwrap();
927        let c1 = rb.column(0).as_any().downcast_ref::<Date32Array>().unwrap();
928        let c2 = rb.column(1).as_any().downcast_ref::<Date64Array>().unwrap();
929        let c3 = rb
930            .column(2)
931            .as_any()
932            .downcast_ref::<TimestampNanosecondArray>()
933            .unwrap();
934
935        let actual = c1.into_iter().collect::<Vec<_>>();
936        let expected = vec![Some(3), Some(2), Some(1)];
937        assert_eq!(actual, expected);
938        let actual = c2.into_iter().collect::<Vec<_>>();
939        let expected = vec![Some(3), Some(2), Some(1)];
940        assert_eq!(actual, expected);
941        let actual = c3.into_iter().collect::<Vec<_>>();
942        let expected = nanoseconds.into_iter().map(Some).collect::<Vec<_>>();
943        assert_eq!(actual, expected);
944    }
945
946    #[test]
947    fn test_write_csv_invalid_cast() {
948        let schema = Schema::new(vec![
949            Field::new("c0", DataType::UInt32, false),
950            Field::new("c1", DataType::Date64, false),
951        ]);
952
953        let c0 = UInt32Array::from(vec![Some(123), Some(234)]);
954        let c1 = Date64Array::from(vec![Some(1926632005177), Some(1926632005177685347)]);
955        let batch =
956            RecordBatch::try_new(Arc::new(schema), vec![Arc::new(c0), Arc::new(c1)]).unwrap();
957
958        let mut file = tempfile::tempfile().unwrap();
959        let mut writer = Writer::new(&mut file);
960        let batches = vec![&batch, &batch];
961
962        for batch in batches {
963            let err = writer.write(batch).unwrap_err().to_string();
964            assert_eq!(
965                err,
966                "Csv error: Error processing row 2, col 2: Cast error: Failed to convert 1926632005177685347 to temporal for Date64"
967            )
968        }
969        drop(writer);
970    }
971
972    #[test]
973    fn test_write_csv_using_rfc3339() {
974        let schema = Schema::new(vec![
975            Field::new(
976                "c1",
977                DataType::Timestamp(TimeUnit::Millisecond, Some("+00:00".into())),
978                true,
979            ),
980            Field::new("c2", DataType::Timestamp(TimeUnit::Millisecond, None), true),
981            Field::new("c3", DataType::Date32, false),
982            Field::new("c4", DataType::Time32(TimeUnit::Second), false),
983        ]);
984
985        let c1 = TimestampMillisecondArray::from(vec![Some(1555584887378), Some(1635577147000)])
986            .with_timezone("+00:00".to_string());
987        let c2 = TimestampMillisecondArray::from(vec![Some(1555584887378), Some(1635577147000)]);
988        let c3 = Date32Array::from(vec![3, 2]);
989        let c4 = Time32SecondArray::from(vec![1234, 24680]);
990
991        let batch = RecordBatch::try_new(
992            Arc::new(schema),
993            vec![Arc::new(c1), Arc::new(c2), Arc::new(c3), Arc::new(c4)],
994        )
995        .unwrap();
996
997        let mut file = tempfile::tempfile().unwrap();
998
999        let builder = WriterBuilder::new();
1000        let mut writer = builder.build(&mut file);
1001        let batches = vec![&batch];
1002        for batch in batches {
1003            writer.write(batch).unwrap();
1004        }
1005        drop(writer);
1006
1007        file.rewind().unwrap();
1008        let mut buffer: Vec<u8> = vec![];
1009        file.read_to_end(&mut buffer).unwrap();
1010
1011        assert_eq!(
1012            "c1,c2,c3,c4
10132019-04-18T10:54:47.378Z,2019-04-18T10:54:47.378,1970-01-04,00:20:34
10142021-10-30T06:59:07Z,2021-10-30T06:59:07,1970-01-03,06:51:20\n",
1015            String::from_utf8(buffer).unwrap()
1016        );
1017    }
1018
1019    #[test]
1020    fn test_write_csv_tz_format() {
1021        let schema = Schema::new(vec![
1022            Field::new(
1023                "c1",
1024                DataType::Timestamp(TimeUnit::Millisecond, Some("+02:00".into())),
1025                true,
1026            ),
1027            Field::new(
1028                "c2",
1029                DataType::Timestamp(TimeUnit::Second, Some("+04:00".into())),
1030                true,
1031            ),
1032        ]);
1033        let c1 = TimestampMillisecondArray::from(vec![Some(1_000), Some(2_000)])
1034            .with_timezone("+02:00".to_string());
1035        let c2 = TimestampSecondArray::from(vec![Some(1_000_000), None])
1036            .with_timezone("+04:00".to_string());
1037        let batch =
1038            RecordBatch::try_new(Arc::new(schema), vec![Arc::new(c1), Arc::new(c2)]).unwrap();
1039
1040        let mut file = tempfile::tempfile().unwrap();
1041        let mut writer = WriterBuilder::new()
1042            .with_timestamp_tz_format("%M:%H".to_string())
1043            .build(&mut file);
1044        writer.write(&batch).unwrap();
1045
1046        drop(writer);
1047        file.rewind().unwrap();
1048        let mut buffer: Vec<u8> = vec![];
1049        file.read_to_end(&mut buffer).unwrap();
1050
1051        let output = String::from_utf8(buffer).unwrap();
1052        assert_eq!(output, "c1,c2\n00:02,46:17\n00:02,\n");
1053    }
1054
1055    #[test]
1056    fn test_write_csv_with_lf_terminator() {
1057        let output = write_batch_with_terminator(Terminator::Any(b'\n'));
1058        assert_eq!(output, "c1,c2\nhello,1\nworld,2\n");
1059    }
1060
1061    #[test]
1062    fn test_write_csv_with_crlf_terminator() {
1063        let output = write_batch_with_terminator(Terminator::CRLF);
1064        assert_eq!(output, "c1,c2\r\nhello,1\r\nworld,2\r\n");
1065    }
1066
1067    #[test]
1068    fn test_write_csv_with_any_terminator() {
1069        let output = write_batch_with_terminator(Terminator::Any(b'|'));
1070        assert_eq!(output, "c1,c2|hello,1|world,2|");
1071    }
1072
1073    fn write_batch_with_terminator(terminator: Terminator) -> String {
1074        let schema = Schema::new(vec![
1075            Field::new("c1", DataType::Utf8, false),
1076            Field::new("c2", DataType::UInt32, false),
1077        ]);
1078
1079        let c1 = StringArray::from(vec!["hello", "world"]);
1080        let c2 = PrimitiveArray::<UInt32Type>::from(vec![1, 2]);
1081
1082        let batch =
1083            RecordBatch::try_new(Arc::new(schema), vec![Arc::new(c1), Arc::new(c2)]).unwrap();
1084
1085        let mut buf = Vec::new();
1086        let mut writer = WriterBuilder::new()
1087            .with_line_terminator(terminator)
1088            .build(&mut buf);
1089        writer.write(&batch).unwrap();
1090        drop(writer);
1091
1092        String::from_utf8(buf).unwrap()
1093    }
1094
1095    #[test]
1096    fn test_write_csv_binary() {
1097        let fixed_size = 8;
1098        let schema = SchemaRef::new(Schema::new(vec![
1099            Field::new("c1", DataType::Binary, true),
1100            Field::new("c2", DataType::FixedSizeBinary(fixed_size), true),
1101            Field::new("c3", DataType::LargeBinary, true),
1102        ]));
1103        let mut c1_builder = BinaryBuilder::new();
1104        c1_builder.append_value(b"Homer");
1105        c1_builder.append_value(b"Bart");
1106        c1_builder.append_null();
1107        c1_builder.append_value(b"Bob");
1108        let mut c2_builder = FixedSizeBinaryBuilder::new(fixed_size);
1109        c2_builder.append_value(b"Simpson ").unwrap();
1110        c2_builder.append_value(b"Simpson ").unwrap();
1111        c2_builder.append_null();
1112        c2_builder.append_value(b"Flanders").unwrap();
1113        let mut c3_builder = LargeBinaryBuilder::new();
1114        c3_builder.append_null();
1115        c3_builder.append_null();
1116        c3_builder.append_value(b"Comic Book Guy");
1117        c3_builder.append_null();
1118
1119        let batch = RecordBatch::try_new(
1120            schema,
1121            vec![
1122                Arc::new(c1_builder.finish()) as ArrayRef,
1123                Arc::new(c2_builder.finish()) as ArrayRef,
1124                Arc::new(c3_builder.finish()) as ArrayRef,
1125            ],
1126        )
1127        .unwrap();
1128
1129        let mut buf = Vec::new();
1130        let builder = WriterBuilder::new();
1131        let mut writer = builder.build(&mut buf);
1132        writer.write(&batch).unwrap();
1133        drop(writer);
1134        assert_eq!(
1135            "\
1136            c1,c2,c3\n\
1137            486f6d6572,53696d70736f6e20,\n\
1138            42617274,53696d70736f6e20,\n\
1139            ,,436f6d696320426f6f6b20477579\n\
1140            426f62,466c616e64657273,\n\
1141            ",
1142            String::from_utf8(buf).unwrap()
1143        );
1144    }
1145
1146    #[test]
1147    fn test_write_csv_whitespace_handling() {
1148        let schema = Schema::new(vec![
1149            Field::new("c1", DataType::Utf8, false),
1150            Field::new("c2", DataType::Float64, true),
1151            Field::new("c3", DataType::Utf8, true),
1152        ]);
1153
1154        let c1 = StringArray::from(vec![
1155            "  leading space",
1156            "trailing space  ",
1157            "  both spaces  ",
1158            "no spaces",
1159        ]);
1160        let c2 = PrimitiveArray::<Float64Type>::from(vec![
1161            Some(123.45),
1162            Some(678.90),
1163            None,
1164            Some(111.22),
1165        ]);
1166        let c3 = StringArray::from(vec![
1167            Some("  test  "),
1168            Some("value  "),
1169            None,
1170            Some("  another"),
1171        ]);
1172
1173        let batch = RecordBatch::try_new(
1174            Arc::new(schema),
1175            vec![Arc::new(c1), Arc::new(c2), Arc::new(c3)],
1176        )
1177        .unwrap();
1178
1179        // Test with no whitespace handling (default)
1180        let mut buf = Vec::new();
1181        let builder = WriterBuilder::new();
1182        let mut writer = builder.build(&mut buf);
1183        writer.write(&batch).unwrap();
1184        drop(writer);
1185        assert_eq!(
1186            "c1,c2,c3\n  leading space,123.45,  test  \ntrailing space  ,678.9,value  \n  both spaces  ,,\nno spaces,111.22,  another\n",
1187            String::from_utf8(buf).unwrap()
1188        );
1189
1190        // Test with ignore leading whitespace only
1191        let mut buf = Vec::new();
1192        let builder = WriterBuilder::new().with_ignore_leading_whitespace(true);
1193        let mut writer = builder.build(&mut buf);
1194        writer.write(&batch).unwrap();
1195        drop(writer);
1196        assert_eq!(
1197            "c1,c2,c3\nleading space,123.45,test  \ntrailing space  ,678.9,value  \nboth spaces  ,,\nno spaces,111.22,another\n",
1198            String::from_utf8(buf).unwrap()
1199        );
1200
1201        // Test with ignore trailing whitespace only
1202        let mut buf = Vec::new();
1203        let builder = WriterBuilder::new().with_ignore_trailing_whitespace(true);
1204        let mut writer = builder.build(&mut buf);
1205        writer.write(&batch).unwrap();
1206        drop(writer);
1207        assert_eq!(
1208            "c1,c2,c3\n  leading space,123.45,  test\ntrailing space,678.9,value\n  both spaces,,\nno spaces,111.22,  another\n",
1209            String::from_utf8(buf).unwrap()
1210        );
1211
1212        // Test with both ignore leading and trailing whitespace
1213        let mut buf = Vec::new();
1214        let builder = WriterBuilder::new()
1215            .with_ignore_leading_whitespace(true)
1216            .with_ignore_trailing_whitespace(true);
1217        let mut writer = builder.build(&mut buf);
1218        writer.write(&batch).unwrap();
1219        drop(writer);
1220        assert_eq!(
1221            "c1,c2,c3\nleading space,123.45,test\ntrailing space,678.9,value\nboth spaces,,\nno spaces,111.22,another\n",
1222            String::from_utf8(buf).unwrap()
1223        );
1224    }
1225
1226    #[test]
1227    fn test_write_csv_whitespace_with_special_chars() {
1228        let schema = Schema::new(vec![Field::new("c1", DataType::Utf8, false)]);
1229
1230        let c1 = StringArray::from(vec![
1231            "  quoted \"value\"  ",
1232            "  new\nline  ",
1233            "  comma,value  ",
1234            "\ttab\tvalue\t",
1235        ]);
1236
1237        let batch = RecordBatch::try_new(Arc::new(schema), vec![Arc::new(c1)]).unwrap();
1238
1239        // Test with both ignore leading and trailing whitespace
1240        let mut buf = Vec::new();
1241        let builder = WriterBuilder::new()
1242            .with_ignore_leading_whitespace(true)
1243            .with_ignore_trailing_whitespace(true);
1244        let mut writer = builder.build(&mut buf);
1245        writer.write(&batch).unwrap();
1246        drop(writer);
1247
1248        // Note: tabs are trimmed as they are whitespace characters
1249        assert_eq!(
1250            "c1\n\"quoted \"\"value\"\"\"\n\"new\nline\"\n\"comma,value\"\ntab\tvalue\n",
1251            String::from_utf8(buf).unwrap()
1252        );
1253    }
1254
1255    #[test]
1256    fn test_write_csv_whitespace_all_string_types() {
1257        use arrow_array::{LargeStringArray, StringViewArray};
1258
1259        let schema = Schema::new(vec![
1260            Field::new("utf8", DataType::Utf8, false),
1261            Field::new("large_utf8", DataType::LargeUtf8, false),
1262            Field::new("utf8_view", DataType::Utf8View, false),
1263        ]);
1264
1265        let utf8 = StringArray::from(vec!["  leading", "trailing  ", "  both  ", "no_spaces"]);
1266
1267        let large_utf8 =
1268            LargeStringArray::from(vec!["  leading", "trailing  ", "  both  ", "no_spaces"]);
1269
1270        let utf8_view =
1271            StringViewArray::from(vec!["  leading", "trailing  ", "  both  ", "no_spaces"]);
1272
1273        let batch = RecordBatch::try_new(
1274            Arc::new(schema),
1275            vec![Arc::new(utf8), Arc::new(large_utf8), Arc::new(utf8_view)],
1276        )
1277        .unwrap();
1278
1279        // Test with no whitespace handling (default)
1280        let mut buf = Vec::new();
1281        let builder = WriterBuilder::new();
1282        let mut writer = builder.build(&mut buf);
1283        writer.write(&batch).unwrap();
1284        drop(writer);
1285        assert_eq!(
1286            "utf8,large_utf8,utf8_view\n  leading,  leading,  leading\ntrailing  ,trailing  ,trailing  \n  both  ,  both  ,  both  \nno_spaces,no_spaces,no_spaces\n",
1287            String::from_utf8(buf).unwrap()
1288        );
1289
1290        // Test with both ignore leading and trailing whitespace
1291        let mut buf = Vec::new();
1292        let builder = WriterBuilder::new()
1293            .with_ignore_leading_whitespace(true)
1294            .with_ignore_trailing_whitespace(true);
1295        let mut writer = builder.build(&mut buf);
1296        writer.write(&batch).unwrap();
1297        drop(writer);
1298        assert_eq!(
1299            "utf8,large_utf8,utf8_view\nleading,leading,leading\ntrailing,trailing,trailing\nboth,both,both\nno_spaces,no_spaces,no_spaces\n",
1300            String::from_utf8(buf).unwrap()
1301        );
1302
1303        // Test with only leading whitespace trimming
1304        let mut buf = Vec::new();
1305        let builder = WriterBuilder::new().with_ignore_leading_whitespace(true);
1306        let mut writer = builder.build(&mut buf);
1307        writer.write(&batch).unwrap();
1308        drop(writer);
1309        assert_eq!(
1310            "utf8,large_utf8,utf8_view\nleading,leading,leading\ntrailing  ,trailing  ,trailing  \nboth  ,both  ,both  \nno_spaces,no_spaces,no_spaces\n",
1311            String::from_utf8(buf).unwrap()
1312        );
1313
1314        // Test with only trailing whitespace trimming
1315        let mut buf = Vec::new();
1316        let builder = WriterBuilder::new().with_ignore_trailing_whitespace(true);
1317        let mut writer = builder.build(&mut buf);
1318        writer.write(&batch).unwrap();
1319        drop(writer);
1320        assert_eq!(
1321            "utf8,large_utf8,utf8_view\n  leading,  leading,  leading\ntrailing,trailing,trailing\n  both,  both,  both\nno_spaces,no_spaces,no_spaces\n",
1322            String::from_utf8(buf).unwrap()
1323        );
1324    }
1325
1326    fn write_quote_style(batch: &RecordBatch, quote_style: QuoteStyle) -> String {
1327        let mut buf = Vec::new();
1328        let mut writer = WriterBuilder::new()
1329            .with_quote_style(quote_style)
1330            .build(&mut buf);
1331        writer.write(batch).unwrap();
1332        drop(writer);
1333        String::from_utf8(buf).unwrap()
1334    }
1335
1336    fn write_quote_style_with_null(
1337        batch: &RecordBatch,
1338        quote_style: QuoteStyle,
1339        null_value: &str,
1340    ) -> String {
1341        let mut buf = Vec::new();
1342        let mut writer = WriterBuilder::new()
1343            .with_quote_style(quote_style)
1344            .with_null(null_value.to_string())
1345            .build(&mut buf);
1346        writer.write(batch).unwrap();
1347        drop(writer);
1348        String::from_utf8(buf).unwrap()
1349    }
1350
1351    #[test]
1352    fn test_write_csv_quote_style() {
1353        let schema = Schema::new(vec![
1354            Field::new("text", DataType::Utf8, false),
1355            Field::new("number", DataType::Int32, false),
1356            Field::new("float", DataType::Float64, false),
1357        ]);
1358
1359        let text = StringArray::from(vec!["hello", "world", "comma,value", "quote\"test"]);
1360        let number = Int32Array::from(vec![1, 2, 3, 4]);
1361        let float = Float64Array::from(vec![1.1, 2.2, 3.3, 4.4]);
1362
1363        let batch = RecordBatch::try_new(
1364            Arc::new(schema),
1365            vec![Arc::new(text), Arc::new(number), Arc::new(float)],
1366        )
1367        .unwrap();
1368
1369        // Test with QuoteStyle::Necessary (default)
1370        assert_eq!(
1371            "text,number,float\nhello,1,1.1\nworld,2,2.2\n\"comma,value\",3,3.3\n\"quote\"\"test\",4,4.4\n",
1372            write_quote_style(&batch, QuoteStyle::Necessary)
1373        );
1374
1375        // Test with QuoteStyle::Always (equivalent to Spark's quoteAll=true)
1376        assert_eq!(
1377            "\"text\",\"number\",\"float\"\n\"hello\",\"1\",\"1.1\"\n\"world\",\"2\",\"2.2\"\n\"comma,value\",\"3\",\"3.3\"\n\"quote\"\"test\",\"4\",\"4.4\"\n",
1378            write_quote_style(&batch, QuoteStyle::Always)
1379        );
1380
1381        // Test with QuoteStyle::NonNumeric
1382        assert_eq!(
1383            "\"text\",\"number\",\"float\"\n\"hello\",1,1.1\n\"world\",2,2.2\n\"comma,value\",3,3.3\n\"quote\"\"test\",4,4.4\n",
1384            write_quote_style(&batch, QuoteStyle::NonNumeric)
1385        );
1386
1387        // Test with QuoteStyle::Never (warning: can produce invalid CSV)
1388        // Note: This produces invalid CSV for fields with commas or quotes
1389        assert_eq!(
1390            "text,number,float\nhello,1,1.1\nworld,2,2.2\ncomma,value,3,3.3\nquote\"test,4,4.4\n",
1391            write_quote_style(&batch, QuoteStyle::Never)
1392        );
1393    }
1394
1395    #[test]
1396    fn test_write_csv_quote_style_with_nulls() {
1397        let schema = Schema::new(vec![
1398            Field::new("text", DataType::Utf8, true),
1399            Field::new("number", DataType::Int32, true),
1400        ]);
1401
1402        let text = StringArray::from(vec![Some("hello"), None, Some("world")]);
1403        let number = Int32Array::from(vec![Some(1), Some(2), None]);
1404
1405        let batch =
1406            RecordBatch::try_new(Arc::new(schema), vec![Arc::new(text), Arc::new(number)]).unwrap();
1407
1408        // Test with QuoteStyle::Always
1409        assert_eq!(
1410            "\"text\",\"number\"\n\"hello\",\"1\"\n\"\",\"2\"\n\"world\",\"\"\n",
1411            write_quote_style(&batch, QuoteStyle::Always)
1412        );
1413
1414        // Test with QuoteStyle::Always and custom null value
1415        assert_eq!(
1416            "\"text\",\"number\"\n\"hello\",\"1\"\n\"NULL\",\"2\"\n\"world\",\"NULL\"\n",
1417            write_quote_style_with_null(&batch, QuoteStyle::Always, "NULL")
1418        );
1419    }
1420}