arrow_ipc/lib.rs
1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4
5//! Support for the [Arrow IPC Format]
6//!
7//! The Arrow IPC format defines how to read and write [`RecordBatch`]es to/from
8//! a file or stream of bytes. This format can be used to serialize and deserialize
9//! data to files and over the network.
10//!
11//! There are two variants of the IPC format:
12//! 1. [IPC Streaming Format]: Supports streaming data sources, implemented by
13//! [StreamReader] and [StreamWriter]
14//!
15//! 2. [IPC File Format]: Supports random access, implemented by [FileReader] and
16//! [FileWriter].
17//!
18//! See the [`reader`] and [`writer`] modules for more information.
19//!
20//! [Arrow IPC Format]: https://arrow.apache.org/docs/format/Columnar.html#serialization-and-interprocess-communication-ipc
21//! [IPC Streaming Format]: https://arrow.apache.org/docs/format/Columnar.html#ipc-streaming-format
22//! [StreamReader]: reader::StreamReader
23//! [StreamWriter]: writer::StreamWriter
24//! [IPC File Format]: https://arrow.apache.org/docs/format/Columnar.html#ipc-file-format
25//! [FileReader]: reader::FileReader
26//! [FileWriter]: writer::FileWriter
27//!
28//! # Platform Support
29//!
30//! Only little-endian platforms are officially supported and tested in CI.
31//! Big-endian platforms are not tested in CI and may not work correctly.
32//! Fixes for big-endian platforms are welcome and handled on a best-effort basis,
33//! but compatibility is not guaranteed.
34
35#![doc(
36 html_logo_url = "https://arrow.apache.org/img/arrow-logo_chevrons_black-txt_white-bg.svg",
37 html_favicon_url = "https://arrow.apache.org/img/arrow-logo_chevrons_black-txt_transparent-bg.svg"
38)]
39#![cfg_attr(docsrs, feature(doc_cfg))]
40#![warn(missing_docs)]
41pub mod convert;
42pub mod reader;
43pub mod writer;
44
45mod compression;
46
47#[cfg(test)]
48mod tests;
49
50// This code is generated so we don't want to fix any lint violations manually
51#[allow(clippy::allow_attributes)]
52#[allow(mismatched_lifetime_syntaxes)]
53#[allow(clippy::redundant_closure)]
54#[allow(clippy::needless_lifetimes)]
55#[allow(clippy::extra_unused_lifetimes)]
56#[allow(clippy::redundant_static_lifetimes)]
57#[allow(clippy::redundant_field_names)]
58#[allow(non_camel_case_types)]
59#[allow(missing_docs)] // Because this is autogenerated
60pub mod r#gen;
61
62pub use self::r#gen::File::*;
63pub use self::r#gen::Message::*;
64pub use self::r#gen::Schema::*;
65pub use self::r#gen::SparseTensor::*;
66pub use self::r#gen::Tensor::*;
67
68const ARROW_MAGIC: [u8; 6] = *b"ARROW1";
69const CONTINUATION_MARKER: [u8; 4] = [0xff; 4];
70
71impl Endianness {
72 /// Returns true if the endianness of the source system matches the endianness of the target system.
73 pub fn equals_to_target_endianness(self) -> bool {
74 match self {
75 Self::Little => cfg!(target_endian = "little"),
76 Self::Big => cfg!(target_endian = "big"),
77 _ => false,
78 }
79 }
80}