parquet/util/
utf8.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4
5//! [`check_valid_utf8`] validation function
6use crate::errors::{ParquetError, Result};
7
8/// Check that `val` is a valid UTF-8 sequence.
9///
10/// If the `simdutf8` feature is enabled, this function will use
11/// SIMD-accelerated validation from the [`simdutf8`] crate. Otherwise, it will use
12/// [`std::str::from_utf8`].
13///
14/// # Errors
15///
16/// Returns `Err::General` with a message compatible with [`std::str::from_utf8`] on failure.
17///
18/// # Example
19/// ```
20/// use parquet::utf8::check_valid_utf8;
21/// assert!(check_valid_utf8(b"hello").is_ok());
22/// assert!(check_valid_utf8(b"hello \xF0\x9F\x98\x8E").is_ok());
23/// // invalid UTF-8
24/// assert!(check_valid_utf8(b"hello \xF0\x9F\x98").is_err());
25/// ```
26///
27/// [`simdutf8`]: https://crates.io/crates/simdutf8
28#[inline(always)]
29pub fn check_valid_utf8(val: &[u8]) -> Result<()> {
30    #[cfg(feature = "simdutf8")]
31    match simdutf8::basic::from_utf8(val) {
32        Ok(_) => Ok(()),
33        Err(_) => {
34            // Use simdutf8::compat to return details about the decoding error
35            let e = simdutf8::compat::from_utf8(val).unwrap_err();
36            Err(general_err!("encountered non UTF-8 data: {}", e))
37        }
38    }
39    #[cfg(not(feature = "simdutf8"))]
40    match std::str::from_utf8(val) {
41        Ok(_) => Ok(()),
42        Err(e) => Err(general_err!("encountered non UTF-8 data: {}", e)),
43    }
44}
parquet/util/utf8.rs

parquet/util/
utf8.rs
