parquet/file/page_index/
offset_index.rs1use std::io::Write;
10
11use crate::parquet_thrift::{
12 ElementType, FieldType, ReadThrift, ThriftCompactInputProtocol, ThriftCompactOutputProtocol,
13 WriteThrift, WriteThriftField, check_list_size, read_thrift_vec, validate_list_type,
14};
15use crate::{
16 errors::{ParquetError, Result},
17 thrift_struct,
18};
19
20thrift_struct!(
21pub struct PageLocation {
23 1: required i64 offset
25 2: required i32 compressed_page_size
27 3: required i64 first_row_index
31}
32);
33
34thrift_struct!(
35pub struct OffsetIndexMetaData {
43 1: required list<PageLocation> page_locations
45 2: optional list<i64> unencoded_byte_array_data_bytes
48}
49);
50
51impl OffsetIndexMetaData {
52 pub fn page_locations(&self) -> &Vec<PageLocation> {
54 &self.page_locations
55 }
56
57 pub fn unencoded_byte_array_data_bytes(&self) -> Option<&Vec<i64>> {
60 self.unencoded_byte_array_data_bytes.as_ref()
61 }
62
63 pub(super) fn try_from_fast<'a, R: ThriftCompactInputProtocol<'a>>(
67 prot: &mut R,
68 ) -> Result<Self> {
69 let (field_type, delta) = prot.read_field_header()?;
74 if delta != 1 || field_type != FieldType::List as u8 {
75 return Err(general_err!("error reading OffsetIndex::page_locations"));
76 }
77
78 let list_ident = prot.read_list_begin()?;
80 validate_list_type(ElementType::Struct, &list_ident)?;
81 let size = list_ident.size as usize;
83 check_list_size(prot.remaining_bytes(), size)?;
84 let mut page_locations = Vec::with_capacity(size);
85 for _ in 0..list_ident.size {
86 page_locations.push(read_page_location(prot)?);
87 }
88
89 let mut unencoded_byte_array_data_bytes: Option<Vec<i64>> = None;
90
91 let (mut field_type, delta) = prot.read_field_header()?;
93 if field_type == FieldType::List as u8 {
94 if delta != 1 {
95 return Err(general_err!(
96 "encountered unknown field while reading OffsetIndex"
97 ));
98 }
99 let vec = read_thrift_vec::<i64, R>(&mut *prot)?;
100 unencoded_byte_array_data_bytes = Some(vec);
101
102 (field_type, _) = prot.read_field_header()?;
104 }
105
106 if field_type != FieldType::Stop as u8 {
107 return Err(general_err!(
108 "encountered unknown field while reading OffsetIndex"
109 ));
110 }
111
112 Ok(Self {
113 page_locations,
114 unencoded_byte_array_data_bytes,
115 })
116 }
117}
118
119fn read_page_location<'a, R: ThriftCompactInputProtocol<'a>>(prot: &mut R) -> Result<PageLocation> {
124 let (field_type, delta) = prot.read_field_header()?;
126 if delta != 1 || field_type != FieldType::I64 as u8 {
127 return Err(general_err!("error reading PageLocation::offset"));
128 }
129 let offset = prot.read_i64()?;
130
131 let (field_type, delta) = prot.read_field_header()?;
132 if delta != 1 || field_type != FieldType::I32 as u8 {
133 return Err(general_err!(
134 "error reading PageLocation::compressed_page_size"
135 ));
136 }
137 let compressed_page_size = prot.read_i32()?;
138
139 let (field_type, delta) = prot.read_field_header()?;
140 if delta != 1 || field_type != FieldType::I64 as u8 {
141 return Err(general_err!("error reading PageLocation::first_row_index"));
142 }
143 let first_row_index = prot.read_i64()?;
144
145 let (field_type, _) = prot.read_field_header()?;
147 if field_type != FieldType::Stop as u8 {
148 return Err(general_err!("unexpected field in PageLocation"));
149 }
150
151 Ok(PageLocation {
152 offset,
153 compressed_page_size,
154 first_row_index,
155 })
156}
157
158#[cfg(test)]
159mod tests {
160 use super::*;
161 use crate::file::page_index::index_reader::decode_offset_index;
162 use crate::parquet_thrift::{ThriftSliceInputProtocol, tests::test_roundtrip};
163
164 #[test]
165 fn test_offset_idx_rejects_count_exceeding_remaining_input() {
166 let data = [0x19, 0xfc, 0x80, 0x20];
168 let mut prot = ThriftSliceInputProtocol::new(&data);
169 let err = OffsetIndexMetaData::try_from_fast(&mut prot).unwrap_err();
170 assert!(
171 err.to_string()
172 .contains("Thrift list size 4096 exceeds remaining input length 0"),
173 "{err}"
174 );
175 assert!(decode_offset_index(&data).is_err());
176 }
177
178 #[test]
179 fn test_offset_idx_fast_path_roundtrip() {
180 for page_locations in [
181 Vec::new(),
182 vec![PageLocation {
183 offset: 0,
184 compressed_page_size: 10,
185 first_row_index: 0,
186 }],
187 ] {
188 let index = OffsetIndexMetaData {
189 page_locations,
190 unencoded_byte_array_data_bytes: None,
191 };
192 let mut data = Vec::new();
193 index
194 .write_thrift(&mut ThriftCompactOutputProtocol::new(&mut data))
195 .unwrap();
196 let mut prot = ThriftSliceInputProtocol::new(&data);
197 assert_eq!(
198 OffsetIndexMetaData::try_from_fast(&mut prot).unwrap(),
199 index
200 );
201 assert_eq!(decode_offset_index(&data).unwrap(), index);
202 }
203 }
204
205 #[test]
206 fn test_offset_idx_roundtrip() {
207 let page_locations = [
208 PageLocation {
209 offset: 0,
210 compressed_page_size: 10,
211 first_row_index: 0,
212 },
213 PageLocation {
214 offset: 10,
215 compressed_page_size: 20,
216 first_row_index: 100,
217 },
218 ]
219 .to_vec();
220 let unenc = [0i64, 100i64].to_vec();
221
222 test_roundtrip(OffsetIndexMetaData {
223 page_locations: page_locations.clone(),
224 unencoded_byte_array_data_bytes: Some(unenc),
225 });
226 test_roundtrip(OffsetIndexMetaData {
227 page_locations,
228 unencoded_byte_array_data_bytes: None,
229 });
230 }
231}