Skip to main content

grib/
decoder.rs

1use std::vec::IntoIter;
2
3use crate::{
4    TryFromSlice as _,
5    context::{SectionBody, SubMessage},
6    decoder::{
7        bitmap::{BitmapDecodeIterator, dummy_bitmap_for_nonnullable_data},
8        complex::ComplexPackingDecoded,
9        simple::SimplePackingDecoder,
10        stream::NBitwiseIterator,
11    },
12    def::grib2::{DataRepresentationTemplate, Section5, SectionHeader},
13    error::*,
14    reader::Grib2Read,
15};
16
17/// Decoder for grid point values of GRIB2 submessages.
18///
19/// # Examples
20/// ```
21/// use grib::Grib2SubmessageDecoder;
22///
23/// fn main() -> Result<(), Box<dyn std::error::Error>> {
24///     let f = std::fs::File::open(
25///         "testdata/Z__C_RJTD_20160822020000_NOWC_GPV_Ggis10km_Pphw10_FH0000-0100_grib2.bin",
26///     )?;
27///     let f = std::io::BufReader::new(f);
28///     let grib2 = grib::from_reader(f)?;
29///     let (_index, first_submessage) = grib2.iter().next().unwrap();
30///
31///     let decoder = Grib2SubmessageDecoder::from(first_submessage)?;
32///     let mut decoded = decoder.dispatch()?;
33///     assert_eq!(decoded.size_hint(), (86016, Some(86016)));
34///
35///     let first_value = decoded.next();
36///     assert_eq!(first_value.map(|f| f.is_nan()), Some(true));
37///
38///     let non_nan_value = decoded.find(|f| !f.is_nan());
39///     assert_eq!(non_nan_value.map(|f| f.round()), Some(1.0_f32));
40///
41///     let last_value = decoded.last();
42///     assert_eq!(last_value.map(|f| f.is_nan()), Some(true));
43///     Ok(())
44/// }
45/// ```
46///
47/// If the byte sequences for Sections 5, 6, and 7 of the GRIB2 data are known,
48/// and the number of grid points (described in Section 3) is also known, it is
49/// also possible to create a decoder instance by passing them to
50/// [`Grib2SubmessageDecoder::new`]. The example above is equivalent to the
51/// following:
52///
53/// ```
54/// use std::io::Read;
55///
56/// use grib::Grib2SubmessageDecoder;
57///
58/// fn main() -> Result<(), Box<dyn std::error::Error>> {
59///     let f = std::fs::File::open(
60///         "testdata/Z__C_RJTD_20160822020000_NOWC_GPV_Ggis10km_Pphw10_FH0000-0100_grib2.bin",
61///     )?;
62///     let mut f = std::io::BufReader::new(f);
63///     let mut buf = Vec::new();
64///     f.read_to_end(&mut buf)?;
65///
66///     let decoder = Grib2SubmessageDecoder::new(
67///         86016,
68///         buf[0x0000008f..0x000000a6].to_vec(),
69///         buf[0x000000a6..0x000000ac].to_vec(),
70///         buf[0x000000ac..0x0000061b].to_vec(),
71///     )?;
72///     let mut decoded = decoder.dispatch()?;
73///     assert_eq!(decoded.size_hint(), (86016, Some(86016)));
74///
75///     let first_value = decoded.next();
76///     assert_eq!(first_value.map(|f| f.is_nan()), Some(true));
77///
78///     let non_nan_value = decoded.find(|f| !f.is_nan());
79///     assert_eq!(non_nan_value.map(|f| f.round()), Some(1.0_f32));
80///
81///     let last_value = decoded.last();
82///     assert_eq!(last_value.map(|f| f.is_nan()), Some(true));
83///     Ok(())
84/// }
85/// ```
86pub struct Grib2SubmessageDecoder {
87    num_points_total: usize,
88    sect5_param: Section5,
89    sect6_bytes: Vec<u8>,
90    sect7_bytes: Vec<u8>,
91    limits: Option<DecodeLimits>,
92}
93
94impl Grib2SubmessageDecoder {
95    /// Creates an instance from the number of grid points (described in Section
96    /// 3) and byte sequences for Sections 5, 6, and 7 of the GRIB2 data.
97    ///
98    /// For code examples, refer to the description of this `struct`.
99    pub fn new(
100        num_points_total: usize,
101        sect5_bytes: Vec<u8>,
102        sect6_bytes: Vec<u8>,
103        sect7_bytes: Vec<u8>,
104    ) -> Result<Self, GribError> {
105        Self::check_section_slice(6, &sect6_bytes)?;
106        Self::check_section_slice(7, &sect7_bytes)?;
107        Self::new_from_checked_slices(num_points_total, sect5_bytes, sect6_bytes, sect7_bytes)
108    }
109
110    fn check_section_slice(sect_num: u8, bytes: &[u8]) -> Result<(), GribError> {
111        let header = SectionHeader::try_from_slice(bytes, &mut 0)
112            .map_err(|e| GribError::DecodeError(DecodeError::from(e)))?;
113        if header.sect_num != sect_num || header.len as usize != bytes.len() {
114            return Err(GribError::DecodeError(DecodeError::from(format!(
115                "invalid section {sect_num} slice"
116            ))));
117        }
118        Ok(())
119    }
120
121    fn new_from_checked_slices(
122        num_points_total: usize,
123        sect5_bytes: Vec<u8>,
124        sect6_bytes: Vec<u8>,
125        sect7_bytes: Vec<u8>,
126    ) -> Result<Self, GribError> {
127        let mut pos = 0;
128        let sect5_param = Section5::try_from_slice(&sect5_bytes, &mut pos)
129            .map_err(|e| GribError::DecodeError(DecodeError::from(e)))?;
130        let sect6_bytes = match sect6_bytes[5] {
131            0x00 => sect6_bytes,
132            0xff => {
133                let mut sect6_bytes = sect6_bytes;
134                sect6_bytes.append(&mut dummy_bitmap_for_nonnullable_data(num_points_total));
135                sect6_bytes
136            }
137            n => {
138                return Err(GribError::DecodeError(DecodeError::NotSupported(
139                    "GRIB2 code table 6.0 (bit map indicator)",
140                    n.into(),
141                )));
142            }
143        };
144
145        Ok(Self {
146            num_points_total,
147            sect5_param,
148            sect6_bytes,
149            sect7_bytes,
150            limits: None,
151        })
152    }
153
154    /// Sets up a decoder for grid point values of `submessage`.
155    pub fn from<R: Grib2Read>(submessage: SubMessage<R>) -> Result<Self, GribError> {
156        let mut reader = submessage.9;
157        let sect5 = submessage.5.body;
158        let sect6 = submessage.6.body;
159        let sect7 = submessage.7.body;
160        let sect3_body = match submessage.3.body.body.as_ref() {
161            Some(SectionBody::Section3(b3)) => b3,
162            _ => return Err(GribError::InternalDataError),
163        };
164        let sect3_num_points = sect3_body.num_points() as usize;
165
166        Self::new_from_checked_slices(
167            sect3_num_points,
168            reader.read_sect_as_slice(sect5)?,
169            reader.read_sect_as_slice(sect6)?,
170            reader.read_sect_as_slice(sect7)?,
171        )
172    }
173
174    pub fn with_limits(self, limits: DecodeLimits) -> Self {
175        Self {
176            limits: Some(limits),
177            ..self
178        }
179    }
180
181    /// Dispatches a decoding process and gets an iterator of decoded values.
182    pub fn dispatch(
183        &self,
184    ) -> Result<Grib2DecodedValues<'_, impl Iterator<Item = f32> + '_>, GribError> {
185        let decoder = match &self.sect5_param.payload.template {
186            DataRepresentationTemplate::_5_0(template) => {
187                Grib2ValueIterator::SigSTNS(simple::Simple(self, template).iter()?)
188            }
189            DataRepresentationTemplate::_5_2(template) => {
190                Grib2ValueIterator::SigSC(complex::Complex(self, template).iter()?)
191            }
192            DataRepresentationTemplate::_5_3(template) => {
193                Grib2ValueIterator::SigSSCI(complex::ComplexSpatial(self, template).iter()?)
194            }
195            #[cfg(any(
196                feature = "jpeg2000-unpack-with-openjpeg",
197                feature = "jpeg2000-unpack-with-hayro"
198            ))]
199            DataRepresentationTemplate::_5_40(template) => {
200                Grib2ValueIterator::SigSIm(jpeg2000::Jpeg2000(self, template).iter()?)
201            }
202            #[cfg(feature = "png-unpack-with-png-crate")]
203            DataRepresentationTemplate::_5_41(template) => {
204                Grib2ValueIterator::SigSNV(png::Png(self, template).iter()?)
205            }
206            #[cfg(any(
207                feature = "ccsds-unpack-with-libaec",
208                feature = "ccsds-unpack-with-rust-aec"
209            ))]
210            DataRepresentationTemplate::_5_42(template) => {
211                Grib2ValueIterator::SigSNV(ccsds::Ccsds(self, template).iter()?)
212            }
213            DataRepresentationTemplate::_5_200(template) => {
214                Grib2ValueIterator::SigI(run_length::RunLength(self, template).iter()?)
215            }
216            #[allow(unreachable_patterns)]
217            _ => {
218                return Err(GribError::DecodeError(DecodeError::NotSupported(
219                    "GRIB2 code table 5.0 (data representation template number)",
220                    self.sect5_param.payload.template_num.0,
221                )));
222            }
223        };
224
225        let bitmap_slice = &self.sect6_bytes[6..];
226        bitmap::check_consistency(
227            self.num_points_total,
228            self.num_encoded_points(),
229            bitmap_slice,
230        )?;
231        let decoder =
232            BitmapDecodeIterator::new(bitmap_slice.iter(), decoder, self.num_points_total);
233        Ok(Grib2DecodedValues(decoder))
234    }
235
236    pub(crate) fn num_encoded_points(&self) -> usize {
237        self.sect5_param.payload.num_encoded_points as usize
238    }
239
240    pub(crate) fn sect7_payload(&self) -> &[u8] {
241        &self.sect7_bytes[5..]
242    }
243
244    /// Provides access to the parameters in Section 5.
245    ///
246    /// # Examples
247    /// ```
248    /// use grib::Grib2SubmessageDecoder;
249    ///
250    /// fn main() -> Result<(), Box<dyn std::error::Error>> {
251    ///     let f = std::fs::File::open(
252    ///         "testdata/Z__C_RJTD_20160822020000_NOWC_GPV_Ggis10km_Pphw10_FH0000-0100_grib2.bin",
253    ///     )?;
254    ///     let f = std::io::BufReader::new(f);
255    ///     let grib2 = grib::from_reader(f)?;
256    ///     let (_index, first_submessage) = grib2.iter().next().unwrap();
257    ///
258    ///     let decoder = Grib2SubmessageDecoder::from(first_submessage)?;
259    ///     let actual = decoder.section5();
260    ///     let expected = grib::def::grib2::Section5 {
261    ///         header: grib::def::grib2::SectionHeader {
262    ///             len: 23,
263    ///             sect_num: 5,
264    ///         },
265    ///         payload: grib::def::grib2::Section5Payload {
266    ///             num_encoded_points: 86016,
267    ///             template_num: grib::def::grib2::table::DataRepresentationTemplateNum(200),
268    ///             template: grib::def::grib2::DataRepresentationTemplate::_5_200(
269    ///                 grib::def::grib2::template::Template5_200 {
270    ///                     num_bits: 8,
271    ///                     max_val: 3,
272    ///                     max_level: 3,
273    ///                     dec: 0,
274    ///                     level_vals: vec![1, 2, 3],
275    ///                 },
276    ///             ),
277    ///         },
278    ///     };
279    ///     assert_eq!(actual, &expected);
280    ///
281    ///     Ok(())
282    /// }
283    /// ```
284    pub fn section5(&self) -> &Section5 {
285        &self.sect5_param
286    }
287}
288
289pub(crate) fn orig_field_type_is_supported(orig_field_type: u8) -> Result<(), DecodeError> {
290    if orig_field_type != 0 {
291        return Err(DecodeError::NotSupported(
292            "GRIB2 code table 5.1 (type of original field values)",
293            orig_field_type.into(),
294        ));
295    }
296    Ok(())
297}
298
299pub struct Grib2DecodedValues<'b, I>(BitmapDecodeIterator<std::slice::Iter<'b, u8>, I>);
300
301impl<I> Iterator for Grib2DecodedValues<'_, I>
302where
303    I: Iterator<Item = f32>,
304{
305    type Item = f32;
306
307    fn next(&mut self) -> Option<Self::Item> {
308        let Self(inner) = self;
309        inner.next()
310    }
311
312    fn size_hint(&self) -> (usize, Option<usize>) {
313        let Self(inner) = self;
314        inner.size_hint()
315    }
316}
317
318enum Grib2ValueIterator<'d> {
319    SigSTNS(SimplePackingDecoder<std::iter::Take<NBitwiseIterator<&'d [u8]>>>),
320    SigSC(SimplePackingDecoder<ComplexPackingDecoded<'d>>),
321    SigSSCI(
322        SimplePackingDecoder<
323            complex::SpatialDifferencingDecodeIterator<ComplexPackingDecoded<'d>, IntoIter<i64>>,
324        >,
325    ),
326    #[allow(dead_code)]
327    SigSI(SimplePackingDecoder<IntoIter<i32>>),
328    #[cfg(any(
329        feature = "jpeg2000-unpack-with-openjpeg",
330        feature = "jpeg2000-unpack-with-hayro"
331    ))]
332    SigSIm(SimplePackingDecoder<self::jpeg2000::ImageIntoIter>),
333    #[allow(dead_code)]
334    SigSNV(SimplePackingDecoder<NBitwiseIterator<Vec<u8>>>),
335    SigI(IntoIter<f32>),
336}
337
338impl<'d> Iterator for Grib2ValueIterator<'d> {
339    type Item = f32;
340
341    fn next(&mut self) -> Option<Self::Item> {
342        match self {
343            Self::SigSTNS(inner) => inner.next(),
344            Self::SigSC(inner) => inner.next(),
345            Self::SigSSCI(inner) => inner.next(),
346            Self::SigSI(inner) => inner.next(),
347            #[cfg(any(
348                feature = "jpeg2000-unpack-with-openjpeg",
349                feature = "jpeg2000-unpack-with-hayro"
350            ))]
351            Self::SigSIm(inner) => inner.next(),
352            Self::SigSNV(inner) => inner.next(),
353            Self::SigI(inner) => inner.next(),
354        }
355    }
356
357    fn size_hint(&self) -> (usize, Option<usize>) {
358        match self {
359            Self::SigSTNS(inner) => inner.size_hint(),
360            Self::SigSC(inner) => inner.size_hint(),
361            Self::SigSSCI(inner) => inner.size_hint(),
362            Self::SigSI(inner) => inner.size_hint(),
363            #[cfg(any(
364                feature = "jpeg2000-unpack-with-openjpeg",
365                feature = "jpeg2000-unpack-with-hayro"
366            ))]
367            Self::SigSIm(inner) => inner.size_hint(),
368            Self::SigSNV(inner) => inner.size_hint(),
369            Self::SigI(inner) => inner.size_hint(),
370        }
371    }
372}
373
374/// User-configurable limits for decoding. Use this struct if your application
375/// requires resource guards.
376///
377/// For example, resource guarding is likely unnecessary for applications
378/// running on a personal computing environment with ample memory. It is also
379/// unnecessary when you know that the input data contains only a small number
380/// of grid points. On the other hand, resource guarding is recommended for
381/// applications running on shared computing environments or when processing
382/// input data containing a large number of grid points from an unknown source.
383#[derive(Clone, Debug, Default)]
384#[non_exhaustive]
385pub struct DecodeLimits {
386    /// Maximum number of grid points to be decoded per submessage. `None` means
387    /// no additional limit.
388    ///
389    /// According to the GRIB2 specification, a single submessage can contain up
390    /// to `u32::MAX` grid points (although the developers of this library have
391    /// never encountered a submessage containing that many grid points).
392    /// Depending on the decoding method, it may be necessary to allocate a
393    /// large buffer to store the values of all grid points. If `u32::MAX` grid
394    /// points are stored as an `f32` array, a 16 GiB buffer is required.
395    ///
396    /// This configuration option is intended to prevent such massive
397    /// allocations. It is currently effective in the Template 5.40/7.40 (JPEG
398    /// 2000 code stream format) decoder when `jpeg2000-unpack-with-hayro` is
399    /// enabled.
400    pub max_grid_points_decoded: Option<u32>,
401    /// Maximum number of JPEG 2000 code stream tiles to be decoded per
402    /// submessage. `None` means no additional limit.
403    ///
404    /// This configuration is currently effective in the Template 5.40/7.40
405    /// (JPEG 2000 code stream format) decoder when `jpeg2000-unpack-with-hayro`
406    /// is enabled.
407    pub max_tiles_decoded: Option<u32>,
408}
409
410#[derive(Debug, Clone, PartialEq, Eq, Hash)]
411#[non_exhaustive]
412pub enum DecodeError {
413    NotSupported(&'static str, u16),
414    LengthMismatch,
415    UnclassifiedError(String),
416}
417
418impl From<String> for DecodeError {
419    fn from(value: String) -> Self {
420        Self::UnclassifiedError(value)
421    }
422}
423
424impl From<&str> for DecodeError {
425    fn from(value: &str) -> Self {
426        Self::UnclassifiedError(value.to_owned())
427    }
428}
429
430pub(crate) trait Grib2GpvUnpack {
431    type Iter<'a>: Iterator<Item = f32>
432    where
433        Self: 'a;
434
435    fn iter<'a>(&'a self) -> Result<Self::Iter<'a>, DecodeError>;
436}
437
438mod bitmap;
439#[cfg(any(
440    feature = "ccsds-unpack-with-libaec",
441    feature = "ccsds-unpack-with-rust-aec"
442))]
443mod ccsds;
444mod complex;
445mod helpers;
446#[cfg(any(
447    feature = "jpeg2000-unpack-with-openjpeg",
448    feature = "jpeg2000-unpack-with-hayro"
449))]
450mod jpeg2000;
451#[cfg(feature = "png-unpack-with-png-crate")]
452mod png;
453mod run_length;
454mod simple;
455mod stream;
456#[cfg(test)]
457mod tests;