Skip to main content

grib/encoder/
simple.rs

1use crate::{
2    WriteToBuffer,
3    def::grib2::template::param_set::SimplePacking,
4    encoder::{Encode, WriteGrib2DataSections, bitmap::Bitmap, helpers::BitsRequired, writer},
5};
6
7// Complex packing decodes group reference values as `i32`.
8const MAX_ENCODED_VALUE: u32 = i32::MAX as u32;
9
10/// Strategies applied when performing simple packing on numerical sequences.
11/// Simple packing is a method for discretizing continuous numerical values as
12/// integers, and various approaches can be taken during this process.
13#[derive(Debug, PartialEq, Eq, Clone)]
14#[non_exhaustive]
15pub enum SimplePackingStrategy {
16    /// A strategy specifying how many decimal places to consider valid for the
17    /// numbers. This strategy is effective for various types of data, such as
18    /// observation data obtained from specific observation instruments, where
19    /// precision is clearly defined. Constant fields are stored with a decimal
20    /// scale factor of zero.
21    Decimal(i16),
22}
23
24pub(crate) struct Encoder<'a> {
25    data: &'a [f64],
26    strategy: SimplePackingStrategy,
27}
28
29impl<'a> Encoder<'a> {
30    pub(crate) fn new(data: &'a [f64], strategy: SimplePackingStrategy) -> Self {
31        Self { data, strategy }
32    }
33}
34
35impl<'a> Encode for Encoder<'a> {
36    type Output = Encoded;
37
38    fn encode(&self) -> Self::Output {
39        match self.strategy {
40            SimplePackingStrategy::Decimal(dec) => {
41                let (params, scaled, bitmap) = determine_simple_packing_params(self.data, dec);
42                let coded = if params.num_bits == 0 {
43                    CodedValues::Unique(scaled.len())
44                } else {
45                    let exp = 2_f64.powf(params.exp as f64);
46                    let coded = scaled
47                        .iter()
48                        .map(|value| ((value - params.ref_val as f64) / exp).round() as u32)
49                        .collect::<Vec<_>>();
50                    CodedValues::NonUnique(coded)
51                };
52                Encoded::new(params, coded, bitmap)
53            }
54        }
55    }
56}
57
58#[derive(Debug)]
59pub(crate) struct Encoded {
60    params: SimplePacking,
61    coded: CodedValues,
62    bitmap: Bitmap,
63}
64
65impl Encoded {
66    fn new(params: SimplePacking, coded: CodedValues, bitmap: Bitmap) -> Self {
67        Self {
68            params,
69            coded,
70            bitmap,
71        }
72    }
73
74    pub(crate) fn params(&self) -> &SimplePacking {
75        &self.params
76    }
77}
78
79impl WriteGrib2DataSections for Encoded {
80    fn section5_len(&self) -> usize {
81        21
82    }
83
84    fn write_section5(&self, buf: &mut [u8]) -> Result<usize, &'static str> {
85        let len = self.section5_len();
86        if buf.len() < len {
87            return Err("destination buffer is too small");
88        }
89
90        let mut pos = 0;
91        pos += super::write_section_header(len as u32, 5, &mut buf[pos..])?;
92        pos += (self.coded.num_values() as u32).write_to_buffer(&mut buf[pos..])?; // payload.num_encoded_points
93        pos += 0_u16.write_to_buffer(&mut buf[pos..])?; // payload.template_num
94        pos += self.params.write_to_buffer(&mut buf[pos..])?;
95        pos += 0_u8.write_to_buffer(&mut buf[pos..])?; // payload.template.orig_field_type
96
97        Ok(pos)
98    }
99
100    fn section6_len(&self) -> usize {
101        let bitmap_size = if self.bitmap.has_nan() {
102            self.bitmap.num_bytes_required()
103        } else {
104            0
105        };
106        6 + bitmap_size
107    }
108
109    fn write_section6(&self, buf: &mut [u8]) -> Result<usize, &'static str> {
110        let len = self.section6_len();
111        if buf.len() < len {
112            return Err("destination buffer is too small");
113        }
114
115        let mut pos = 0;
116        pos += super::write_section_header(len as u32, 6, &mut buf[pos..])?;
117        if self.bitmap.has_nan() {
118            pos += 0_u8.write_to_buffer(&mut buf[pos..])?;
119            pos += self.bitmap.write_to_buffer(&mut buf[pos..])?;
120        } else {
121            pos += 255_u8.write_to_buffer(&mut buf[pos..])?;
122        }
123
124        Ok(pos)
125    }
126
127    fn section7_len(&self) -> usize {
128        let len = match &self.coded {
129            CodedValues::NonUnique(vec) => {
130                let nbitwise = writer::NBitwise::new(&vec, self.params.num_bits as usize);
131                nbitwise.num_bytes_required()
132            }
133            CodedValues::Unique(_) => 0,
134        };
135        5 + len
136    }
137
138    fn write_section7(&self, buf: &mut [u8]) -> Result<usize, &'static str> {
139        let len = self.section7_len();
140        if buf.len() < len {
141            return Err("destination buffer is too small");
142        }
143
144        let mut pos = 0;
145        pos += super::write_section_header(len as u32, 7, &mut buf[pos..])?;
146        match &self.coded {
147            CodedValues::NonUnique(vec) => {
148                let nbitwise = writer::NBitwise::new(&vec, self.params.num_bits as usize);
149                pos += nbitwise.write_to_buffer(&mut buf[pos..])?;
150            }
151            CodedValues::Unique(_) => {}
152        }
153
154        Ok(pos)
155    }
156}
157
158pub(crate) fn determine_simple_packing_params(
159    values: &[f64],
160    dec: i16,
161) -> (SimplePacking, Vec<f64>, Bitmap) {
162    // Store constant fields without decimal scaling for compatibility with
163    // decoders that return the reference value directly for zero-bit fields.
164    // wgrib2 documents the same encoding policy for compatibility with both
165    // WMO-compliant decoding and g2clib/g2lib:
166    // https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/g2clib.html
167    // For the decoding implementations, see
168    // SimplePacking::zero_bit_reference_value.
169    let mut present = values.iter().copied().filter(|value| !value.is_nan());
170    let dec = match present.next() {
171        Some(first) if present.all(|value| value == first) => 0,
172        _ => dec,
173    };
174    let mut min = f64::MAX;
175    let mut max = f64::MIN;
176    let mut bitmap = Bitmap::for_values(values);
177    let scaled = values
178        .iter()
179        .filter_map(|value| {
180            if value.is_nan() {
181                bitmap.push(false);
182                None
183            } else {
184                bitmap.push(true);
185                let scaled = value * 10_f64.powf(dec as f64);
186                (min, max) = (scaled.min(min), scaled.max(max));
187                Some(scaled)
188            }
189        })
190        .collect::<Vec<_>>();
191    let ref_val = min as f32;
192    let params = if min == max {
193        SimplePacking {
194            ref_val,
195            exp: 0,
196            dec,
197            num_bits: 0,
198        }
199    } else {
200        let max_diff = max - f64::from(ref_val);
201        let exp = if max_diff <= f64::from(MAX_ENCODED_VALUE) {
202            0
203        } else {
204            (max_diff / f64::from(MAX_ENCODED_VALUE)).log2().ceil() as i16
205        };
206        let max_code = (max_diff / 2_f64.powi(i32::from(exp))).round() as u32;
207        let num_bits = max_code.bits_required();
208        SimplePacking {
209            ref_val,
210            exp,
211            dec,
212            num_bits,
213        }
214    };
215    (params, scaled, bitmap)
216}
217
218#[derive(Debug)]
219enum CodedValues {
220    NonUnique(Vec<u32>),
221    Unique(usize),
222}
223
224impl CodedValues {
225    pub(crate) fn num_values(&self) -> usize {
226        match self {
227            Self::NonUnique(vec) => vec.len(),
228            Self::Unique(size) => *size,
229        }
230    }
231}
232
233#[cfg(test)]
234mod tests {
235    use super::*;
236
237    macro_rules! test_decimal_strategy {
238        ($((
239            $name:ident,
240            $input:expr,
241            $decimal:expr,
242            $expected_params:expr $(,)?
243        )),* $(,)?) => ($(
244            #[test]
245            fn $name() {
246                let values = $input;
247                let encoder = Encoder::new(&values, SimplePackingStrategy::Decimal($decimal));
248                let encoded = encoder.encode();
249                let actual_params = encoded.params();
250                let expected_params = $expected_params;
251                assert_eq!(actual_params, &expected_params);
252                let actual_num_values = encoded.coded.num_values();
253                assert_eq!(actual_num_values, values.len());
254            }
255        )*);
256    }
257
258    test_decimal_strategy! {
259        (
260            decimal_strategy_with_decimal_0,
261            (2..11).map(|val| val as f64).collect::<Vec<_>>(),
262            0,
263            SimplePacking {
264                ref_val: 2.,
265                exp: 0,
266                dec: 0,
267                num_bits: 4,
268            },
269        ),
270        (
271            decimal_strategy_with_decimal_1,
272            (2..11).map(|val| val as f64).collect::<Vec<_>>(),
273            1,
274            SimplePacking {
275                ref_val: 20.,
276                exp: 0,
277                dec: 1,
278                num_bits: 7,
279            },
280        ),
281        (
282            decimal_strategy_with_decimal_0_for_unique_values,
283            vec![10.0_f64; 256],
284            0,
285            SimplePacking {
286                ref_val: 10.0,
287                exp: 0,
288                dec: 0,
289                num_bits: 0,
290            },
291        ),
292    }
293
294    #[test]
295    fn constant_fields_ignore_requested_decimal_scale() {
296        for dec in [-32768, -2, 0, 2, 32767] {
297            let values = [f64::NAN, 1.25, 1.25, f64::NAN];
298            let encoded = Encoder::new(&values, SimplePackingStrategy::Decimal(dec)).encode();
299            assert_eq!(
300                encoded.params(),
301                &SimplePacking {
302                    ref_val: 1.25,
303                    exp: 0,
304                    dec: 0,
305                    num_bits: 0,
306                }
307            );
308            assert_eq!(encoded.coded.num_values(), 2);
309        }
310    }
311
312    #[test]
313    fn simple_packing_scales_large_finite_ranges_to_u32() -> Result<(), &'static str> {
314        let values = [0.0, 1.0e20];
315        let encoder = Encoder::new(&values, SimplePackingStrategy::Decimal(0));
316        let encoded = encoder.encode();
317
318        assert!(encoded.params().num_bits <= i32::BITS as u8);
319
320        let mut sect7 = vec![0; encoded.section7_len()];
321        let pos = encoded.write_section7(&mut sect7)?;
322        assert_eq!(pos, sect7.len());
323
324        Ok(())
325    }
326
327    macro_rules! grib2_coded_values_roundtrip_tests {
328        ($(($name:ident, $input:expr, $decimal:expr $(,)?)),* $(,)?) => ($(
329            #[test]
330            fn $name() -> Result<(), Box<dyn std::error::Error>> {
331                let values = $input;
332                let encoder = Encoder::new(&values, SimplePackingStrategy::Decimal($decimal));
333                let encoded = encoder.encode();
334                let mut sect5 = vec![0; encoded.section5_len()];
335                let pos = encoded.write_section5(&mut sect5)?;
336                assert_eq!(pos, sect5.len());
337                let mut sect6 = vec![0; encoded.section6_len()];
338                let pos = encoded.write_section6(&mut sect6)?;
339                assert_eq!(pos, sect6.len());
340                let mut sect7 = vec![0; encoded.section7_len()];
341                let pos = encoded.write_section7(&mut sect7)?;
342                assert_eq!(pos, sect7.len());
343                let decoder = crate::Grib2SubmessageDecoder::new(values.len(), sect5, sect6, sect7)?;
344                let actual = decoder.dispatch()?.collect::<Vec<_>>();
345                let expected = values.iter().map(|val| *val as f32).collect::<Vec<_>>();
346                assert_eq!(actual.len(), expected.len());
347                assert!(actual
348                    .iter()
349                    .zip(expected.iter())
350                    .all(|(a, b)| (a.is_nan() && b.is_nan()) || (a == b)));
351                Ok(())
352            }
353        )*);
354    }
355
356    grib2_coded_values_roundtrip_tests! {
357        (constant_positive_decimal, vec![1.25; 4], 2),
358        (constant_negative_decimal, vec![1.25; 4], -2),
359        (constant_with_bitmap, vec![f64::NAN, 1.25, 1.25], 2),
360        (
361            grib2_coded_values_roundtrip_test_with_decimal_0_and_nonunique_values,
362            (2..11).map(|val| val as f64).collect::<Vec<_>>(),
363            0,
364        ),
365        (
366            grib2_coded_values_roundtrip_test_with_decimal_0_and_unique_values,
367            vec![10.0_f64; 256],
368            0,
369        ),
370        (
371            grib2_coded_values_roundtrip_test_with_data_containing_nan_values,
372            [f64::NAN; 32]
373                .into_iter()
374                .chain([1., 2., 3.].into_iter())
375                .chain([f64::NAN; 32].into_iter())
376                .chain([4., 5., 6., 7.].into_iter())
377                .chain([f64::NAN; 32].into_iter())
378                .chain([8., 9., 10., 11.].into_iter())
379                .collect::<Vec<_>>(),
380            0,
381        ),
382    }
383}