grib/encoder/
simple.rs

1use crate::{
2    WriteToBuffer,
3    def::grib2::template::param_set::SimplePacking,
4    encoder::{Encode, WriteGrib2DataSections, bitmap::Bitmap, helpers::BitsRequired, writer},
5};
6
7// Complex packing decodes group reference values as `i32`.
8const MAX_ENCODED_VALUE: u32 = i32::MAX as u32;
9
10/// Strategies applied when performing simple packing on numerical sequences.
11/// Simple packing is a method for discretizing continuous numerical values as
12/// integers, and various approaches can be taken during this process.
13#[derive(Debug, PartialEq, Eq, Clone)]
14#[non_exhaustive]
15pub enum SimplePackingStrategy {
16    /// A strategy specifying how many decimal places to consider valid for the
17    /// numbers. This strategy is effective for various types of data, such as
18    /// observation data obtained from specific observation instruments, where
19    /// precision is clearly defined.
20    Decimal(i16),
21}
22
23pub(crate) struct Encoder<'a> {
24    data: &'a [f64],
25    strategy: SimplePackingStrategy,
26}
27
28impl<'a> Encoder<'a> {
29    pub(crate) fn new(data: &'a [f64], strategy: SimplePackingStrategy) -> Self {
30        Self { data, strategy }
31    }
32}
33
34impl<'a> Encode for Encoder<'a> {
35    type Output = Encoded;
36
37    fn encode(&self) -> Self::Output {
38        match self.strategy {
39            SimplePackingStrategy::Decimal(dec) => {
40                let (params, scaled, bitmap) = determine_simple_packing_params(self.data, dec);
41                let coded = if params.num_bits == 0 {
42                    CodedValues::Unique(scaled.len())
43                } else {
44                    let exp = 2_f64.powf(params.exp as f64);
45                    let coded = scaled
46                        .iter()
47                        .map(|value| ((value - params.ref_val as f64) / exp).round() as u32)
48                        .collect::<Vec<_>>();
49                    CodedValues::NonUnique(coded)
50                };
51                Encoded::new(params, coded, bitmap)
52            }
53        }
54    }
55}
56
57#[derive(Debug)]
58pub(crate) struct Encoded {
59    params: SimplePacking,
60    coded: CodedValues,
61    bitmap: Bitmap,
62}
63
64impl Encoded {
65    fn new(params: SimplePacking, coded: CodedValues, bitmap: Bitmap) -> Self {
66        Self {
67            params,
68            coded,
69            bitmap,
70        }
71    }
72
73    pub(crate) fn params(&self) -> &SimplePacking {
74        &self.params
75    }
76}
77
78impl WriteGrib2DataSections for Encoded {
79    fn section5_len(&self) -> usize {
80        21
81    }
82
83    fn write_section5(&self, buf: &mut [u8]) -> Result<usize, &'static str> {
84        let len = self.section5_len();
85        if buf.len() < len {
86            return Err("destination buffer is too small");
87        }
88
89        let mut pos = 0;
90        pos += super::write_section_header(len as u32, 5, &mut buf[pos..])?;
91        pos += (self.coded.num_values() as u32).write_to_buffer(&mut buf[pos..])?; // payload.num_encoded_points
92        pos += 0_u16.write_to_buffer(&mut buf[pos..])?; // payload.template_num
93        pos += self.params.write_to_buffer(&mut buf[pos..])?;
94        pos += 0_u8.write_to_buffer(&mut buf[pos..])?; // payload.template.orig_field_type
95
96        Ok(pos)
97    }
98
99    fn section6_len(&self) -> usize {
100        let bitmap_size = if self.bitmap.has_nan() {
101            self.bitmap.num_bytes_required()
102        } else {
103            0
104        };
105        6 + bitmap_size
106    }
107
108    fn write_section6(&self, buf: &mut [u8]) -> Result<usize, &'static str> {
109        let len = self.section6_len();
110        if buf.len() < len {
111            return Err("destination buffer is too small");
112        }
113
114        let mut pos = 0;
115        pos += super::write_section_header(len as u32, 6, &mut buf[pos..])?;
116        if self.bitmap.has_nan() {
117            pos += 0_u8.write_to_buffer(&mut buf[pos..])?;
118            pos += self.bitmap.write_to_buffer(&mut buf[pos..])?;
119        } else {
120            pos += 255_u8.write_to_buffer(&mut buf[pos..])?;
121        }
122
123        Ok(pos)
124    }
125
126    fn section7_len(&self) -> usize {
127        let len = match &self.coded {
128            CodedValues::NonUnique(vec) => {
129                let nbitwise = writer::NBitwise::new(&vec, self.params.num_bits as usize);
130                nbitwise.num_bytes_required()
131            }
132            CodedValues::Unique(_) => 0,
133        };
134        5 + len
135    }
136
137    fn write_section7(&self, buf: &mut [u8]) -> Result<usize, &'static str> {
138        let len = self.section7_len();
139        if buf.len() < len {
140            return Err("destination buffer is too small");
141        }
142
143        let mut pos = 0;
144        pos += super::write_section_header(len as u32, 7, &mut buf[pos..])?;
145        match &self.coded {
146            CodedValues::NonUnique(vec) => {
147                let nbitwise = writer::NBitwise::new(&vec, self.params.num_bits as usize);
148                pos += nbitwise.write_to_buffer(&mut buf[pos..])?;
149            }
150            CodedValues::Unique(_) => {}
151        }
152
153        Ok(pos)
154    }
155}
156
157pub(crate) fn determine_simple_packing_params(
158    values: &[f64],
159    dec: i16,
160) -> (SimplePacking, Vec<f64>, Bitmap) {
161    let mut min = f64::MAX;
162    let mut max = f64::MIN;
163    let mut bitmap = Bitmap::for_values(values);
164    let scaled = values
165        .iter()
166        .filter_map(|value| {
167            if value.is_nan() {
168                bitmap.push(false);
169                None
170            } else {
171                bitmap.push(true);
172                let scaled = value * 10_f64.powf(dec as f64);
173                (min, max) = (scaled.min(min), scaled.max(max));
174                Some(scaled)
175            }
176        })
177        .collect::<Vec<_>>();
178    let ref_val = min as f32;
179    let params = if min == max {
180        SimplePacking {
181            ref_val,
182            exp: 0,
183            dec,
184            num_bits: 0,
185        }
186    } else {
187        let max_diff = max - f64::from(ref_val);
188        let exp = if max_diff <= f64::from(MAX_ENCODED_VALUE) {
189            0
190        } else {
191            (max_diff / f64::from(MAX_ENCODED_VALUE)).log2().ceil() as i16
192        };
193        let max_code = (max_diff / 2_f64.powi(i32::from(exp))).round() as u32;
194        let num_bits = max_code.bits_required();
195        SimplePacking {
196            ref_val,
197            exp,
198            dec,
199            num_bits,
200        }
201    };
202    (params, scaled, bitmap)
203}
204
205#[derive(Debug)]
206enum CodedValues {
207    NonUnique(Vec<u32>),
208    Unique(usize),
209}
210
211impl CodedValues {
212    pub(crate) fn num_values(&self) -> usize {
213        match self {
214            Self::NonUnique(vec) => vec.len(),
215            Self::Unique(size) => *size,
216        }
217    }
218}
219
220#[cfg(test)]
221mod tests {
222    use super::*;
223
224    macro_rules! test_decimal_strategy {
225        ($((
226            $name:ident,
227            $input:expr,
228            $decimal:expr,
229            $expected_params:expr,
230        ),)*) => ($(
231            #[test]
232            fn $name() {
233                let values = $input;
234                let encoder = Encoder::new(&values, SimplePackingStrategy::Decimal($decimal));
235                let encoded = encoder.encode();
236                let actual_params = encoded.params();
237                let expected_params = $expected_params;
238                assert_eq!(actual_params, &expected_params);
239                let actual_num_values = encoded.coded.num_values();
240                assert_eq!(actual_num_values, values.len());
241            }
242        )*);
243    }
244
245    test_decimal_strategy! {
246        (
247            decimal_strategy_with_decimal_0,
248            (2..11).map(|val| val as f64).collect::<Vec<_>>(),
249            0,
250            SimplePacking {
251                ref_val: 2.,
252                exp: 0,
253                dec: 0,
254                num_bits: 4,
255            },
256        ),
257        (
258            decimal_strategy_with_decimal_1,
259            (2..11).map(|val| val as f64).collect::<Vec<_>>(),
260            1,
261            SimplePacking {
262                ref_val: 20.,
263                exp: 0,
264                dec: 1,
265                num_bits: 7,
266            },
267        ),
268        (
269            decimal_strategy_with_decimal_0_for_unique_values,
270            vec![10.0_f64; 256],
271            0,
272            SimplePacking {
273                ref_val: 10.0,
274                exp: 0,
275                dec: 0,
276                num_bits: 0,
277            },
278        ),
279    }
280
281    #[test]
282    fn simple_packing_scales_large_finite_ranges_to_u32() -> Result<(), &'static str> {
283        let values = [0.0, 1.0e20];
284        let encoder = Encoder::new(&values, SimplePackingStrategy::Decimal(0));
285        let encoded = encoder.encode();
286
287        assert!(encoded.params().num_bits <= i32::BITS as u8);
288
289        let mut sect7 = vec![0; encoded.section7_len()];
290        let pos = encoded.write_section7(&mut sect7)?;
291        assert_eq!(pos, sect7.len());
292
293        Ok(())
294    }
295
296    macro_rules! grib2_coded_values_roundtrip_tests {
297        ($(($name:ident, $input:expr, $decimal:expr),)*) => ($(
298            #[test]
299            fn $name() -> Result<(), Box<dyn std::error::Error>> {
300                let values = $input;
301                let encoder = Encoder::new(&values, SimplePackingStrategy::Decimal($decimal));
302                let encoded = encoder.encode();
303                let mut sect5 = vec![0; encoded.section5_len()];
304                let pos = encoded.write_section5(&mut sect5)?;
305                assert_eq!(pos, sect5.len());
306                let mut sect6 = vec![0; encoded.section6_len()];
307                let pos = encoded.write_section6(&mut sect6)?;
308                assert_eq!(pos, sect6.len());
309                let mut sect7 = vec![0; encoded.section7_len()];
310                let pos = encoded.write_section7(&mut sect7)?;
311                assert_eq!(pos, sect7.len());
312                let decoder = crate::Grib2SubmessageDecoder::new(values.len(), sect5, sect6, sect7)?;
313                let actual = decoder.dispatch()?.collect::<Vec<_>>();
314                let expected = values.iter().map(|val| *val as f32).collect::<Vec<_>>();
315                assert_eq!(actual.len(), expected.len());
316                actual
317                    .iter()
318                    .zip(expected.iter())
319                    .all(|(a, b)| (a.is_nan() && b.is_nan()) || (a == b));
320                Ok(())
321            }
322        )*);
323    }
324
325    grib2_coded_values_roundtrip_tests! {
326        (
327            grib2_coded_values_roundtrip_test_with_decimal_0_and_nonunique_values,
328            (2..11).map(|val| val as f64).collect::<Vec<_>>(),
329            0
330        ),
331        (
332            grib2_coded_values_roundtrip_test_with_decimal_0_and_unique_values,
333            vec![10.0_f64; 256],
334            0
335        ),
336        (
337            grib2_coded_values_roundtrip_test_with_data_containing_nan_values,
338            [f64::NAN; 32]
339                .into_iter()
340                .chain([1., 2., 3.].into_iter())
341                .chain([f64::NAN; 32].into_iter())
342                .chain([4., 5., 6., 7.].into_iter())
343                .chain([f64::NAN; 32].into_iter())
344                .chain([8., 9., 10., 11.].into_iter())
345                .collect::<Vec<_>>(),
346            0
347        ),
348    }
349}
grib/encoder/simple.rs

grib/encoder/
simple.rs
