gibbon/examples/csv_to_packed.rs at master · johshoff/gibbon · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
extern crate csv;
extern crate gibbon;
extern crate time;

use gibbon::*;
use std::{f64, u64};
use std::cmp;

struct TimeAndValue {
    timestamp: u64,
    value: f64,
}

fn main() {
    let mut rdr = csv::Reader::from_file("./examples/test_data.csv").unwrap();
    let mut w = vec_stream::VecWriter::new();
    let header_time = (1496366523 / 3600) * 3600;
    let mut c = TimeAndValueStream::new(header_time);

    let mut uncompressed = Vec::<TimeAndValue>::new();

    let mut start = time::precise_time_ns();
    for record in rdr.decode() {
        let (timestamp, value): (u64, f64) = record.unwrap();
        c.push(timestamp, value, &mut w);
        uncompressed.push(TimeAndValue { timestamp: timestamp, value: value });
    }
    let now = time::precise_time_ns();
    println!("Read data in {} ms", (now - start) as f64 / 1_000_000f64);
    start = now;

    //------------------------------------------------------

    println!("\nCompressed:");

    {
        let i = TimeAndValueIterator::new(vec_stream::VecReader::new(&w.bit_vector, w.used_bits_last_elm), header_time);
        print!("Max: {}", i.map(|(_timestamp, value)| value).fold(f64::NEG_INFINITY, f64::max));
    }
    let now = time::precise_time_ns();
    println!(" ({} ms)", (now - start) as f64 / 1_000_000f64);
    start = now;

    {
        let i = TimeAndValueIterator::new(vec_stream::VecReader::new(&w.bit_vector, w.used_bits_last_elm), header_time);
        print!("Min: {}", i.map(|(_timestamp, value)| value).fold(f64::INFINITY, f64::min));
    }
    let now = time::precise_time_ns();
    println!(" ({} ms)", (now - start) as f64 / 1_000_000f64);
    start = now;

    let samples = {
        let i = TimeAndValueIterator::new(vec_stream::VecReader::new(&w.bit_vector, w.used_bits_last_elm), header_time);
        let samples = i.count();
        print!("Samples: {}", samples);
        samples
    };
    let now = time::precise_time_ns();
    println!(" ({} ms)", (now - start) as f64 / 1_000_000f64);
    start = now;

    {
        let i = TimeAndValueIterator::new(vec_stream::VecReader::new(&w.bit_vector, w.used_bits_last_elm), header_time);
        print!("Average: {}", i.map(|(_timestamp, value)| value).sum::<f64>() / (samples as f64));
    }
    let now = time::precise_time_ns();
    println!(" ({} ms)", (now - start) as f64 / 1_000_000f64);
    start = now;

    {
        let i = TimeAndValueIterator::new(vec_stream::VecReader::new(&w.bit_vector, w.used_bits_last_elm), header_time);
        print!("Max timestamp: {}", i.map(|(timestamp, _value)| timestamp).fold(u64::MIN, cmp::max));
    }
    let now = time::precise_time_ns();
    println!(" ({} ms)", (now - start) as f64 / 1_000_000f64);

    //------------------------------------------------------

    println!("\nUncompressed:");

    print!("Max: {}", uncompressed.iter().map(|v| v.value).fold(f64::NEG_INFINITY, f64::max));
    let now = time::precise_time_ns();
    println!(" ({} ms)", (now - start) as f64 / 1_000_000f64);
    start = now;

    print!("Min: {}", uncompressed.iter().map(|v| v.value).fold(f64::INFINITY, f64::min));
    let now = time::precise_time_ns();
    println!(" ({} ms)", (now - start) as f64 / 1_000_000f64);
    start = now;

    print!("Samples: {}", uncompressed.iter().count());
    let now = time::precise_time_ns();
    println!(" ({} ms)", (now - start) as f64 / 1_000_000f64);
    start = now;

    print!("Average: {}", uncompressed.iter().map(|v| v.value).sum::<f64>() / (samples as f64));
    let now = time::precise_time_ns();
    println!(" ({} ms)", (now - start) as f64 / 1_000_000f64);
    start = now;

    print!("Max timestamp: {}", uncompressed.iter().map(|v| v.timestamp).fold(u64::MIN, cmp::max));
    let now = time::precise_time_ns();
    println!(" ({} ms)", (now - start) as f64 / 1_000_000f64);

    //------------------------------------------------------

    println!("\nStats:");

    let bytes_compressed = w.bit_vector.len() * 8;
    let bytes_uncompressed = uncompressed.len() * (8 + 8);
    println!("Bytes consumed             {:10}", bytes_compressed);
    println!("Bytes consumed uncompressed{:10}", bytes_uncompressed);
    println!("Compression ratio          {:10.2}%", (100f64 * (bytes_compressed as f64) / (bytes_uncompressed as f64)));
}