Skip to content

Commit b16d96d

Browse files
author
Michael Friedrich
authored
Merge pull request #6489 from Icinga/feature/object-packer
Implement object packer for consistent hashing
2 parents 56bff13 + dd8cb42 commit b16d96d

File tree

3 files changed

+304
-0
lines changed

3 files changed

+304
-0
lines changed

‎lib/base/CMakeLists.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@ set(base_SOURCES
5757
number.cpp number.hpp number-script.cpp
5858
object.cpp object.hpp object-script.cpp
5959
objectlock.cpp objectlock.hpp
60+
object-packer.cpp object-packer.hpp
6061
objecttype.cpp objecttype.hpp
6162
perfdatavalue.cpp perfdatavalue.hpp perfdatavalue-ti.hpp
6263
primitivetype.cpp primitivetype.hpp

‎lib/base/object-packer.cpp

Lines changed: 268 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,268 @@
1+
/******************************************************************************
2+
* Icinga 2 *
3+
* Copyright (C) 2012-2018 Icinga Development Team (https://www.icinga.com/) *
4+
* *
5+
* This program is free software; you can redistribute it and/or *
6+
* modify it under the terms of the GNU General Public License *
7+
* as published by the Free Software Foundation; either version 2 *
8+
* of the License, or (at your option) any later version. *
9+
* *
10+
* This program is distributed in the hope that it will be useful, *
11+
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
12+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
13+
* GNU General Public License for more details. *
14+
* *
15+
* You should have received a copy of the GNU General Public License *
16+
* along with this program; if not, write to the Free Software Foundation *
17+
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA. *
18+
******************************************************************************/
19+
20+
#include "base/object-packer.hpp"
21+
#include "base/debug.hpp"
22+
#include "base/dictionary.hpp"
23+
#include "base/array.hpp"
24+
#include "base/objectlock.hpp"
25+
#include <algorithm>
26+
#include <climits>
27+
#include <cstdint>
28+
#include <utility>
29+
#include <vector>
30+
31+
using namespace icinga;
32+
33+
// Just for the sake of code readability
34+
typedef std::vector<char> StringBuilder;
35+
36+
union EndiannessDetector
37+
{
38+
EndiannessDetector()
39+
{
40+
i = 1;
41+
}
42+
43+
int i;
44+
char buf[sizeof(int)];
45+
};
46+
47+
static const EndiannessDetector l_EndiannessDetector;
48+
49+
// Assumption: The compiler will optimize (away) if/else statements using this.
50+
#define MACHINE_LITTLE_ENDIAN (l_EndiannessDetector.buf[0])
51+
52+
static void PackAny(const Value& value, StringBuilder& builder);
53+
54+
/**
55+
* std::swap() seems not to work
56+
*/
57+
static inline void SwapBytes(char& a, char& b)
58+
{
59+
char c = a;
60+
a = b;
61+
b = c;
62+
}
63+
64+
#if CHAR_MIN != 0
65+
union CharU2SConverter
66+
{
67+
CharU2SConverter()
68+
{
69+
s = 0;
70+
}
71+
72+
unsigned char u;
73+
signed char s;
74+
};
75+
#endif
76+
77+
/**
78+
* Avoid implementation-defined overflows during unsigned to signed casts
79+
*/
80+
static inline char UIntToByte(unsigned i)
81+
{
82+
#if CHAR_MIN == 0
83+
return i;
84+
#else
85+
CharU2SConverter converter;
86+
87+
converter.u = i;
88+
return converter.s;
89+
#endif
90+
}
91+
92+
/**
93+
* Append the given int as big-endian 64-bit unsigned int
94+
*/
95+
static inline void PackUInt64BE(uint_least64_t i, StringBuilder& builder)
96+
{
97+
char buf[8] = {
98+
UIntToByte(i >> 56u),
99+
UIntToByte((i >> 48u) & 255u),
100+
UIntToByte((i >> 40u) & 255u),
101+
UIntToByte((i >> 32u) & 255u),
102+
UIntToByte((i >> 24u) & 255u),
103+
UIntToByte((i >> 16u) & 255u),
104+
UIntToByte((i >> 8u) & 255u),
105+
UIntToByte(i & 255u)
106+
};
107+
108+
builder.insert(builder.end(), (char*)buf, (char*)buf + 8);
109+
}
110+
111+
union Double2BytesConverter
112+
{
113+
Double2BytesConverter()
114+
{
115+
buf[0] = 0;
116+
buf[1] = 0;
117+
buf[2] = 0;
118+
buf[3] = 0;
119+
buf[4] = 0;
120+
buf[5] = 0;
121+
buf[6] = 0;
122+
buf[7] = 0;
123+
}
124+
125+
double f;
126+
char buf[8];
127+
};
128+
129+
/**
130+
* Append the given double as big-endian IEEE 754 binary64
131+
*/
132+
static inline void PackFloat64BE(double f, StringBuilder& builder)
133+
{
134+
Double2BytesConverter converter;
135+
136+
converter.f = f;
137+
138+
if (MACHINE_LITTLE_ENDIAN) {
139+
SwapBytes(converter.buf[0], converter.buf[7]);
140+
SwapBytes(converter.buf[1], converter.buf[6]);
141+
SwapBytes(converter.buf[2], converter.buf[5]);
142+
SwapBytes(converter.buf[3], converter.buf[4]);
143+
}
144+
145+
builder.insert(builder.end(), (char*)converter.buf, (char*)converter.buf + 8);
146+
}
147+
148+
/**
149+
* Append the given string's length (BE uint64) and the string itself
150+
*/
151+
static inline void PackString(const String& string, StringBuilder& builder)
152+
{
153+
PackUInt64BE(string.GetLength(), builder);
154+
builder.insert(builder.end(), string.Begin(), string.End());
155+
}
156+
157+
/**
158+
* Append the given array
159+
*/
160+
static inline void PackArray(const Array::Ptr& arr, StringBuilder& builder)
161+
{
162+
ObjectLock olock(arr);
163+
164+
builder.emplace_back(5);
165+
PackUInt64BE(arr->GetLength(), builder);
166+
167+
for (const Value& value : arr) {
168+
PackAny(value, builder);
169+
}
170+
}
171+
172+
/**
173+
* Append the given dictionary
174+
*/
175+
static inline void PackDictionary(const Dictionary::Ptr& dict, StringBuilder& builder)
176+
{
177+
ObjectLock olock(dict);
178+
179+
builder.emplace_back(6);
180+
PackUInt64BE(dict->GetLength(), builder);
181+
182+
for (const Dictionary::Pair& kv : dict) {
183+
PackString(kv.first, builder);
184+
PackAny(kv.second, builder);
185+
}
186+
}
187+
188+
/**
189+
* Append any JSON-encodable value
190+
*/
191+
static void PackAny(const Value& value, StringBuilder& builder)
192+
{
193+
switch (value.GetType()) {
194+
case ValueString:
195+
builder.emplace_back(4);
196+
PackString(value.Get<String>(), builder);
197+
break;
198+
199+
case ValueNumber:
200+
builder.emplace_back(3);
201+
PackFloat64BE(value.Get<double>(), builder);
202+
break;
203+
204+
case ValueBoolean:
205+
builder.emplace_back(value.ToBool() ? 2 : 1);
206+
break;
207+
208+
case ValueEmpty:
209+
builder.emplace_back(0);
210+
break;
211+
212+
case ValueObject:
213+
{
214+
const Object::Ptr& obj = value.Get<Object::Ptr>();
215+
216+
Dictionary::Ptr dict = dynamic_pointer_cast<Dictionary>(obj);
217+
if (dict) {
218+
PackDictionary(dict, builder);
219+
break;
220+
}
221+
222+
Array::Ptr arr = dynamic_pointer_cast<Array>(obj);
223+
if (arr) {
224+
PackArray(arr, builder);
225+
break;
226+
}
227+
}
228+
229+
builder.emplace_back(0);
230+
break;
231+
232+
default:
233+
VERIFY(!"Invalid variant type.");
234+
}
235+
}
236+
237+
/**
238+
* Pack any JSON-encodable value to a BSON-similar structure suitable for consistent hashing
239+
*
240+
* Spec:
241+
* null: 0x00
242+
* false: 0x01
243+
* true: 0x02
244+
* number: 0x03 (ieee754_binary64_bigendian)payload
245+
* string: 0x04 (uint64_bigendian)payload.length (char[])payload
246+
* array: 0x05 (uint64_bigendian)payload.length (any[])payload
247+
* object: 0x06 (uint64_bigendian)payload.length (keyvalue[])payload.sort()
248+
*
249+
* any: null|false|true|number|string|array|object
250+
* keyvalue: (uint64_bigendian)key.length (char[])key (any)value
251+
*
252+
* Assumptions:
253+
* - double is IEEE 754 binary64
254+
* - all int types (signed and unsigned) and all float types share the same endianness
255+
* - char is exactly 8 bits wide and one char is exactly one byte affected by the machine endianness
256+
* - all input strings, arrays and dictionaries are at most 2^64-1 long
257+
*
258+
* If not, this function will silently produce invalid results.
259+
*/
260+
String icinga::PackObject(const Value& value)
261+
{
262+
StringBuilder builder;
263+
PackAny(value, builder);
264+
265+
String result;
266+
result.insert(result.End(), builder.begin(), builder.end());
267+
return std::move(result);
268+
}

‎lib/base/object-packer.hpp

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
/******************************************************************************
2+
* Icinga 2 *
3+
* Copyright (C) 2012-2018 Icinga Development Team (https://www.icinga.com/) *
4+
* *
5+
* This program is free software; you can redistribute it and/or *
6+
* modify it under the terms of the GNU General Public License *
7+
* as published by the Free Software Foundation; either version 2 *
8+
* of the License, or (at your option) any later version. *
9+
* *
10+
* This program is distributed in the hope that it will be useful, *
11+
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
12+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
13+
* GNU General Public License for more details. *
14+
* *
15+
* You should have received a copy of the GNU General Public License *
16+
* along with this program; if not, write to the Free Software Foundation *
17+
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA. *
18+
******************************************************************************/
19+
20+
#ifndef OBJECT_PACKER
21+
#define OBJECT_PACKER
22+
23+
#include "base/i2-base.hpp"
24+
25+
namespace icinga
26+
{
27+
28+
class String;
29+
class Value;
30+
31+
String PackObject(const Value& value);
32+
33+
}
34+
35+
#endif /* OBJECT_PACKER */

0 commit comments

Comments
 (0)