llvm.org GIT mirror llvm / 8d164e7
[Support] Make JSON handle doubles and int64s losslessly Summary: This patch adds a new "integer" ValueType, and renames Number -> Double. This allows us to preserve the full precision of int64_t when parsing integers from the wire, or constructing from an integer. The API is unchanged, other than giving asInteger() a clearer contract. In addition, always output doubles with enough precision that parsing will reconstruct the same double. Reviewers: simon_tatham Subscribers: llvm-commits Differential Revision: https://reviews.llvm.org/D46209 git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@336541 91177308-0d34-0410-b5e6-96231b3b80d8 Sam McCall 1 year, 3 months ago
3 changed file(s) with 129 addition(s) and 33 deletion(s). Raw diff Collapse all Expand all
203203 /// Each Value is one of the JSON kinds:
204204 /// null (nullptr_t)
205205 /// boolean (bool)
206 /// number (double)
206 /// number (double or int64)
207207 /// string (StringRef)
208208 /// array (json::Array)
209209 /// object (json::Object)
225225 /// fromJSON(const json::Value&, T&)->bool
226226 /// Deserializers are provided for:
227227 /// - bool
228 /// - int
228 /// - int and int64_t
229229 /// - double
230230 /// - std::string
231231 /// - vector, where T is deserializable
253253 enum Kind {
254254 Null,
255255 Boolean,
256 /// Number values can store both int64s and doubles at full precision,
257 /// depending on what they were constructed/parsed from.
256258 Number,
257259 String,
258260 Array,
280282 Value(llvm::StringRef V) : Type(T_StringRef) { create(V); }
281283 Value(const char *V) : Type(T_StringRef) { create(V); }
282284 Value(std::nullptr_t) : Type(T_Null) {}
283 // Prevent implicit conversions to boolean.
284 template
285 std::is_same::value>::type>
285 // Boolean (disallow implicit conversions).
286 // (The last template parameter is a dummy to keep templates distinct.)
287 template <
288 typename T,
289 typename = typename std::enable_if::value>::type,
290 bool = false>
286291 Value(T B) : Type(T_Boolean) {
287292 create(B);
288293 }
289 // Numbers: arithmetic types that are not boolean.
294 // Integers (except boolean). Must be non-narrowing convertible to int64_t.
290295 template <
291296 typename T,
292 typename = typename std::enable_ifarithmetic::value>::type,
297 typename = typename std::enable_ifintegral::value>::type,
293298 typename = typename std::enable_if::value>::type>
294 Value(T D) : Type(T_Number) {
295 create(D);
299 Value(T I) : Type(T_Integer) {
300 create(int64_t{I});
301 }
302 // Floating point. Must be non-narrowing convertible to double.
303 template
304 typename =
305 typename std::enable_if::value>::type,
306 double * = nullptr>
307 Value(T D) : Type(T_Double) {
308 create(double{D});
296309 }
297310 // Serializable types: with a toJSON(const T&)->Value function, found by ADL.
298311 template
299312 typename = typename std::enable_if
300 Value, decltype(toJSON(*(const T *)nullptr))>::value>>
313 Value, decltype(toJSON(*(const T *)nullptr))>::value>,
314 Value * = nullptr>
301315 Value(const T &V) : Value(toJSON(V)) {}
302316
303317 Value &operator=(const Value &M) {
318332 return Null;
319333 case T_Boolean:
320334 return Boolean;
321 case T_Number:
335 case T_Double:
336 case T_Integer:
322337 return Number;
323338 case T_String:
324339 case T_StringRef:
343358 return llvm::None;
344359 }
345360 llvm::Optional getAsNumber() const {
346 if (LLVM_LIKELY(Type == T_Number))
361 if (LLVM_LIKELY(Type == T_Double))
347362 return as();
363 if (LLVM_LIKELY(Type == T_Integer))
364 return as();
348365 return llvm::None;
349366 }
367 // Succeeds if the Value is a Number, and exactly representable as int64_t.
350368 llvm::Optional getAsInteger() const {
351 if (LLVM_LIKELY(Type == T_Number)) {
369 if (LLVM_LIKELY(Type == T_Integer))
370 return as();
371 if (LLVM_LIKELY(Type == T_Double)) {
352372 double D = as();
353373 if (LLVM_LIKELY(std::modf(D, &D) == 0.0 &&
354374 D >= double(std::numeric_limits::min()) &&
406426 enum ValueType : char {
407427 T_Null,
408428 T_Boolean,
409 // FIXME: splitting Number into Double and Integer would allow us to
410 // round-trip 64-bit integers.
411 T_Number,
429 T_Double,
430 T_Integer,
412431 T_StringRef,
413432 T_String,
414433 T_Object,
416435 };
417436 // All members mutable, see moveFrom().
418437 mutable ValueType Type;
419 mutable llvm::AlignedCharArrayUnionllvm::StringRef,
438 mutable llvm::AlignedCharArrayUnionint64_t, llvm::StringRef,
420439 std::string, json::Array, json::Object>
421440 Union;
422441 };
504523 }
505524 return false;
506525 }
526 inline bool fromJSON(const Value &E, int64_t &Out) {
527 if (auto S = E.getAsInteger()) {
528 Out = *S;
529 return true;
530 }
531 return false;
532 }
507533 inline bool fromJSON(const Value &E, double &Out) {
508534 if (auto S = E.getAsNumber()) {
509535 Out = *S;
103103 switch (Type) {
104104 case T_Null:
105105 case T_Boolean:
106 case T_Number:
106 case T_Double:
107 case T_Integer:
107108 memcpy(Union.buffer, M.Union.buffer, sizeof(Union.buffer));
108109 break;
109110 case T_StringRef:
126127 switch (Type) {
127128 case T_Null:
128129 case T_Boolean:
129 case T_Number:
130 case T_Double:
131 case T_Integer:
130132 memcpy(Union.buffer, M.Union.buffer, sizeof(Union.buffer));
131133 break;
132134 case T_StringRef:
151153 switch (Type) {
152154 case T_Null:
153155 case T_Boolean:
154 case T_Number:
156 case T_Double:
157 case T_Integer:
155158 break;
156159 case T_StringRef:
157160 as().~StringRef();
216219 }
217220
218221 // On invalid syntax, parseX() functions return false and set Err.
219 bool parseNumber(char First, double &Out);
222 bool parseNumber(char First, Value &Out);
220223 bool parseString(std::string &Out);
221224 bool parseUnicode(std::string &Out);
222225 bool parseError(const char *Msg); // always returns false
316319 }
317320 }
318321 default:
319 if (isNumber(C)) {
320 double Num;
321 if (parseNumber(C, Num)) {
322 Out = Num;
323 return true;
324 } else {
325 return false;
326 }
327 }
322 if (isNumber(C))
323 return parseNumber(C, Out);
328324 return parseError("Invalid JSON value");
329325 }
330326 }
331327
332 bool Parser::parseNumber(char First, double &Out) {
328 bool Parser::parseNumber(char First, Value &Out) {
329 // Read the number into a string. (Must be null-terminated for strto*).
333330 SmallString<24> S;
334331 S.push_back(First);
335332 while (isNumber(peek()))
336333 S.push_back(next());
337334 char *End;
335 // Try first to parse as integer, and if so preserve full 64 bits.
336 // strtoll returns long long >= 64 bits, so check it's in range too.
337 auto I = std::strtoll(S.c_str(), &End, 10);
338 if (End == S.end() && I >= std::numeric_limits::min() &&
339 I <= std::numeric_limits::max()) {
340 Out = int64_t(I);
341 return true;
342 }
343 // If it's not an integer
338344 Out = std::strtod(S.c_str(), &End);
339345 return End == S.end() || parseError("Invalid JSON value (number?)");
340346 }
557563 case T_Boolean:
558564 OS << (as() ? "true" : "false");
559565 break;
560 case T_Number:
561 OS << format("%g", as());
566 case T_Double:
567 OS << format("%.*g", std::numeric_limits::max_digits10,
568 as());
569 break;
570 case T_Integer:
571 OS << as();
562572 break;
563573 case T_StringRef:
564574 quote(OS, as());
223223 llvm::Optional("arrow"));
224224 } else
225225 EXPECT_FALSE(E.getAsObject());
226 }
227 }
228
229 // Verify special integer handling - we try to preserve exact int64 values.
230 TEST(JSONTest, Integers) {
231 struct {
232 const char *Desc;
233 Value Val;
234 const char *Str;
235 llvm::Optional AsInt;
236 llvm::Optional AsNumber;
237 } TestCases[] = {
238 {
239 "Non-integer. Stored as double, not convertible.",
240 double{1.5},
241 "1.5",
242 llvm::None,
243 1.5,
244 },
245
246 {
247 "Integer, not exact double. Stored as int64, convertible.",
248 int64_t{0x4000000000000001},
249 "4611686018427387905",
250 int64_t{0x4000000000000001},
251 double{0x4000000000000000},
252 },
253
254 {
255 "Negative integer, not exact double. Stored as int64, convertible.",
256 int64_t{-0x4000000000000001},
257 "-4611686018427387905",
258 int64_t{-0x4000000000000001},
259 double{-0x4000000000000000},
260 },
261
262 {
263 "Dynamically exact integer. Stored as double, convertible.",
264 double{0x6000000000000000},
265 "6.9175290276410819e+18",
266 int64_t{0x6000000000000000},
267 double{0x6000000000000000},
268 },
269
270 {
271 "Dynamically integer, >64 bits. Stored as double, not convertible.",
272 1.5 * double{0x8000000000000000},
273 "1.3835058055282164e+19",
274 llvm::None,
275 1.5 * double{0x8000000000000000},
276 },
277 };
278 for (const auto &T : TestCases) {
279 EXPECT_EQ(T.Str, s(T.Val)) << T.Desc;
280 llvm::Expected Doc = parse(T.Str);
281 EXPECT_TRUE(!!Doc) << T.Desc;
282 EXPECT_EQ(Doc->getAsInteger(), T.AsInt) << T.Desc;
283 EXPECT_EQ(Doc->getAsNumber(), T.AsNumber) << T.Desc;
284 EXPECT_EQ(T.Val, *Doc) << T.Desc;
285 EXPECT_EQ(T.Str, s(*Doc)) << T.Desc;
226286 }
227287 }
228288