diff --git a/.gitignore b/.gitignore index c319815..eab1c20 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ target/ .DS_Store .vscode .java-version +/.idea/ \ No newline at end of file diff --git a/README.md b/README.md index 71adf88..0b4911b 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,16 @@ +# JSONata Java (BigDecimal Fork) + +> **Note:** This is a fork of [jsonata-java](https://github.com/dashjoin/jsonata-java) that uses `BigDecimal` for all numeric operations instead of floating-point types. This change addresses binary arithmetic precision issues (e.g., 0.1 + 0.2 != 0.3). +> +> See +> +> * [ArbitraryPrecisionTest.java](src/test/java/com/dashjoin/jsonata/ArbitraryPrecisionTest.java) +> * [https://medium.com/@muhamad99t/unraveling-the-binary-mystery-0-1-0-2-0-3-364a3eff4062](Unraveling the Binary Mystery: 0.1 + 0.2 ≠ 0.3) +> * [https://levelup.gitconnected.com/why-computers-fail-at-simple-math-0-1-0-2-0-3-928a7f884d9d](Why computers fail at simple math — 0.1 + 0.2 != 0.3) +> +> This hasn't been fully tested but is simply a proof of concept and a familiarisation exercise. +--- + # jsonata-java is the JSONata Java reference port JSONata reference ported to Java diff --git a/change-float-to-bigdecimal.patch b/change-float-to-bigdecimal.patch new file mode 100644 index 0000000..4ba8d40 --- /dev/null +++ b/change-float-to-bigdecimal.patch @@ -0,0 +1,566 @@ +Index: src/main/java/com/dashjoin/jsonata/Functions.java +IDEA additional info: +Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP +<+>UTF-8 +=================================================================== +diff --git a/src/main/java/com/dashjoin/jsonata/Functions.java b/src/main/java/com/dashjoin/jsonata/Functions.java +--- a/src/main/java/com/dashjoin/jsonata/Functions.java (revision 0645e7bc78d34ad1afcb67731b57e433b788cd1f) ++++ b/src/main/java/com/dashjoin/jsonata/Functions.java (date 1761447923422) +@@ -1321,14 +1321,15 @@ + result = (Number)arg; + else if (arg instanceof String) { + String s = (String)arg; ++ // todo is it okay to build big decimal like this with radix + if (s.startsWith("0x")) +- result = Long.parseLong(s.substring(2), 16); ++ result = BigDecimal.valueOf(Long.parseLong(s.substring(2), 16)); + else if (s.startsWith("0B")) +- result = Long.parseLong(s.substring(2), 2); ++ result = BigDecimal.valueOf(Long.parseLong(s.substring(2), 2)); + else if (s.startsWith("0O")) +- result = Long.parseLong(s.substring(2), 8); ++ result = BigDecimal.valueOf(Long.parseLong(s.substring(2), 8)); + else +- result = Double.valueOf((String)arg); ++ result = new BigDecimal((String)arg); + } else if (arg instanceof Boolean) { + result = ((boolean)arg) ? 1:0; + } +@@ -2134,6 +2135,13 @@ + if (result==null && ((java.util.Map)input).containsKey(key)) + result = Jsonata.NULL_VALUE; + } ++ ++ // automatically convert numbers in the context to BigDecimals ++ if (result instanceof Number) { ++ if (!(result instanceof BigDecimal)) { ++ result = BigDecimal.valueOf(((Number) result).doubleValue()); ++ } ++ } + return result; + } + +Index: src/main/java/com/dashjoin/jsonata/Jsonata.java +IDEA additional info: +Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP +<+>UTF-8 +=================================================================== +diff --git a/src/main/java/com/dashjoin/jsonata/Jsonata.java b/src/main/java/com/dashjoin/jsonata/Jsonata.java +--- a/src/main/java/com/dashjoin/jsonata/Jsonata.java (revision 0645e7bc78d34ad1afcb67731b57e433b788cd1f) ++++ b/src/main/java/com/dashjoin/jsonata/Jsonata.java (date 1761448415983) +@@ -26,6 +26,7 @@ + + import java.lang.reflect.InvocationTargetException; + import java.lang.reflect.Method; ++import java.math.BigDecimal; + import java.util.ArrayList; + import java.util.Comparator; + import java.util.HashMap; +@@ -804,14 +805,31 @@ + * @returns {*} Result + */ + Object evaluateNumericExpression(Object _lhs, Object _rhs, String op) { +- double result = 0; ++ BigDecimal result = BigDecimal.ZERO; ++ ++ if(_lhs != null) { ++ if (_lhs instanceof Number) { ++ if (!(_lhs instanceof BigDecimal)) { ++ _lhs = new BigDecimal(((Number) _lhs).doubleValue()); ++ } ++ } ++ } + +- if (_lhs!=null && !Utils.isNumeric(_lhs)) { ++ if(_rhs != null) { ++ if (_rhs instanceof Number) { ++ if (!(_rhs instanceof BigDecimal)) { ++ _rhs = new BigDecimal(((Number) _rhs).doubleValue()); ++ } ++ } ++ } ++ ++ if (_lhs!=null && !(_lhs instanceof BigDecimal)) { + throw new JException("T2001", -1, + op, _lhs + ); + } +- if (_rhs!=null && !Utils.isNumeric(_rhs)) { ++ ++ if (_rhs!=null && !(_rhs instanceof BigDecimal)) { + throw new JException("T2002", -1, + op, _rhs + ); +@@ -823,27 +841,33 @@ + } + + //System.out.println("op22 "+op+" "+_lhs+" "+_rhs); +- double lhs = ((Number)_lhs).doubleValue(); +- double rhs = ((Number)_rhs).doubleValue(); ++ BigDecimal lhs = ((BigDecimal)_lhs); ++ BigDecimal rhs = ((BigDecimal)_rhs); + + switch (op) { + case "+": +- result = lhs + rhs; ++ result = lhs.add(rhs); + break; + case "-": +- result = lhs - rhs; ++ result = lhs.subtract(rhs); + break; + case "*": +- result = lhs * rhs; ++ result = lhs.multiply(rhs); + break; + case "/": +- result = lhs / rhs; ++ try { ++ result = lhs.divide(rhs); ++ } catch (ArithmeticException e) { ++ if (e.getMessage().contains("Division by zero")) { ++ throw new JException("D1001", 0, "Infinity"); ++ } ++ } + break; + case "%": +- result = lhs % rhs; ++ result = lhs.remainder(rhs); + break; + } +- return Utils.convertNumber(result); ++ return result; + } + + /** +@@ -868,10 +892,10 @@ + // JSON might come with integers, + // convert all to double... + // FIXME: semantically OK? +- if (lhs instanceof Number) +- lhs = ((Number)lhs).doubleValue(); +- if (rhs instanceof Number) +- rhs = ((Number)rhs).doubleValue(); ++// if (lhs instanceof Number) ++// lhs = ((Number)lhs).doubleValue(); ++// if (rhs instanceof Number) ++// rhs = ((Number)rhs).doubleValue(); + + switch (op) { + case "=": +@@ -1165,6 +1189,32 @@ + Object evaluateRangeExpression(Object lhs, Object rhs) { + Object result = null; + ++ // convert to long, and error if there is any fractional part ++ if (lhs instanceof BigDecimal) { ++ try { ++ lhs = ((BigDecimal) lhs).longValueExact(); ++ } catch (ArithmeticException e) { ++ throw new JException("T2003", ++ //stack: (new Error()).stack, ++ -1, ++ lhs ++ ); ++ } ++ } ++ ++ // convert to long, and error if there is any fractional part ++ if (rhs instanceof BigDecimal) { ++ try { ++ rhs = ((BigDecimal) rhs).longValueExact(); ++ } catch (ArithmeticException e) { ++ throw new JException("T2003", ++ //stack: (new Error()).stack, ++ -1, ++ rhs ++ ); ++ } ++ } ++ + if (lhs != null && (!(lhs instanceof Long) && !(lhs instanceof Integer))) { + throw new JException("T2003", + //stack: (new Error()).stack, +Index: src/main/java/com/dashjoin/jsonata/Parser.java +IDEA additional info: +Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP +<+>UTF-8 +=================================================================== +diff --git a/src/main/java/com/dashjoin/jsonata/Parser.java b/src/main/java/com/dashjoin/jsonata/Parser.java +--- a/src/main/java/com/dashjoin/jsonata/Parser.java (revision 0645e7bc78d34ad1afcb67731b57e433b788cd1f) ++++ b/src/main/java/com/dashjoin/jsonata/Parser.java (date 1761353839058) +@@ -28,6 +28,7 @@ + import java.io.ByteArrayOutputStream; + import java.io.ObjectInputStream; + import java.io.ObjectOutputStream; ++import java.math.BigDecimal; + import java.util.ArrayList; + import java.util.Arrays; + import java.util.HashMap; +@@ -1255,7 +1256,7 @@ + // if unary minus on a number, then pre-process + if (exprValue.equals("-") && result.expression.type.equals("number")) { + result = result.expression; +- result.value = Utils.convertNumber( -((Number)result.value).doubleValue() ); ++ result.value = ((BigDecimal)result.value).negate(); + if (dbg) System.out.println("unary - value="+result.value); + } else { + pushAncestry(result, result.expression); +Index: src/main/java/com/dashjoin/jsonata/Tokenizer.java +IDEA additional info: +Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP +<+>UTF-8 +=================================================================== +diff --git a/src/main/java/com/dashjoin/jsonata/Tokenizer.java b/src/main/java/com/dashjoin/jsonata/Tokenizer.java +--- a/src/main/java/com/dashjoin/jsonata/Tokenizer.java (revision 0645e7bc78d34ad1afcb67731b57e433b788cd1f) ++++ b/src/main/java/com/dashjoin/jsonata/Tokenizer.java (date 1761447923438) +@@ -1,8 +1,8 @@ + /** + * jsonata-java is the JSONata Java reference port +- * ++ * + * Copyright Dashjoin GmbH. https://dashjoin.com +- * ++ * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at +@@ -24,6 +24,7 @@ + */ + package com.dashjoin.jsonata; + ++import java.math.BigDecimal; + import java.util.HashMap; + import java.util.regex.Matcher; + import java.util.regex.Pattern; +@@ -151,13 +152,13 @@ + } + } + flags = path.substring(start, position) + 'g'; +- ++ + // Convert flags to Java Pattern flags + int _flags = 0; + if (flags.contains("i")) + _flags |= Pattern.CASE_INSENSITIVE; + if (flags.contains("m")) +- _flags |= Pattern.MULTILINE; ++ _flags |= Pattern.MULTILINE; + return Pattern.compile(pattern, _flags); // Pattern.CASE_INSENSITIVE | Pattern.MULTILINE | Pattern.DOTALL); + } + if ((currentChar == '(' || currentChar == '[' || currentChar == '{') && path.charAt(position - 1) != '\\') { +@@ -295,12 +296,12 @@ + Pattern numregex = Pattern.compile("^-?(0|([1-9][0-9]*))(\\.[0-9]+)?([Ee][-+]?[0-9]+)?"); + Matcher match = numregex.matcher(path.substring(position)); + if (match.find()) { +- double num = Double.parseDouble(match.group(0)); +- if (!Double.isNaN(num) && Double.isFinite(num)) { ++ try { ++ BigDecimal num = new BigDecimal(match.group(0)); + position += match.group(0).length(); + // If the number is integral, use long as type +- return create("number", Utils.convertNumber(num)); +- } else { ++ return create("number", num); ++ } catch (NumberFormatException e) { + throw new JException("S0102", position); //, match.group[0]); + } + } +Index: src/main/java/com/dashjoin/jsonata/Utils.java +IDEA additional info: +Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP +<+>UTF-8 +=================================================================== +diff --git a/src/main/java/com/dashjoin/jsonata/Utils.java b/src/main/java/com/dashjoin/jsonata/Utils.java +--- a/src/main/java/com/dashjoin/jsonata/Utils.java (revision 0645e7bc78d34ad1afcb67731b57e433b788cd1f) ++++ b/src/main/java/com/dashjoin/jsonata/Utils.java (date 1761355550639) +@@ -17,6 +17,7 @@ + */ + package com.dashjoin.jsonata; + ++import java.math.BigDecimal; + import java.util.AbstractList; + import java.util.ArrayList; + import java.util.Collection; +@@ -151,6 +152,9 @@ + } + + public static Number convertNumber(Number n) { ++ if (n instanceof BigDecimal) { ++ return n; ++ } + // Use long if the number is not fractional + if (!isNumeric(n)) return null; + if (n.longValue()==n.doubleValue()) { +Index: src/test/java/com/dashjoin/jsonata/ArbitraryPrecisionTest.java +IDEA additional info: +Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP +<+>UTF-8 +=================================================================== +diff --git a/src/test/java/com/dashjoin/jsonata/ArbitraryPrecisionTest.java b/src/test/java/com/dashjoin/jsonata/ArbitraryPrecisionTest.java +new file mode 100644 +--- /dev/null (date 1761448479257) ++++ b/src/test/java/com/dashjoin/jsonata/ArbitraryPrecisionTest.java (date 1761448479257) +@@ -0,0 +1,40 @@ ++package com.dashjoin.jsonata; ++ ++import org.junit.jupiter.params.ParameterizedTest; ++import org.junit.jupiter.params.provider.Arguments; ++import org.junit.jupiter.params.provider.MethodSource; ++ ++import java.math.BigDecimal; ++import java.util.Map; ++import java.util.stream.Stream; ++ ++import static com.dashjoin.jsonata.Jsonata.jsonata; ++import static org.junit.jupiter.api.Assertions.assertEquals; ++ ++public class ArbitraryPrecisionTest { ++ ++ static Stream cases() { ++ return Stream.of( ++ // subtraction ++ Arguments.of("0.8 - 0.1", new BigDecimal("0.7"), Map.of()), ++ Arguments.of("a - b", new BigDecimal("0.7"), Map.of("a", 0.8, "b", 0.1)), ++ // addition ++ Arguments.of("0.8 + 0.1", new BigDecimal("0.9"), Map.of()), ++ Arguments.of("a + b", new BigDecimal("0.9"), Map.of("a", 0.8, "b", 0.1)), ++ // multiplication ++ Arguments.of("0.8 * 0.1", new BigDecimal("0.08"), Map.of()), ++ Arguments.of("a * b", new BigDecimal("0.08"), Map.of("a", 0.8, "b", 0.1)), ++ // division ++ Arguments.of("0.8 / 0.1", new BigDecimal("8"), Map.of()), ++ Arguments.of("a / b", new BigDecimal("8"), Map.of("a", 0.8, "b", 0.1)) ++ ); ++ } ++ ++ @ParameterizedTest ++ @MethodSource("cases") ++ void test(String expression, BigDecimal expected, Map input) { ++ Jsonata expr = jsonata(expression); ++ var res = expr.evaluate(input); ++ assertEquals(expected, res); ++ } ++} +Index: src/test/java/com/dashjoin/jsonata/ArrayTest.java +IDEA additional info: +Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP +<+>UTF-8 +=================================================================== +diff --git a/src/test/java/com/dashjoin/jsonata/ArrayTest.java b/src/test/java/com/dashjoin/jsonata/ArrayTest.java +--- a/src/test/java/com/dashjoin/jsonata/ArrayTest.java (revision 0645e7bc78d34ad1afcb67731b57e433b788cd1f) ++++ b/src/test/java/com/dashjoin/jsonata/ArrayTest.java (date 1761355890955) +@@ -4,6 +4,8 @@ + import static java.util.Arrays.asList; + import static java.util.Map.of; + import static org.junit.jupiter.api.Assertions.assertEquals; ++ ++import java.math.BigDecimal; + import java.util.Arrays; + import java.util.List; + import java.util.Map; +@@ -39,13 +41,15 @@ + @Test + public void testSort() { + Jsonata expr = jsonata("$sort([{'x': 2}, {'x': 1}], function($l, $r){$l.x > $r.x})"); +- Assertions.assertEquals(Arrays.asList(Map.of("x", 1), Map.of("x", 2)), expr.evaluate(null)); ++ Assertions.assertEquals( ++ Arrays.asList(Map.of("x", BigDecimal.valueOf(1)), Map.of("x", BigDecimal.valueOf(2))), ++ expr.evaluate(null)); + } + + @Test + public void testSortNull() { + Jsonata expr = jsonata("$sort([{'x': 2}, {'x': 1}], function($l, $r){$l.y > $r.y})"); +- Assertions.assertEquals(Arrays.asList(Map.of("x", 2), Map.of("x", 1)), expr.evaluate(null)); ++ Assertions.assertEquals(Arrays.asList(Map.of("x", BigDecimal.valueOf(2)), Map.of("x", BigDecimal.valueOf(1))), expr.evaluate(null)); + } + + @Test +Index: src/test/java/com/dashjoin/jsonata/CustomFunctionTest.java +IDEA additional info: +Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP +<+>UTF-8 +=================================================================== +diff --git a/src/test/java/com/dashjoin/jsonata/CustomFunctionTest.java b/src/test/java/com/dashjoin/jsonata/CustomFunctionTest.java +--- a/src/test/java/com/dashjoin/jsonata/CustomFunctionTest.java (revision 0645e7bc78d34ad1afcb67731b57e433b788cd1f) ++++ b/src/test/java/com/dashjoin/jsonata/CustomFunctionTest.java (date 1761371810454) +@@ -1,5 +1,6 @@ + package com.dashjoin.jsonata; + ++import java.math.BigDecimal; + import java.util.List; + import java.util.Map; + import org.junit.jupiter.api.Assertions; +@@ -35,14 +36,14 @@ + public void testUnary() { + var expression = Jsonata.jsonata("$echo(123)"); + expression.registerFunction("echo", (x) -> x); +- Assertions.assertEquals(123, expression.evaluate(null)); ++ Assertions.assertEquals(BigDecimal.valueOf(123), expression.evaluate(null)); + } + + @Test + public void testBinary() { + var expression = Jsonata.jsonata("$add(21, 21)"); +- expression.registerFunction("add", (Integer a, Integer b) -> a + b); +- Assertions.assertEquals(42, expression.evaluate(null)); ++ expression.registerFunction("add", (BigDecimal a, BigDecimal b) -> a.add(b)); ++ Assertions.assertEquals(BigDecimal.valueOf(42), expression.evaluate(null)); + } + + @Test +@@ -64,7 +65,7 @@ + @Test + public void testLambdaSignatureError() { + var expression = Jsonata.jsonata("$append(1, 2)"); +- expression.registerFunction("append", (Integer a, Boolean b) -> "" + a + b); ++ expression.registerFunction("append", (BigDecimal a, Boolean b) -> "" + a + b); + Assertions.assertThrowsExactly(ClassCastException.class, () -> expression.evaluate(null)); + } + +Index: src/test/java/com/dashjoin/jsonata/NumberTest.java +IDEA additional info: +Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP +<+>UTF-8 +=================================================================== +diff --git a/src/test/java/com/dashjoin/jsonata/NumberTest.java b/src/test/java/com/dashjoin/jsonata/NumberTest.java +--- a/src/test/java/com/dashjoin/jsonata/NumberTest.java (revision 0645e7bc78d34ad1afcb67731b57e433b788cd1f) ++++ b/src/test/java/com/dashjoin/jsonata/NumberTest.java (date 1761447923413) +@@ -3,10 +3,14 @@ + import static com.dashjoin.jsonata.Jsonata.jsonata; + import static java.util.Map.of; + import static org.junit.jupiter.api.Assertions.assertEquals; ++import static org.junit.jupiter.api.Assertions.assertTrue; ++ + import org.junit.jupiter.api.Disabled; + import org.junit.jupiter.api.Test; + import com.dashjoin.jsonata.json.Json; + ++import java.math.BigDecimal; ++ + public class NumberTest { + + /** +@@ -19,7 +23,7 @@ + var res = expr1.evaluate(of("x", 1.0)); + assertEquals(1, res); + } +- ++ + /** + * a computation is applied, and com.dashjoin.jsonata.Utils.convertNumber(Number) casts the double to int + */ +@@ -27,7 +31,7 @@ + public void testDouble2() { + Jsonata expr1 = jsonata("x+0"); + var res = expr1.evaluate(of("x", 1.0)); +- assertEquals(1, res); ++ assertTrue(BigDecimal.valueOf(1.0).compareTo((BigDecimal) res)==0); + } + + /** +@@ -39,27 +43,27 @@ + var res = expr1.evaluate(Json.parseJson("{\"x\":1.0}")); + assertEquals(1, res); + } +- ++ + /** + * "clean" the input using com.dashjoin.jsonata.Utils.convertNumber(Number) + */ + @Test + public void testDouble4() { + Jsonata expr1 = jsonata("x"); +- var res = expr1.evaluate(of("x", Utils.convertNumber(1.0))); +- assertEquals(1, res); ++ var res = expr1.evaluate(of("x", BigDecimal.valueOf(1.0))); ++ assertTrue(BigDecimal.valueOf(1).compareTo((BigDecimal) res)==0); + } +- ++ + /** + * int 1 is converted to double when divided by 2 + */ + @Test + public void testInt() { + Jsonata expr1 = jsonata("$ / 2"); +- var res = expr1.evaluate(1); +- assertEquals(0.5, res); ++ var res = expr1.evaluate(BigDecimal.valueOf(1)); ++ assertEquals(BigDecimal.valueOf(0.5), res); + } +- ++ + /** + * JSONata constant 1.0 evaluates to 1 + */ +@@ -67,6 +71,6 @@ + public void testConst() { + Jsonata expr1 = jsonata("1.0"); + var res = expr1.evaluate(null); +- assertEquals(1, res); ++ assertEquals(BigDecimal.valueOf(1.0), res); + } + } +Index: src/test/java/com/dashjoin/jsonata/SignatureTest.java +IDEA additional info: +Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP +<+>UTF-8 +=================================================================== +diff --git a/src/test/java/com/dashjoin/jsonata/SignatureTest.java b/src/test/java/com/dashjoin/jsonata/SignatureTest.java +--- a/src/test/java/com/dashjoin/jsonata/SignatureTest.java (revision 0645e7bc78d34ad1afcb67731b57e433b788cd1f) ++++ b/src/test/java/com/dashjoin/jsonata/SignatureTest.java (date 1761373516670) +@@ -1,6 +1,8 @@ + package com.dashjoin.jsonata; + + import static com.dashjoin.jsonata.Jsonata.jsonata; ++ ++import java.math.BigDecimal; + import java.util.List; + import org.junit.jupiter.api.Assertions; + import org.junit.jupiter.api.Test; +@@ -47,13 +49,13 @@ + @SuppressWarnings("rawtypes") + @Override + public Object call(Object input, List args) throws Throwable { +- int sum = 0; ++ BigDecimal sum = BigDecimal.ZERO; + for (Object i : args) +- sum += (int) i; ++ sum = sum.add((BigDecimal) i); + return sum; + } + }, "")); +- Assertions.assertEquals(6, expression.evaluate(null)); ++ Assertions.assertEquals(BigDecimal.valueOf(6), expression.evaluate(null)); + } + + @Test +Index: .gitignore +IDEA additional info: +Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP +<+>UTF-8 +=================================================================== +diff --git a/.gitignore b/.gitignore +--- a/.gitignore (revision 0645e7bc78d34ad1afcb67731b57e433b788cd1f) ++++ b/.gitignore (date 1761386426252) +@@ -2,3 +2,4 @@ + .DS_Store + .vscode + .java-version ++/.idea/ +\ No newline at end of file +Index: jsonata +IDEA additional info: +Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP +<+>UTF-8 +=================================================================== +diff --git a/jsonata b/jsonata +--- a/jsonata (revision 0645e7bc78d34ad1afcb67731b57e433b788cd1f) ++++ b/jsonata +@@ -1,1 +1,1 @@ +-2bb258933b509b97ed173a48e0d5661033b1df19 +\ No newline at end of file ++087d63314d4e7ee8ac342b139fbec8594eb391c3 +\ No newline at end of file diff --git a/jsonata b/jsonata index 2bb2589..087d633 160000 --- a/jsonata +++ b/jsonata @@ -1 +1 @@ -Subproject commit 2bb258933b509b97ed173a48e0d5661033b1df19 +Subproject commit 087d63314d4e7ee8ac342b139fbec8594eb391c3 diff --git a/src/main/java/com/dashjoin/jsonata/Functions.java b/src/main/java/com/dashjoin/jsonata/Functions.java index 8dc7b7a..b35db7b 100644 --- a/src/main/java/com/dashjoin/jsonata/Functions.java +++ b/src/main/java/com/dashjoin/jsonata/Functions.java @@ -1321,14 +1321,15 @@ public static Number number(Object arg) throws NumberFormatException, JException result = (Number)arg; else if (arg instanceof String) { String s = (String)arg; + // todo is it okay to build big decimal like this with radix if (s.startsWith("0x")) - result = Long.parseLong(s.substring(2), 16); + result = BigDecimal.valueOf(Long.parseLong(s.substring(2), 16)); else if (s.startsWith("0B")) - result = Long.parseLong(s.substring(2), 2); + result = BigDecimal.valueOf(Long.parseLong(s.substring(2), 2)); else if (s.startsWith("0O")) - result = Long.parseLong(s.substring(2), 8); + result = BigDecimal.valueOf(Long.parseLong(s.substring(2), 8)); else - result = Double.valueOf((String)arg); + result = new BigDecimal((String)arg); } else if (arg instanceof Boolean) { result = ((boolean)arg) ? 1:0; } @@ -2134,6 +2135,13 @@ public static Object lookup(Object input, String key) { if (result==null && ((java.util.Map)input).containsKey(key)) result = Jsonata.NULL_VALUE; } + + // automatically convert numbers in the context to BigDecimals + if (result instanceof Number) { + if (!(result instanceof BigDecimal)) { + result = BigDecimal.valueOf(((Number) result).doubleValue()); + } + } return result; } diff --git a/src/main/java/com/dashjoin/jsonata/Jsonata.java b/src/main/java/com/dashjoin/jsonata/Jsonata.java index 7a78177..8d5b8b8 100644 --- a/src/main/java/com/dashjoin/jsonata/Jsonata.java +++ b/src/main/java/com/dashjoin/jsonata/Jsonata.java @@ -26,6 +26,7 @@ import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; +import java.math.BigDecimal; import java.util.ArrayList; import java.util.Comparator; import java.util.HashMap; @@ -804,14 +805,31 @@ void recurseDescendants(Object input, List results) { * @returns {*} Result */ Object evaluateNumericExpression(Object _lhs, Object _rhs, String op) { - double result = 0; + BigDecimal result = BigDecimal.ZERO; - if (_lhs!=null && !Utils.isNumeric(_lhs)) { + if(_lhs != null) { + if (_lhs instanceof Number) { + if (!(_lhs instanceof BigDecimal)) { + _lhs = new BigDecimal(((Number) _lhs).doubleValue()); + } + } + } + + if(_rhs != null) { + if (_rhs instanceof Number) { + if (!(_rhs instanceof BigDecimal)) { + _rhs = new BigDecimal(((Number) _rhs).doubleValue()); + } + } + } + + if (_lhs!=null && !(_lhs instanceof BigDecimal)) { throw new JException("T2001", -1, op, _lhs ); } - if (_rhs!=null && !Utils.isNumeric(_rhs)) { + + if (_rhs!=null && !(_rhs instanceof BigDecimal)) { throw new JException("T2002", -1, op, _rhs ); @@ -823,27 +841,33 @@ Object evaluateNumericExpression(Object _lhs, Object _rhs, String op) { } //System.out.println("op22 "+op+" "+_lhs+" "+_rhs); - double lhs = ((Number)_lhs).doubleValue(); - double rhs = ((Number)_rhs).doubleValue(); + BigDecimal lhs = ((BigDecimal)_lhs); + BigDecimal rhs = ((BigDecimal)_rhs); switch (op) { case "+": - result = lhs + rhs; + result = lhs.add(rhs); break; case "-": - result = lhs - rhs; + result = lhs.subtract(rhs); break; case "*": - result = lhs * rhs; + result = lhs.multiply(rhs); break; case "/": - result = lhs / rhs; + try { + result = lhs.divide(rhs); + } catch (ArithmeticException e) { + if (e.getMessage().contains("Division by zero")) { + throw new JException("D1001", 0, "Infinity"); + } + } break; case "%": - result = lhs % rhs; + result = lhs.remainder(rhs); break; } - return Utils.convertNumber(result); + return result; } /** @@ -868,10 +892,10 @@ Object evaluateEqualityExpression(Object lhs, Object rhs, String op) { // JSON might come with integers, // convert all to double... // FIXME: semantically OK? - if (lhs instanceof Number) - lhs = ((Number)lhs).doubleValue(); - if (rhs instanceof Number) - rhs = ((Number)rhs).doubleValue(); +// if (lhs instanceof Number) +// lhs = ((Number)lhs).doubleValue(); +// if (rhs instanceof Number) +// rhs = ((Number)rhs).doubleValue(); switch (op) { case "=": @@ -1165,6 +1189,32 @@ Object reduceTupleStream(Object _tupleStream) { Object evaluateRangeExpression(Object lhs, Object rhs) { Object result = null; + // convert to long, and error if there is any fractional part + if (lhs instanceof BigDecimal) { + try { + lhs = ((BigDecimal) lhs).longValueExact(); + } catch (ArithmeticException e) { + throw new JException("T2003", + //stack: (new Error()).stack, + -1, + lhs + ); + } + } + + // convert to long, and error if there is any fractional part + if (rhs instanceof BigDecimal) { + try { + rhs = ((BigDecimal) rhs).longValueExact(); + } catch (ArithmeticException e) { + throw new JException("T2003", + //stack: (new Error()).stack, + -1, + rhs + ); + } + } + if (lhs != null && (!(lhs instanceof Long) && !(lhs instanceof Integer))) { throw new JException("T2003", //stack: (new Error()).stack, diff --git a/src/main/java/com/dashjoin/jsonata/Parser.java b/src/main/java/com/dashjoin/jsonata/Parser.java index ccb05d9..57535f5 100644 --- a/src/main/java/com/dashjoin/jsonata/Parser.java +++ b/src/main/java/com/dashjoin/jsonata/Parser.java @@ -28,6 +28,7 @@ import java.io.ByteArrayOutputStream; import java.io.ObjectInputStream; import java.io.ObjectOutputStream; +import java.math.BigDecimal; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -1255,7 +1256,7 @@ Symbol processAST(Symbol expr) { // if unary minus on a number, then pre-process if (exprValue.equals("-") && result.expression.type.equals("number")) { result = result.expression; - result.value = Utils.convertNumber( -((Number)result.value).doubleValue() ); + result.value = ((BigDecimal)result.value).negate(); if (dbg) System.out.println("unary - value="+result.value); } else { pushAncestry(result, result.expression); diff --git a/src/main/java/com/dashjoin/jsonata/Tokenizer.java b/src/main/java/com/dashjoin/jsonata/Tokenizer.java index 308b006..1dda887 100644 --- a/src/main/java/com/dashjoin/jsonata/Tokenizer.java +++ b/src/main/java/com/dashjoin/jsonata/Tokenizer.java @@ -1,8 +1,8 @@ /** * jsonata-java is the JSONata Java reference port - * + * * Copyright Dashjoin GmbH. https://dashjoin.com - * + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -24,6 +24,7 @@ */ package com.dashjoin.jsonata; +import java.math.BigDecimal; import java.util.HashMap; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -151,13 +152,13 @@ Pattern scanRegex() { } } flags = path.substring(start, position) + 'g'; - + // Convert flags to Java Pattern flags int _flags = 0; if (flags.contains("i")) _flags |= Pattern.CASE_INSENSITIVE; if (flags.contains("m")) - _flags |= Pattern.MULTILINE; + _flags |= Pattern.MULTILINE; return Pattern.compile(pattern, _flags); // Pattern.CASE_INSENSITIVE | Pattern.MULTILINE | Pattern.DOTALL); } if ((currentChar == '(' || currentChar == '[' || currentChar == '{') && path.charAt(position - 1) != '\\') { @@ -295,12 +296,12 @@ Token next(boolean prefix) { Pattern numregex = Pattern.compile("^-?(0|([1-9][0-9]*))(\\.[0-9]+)?([Ee][-+]?[0-9]+)?"); Matcher match = numregex.matcher(path.substring(position)); if (match.find()) { - double num = Double.parseDouble(match.group(0)); - if (!Double.isNaN(num) && Double.isFinite(num)) { + try { + BigDecimal num = new BigDecimal(match.group(0)); position += match.group(0).length(); // If the number is integral, use long as type - return create("number", Utils.convertNumber(num)); - } else { + return create("number", num); + } catch (NumberFormatException e) { throw new JException("S0102", position); //, match.group[0]); } } diff --git a/src/main/java/com/dashjoin/jsonata/Utils.java b/src/main/java/com/dashjoin/jsonata/Utils.java index 86cbb0c..7f91298 100644 --- a/src/main/java/com/dashjoin/jsonata/Utils.java +++ b/src/main/java/com/dashjoin/jsonata/Utils.java @@ -17,6 +17,7 @@ */ package com.dashjoin.jsonata; +import java.math.BigDecimal; import java.util.AbstractList; import java.util.ArrayList; import java.util.Collection; @@ -151,6 +152,9 @@ public Number get(int index) { } public static Number convertNumber(Number n) { + if (n instanceof BigDecimal) { + return n; + } // Use long if the number is not fractional if (!isNumeric(n)) return null; if (n.longValue()==n.doubleValue()) { diff --git a/src/test/java/com/dashjoin/jsonata/ArbitraryPrecisionTest.java b/src/test/java/com/dashjoin/jsonata/ArbitraryPrecisionTest.java new file mode 100644 index 0000000..13c958b --- /dev/null +++ b/src/test/java/com/dashjoin/jsonata/ArbitraryPrecisionTest.java @@ -0,0 +1,41 @@ +package com.dashjoin.jsonata; + +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.math.BigDecimal; +import java.util.Map; +import java.util.stream.Stream; + +import static com.dashjoin.jsonata.Jsonata.jsonata; +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class ArbitraryPrecisionTest { + + static Stream cases() { + return Stream.of( + // subtraction + Arguments.of("0.8 - 0.1", new BigDecimal("0.7"), Map.of()), + Arguments.of("a - b", new BigDecimal("0.7"), Map.of("a", 0.8, "b", 0.1)), + Arguments.of("a - b", new BigDecimal("0.7"), Map.of("a", new BigDecimal("0.8"), "b", new BigDecimal("0.1"))), + // addition + Arguments.of("0.8 + 0.1", new BigDecimal("0.9"), Map.of()), + Arguments.of("a + b", new BigDecimal("0.9"), Map.of("a", 0.8, "b", 0.1)), + // multiplication + Arguments.of("0.8 * 0.1", new BigDecimal("0.08"), Map.of()), + Arguments.of("a * b", new BigDecimal("0.08"), Map.of("a", 0.8, "b", 0.1)), + // division + Arguments.of("0.8 / 0.1", new BigDecimal("8"), Map.of()), + Arguments.of("a / b", new BigDecimal("8"), Map.of("a", 0.8, "b", 0.1)) + ); + } + + @ParameterizedTest + @MethodSource("cases") + void test(String expression, BigDecimal expected, Map input) { + Jsonata expr = jsonata(expression); + var res = expr.evaluate(input); + assertEquals(expected, res); + } +} diff --git a/src/test/java/com/dashjoin/jsonata/ArrayTest.java b/src/test/java/com/dashjoin/jsonata/ArrayTest.java index 660256c..9cde079 100644 --- a/src/test/java/com/dashjoin/jsonata/ArrayTest.java +++ b/src/test/java/com/dashjoin/jsonata/ArrayTest.java @@ -4,6 +4,8 @@ import static java.util.Arrays.asList; import static java.util.Map.of; import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.math.BigDecimal; import java.util.Arrays; import java.util.List; import java.util.Map; @@ -39,13 +41,15 @@ public void testIndex() { @Test public void testSort() { Jsonata expr = jsonata("$sort([{'x': 2}, {'x': 1}], function($l, $r){$l.x > $r.x})"); - Assertions.assertEquals(Arrays.asList(Map.of("x", 1), Map.of("x", 2)), expr.evaluate(null)); + Assertions.assertEquals( + Arrays.asList(Map.of("x", BigDecimal.valueOf(1)), Map.of("x", BigDecimal.valueOf(2))), + expr.evaluate(null)); } @Test public void testSortNull() { Jsonata expr = jsonata("$sort([{'x': 2}, {'x': 1}], function($l, $r){$l.y > $r.y})"); - Assertions.assertEquals(Arrays.asList(Map.of("x", 2), Map.of("x", 1)), expr.evaluate(null)); + Assertions.assertEquals(Arrays.asList(Map.of("x", BigDecimal.valueOf(2)), Map.of("x", BigDecimal.valueOf(1))), expr.evaluate(null)); } @Test diff --git a/src/test/java/com/dashjoin/jsonata/CustomFunctionTest.java b/src/test/java/com/dashjoin/jsonata/CustomFunctionTest.java index 3005e06..0513713 100644 --- a/src/test/java/com/dashjoin/jsonata/CustomFunctionTest.java +++ b/src/test/java/com/dashjoin/jsonata/CustomFunctionTest.java @@ -1,5 +1,6 @@ package com.dashjoin.jsonata; +import java.math.BigDecimal; import java.util.List; import java.util.Map; import org.junit.jupiter.api.Assertions; @@ -35,14 +36,14 @@ public void testEvalWithParams() { public void testUnary() { var expression = Jsonata.jsonata("$echo(123)"); expression.registerFunction("echo", (x) -> x); - Assertions.assertEquals(123, expression.evaluate(null)); + Assertions.assertEquals(BigDecimal.valueOf(123), expression.evaluate(null)); } @Test public void testBinary() { var expression = Jsonata.jsonata("$add(21, 21)"); - expression.registerFunction("add", (Integer a, Integer b) -> a + b); - Assertions.assertEquals(42, expression.evaluate(null)); + expression.registerFunction("add", (BigDecimal a, BigDecimal b) -> a.add(b)); + Assertions.assertEquals(BigDecimal.valueOf(42), expression.evaluate(null)); } @Test @@ -64,7 +65,7 @@ public Object call(Object input, List args) throws Throwable { @Test public void testLambdaSignatureError() { var expression = Jsonata.jsonata("$append(1, 2)"); - expression.registerFunction("append", (Integer a, Boolean b) -> "" + a + b); + expression.registerFunction("append", (BigDecimal a, Boolean b) -> "" + a + b); Assertions.assertThrowsExactly(ClassCastException.class, () -> expression.evaluate(null)); } diff --git a/src/test/java/com/dashjoin/jsonata/NumberTest.java b/src/test/java/com/dashjoin/jsonata/NumberTest.java index 0cde1b9..76ba938 100644 --- a/src/test/java/com/dashjoin/jsonata/NumberTest.java +++ b/src/test/java/com/dashjoin/jsonata/NumberTest.java @@ -3,10 +3,14 @@ import static com.dashjoin.jsonata.Jsonata.jsonata; import static java.util.Map.of; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import com.dashjoin.jsonata.json.Json; +import java.math.BigDecimal; + public class NumberTest { /** @@ -19,7 +23,7 @@ public void testDouble() { var res = expr1.evaluate(of("x", 1.0)); assertEquals(1, res); } - + /** * a computation is applied, and com.dashjoin.jsonata.Utils.convertNumber(Number) casts the double to int */ @@ -27,7 +31,7 @@ public void testDouble() { public void testDouble2() { Jsonata expr1 = jsonata("x+0"); var res = expr1.evaluate(of("x", 1.0)); - assertEquals(1, res); + assertTrue(BigDecimal.valueOf(1.0).compareTo((BigDecimal) res)==0); } /** @@ -39,27 +43,27 @@ public void testDouble3() { var res = expr1.evaluate(Json.parseJson("{\"x\":1.0}")); assertEquals(1, res); } - + /** * "clean" the input using com.dashjoin.jsonata.Utils.convertNumber(Number) */ @Test public void testDouble4() { Jsonata expr1 = jsonata("x"); - var res = expr1.evaluate(of("x", Utils.convertNumber(1.0))); - assertEquals(1, res); + var res = expr1.evaluate(of("x", BigDecimal.valueOf(1.0))); + assertTrue(BigDecimal.valueOf(1).compareTo((BigDecimal) res)==0); } - + /** - * int 1 is converted to double when divided by 2 + * int 1 is converted to BigDecimal when divided by 2 */ @Test public void testInt() { Jsonata expr1 = jsonata("$ / 2"); - var res = expr1.evaluate(1); - assertEquals(0.5, res); + var res = expr1.evaluate(BigDecimal.valueOf(1)); + assertEquals(BigDecimal.valueOf(0.5), res); } - + /** * JSONata constant 1.0 evaluates to 1 */ @@ -67,6 +71,6 @@ public void testInt() { public void testConst() { Jsonata expr1 = jsonata("1.0"); var res = expr1.evaluate(null); - assertEquals(1, res); + assertEquals(BigDecimal.valueOf(1.0), res); } } diff --git a/src/test/java/com/dashjoin/jsonata/SignatureTest.java b/src/test/java/com/dashjoin/jsonata/SignatureTest.java index 9d2eed0..e265bf5 100644 --- a/src/test/java/com/dashjoin/jsonata/SignatureTest.java +++ b/src/test/java/com/dashjoin/jsonata/SignatureTest.java @@ -1,6 +1,8 @@ package com.dashjoin.jsonata; import static com.dashjoin.jsonata.Jsonata.jsonata; + +import java.math.BigDecimal; import java.util.List; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -47,13 +49,13 @@ public void testVarArg() { @SuppressWarnings("rawtypes") @Override public Object call(Object input, List args) throws Throwable { - int sum = 0; + BigDecimal sum = BigDecimal.ZERO; for (Object i : args) - sum += (int) i; + sum = sum.add((BigDecimal) i); return sum; } }, "")); - Assertions.assertEquals(6, expression.evaluate(null)); + Assertions.assertEquals(BigDecimal.valueOf(6), expression.evaluate(null)); } @Test