Skip to content

Commit 293bbf6

Browse files
Share repeated object keys within a parse (stleary#1091)
1 parent 69383f7 commit 293bbf6

4 files changed

Lines changed: 214 additions & 1 deletion

File tree

‎src/main/java/org/json/JSONObject.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -276,7 +276,7 @@ private boolean parseJSONObject(JSONTokener jsonTokener, JSONParserConfiguration
276276
return true;
277277
default:
278278
obj = jsonTokener.nextSimpleValue(c);
279-
key = obj.toString();
279+
key = jsonTokener.shareKey(obj.toString(), jsonParserConfiguration.getMaxSharedKeys());
280280
}
281281

282282
checkKeyForStrictMode(jsonTokener, jsonParserConfiguration, obj);

‎src/main/java/org/json/JSONParserConfiguration.java‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,16 @@
44
* Configuration object for the JSON parser. The configuration is immutable.
55
*/
66
public class JSONParserConfiguration extends ParserConfiguration {
7+
/**
8+
* The default maximum number of distinct object keys shared within one parse.
9+
*/
10+
public static final int DEFAULT_MAXIMUM_SHARED_KEYS = 1000;
11+
12+
/**
13+
* Used to indicate that there is no limit on the number of distinct object keys shared within one parse.
14+
*/
15+
public static final int UNLIMITED_SHARED_KEYS = -1;
16+
717
/**
818
* Used to indicate whether to overwrite duplicate key or not.
919
*/
@@ -14,12 +24,18 @@ public class JSONParserConfiguration extends ParserConfiguration {
1424
*/
1525
private boolean useNativeNulls;
1626

27+
/**
28+
* The maximum number of distinct object keys shared within one parse.
29+
*/
30+
private int maxSharedKeys;
31+
1732
/**
1833
* Configuration with the default values.
1934
*/
2035
public JSONParserConfiguration() {
2136
super();
2237
this.overwriteDuplicateKey = false;
38+
this.maxSharedKeys = DEFAULT_MAXIMUM_SHARED_KEYS;
2339
// DO NOT DELETE THE FOLLOWING LINE -- it is used for strictMode testing
2440
// this.strictMode = true;
2541
}
@@ -38,6 +54,7 @@ protected JSONParserConfiguration clone() {
3854
clone.maxNestingDepth = maxNestingDepth;
3955
clone.keepStrings = keepStrings;
4056
clone.useNativeNulls = useNativeNulls;
57+
clone.maxSharedKeys = maxSharedKeys;
4158
return clone;
4259
}
4360

@@ -89,6 +106,24 @@ public JSONParserConfiguration withUseNativeNulls(final boolean useNativeNulls)
89106
return clone;
90107
}
91108

109+
/**
110+
* Defines how many distinct object keys the parser shares within one parse. When a key repeats, for example
111+
* in each element of an array of objects, the parser reuses the first String instance read for that key
112+
* instead of keeping a separate copy in every object. Only the first <code>maxSharedKeys</code> distinct keys
113+
* are shared, so input that begins with many unique keys may need a higher limit. The default is 1000.
114+
* Parsed values are the same with any setting.
115+
*
116+
* @param maxSharedKeys the maximum number of distinct keys to share; 0 disables sharing, and
117+
* {@link #UNLIMITED_SHARED_KEYS} (or any negative value) removes the limit
118+
* @return The existing configuration will not be modified. A new configuration is returned.
119+
*/
120+
public JSONParserConfiguration withMaxSharedKeys(final int maxSharedKeys) {
121+
JSONParserConfiguration clone = this.clone();
122+
clone.maxSharedKeys = Math.max(maxSharedKeys, UNLIMITED_SHARED_KEYS);
123+
124+
return clone;
125+
}
126+
92127
/**
93128
* Sets the strict mode configuration for the JSON parser with default true value
94129
* <p>
@@ -140,6 +175,16 @@ public boolean isUseNativeNulls() {
140175
}
141176

142177

178+
/**
179+
* The maximum number of distinct object keys shared within one parse. 0 means keys are not shared,
180+
* and {@link #UNLIMITED_SHARED_KEYS} means there is no limit.
181+
*
182+
* @return The <code>maxSharedKeys</code> configuration value.
183+
*/
184+
public int getMaxSharedKeys() {
185+
return this.maxSharedKeys;
186+
}
187+
143188
/**
144189
* The parser throws an Exception when strict mode is true and tries to parse invalid JSON characters.
145190
* Otherwise, the parser is more relaxed and might tolerate some invalid characters.

‎src/main/java/org/json/JSONTokener.java‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@
22

33
import java.io.*;
44
import java.nio.charset.Charset;
5+
import java.util.HashMap;
6+
import java.util.Map;
57

68
/*
79
Public Domain.
@@ -35,6 +37,9 @@ public class JSONTokener {
3537
// access to this object is required for strict mode checking
3638
private JSONParserConfiguration jsonParserConfiguration;
3739

40+
/** object keys already read, so repeated keys can share one String instance. */
41+
private Map<String, String> sharedKeys;
42+
3843
/**
3944
* Construct a JSONTokener from a Reader. The caller must close the Reader.
4045
*
@@ -124,6 +129,31 @@ void setJsonParserConfigurationInternal(JSONParserConfiguration jsonParserConfig
124129
this.jsonParserConfiguration = jsonParserConfiguration;
125130
}
126131

132+
/**
133+
* Returns an earlier String instance equal to the key if this tokener has read one, otherwise the key itself.
134+
*
135+
* @param key an object key that was just read
136+
* @param maxSharedKeys the maximum number of distinct keys to remember; 0 disables sharing and a negative
137+
* value means no limit
138+
* @return a String equal to <code>key</code>
139+
*/
140+
String shareKey(String key, int maxSharedKeys) {
141+
if (maxSharedKeys == 0) {
142+
return key;
143+
}
144+
if (this.sharedKeys == null) {
145+
this.sharedKeys = new HashMap<String, String>();
146+
}
147+
String shared = this.sharedKeys.get(key);
148+
if (shared != null) {
149+
return shared;
150+
}
151+
if (maxSharedKeys < 0 || this.sharedKeys.size() < maxSharedKeys) {
152+
this.sharedKeys.put(key, key);
153+
}
154+
return key;
155+
}
156+
127157
/**
128158
* Back up one character. This provides a sort of lookahead capability,
129159
* so that you can test for a digit or letter before attempting to parse

‎src/test/java/org/json/junit/JSONParserConfigurationTest.java‎

Lines changed: 138 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,9 +16,11 @@
1616
import java.util.stream.Stream;
1717

1818
import static org.junit.Assert.assertEquals;
19+
import static org.junit.Assert.assertNotSame;
1920
import static org.junit.Assert.assertSame;
2021
import static org.junit.Assert.assertThrows;
2122
import static org.junit.Assert.assertTrue;
23+
import static org.junit.Assert.assertFalse;
2224
import static org.junit.Assert.fail;
2325

2426
public class JSONParserConfigurationTest {
@@ -708,4 +710,140 @@ private List<String> getNonCompliantJSONObjectList() {
708710
);
709711
}
710712

713+
private static final String SHARED_KEYS_SOURCE = "{\"rows\": ["
714+
+ "{\"name\": \"a\", \"count\": 1, \"big\": 12345678901, \"ratio\": 1.5, \"flag\": true, \"missing\": null,"
715+
+ " \"nested\": {\"name\": \"inner\", \"list\": [1, \"name\", null]}},"
716+
+ "{\"name\": \"name\", \"count\": 2, \"nested\": {\"name\": \"inner2\", \"list\": []}},"
717+
+ "{\"name\": \"third\", \"count\": -3}"
718+
+ "]}";
719+
720+
@Test
721+
public void testMaxSharedKeysConfig() {
722+
JSONParserConfiguration config = new JSONParserConfiguration();
723+
assertEquals(JSONParserConfiguration.DEFAULT_MAXIMUM_SHARED_KEYS, config.getMaxSharedKeys());
724+
assertEquals(1000, config.getMaxSharedKeys());
725+
assertEquals(0, config.withMaxSharedKeys(0).getMaxSharedKeys());
726+
assertEquals(JSONParserConfiguration.UNLIMITED_SHARED_KEYS, config.withMaxSharedKeys(-1).getMaxSharedKeys());
727+
assertEquals(JSONParserConfiguration.UNLIMITED_SHARED_KEYS, config.withMaxSharedKeys(-5).getMaxSharedKeys());
728+
729+
JSONParserConfiguration chained = config.withMaxSharedKeys(7).withOverwriteDuplicateKey(true).withStrictMode(false);
730+
assertEquals(7, chained.getMaxSharedKeys());
731+
assertTrue(chained.isOverwriteDuplicateKey());
732+
assertEquals(1000, config.getMaxSharedKeys());
733+
}
734+
735+
@Test
736+
public void testSharedKeysByDefault() {
737+
JSONArray rows = new JSONObject(SHARED_KEYS_SOURCE).getJSONArray("rows");
738+
String nameKey = key(rows.getJSONObject(0), "name");
739+
for (int i = 1; i < rows.length(); i++) {
740+
assertSame(nameKey, key(rows.getJSONObject(i), "name"));
741+
}
742+
assertSame(nameKey, key(rows.getJSONObject(0).getJSONObject("nested"), "name"));
743+
assertSame(key(rows.getJSONObject(0), "count"), key(rows.getJSONObject(2), "count"));
744+
assertSame(key(rows.getJSONObject(0).getJSONObject("nested"), "list"),
745+
key(rows.getJSONObject(1).getJSONObject("nested"), "list"));
746+
747+
// string values equal to a key are not shared
748+
assertNotSame(nameKey, rows.getJSONObject(1).get("name"));
749+
assertNotSame(nameKey, rows.getJSONObject(0).getJSONObject("nested").getJSONArray("list").get(1));
750+
}
751+
752+
@Test
753+
public void testSharedKeysDisabled() {
754+
JSONArray rows = new JSONObject(SHARED_KEYS_SOURCE, new JSONParserConfiguration().withMaxSharedKeys(0))
755+
.getJSONArray("rows");
756+
assertNotSame(key(rows.getJSONObject(0), "name"), key(rows.getJSONObject(1), "name"));
757+
assertNotSame(key(rows.getJSONObject(0), "count"), key(rows.getJSONObject(1), "count"));
758+
}
759+
760+
@Test
761+
public void testSharedKeysLimit() {
762+
JSONArray rows = new JSONArray("[{\"a\": 1, \"b\": 2, \"c\": 3}, {\"a\": 4, \"b\": 5, \"c\": 6}]",
763+
new JSONParserConfiguration().withMaxSharedKeys(2));
764+
assertSame(key(rows.getJSONObject(0), "a"), key(rows.getJSONObject(1), "a"));
765+
assertSame(key(rows.getJSONObject(0), "b"), key(rows.getJSONObject(1), "b"));
766+
assertNotSame(key(rows.getJSONObject(0), "c"), key(rows.getJSONObject(1), "c"));
767+
}
768+
769+
@Test
770+
public void testSharedKeysUnlimited() {
771+
int keyCount = JSONParserConfiguration.DEFAULT_MAXIMUM_SHARED_KEYS + 500;
772+
StringBuilder row = new StringBuilder("{");
773+
for (int i = 0; i < keyCount; i++) {
774+
row.append(i == 0 ? "" : ",").append("\"k").append(i).append("\":").append(i);
775+
}
776+
row.append('}');
777+
String source = "[" + row + "," + row + "]";
778+
779+
JSONArray unlimited = new JSONArray(source,
780+
new JSONParserConfiguration().withMaxSharedKeys(JSONParserConfiguration.UNLIMITED_SHARED_KEYS));
781+
for (int i = 0; i < keyCount; i++) {
782+
assertSame(key(unlimited.getJSONObject(0), "k" + i), key(unlimited.getJSONObject(1), "k" + i));
783+
}
784+
785+
JSONArray limited = new JSONArray(source);
786+
String lastKey = "k" + (keyCount - 1);
787+
assertSame(key(limited.getJSONObject(0), "k0"), key(limited.getJSONObject(1), "k0"));
788+
assertNotSame(key(limited.getJSONObject(0), lastKey), key(limited.getJSONObject(1), lastKey));
789+
assertTrue(unlimited.similar(limited));
790+
}
791+
792+
@Test
793+
public void testSharedKeysNotSharedAcrossParses() {
794+
JSONObject first = new JSONObject("{\"name\": 1}");
795+
JSONObject second = new JSONObject("{\"name\": 2}");
796+
assertNotSame(key(first, "name"), key(second, "name"));
797+
}
798+
799+
@Test
800+
public void testSharedKeysParseUnchanged() {
801+
JSONObject unshared = new JSONObject(SHARED_KEYS_SOURCE, new JSONParserConfiguration().withMaxSharedKeys(0));
802+
JSONObject shared = new JSONObject(SHARED_KEYS_SOURCE);
803+
JSONObject unlimited = new JSONObject(SHARED_KEYS_SOURCE,
804+
new JSONParserConfiguration().withMaxSharedKeys(JSONParserConfiguration.UNLIMITED_SHARED_KEYS));
805+
assertTrue(unshared.similar(shared));
806+
assertTrue(unshared.similar(unlimited));
807+
assertEquals(unshared.toString(), shared.toString());
808+
assertEquals(unshared.toString(), unlimited.toString());
809+
810+
JSONObject row = shared.getJSONArray("rows").getJSONObject(0);
811+
assertEquals(Integer.class, row.get("count").getClass());
812+
assertEquals(Long.class, row.get("big").getClass());
813+
assertEquals(java.math.BigDecimal.class, row.get("ratio").getClass());
814+
assertEquals(Boolean.TRUE, row.get("flag"));
815+
assertSame(JSONObject.NULL, row.get("missing"));
816+
assertSame(JSONObject.NULL, row.getJSONObject("nested").getJSONArray("list").get(2));
817+
assertEquals("third", shared.getJSONArray("rows").getJSONObject(2).get("name"));
818+
}
819+
820+
@Test
821+
public void testSharedKeysLenientSyntax() {
822+
JSONArray rows = new JSONArray("[{\"name\": 1, \"count\": 2}, {'name': 'single', count: 3}]",
823+
new JSONParserConfiguration().withStrictMode(false));
824+
assertSame(key(rows.getJSONObject(0), "name"), key(rows.getJSONObject(1), "name"));
825+
assertSame(key(rows.getJSONObject(0), "count"), key(rows.getJSONObject(1), "count"));
826+
assertEquals("single", rows.getJSONObject(1).get("name"));
827+
}
828+
829+
@Test
830+
public void testSharedKeysStrictMode() {
831+
JSONArray rows = new JSONArray("[{\"a\": 1}, {\"a\": 2}]", new JSONParserConfiguration().withStrictMode());
832+
assertSame(key(rows.getJSONObject(0), "a"), key(rows.getJSONObject(1), "a"));
833+
assertFalse(rows.getJSONObject(0).similar(rows.getJSONObject(1)));
834+
}
835+
836+
/**
837+
* Returns the key instance a JSONObject holds, rather than the caller's equal String.
838+
*/
839+
private static String key(JSONObject jsonObject, String name) {
840+
for (String key : jsonObject.keySet()) {
841+
if (key.equals(name)) {
842+
return key;
843+
}
844+
}
845+
fail("missing key " + name);
846+
return null;
847+
}
848+
711849
}

0 commit comments

Comments
 (0)