From 2e947132eddabada288392809a601a3de39cbc36 Mon Sep 17 00:00:00 2001 From: Adam Dickmeiss Date: Fri, 20 Nov 2026 16:06:16 +0100 Subject: [PATCH] Preserve source positions in parsed CQL trees Track exact token boundaries in the lexer and compose source ranges in the parser, preserving original whitespace, quotes and escapes. Expose getStart() and exclusive getEnd() offsets on nodes, modifiers and sort keys. Include subtrees and enclosing parentheses in node ranges; use -1 for objects without a source location. Preserve existing public signatures and the CQLTokenizer interface. Add position regression tests, document range semantics and credit the proposal in PR #19. All 22 tests pass. Verify binary compatibility against existing API descriptors and a client compiled against the previous implementation. --- README.md | 20 ++++ .../java/org/z3950/zing/cql/CQLLexer.java | 23 ++++ src/main/java/org/z3950/zing/cql/CQLNode.java | 33 ++++++ .../java/org/z3950/zing/cql/CQLParser.java | 37 ++++-- .../java/org/z3950/zing/cql/Modifier.java | 33 ++++++ .../java/org/z3950/zing/cql/ModifierSet.java | 33 ++++++ .../zing/cql/CQLSourcePositionsTest.java | 109 ++++++++++++++++++ 7 files changed, 281 insertions(+), 7 deletions(-) create mode 100644 src/test/java/org/z3950/zing/cql/CQLSourcePositionsTest.java diff --git a/README.md b/README.md index df16dca..9eadf89 100644 --- a/README.md +++ b/README.md @@ -105,6 +105,26 @@ Please email us with bug-reports, wishlist items, patches, deployment stories and, of course, large cash donations. +## Source positions + +Parsed nodes expose `getStart()` and `getEnd()`: zero-based UTF-16 offsets +into the original query, with an exclusive end. For example, +`query.substring(node.getStart(), node.getEnd())` retrieves the original +source, preserving whitespace, quotes and escapes. + +A node's range includes its subtree, enclosing parentheses and any index +and relation introducing a grouped expression. Surrounding whitespace is +excluded. Relations, modifiers (including the slash), and sort keys +(`ModifierSet`) also expose ranges. An inherited relation retains the +location where it was written, which can be outside the inheriting term. +Implicit defaults and manually constructed objects return `-1` for both +offsets. Positions describe the original parse; editing the tree does not +update them. Individual index and term-value ranges are not exposed. + +This implementation was inspired by Querela's source-position proposal in +[PR #19](https://github.com/indexdata/cql-java/pull/19). +Existing method signatures and the `CQLTokenizer` interface are unchanged. + ## Licence The CQL-Java suite is Free Software, which is pretty much legally diff --git a/src/main/java/org/z3950/zing/cql/CQLLexer.java b/src/main/java/org/z3950/zing/cql/CQLLexer.java index a598b01..daead86 100644 --- a/src/main/java/org/z3950/zing/cql/CQLLexer.java +++ b/src/main/java/org/z3950/zing/cql/CQLLexer.java @@ -12,6 +12,7 @@ public class CQLLexer implements CQLTokenizer { private String qs; private int qi; + private int tokenStart = -1; private int ql; private int what = TT_NOTHING; private String val; @@ -30,6 +31,7 @@ public void move() { // eat whitespace while (qi < ql && strchr(" \t\r\n", qs.charAt(qi))) qi++; + tokenStart = qi; // eof if (qi == ql) { what = TT_EOF; @@ -165,6 +167,27 @@ public String render(int token, boolean quoteChars) { } } + /** + * Returns the current token's inclusive UTF-16 source start, or -1 before + * the first move. Quotes are included; leading whitespace is excluded. + * At end of input the span is empty at the input length. + * + * @return token start + */ + public int getStart() { + return tokenStart; + } + + /** + * Returns the current token's exclusive UTF-16 source end, or -1 before + * the first move. + * + * @return token end + */ + public int getEnd() { + return tokenStart < 0 ? -1 : qi; + } + @Override public int pos() { return qi; diff --git a/src/main/java/org/z3950/zing/cql/CQLNode.java b/src/main/java/org/z3950/zing/cql/CQLNode.java index 2b33b0e..2fa3263 100644 --- a/src/main/java/org/z3950/zing/cql/CQLNode.java +++ b/src/main/java/org/z3950/zing/cql/CQLNode.java @@ -11,6 +11,39 @@ */ public abstract class CQLNode { + private int start = -1; + private int end = -1; + + /** + * Returns the inclusive source start of this node (including its subtree and enclosing parentheses). + * Offsets are zero-based UTF-16 indexes into the original query, excluding + * surrounding whitespace. Manually constructed objects and implicit defaults + * have no source location and return -1. Locations describe the original + * parse and are not updated after tree modifications. + * + * @return source start, or -1 when unavailable + */ + public int getStart() { + return start; + } + + /** + * Returns the exclusive source end, suitable for + * {@code query.substring(getStart(), getEnd())}. + * + * @return source end, or -1 when unavailable + * @see #getStart() + */ + public int getEnd() { + return end; + } + + void setSourceRange(int start, int end) { + this.start = start; + this.end = end; + } + + public abstract void traverse(CQLNodeVisitor visitor); /** diff --git a/src/main/java/org/z3950/zing/cql/CQLParser.java b/src/main/java/org/z3950/zing/cql/CQLParser.java index 27456ce..2df8cc1 100644 --- a/src/main/java/org/z3950/zing/cql/CQLParser.java +++ b/src/main/java/org/z3950/zing/cql/CQLParser.java @@ -18,7 +18,9 @@ * >https://zing.z3950.org/cql/index.html */ public class CQLParser { - private CQLTokenizer lexer; + private CQLLexer lexer; + // End of the last consumed token; the lexer already holds the lookahead. + private int consumedEnd; private final int compat; // When false, implement CQL 1.2 private final Set customRelations = new HashSet(); @@ -119,6 +121,7 @@ public boolean unregisterCustomRelation(String relation) { public CQLNode parse(String cql) throws CQLParseException, IOException { lexer = new CQLLexer(cql, LEXDEBUG); + consumedEnd = -1; lexer.move(); debug("about to parseQuery()"); @@ -147,8 +150,9 @@ private CQLNode parseTopLevelPrefixes(String index, CQLRelation relation) CQLSortNode sortnode = new CQLSortNode(node); while (lexer.what() != CQLTokenizer.TT_EOF) { + int keyStart = lexer.getStart(); String sortindex = matchSymbol("sort index"); - ModifierSet ms = gatherModifiers(sortindex); + ModifierSet ms = gatherModifiers(sortindex, keyStart); sortnode.addSortIndex(ms); } @@ -156,6 +160,7 @@ private CQLNode parseTopLevelPrefixes(String index, CQLRelation relation) throw new CQLParseException("no sort keys", lexer.pos()); } + sortnode.setSourceRange(node.getStart(), consumedEnd); node = sortnode; } @@ -174,15 +179,18 @@ private CQLNode parseQuery(String index, CQLRelation relation) lexer.what() == CQLTokenizer.TT_OR || lexer.what() == CQLTokenizer.TT_NOT || lexer.what() == CQLTokenizer.TT_PROX) { + int start = term.getStart(); + int operatorStart = lexer.getStart(); int type = lexer.what(); String val = lexer.value(); match(type); - ModifierSet ms = gatherModifiers(val); + ModifierSet ms = gatherModifiers(val, operatorStart); CQLNode term2 = parseTerm(index, relation); term = ((type == CQLTokenizer.TT_AND) ? new CQLAndNode(term, term2, ms) : (type == CQLTokenizer.TT_OR) ? new CQLOrNode(term, term2, ms) : (type == CQLTokenizer.TT_NOT) ? new CQLNotNode(term, term2, ms) : new CQLProxNode(term, term2, ms)); + term.setSourceRange(start, consumedEnd); } else { throw new CQLParseException("expected boolean, got " + lexer.render(), lexer.pos()); @@ -193,12 +201,13 @@ private CQLNode parseQuery(String index, CQLRelation relation) return term; } - private ModifierSet gatherModifiers(String base) + private ModifierSet gatherModifiers(String base, int start) throws CQLParseException, IOException { debug("in gatherModifiers()"); ModifierSet ms = new ModifierSet(base); while (lexer.what() == '/') { + int modifierStart = lexer.getStart(); match('/'); if (lexer.what() != CQLTokenizer.TT_WORD) throw new CQLParseException("expected modifier, " @@ -216,7 +225,10 @@ private ModifierSet gatherModifiers(String base) String value = matchSymbol("modifier value"); ms.addModifier(type, comparision, value); } + ms.getModifiers().get(ms.getModifiers().size() - 1) + .setSourceRange(modifierStart, consumedEnd); } + ms.setSourceRange(start, consumedEnd); return ms; } @@ -225,6 +237,7 @@ private CQLNode parseTerm(String index, CQLRelation relation) throws CQLParseException, IOException { debug("in parseTerm()"); + int start = lexer.getStart(); String first; StringBuilder all; while (true) { @@ -233,9 +246,12 @@ private CQLNode parseTerm(String index, CQLRelation relation) match('('); CQLNode expr = parseQuery(index, relation); match(')'); + expr.setSourceRange(start, consumedEnd); return expr; } else if (lexer.what() == '>') { - return parsePrefix(index, relation, false); + CQLNode expr = parsePrefix(index, relation, false); + expr.setSourceRange(start, consumedEnd); + return expr; } debug("non-parenthesised term"); @@ -257,14 +273,17 @@ private CQLNode parseTerm(String index, CQLRelation relation) throw new CQLParseException("unexpected relation '" + relstr + "'", lexer.pos()); } index = first; + int relationStart = lexer.getStart(); relation = new CQLRelation(relstr); match(lexer.what()); - ModifierSet ms = gatherModifiers(relstr); + ModifierSet ms = gatherModifiers(relstr, relationStart); relation.ms = ms; + relation.setSourceRange(relationStart, consumedEnd); debug("index='" + index + ", " + "relation='" + relation.toCQL() + "'"); } CQLTermNode node = new CQLTermNode(index, relation, all.toString()); + node.setSourceRange(start, consumedEnd); debug("made term node " + node.toCQL()); return node; } @@ -274,6 +293,7 @@ private CQLNode parsePrefix(String index, CQLRelation relation, throws CQLParseException, IOException { debug("prefix mapping"); + int start = lexer.getStart(); match('>'); String name = null; String identifier = matchSymbol("prefix-name"); @@ -284,7 +304,9 @@ private CQLNode parsePrefix(String index, CQLRelation relation, } CQLNode node = topLevel ? parseTopLevelPrefixes(index, relation) : parseQuery(index, relation); - return new CQLPrefixNode(name, identifier, node); + CQLPrefixNode prefix = new CQLPrefixNode(name, identifier, node); + prefix.setSourceRange(start, consumedEnd); + return prefix; } private boolean isWordOrString() { @@ -329,6 +351,7 @@ private void match(int token) lexer.render(token, true) + ", " + "got " + lexer.render(), lexer.pos()); + consumedEnd = lexer.getEnd(); lexer.move(); debug("match() got token=" + lexer.what() + ", value()='" + lexer.value() + "'"); } diff --git a/src/main/java/org/z3950/zing/cql/Modifier.java b/src/main/java/org/z3950/zing/cql/Modifier.java index 37b21d8..469ea0c 100644 --- a/src/main/java/org/z3950/zing/cql/Modifier.java +++ b/src/main/java/org/z3950/zing/cql/Modifier.java @@ -10,6 +10,39 @@ * */ public class Modifier { + + private int start = -1; + private int end = -1; + + /** + * Returns the inclusive source start of this modifier (including the leading slash). + * Offsets are zero-based UTF-16 indexes into the original query, excluding + * surrounding whitespace. Manually constructed objects and implicit defaults + * have no source location and return -1. Locations describe the original + * parse and are not updated after tree modifications. + * + * @return source start, or -1 when unavailable + */ + public int getStart() { + return start; + } + + /** + * Returns the exclusive source end, suitable for + * {@code query.substring(getStart(), getEnd())}. + * + * @return source end, or -1 when unavailable + * @see #getStart() + */ + public int getEnd() { + return end; + } + + void setSourceRange(int start, int end) { + this.start = start; + this.end = end; + } + String type; String comparison; String value; diff --git a/src/main/java/org/z3950/zing/cql/ModifierSet.java b/src/main/java/org/z3950/zing/cql/ModifierSet.java index 3456155..1c7ead1 100644 --- a/src/main/java/org/z3950/zing/cql/ModifierSet.java +++ b/src/main/java/org/z3950/zing/cql/ModifierSet.java @@ -18,6 +18,39 @@ * */ public class ModifierSet { + + private int start = -1; + private int end = -1; + + /** + * Returns the inclusive source start of this base and its modifiers. + * Offsets are zero-based UTF-16 indexes into the original query, excluding + * surrounding whitespace. Manually constructed objects and implicit defaults + * have no source location and return -1. Locations describe the original + * parse and are not updated after tree modifications. + * + * @return source start, or -1 when unavailable + */ + public int getStart() { + return start; + } + + /** + * Returns the exclusive source end, suitable for + * {@code query.substring(getStart(), getEnd())}. + * + * @return source end, or -1 when unavailable + * @see #getStart() + */ + public int getEnd() { + return end; + } + + void setSourceRange(int start, int end) { + this.start = start; + this.end = end; + } + private String base; private List modifiers; diff --git a/src/test/java/org/z3950/zing/cql/CQLSourcePositionsTest.java b/src/test/java/org/z3950/zing/cql/CQLSourcePositionsTest.java new file mode 100644 index 0000000..9f88fbb --- /dev/null +++ b/src/test/java/org/z3950/zing/cql/CQLSourcePositionsTest.java @@ -0,0 +1,109 @@ +package org.z3950.zing.cql; + +import org.junit.Test; +import static org.junit.Assert.*; + +public class CQLSourcePositionsTest { + private static void span(String query, String expected, CQLNode node) { + assertEquals(expected, query.substring(node.getStart(), node.getEnd())); + assertEquals(query.indexOf(expected), node.getStart()); + } + + @Test + public void preservesOriginalText() throws Exception { + CQLParser parser = new CQLParser(); + for (String text : new String[] { + "cat dog", "\"cat\" \"dog\"", "\"\"", "\"a\\\"b\"", + "cat\t\r\ndog", "\uD83D\uDE00 fish", "((cat))", + "title = (cat or dog)", "cat sortBy title", + ">dc=uri cat", "cat or >dc=uri dog", "title = >dc=uri dog"}) { + String query = " \t" + text + " \r\n"; + span(query, text, parser.parse(query)); + } + } + + @Test + public void nestedOperandsAndInheritedRelation() throws Exception { + String query = " title = (cat or (dog and fish)) "; + CQLBooleanNode root = (CQLBooleanNode) new CQLParser().parse(query); + span(query, query.trim(), root); + CQLTermNode cat = (CQLTermNode) root.getLeftOperand(); + span(query, "cat", cat); + span(query, "=", cat.getRelation()); + CQLBooleanNode right = (CQLBooleanNode) root.getRightOperand(); + span(query, "(dog and fish)", right); + span(query, "dog", right.getLeftOperand()); + span(query, "fish", right.getRightOperand()); + } + + @Test + public void relationsAndModifiers() throws Exception { + for (String relation : new String[] {"=", "==", "<", ">", "<=", ">=", "<>", "any", "cql.any"}) { + String query = "title " + relation + "/relevant /distance <= \"3\" fish "; + CQLTermNode term = (CQLTermNode) new CQLParser().parse(query); + span(query, relation + "/relevant /distance <= \"3\"", term.getRelation()); + Modifier simple = term.getRelation().getModifiers().get(0); + assertEquals("/relevant", query.substring(simple.getStart(), simple.getEnd())); + Modifier complex = term.getRelation().getModifiers().get(1); + assertEquals("/distance <= \"3\"", query.substring(complex.getStart(), complex.getEnd())); + } + } + + @Test + public void prefixSortAndBooleanModifiers() throws Exception { + String query = " >dc=uri cat prox/distance<=3 dog sortBy title/sort.ascending date "; + CQLPrefixNode prefix = (CQLPrefixNode) new CQLParser().parse(query); + span(query, query.trim(), prefix); + CQLSortNode sort = (CQLSortNode) prefix.getSubtree(); + span(query, "cat prox/distance<=3 dog sortBy title/sort.ascending date", sort); + CQLBooleanNode bool = (CQLBooleanNode) sort.getSubtree(); + span(query, "cat prox/distance<=3 dog", bool); + Modifier modifier = bool.getModifiers().get(0); + assertEquals("/distance<=3", query.substring(modifier.getStart(), modifier.getEnd())); + ModifierSet key = sort.getSortIndexes().get(0); + assertEquals("title/sort.ascending", query.substring(key.getStart(), key.getEnd())); + key = sort.getSortIndexes().get(1); + assertEquals("date", query.substring(key.getStart(), key.getEnd())); + } + + @Test + public void unknownLocations() throws Exception { + CQLTermNode parsed = (CQLTermNode) new CQLParser().parse("fish"); + CQLNode[] nodes = {new CQLTermNode("title", new CQLRelation("="), "fish"), + parsed.getRelation()}; + for (CQLNode node : nodes) { + assertEquals(-1, node.getStart()); + assertEquals(-1, node.getEnd()); + } + ModifierSet set = new ModifierSet("any"); + set.addModifier("relevant"); + set.addModifier("distance", "<", "3"); + assertEquals(-1, set.getStart()); + assertEquals(-1, set.getEnd()); + for (Modifier modifier : set.getModifiers()) { + assertEquals(-1, modifier.getStart()); + assertEquals(-1, modifier.getEnd()); + } + } + + @Test + public void lexerOffsetsAndLegacyPosition() { + String query = " \"\uD83D\uDE00\" <= cat "; + CQLLexer lexer = new CQLLexer(query, false); + assertEquals(-1, lexer.getStart()); + assertEquals(-1, lexer.getEnd()); + assertEquals(0, lexer.pos()); + for (String token : new String[] {"\"\uD83D\uDE00\"", "<=", "cat"}) { + lexer.move(); + assertEquals(token, query.substring(lexer.getStart(), lexer.getEnd())); + assertEquals(query.indexOf(token), lexer.getStart()); + assertEquals(lexer.pos(), lexer.getEnd()); + } + lexer.move(); + assertEquals(CQLTokenizer.TT_EOF, lexer.what()); + assertEquals(query.length(), lexer.getStart()); + assertEquals(query.length(), lexer.getEnd()); + lexer.move(); + assertEquals(query.length(), lexer.getEnd()); + } +}