diff --git a/README.md b/README.md index df16dca..9eadf89 100644 --- a/README.md +++ b/README.md @@ -105,6 +105,26 @@ Please email us with bug-reports, wishlist items, patches, deployment stories and, of course, large cash donations. +## Source positions + +Parsed nodes expose `getStart()` and `getEnd()`: zero-based UTF-16 offsets +into the original query, with an exclusive end. For example, +`query.substring(node.getStart(), node.getEnd())` retrieves the original +source, preserving whitespace, quotes and escapes. + +A node's range includes its subtree, enclosing parentheses and any index +and relation introducing a grouped expression. Surrounding whitespace is +excluded. Relations, modifiers (including the slash), and sort keys +(`ModifierSet`) also expose ranges. An inherited relation retains the +location where it was written, which can be outside the inheriting term. +Implicit defaults and manually constructed objects return `-1` for both +offsets. Positions describe the original parse; editing the tree does not +update them. Individual index and term-value ranges are not exposed. + +This implementation was inspired by Querela's source-position proposal in +[PR #19](https://github.com/indexdata/cql-java/pull/19). +Existing method signatures and the `CQLTokenizer` interface are unchanged. + ## Licence The CQL-Java suite is Free Software, which is pretty much legally diff --git a/src/main/java/org/z3950/zing/cql/CQLLexer.java b/src/main/java/org/z3950/zing/cql/CQLLexer.java index a598b01..daead86 100644 --- a/src/main/java/org/z3950/zing/cql/CQLLexer.java +++ b/src/main/java/org/z3950/zing/cql/CQLLexer.java @@ -12,6 +12,7 @@ public class CQLLexer implements CQLTokenizer { private String qs; private int qi; + private int tokenStart = -1; private int ql; private int what = TT_NOTHING; private String val; @@ -30,6 +31,7 @@ public void move() { // eat whitespace while (qi < ql && strchr(" \t\r\n", qs.charAt(qi))) qi++; + tokenStart = qi; // eof if (qi == ql) { what = TT_EOF; @@ -165,6 +167,27 @@ public String render(int token, boolean quoteChars) { } } + /** + * Returns the current token's inclusive UTF-16 source start, or -1 before + * the first move. Quotes are included; leading whitespace is excluded. + * At end of input the span is empty at the input length. + * + * @return token start + */ + public int getStart() { + return tokenStart; + } + + /** + * Returns the current token's exclusive UTF-16 source end, or -1 before + * the first move. + * + * @return token end + */ + public int getEnd() { + return tokenStart < 0 ? -1 : qi; + } + @Override public int pos() { return qi; diff --git a/src/main/java/org/z3950/zing/cql/CQLNode.java b/src/main/java/org/z3950/zing/cql/CQLNode.java index 2b33b0e..2fa3263 100644 --- a/src/main/java/org/z3950/zing/cql/CQLNode.java +++ b/src/main/java/org/z3950/zing/cql/CQLNode.java @@ -11,6 +11,39 @@ */ public abstract class CQLNode { + private int start = -1; + private int end = -1; + + /** + * Returns the inclusive source start of this node (including its subtree and enclosing parentheses). + * Offsets are zero-based UTF-16 indexes into the original query, excluding + * surrounding whitespace. Manually constructed objects and implicit defaults + * have no source location and return -1. Locations describe the original + * parse and are not updated after tree modifications. + * + * @return source start, or -1 when unavailable + */ + public int getStart() { + return start; + } + + /** + * Returns the exclusive source end, suitable for + * {@code query.substring(getStart(), getEnd())}. + * + * @return source end, or -1 when unavailable + * @see #getStart() + */ + public int getEnd() { + return end; + } + + void setSourceRange(int start, int end) { + this.start = start; + this.end = end; + } + + public abstract void traverse(CQLNodeVisitor visitor); /** diff --git a/src/main/java/org/z3950/zing/cql/CQLParser.java b/src/main/java/org/z3950/zing/cql/CQLParser.java index 27456ce..2df8cc1 100644 --- a/src/main/java/org/z3950/zing/cql/CQLParser.java +++ b/src/main/java/org/z3950/zing/cql/CQLParser.java @@ -18,7 +18,9 @@ * >https://zing.z3950.org/cql/index.html */ public class CQLParser { - private CQLTokenizer lexer; + private CQLLexer lexer; + // End of the last consumed token; the lexer already holds the lookahead. + private int consumedEnd; private final int compat; // When false, implement CQL 1.2 private final Set customRelations = new HashSet(); @@ -119,6 +121,7 @@ public boolean unregisterCustomRelation(String relation) { public CQLNode parse(String cql) throws CQLParseException, IOException { lexer = new CQLLexer(cql, LEXDEBUG); + consumedEnd = -1; lexer.move(); debug("about to parseQuery()"); @@ -147,8 +150,9 @@ private CQLNode parseTopLevelPrefixes(String index, CQLRelation relation) CQLSortNode sortnode = new CQLSortNode(node); while (lexer.what() != CQLTokenizer.TT_EOF) { + int keyStart = lexer.getStart(); String sortindex = matchSymbol("sort index"); - ModifierSet ms = gatherModifiers(sortindex); + ModifierSet ms = gatherModifiers(sortindex, keyStart); sortnode.addSortIndex(ms); } @@ -156,6 +160,7 @@ private CQLNode parseTopLevelPrefixes(String index, CQLRelation relation) throw new CQLParseException("no sort keys", lexer.pos()); } + sortnode.setSourceRange(node.getStart(), consumedEnd); node = sortnode; } @@ -174,15 +179,18 @@ private CQLNode parseQuery(String index, CQLRelation relation) lexer.what() == CQLTokenizer.TT_OR || lexer.what() == CQLTokenizer.TT_NOT || lexer.what() == CQLTokenizer.TT_PROX) { + int start = term.getStart(); + int operatorStart = lexer.getStart(); int type = lexer.what(); String val = lexer.value(); match(type); - ModifierSet ms = gatherModifiers(val); + ModifierSet ms = gatherModifiers(val, operatorStart); CQLNode term2 = parseTerm(index, relation); term = ((type == CQLTokenizer.TT_AND) ? new CQLAndNode(term, term2, ms) : (type == CQLTokenizer.TT_OR) ? new CQLOrNode(term, term2, ms) : (type == CQLTokenizer.TT_NOT) ? new CQLNotNode(term, term2, ms) : new CQLProxNode(term, term2, ms)); + term.setSourceRange(start, consumedEnd); } else { throw new CQLParseException("expected boolean, got " + lexer.render(), lexer.pos()); @@ -193,12 +201,13 @@ private CQLNode parseQuery(String index, CQLRelation relation) return term; } - private ModifierSet gatherModifiers(String base) + private ModifierSet gatherModifiers(String base, int start) throws CQLParseException, IOException { debug("in gatherModifiers()"); ModifierSet ms = new ModifierSet(base); while (lexer.what() == '/') { + int modifierStart = lexer.getStart(); match('/'); if (lexer.what() != CQLTokenizer.TT_WORD) throw new CQLParseException("expected modifier, " @@ -216,7 +225,10 @@ private ModifierSet gatherModifiers(String base) String value = matchSymbol("modifier value"); ms.addModifier(type, comparision, value); } + ms.getModifiers().get(ms.getModifiers().size() - 1) + .setSourceRange(modifierStart, consumedEnd); } + ms.setSourceRange(start, consumedEnd); return ms; } @@ -225,6 +237,7 @@ private CQLNode parseTerm(String index, CQLRelation relation) throws CQLParseException, IOException { debug("in parseTerm()"); + int start = lexer.getStart(); String first; StringBuilder all; while (true) { @@ -233,9 +246,12 @@ private CQLNode parseTerm(String index, CQLRelation relation) match('('); CQLNode expr = parseQuery(index, relation); match(')'); + expr.setSourceRange(start, consumedEnd); return expr; } else if (lexer.what() == '>') { - return parsePrefix(index, relation, false); + CQLNode expr = parsePrefix(index, relation, false); + expr.setSourceRange(start, consumedEnd); + return expr; } debug("non-parenthesised term"); @@ -257,14 +273,17 @@ private CQLNode parseTerm(String index, CQLRelation relation) throw new CQLParseException("unexpected relation '" + relstr + "'", lexer.pos()); } index = first; + int relationStart = lexer.getStart(); relation = new CQLRelation(relstr); match(lexer.what()); - ModifierSet ms = gatherModifiers(relstr); + ModifierSet ms = gatherModifiers(relstr, relationStart); relation.ms = ms; + relation.setSourceRange(relationStart, consumedEnd); debug("index='" + index + ", " + "relation='" + relation.toCQL() + "'"); } CQLTermNode node = new CQLTermNode(index, relation, all.toString()); + node.setSourceRange(start, consumedEnd); debug("made term node " + node.toCQL()); return node; } @@ -274,6 +293,7 @@ private CQLNode parsePrefix(String index, CQLRelation relation, throws CQLParseException, IOException { debug("prefix mapping"); + int start = lexer.getStart(); match('>'); String name = null; String identifier = matchSymbol("prefix-name"); @@ -284,7 +304,9 @@ private CQLNode parsePrefix(String index, CQLRelation relation, } CQLNode node = topLevel ? parseTopLevelPrefixes(index, relation) : parseQuery(index, relation); - return new CQLPrefixNode(name, identifier, node); + CQLPrefixNode prefix = new CQLPrefixNode(name, identifier, node); + prefix.setSourceRange(start, consumedEnd); + return prefix; } private boolean isWordOrString() { @@ -329,6 +351,7 @@ private void match(int token) lexer.render(token, true) + ", " + "got " + lexer.render(), lexer.pos()); + consumedEnd = lexer.getEnd(); lexer.move(); debug("match() got token=" + lexer.what() + ", value()='" + lexer.value() + "'"); } diff --git a/src/main/java/org/z3950/zing/cql/Modifier.java b/src/main/java/org/z3950/zing/cql/Modifier.java index 37b21d8..469ea0c 100644 --- a/src/main/java/org/z3950/zing/cql/Modifier.java +++ b/src/main/java/org/z3950/zing/cql/Modifier.java @@ -10,6 +10,39 @@ * */ public class Modifier { + + private int start = -1; + private int end = -1; + + /** + * Returns the inclusive source start of this modifier (including the leading slash). + * Offsets are zero-based UTF-16 indexes into the original query, excluding + * surrounding whitespace. Manually constructed objects and implicit defaults + * have no source location and return -1. Locations describe the original + * parse and are not updated after tree modifications. + * + * @return source start, or -1 when unavailable + */ + public int getStart() { + return start; + } + + /** + * Returns the exclusive source end, suitable for + * {@code query.substring(getStart(), getEnd())}. + * + * @return source end, or -1 when unavailable + * @see #getStart() + */ + public int getEnd() { + return end; + } + + void setSourceRange(int start, int end) { + this.start = start; + this.end = end; + } + String type; String comparison; String value; diff --git a/src/main/java/org/z3950/zing/cql/ModifierSet.java b/src/main/java/org/z3950/zing/cql/ModifierSet.java index 3456155..1c7ead1 100644 --- a/src/main/java/org/z3950/zing/cql/ModifierSet.java +++ b/src/main/java/org/z3950/zing/cql/ModifierSet.java @@ -18,6 +18,39 @@ * */ public class ModifierSet { + + private int start = -1; + private int end = -1; + + /** + * Returns the inclusive source start of this base and its modifiers. + * Offsets are zero-based UTF-16 indexes into the original query, excluding + * surrounding whitespace. Manually constructed objects and implicit defaults + * have no source location and return -1. Locations describe the original + * parse and are not updated after tree modifications. + * + * @return source start, or -1 when unavailable + */ + public int getStart() { + return start; + } + + /** + * Returns the exclusive source end, suitable for + * {@code query.substring(getStart(), getEnd())}. + * + * @return source end, or -1 when unavailable + * @see #getStart() + */ + public int getEnd() { + return end; + } + + void setSourceRange(int start, int end) { + this.start = start; + this.end = end; + } + private String base; private List modifiers; diff --git a/src/test/java/org/z3950/zing/cql/CQLSourcePositionsTest.java b/src/test/java/org/z3950/zing/cql/CQLSourcePositionsTest.java new file mode 100644 index 0000000..9f88fbb --- /dev/null +++ b/src/test/java/org/z3950/zing/cql/CQLSourcePositionsTest.java @@ -0,0 +1,109 @@ +package org.z3950.zing.cql; + +import org.junit.Test; +import static org.junit.Assert.*; + +public class CQLSourcePositionsTest { + private static void span(String query, String expected, CQLNode node) { + assertEquals(expected, query.substring(node.getStart(), node.getEnd())); + assertEquals(query.indexOf(expected), node.getStart()); + } + + @Test + public void preservesOriginalText() throws Exception { + CQLParser parser = new CQLParser(); + for (String text : new String[] { + "cat dog", "\"cat\" \"dog\"", "\"\"", "\"a\\\"b\"", + "cat\t\r\ndog", "\uD83D\uDE00 fish", "((cat))", + "title = (cat or dog)", "cat sortBy title", + ">dc=uri cat", "cat or >dc=uri dog", "title = >dc=uri dog"}) { + String query = " \t" + text + " \r\n"; + span(query, text, parser.parse(query)); + } + } + + @Test + public void nestedOperandsAndInheritedRelation() throws Exception { + String query = " title = (cat or (dog and fish)) "; + CQLBooleanNode root = (CQLBooleanNode) new CQLParser().parse(query); + span(query, query.trim(), root); + CQLTermNode cat = (CQLTermNode) root.getLeftOperand(); + span(query, "cat", cat); + span(query, "=", cat.getRelation()); + CQLBooleanNode right = (CQLBooleanNode) root.getRightOperand(); + span(query, "(dog and fish)", right); + span(query, "dog", right.getLeftOperand()); + span(query, "fish", right.getRightOperand()); + } + + @Test + public void relationsAndModifiers() throws Exception { + for (String relation : new String[] {"=", "==", "<", ">", "<=", ">=", "<>", "any", "cql.any"}) { + String query = "title " + relation + "/relevant /distance <= \"3\" fish "; + CQLTermNode term = (CQLTermNode) new CQLParser().parse(query); + span(query, relation + "/relevant /distance <= \"3\"", term.getRelation()); + Modifier simple = term.getRelation().getModifiers().get(0); + assertEquals("/relevant", query.substring(simple.getStart(), simple.getEnd())); + Modifier complex = term.getRelation().getModifiers().get(1); + assertEquals("/distance <= \"3\"", query.substring(complex.getStart(), complex.getEnd())); + } + } + + @Test + public void prefixSortAndBooleanModifiers() throws Exception { + String query = " >dc=uri cat prox/distance<=3 dog sortBy title/sort.ascending date "; + CQLPrefixNode prefix = (CQLPrefixNode) new CQLParser().parse(query); + span(query, query.trim(), prefix); + CQLSortNode sort = (CQLSortNode) prefix.getSubtree(); + span(query, "cat prox/distance<=3 dog sortBy title/sort.ascending date", sort); + CQLBooleanNode bool = (CQLBooleanNode) sort.getSubtree(); + span(query, "cat prox/distance<=3 dog", bool); + Modifier modifier = bool.getModifiers().get(0); + assertEquals("/distance<=3", query.substring(modifier.getStart(), modifier.getEnd())); + ModifierSet key = sort.getSortIndexes().get(0); + assertEquals("title/sort.ascending", query.substring(key.getStart(), key.getEnd())); + key = sort.getSortIndexes().get(1); + assertEquals("date", query.substring(key.getStart(), key.getEnd())); + } + + @Test + public void unknownLocations() throws Exception { + CQLTermNode parsed = (CQLTermNode) new CQLParser().parse("fish"); + CQLNode[] nodes = {new CQLTermNode("title", new CQLRelation("="), "fish"), + parsed.getRelation()}; + for (CQLNode node : nodes) { + assertEquals(-1, node.getStart()); + assertEquals(-1, node.getEnd()); + } + ModifierSet set = new ModifierSet("any"); + set.addModifier("relevant"); + set.addModifier("distance", "<", "3"); + assertEquals(-1, set.getStart()); + assertEquals(-1, set.getEnd()); + for (Modifier modifier : set.getModifiers()) { + assertEquals(-1, modifier.getStart()); + assertEquals(-1, modifier.getEnd()); + } + } + + @Test + public void lexerOffsetsAndLegacyPosition() { + String query = " \"\uD83D\uDE00\" <= cat "; + CQLLexer lexer = new CQLLexer(query, false); + assertEquals(-1, lexer.getStart()); + assertEquals(-1, lexer.getEnd()); + assertEquals(0, lexer.pos()); + for (String token : new String[] {"\"\uD83D\uDE00\"", "<=", "cat"}) { + lexer.move(); + assertEquals(token, query.substring(lexer.getStart(), lexer.getEnd())); + assertEquals(query.indexOf(token), lexer.getStart()); + assertEquals(lexer.pos(), lexer.getEnd()); + } + lexer.move(); + assertEquals(CQLTokenizer.TT_EOF, lexer.what()); + assertEquals(query.length(), lexer.getStart()); + assertEquals(query.length(), lexer.getEnd()); + lexer.move(); + assertEquals(query.length(), lexer.getEnd()); + } +}