Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 20 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,26 @@ Please email us with bug-reports, wishlist items, patches, deployment
stories and, of course, large cash donations.


## Source positions

Parsed nodes expose `getStart()` and `getEnd()`: zero-based UTF-16 offsets
into the original query, with an exclusive end. For example,
`query.substring(node.getStart(), node.getEnd())` retrieves the original
source, preserving whitespace, quotes and escapes.

A node's range includes its subtree, enclosing parentheses and any index
and relation introducing a grouped expression. Surrounding whitespace is
excluded. Relations, modifiers (including the slash), and sort keys
(`ModifierSet`) also expose ranges. An inherited relation retains the
location where it was written, which can be outside the inheriting term.
Implicit defaults and manually constructed objects return `-1` for both
offsets. Positions describe the original parse; editing the tree does not
update them. Individual index and term-value ranges are not exposed.

This implementation was inspired by Querela's source-position proposal in
[PR #19](https://github.com/indexdata/cql-java/pull/19).
Existing method signatures and the `CQLTokenizer` interface are unchanged.

## Licence

The CQL-Java suite is Free Software, which is pretty much legally
Expand Down
23 changes: 23 additions & 0 deletions src/main/java/org/z3950/zing/cql/CQLLexer.java
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
public class CQLLexer implements CQLTokenizer {
private String qs;
private int qi;
private int tokenStart = -1;
private int ql;
private int what = TT_NOTHING;
private String val;
Expand All @@ -30,6 +31,7 @@ public void move() {
// eat whitespace
while (qi < ql && strchr(" \t\r\n", qs.charAt(qi)))
qi++;
tokenStart = qi;
// eof
if (qi == ql) {
what = TT_EOF;
Expand Down Expand Up @@ -165,6 +167,27 @@ public String render(int token, boolean quoteChars) {
}
}

/**
* Returns the current token's inclusive UTF-16 source start, or -1 before
* the first move. Quotes are included; leading whitespace is excluded.
* At end of input the span is empty at the input length.
*
* @return token start
*/
public int getStart() {
return tokenStart;
}

/**
* Returns the current token's exclusive UTF-16 source end, or -1 before
* the first move.
*
* @return token end
*/
public int getEnd() {
return tokenStart < 0 ? -1 : qi;
}

@Override
public int pos() {
return qi;
Expand Down
33 changes: 33 additions & 0 deletions src/main/java/org/z3950/zing/cql/CQLNode.java
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,39 @@
*/
public abstract class CQLNode {

private int start = -1;
private int end = -1;

/**
* Returns the inclusive source start of this node (including its subtree and enclosing parentheses).
* Offsets are zero-based UTF-16 indexes into the original query, excluding
* surrounding whitespace. Manually constructed objects and implicit defaults
* have no source location and return -1. Locations describe the original
* parse and are not updated after tree modifications.
*
* @return source start, or -1 when unavailable
*/
public int getStart() {
return start;
}

/**
* Returns the exclusive source end, suitable for
* {@code query.substring(getStart(), getEnd())}.
*
* @return source end, or -1 when unavailable
* @see #getStart()
*/
public int getEnd() {
return end;
}

void setSourceRange(int start, int end) {
this.start = start;
this.end = end;
}


public abstract void traverse(CQLNodeVisitor visitor);

/**
Expand Down
37 changes: 30 additions & 7 deletions src/main/java/org/z3950/zing/cql/CQLParser.java
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,9 @@
* >https://zing.z3950.org/cql/index.html</A>
*/
public class CQLParser {
private CQLTokenizer lexer;
private CQLLexer lexer;
// End of the last consumed token; the lexer already holds the lookahead.
private int consumedEnd;
private final int compat; // When false, implement CQL 1.2
private final Set<String> customRelations = new HashSet<String>();

Expand Down Expand Up @@ -119,6 +121,7 @@ public boolean unregisterCustomRelation(String relation) {
public CQLNode parse(String cql)
throws CQLParseException, IOException {
lexer = new CQLLexer(cql, LEXDEBUG);
consumedEnd = -1;

lexer.move();
debug("about to parseQuery()");
Expand Down Expand Up @@ -147,15 +150,17 @@ private CQLNode parseTopLevelPrefixes(String index, CQLRelation relation)

CQLSortNode sortnode = new CQLSortNode(node);
while (lexer.what() != CQLTokenizer.TT_EOF) {
int keyStart = lexer.getStart();
String sortindex = matchSymbol("sort index");
ModifierSet ms = gatherModifiers(sortindex);
ModifierSet ms = gatherModifiers(sortindex, keyStart);
sortnode.addSortIndex(ms);
}

if (sortnode.keys.size() == 0) {
throw new CQLParseException("no sort keys", lexer.pos());
}

sortnode.setSourceRange(node.getStart(), consumedEnd);
node = sortnode;
}

Expand All @@ -174,15 +179,18 @@ private CQLNode parseQuery(String index, CQLRelation relation)
lexer.what() == CQLTokenizer.TT_OR ||
lexer.what() == CQLTokenizer.TT_NOT ||
lexer.what() == CQLTokenizer.TT_PROX) {
int start = term.getStart();
int operatorStart = lexer.getStart();
int type = lexer.what();
String val = lexer.value();
match(type);
ModifierSet ms = gatherModifiers(val);
ModifierSet ms = gatherModifiers(val, operatorStart);
CQLNode term2 = parseTerm(index, relation);
term = ((type == CQLTokenizer.TT_AND) ? new CQLAndNode(term, term2, ms)
: (type == CQLTokenizer.TT_OR) ? new CQLOrNode(term, term2, ms)
: (type == CQLTokenizer.TT_NOT) ? new CQLNotNode(term, term2, ms)
: new CQLProxNode(term, term2, ms));
term.setSourceRange(start, consumedEnd);
} else {
throw new CQLParseException("expected boolean, got " +
lexer.render(), lexer.pos());
Expand All @@ -193,12 +201,13 @@ private CQLNode parseQuery(String index, CQLRelation relation)
return term;
}

private ModifierSet gatherModifiers(String base)
private ModifierSet gatherModifiers(String base, int start)
throws CQLParseException, IOException {
debug("in gatherModifiers()");

ModifierSet ms = new ModifierSet(base);
while (lexer.what() == '/') {
int modifierStart = lexer.getStart();
match('/');
if (lexer.what() != CQLTokenizer.TT_WORD)
throw new CQLParseException("expected modifier, "
Expand All @@ -216,7 +225,10 @@ private ModifierSet gatherModifiers(String base)
String value = matchSymbol("modifier value");
ms.addModifier(type, comparision, value);
}
ms.getModifiers().get(ms.getModifiers().size() - 1)
.setSourceRange(modifierStart, consumedEnd);
}
ms.setSourceRange(start, consumedEnd);

return ms;
}
Expand All @@ -225,6 +237,7 @@ private CQLNode parseTerm(String index, CQLRelation relation)
throws CQLParseException, IOException {
debug("in parseTerm()");

int start = lexer.getStart();
String first;
StringBuilder all;
while (true) {
Expand All @@ -233,9 +246,12 @@ private CQLNode parseTerm(String index, CQLRelation relation)
match('(');
CQLNode expr = parseQuery(index, relation);
match(')');
expr.setSourceRange(start, consumedEnd);
return expr;
} else if (lexer.what() == '>') {
return parsePrefix(index, relation, false);
CQLNode expr = parsePrefix(index, relation, false);
expr.setSourceRange(start, consumedEnd);
return expr;
}

debug("non-parenthesised term");
Expand All @@ -257,14 +273,17 @@ private CQLNode parseTerm(String index, CQLRelation relation)
throw new CQLParseException("unexpected relation '" + relstr + "'", lexer.pos());
}
index = first;
int relationStart = lexer.getStart();
relation = new CQLRelation(relstr);
match(lexer.what());
ModifierSet ms = gatherModifiers(relstr);
ModifierSet ms = gatherModifiers(relstr, relationStart);
relation.ms = ms;
relation.setSourceRange(relationStart, consumedEnd);
debug("index='" + index + ", " +
"relation='" + relation.toCQL() + "'");
}
CQLTermNode node = new CQLTermNode(index, relation, all.toString());
node.setSourceRange(start, consumedEnd);
debug("made term node " + node.toCQL());
return node;
}
Expand All @@ -274,6 +293,7 @@ private CQLNode parsePrefix(String index, CQLRelation relation,
throws CQLParseException, IOException {
debug("prefix mapping");

int start = lexer.getStart();
match('>');
String name = null;
String identifier = matchSymbol("prefix-name");
Expand All @@ -284,7 +304,9 @@ private CQLNode parsePrefix(String index, CQLRelation relation,
}
CQLNode node = topLevel ? parseTopLevelPrefixes(index, relation) : parseQuery(index, relation);

return new CQLPrefixNode(name, identifier, node);
CQLPrefixNode prefix = new CQLPrefixNode(name, identifier, node);
prefix.setSourceRange(start, consumedEnd);
return prefix;
}

private boolean isWordOrString() {
Expand Down Expand Up @@ -329,6 +351,7 @@ private void match(int token)
lexer.render(token, true) +
", " + "got " + lexer.render(),
lexer.pos());
consumedEnd = lexer.getEnd();
lexer.move();
debug("match() got token=" + lexer.what() + ", value()='" + lexer.value() + "'");
}
Expand Down
33 changes: 33 additions & 0 deletions src/main/java/org/z3950/zing/cql/Modifier.java
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,39 @@
*
*/
public class Modifier {

private int start = -1;
private int end = -1;

/**
* Returns the inclusive source start of this modifier (including the leading slash).
* Offsets are zero-based UTF-16 indexes into the original query, excluding
* surrounding whitespace. Manually constructed objects and implicit defaults
* have no source location and return -1. Locations describe the original
* parse and are not updated after tree modifications.
*
* @return source start, or -1 when unavailable
*/
public int getStart() {
return start;
}

/**
* Returns the exclusive source end, suitable for
* {@code query.substring(getStart(), getEnd())}.
*
* @return source end, or -1 when unavailable
* @see #getStart()
*/
public int getEnd() {
return end;
}

void setSourceRange(int start, int end) {
this.start = start;
this.end = end;
}

String type;
String comparison;
String value;
Expand Down
33 changes: 33 additions & 0 deletions src/main/java/org/z3950/zing/cql/ModifierSet.java
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,39 @@
*
*/
public class ModifierSet {

private int start = -1;
private int end = -1;

/**
* Returns the inclusive source start of this base and its modifiers.
* Offsets are zero-based UTF-16 indexes into the original query, excluding
* surrounding whitespace. Manually constructed objects and implicit defaults
* have no source location and return -1. Locations describe the original
* parse and are not updated after tree modifications.
*
* @return source start, or -1 when unavailable
*/
public int getStart() {
return start;
}

/**
* Returns the exclusive source end, suitable for
* {@code query.substring(getStart(), getEnd())}.
*
* @return source end, or -1 when unavailable
* @see #getStart()
*/
public int getEnd() {
return end;
}

void setSourceRange(int start, int end) {
this.start = start;
this.end = end;
}

private String base;
private List<Modifier> modifiers;

Expand Down
Loading