Make token list manipulatable

10 files changed, 165 insertions, 38 deletions

diff --git a/javaparser-core/src/main/java/com/github/javaparser/JavaToken.java b/javaparser-core/src/main/java/com/github/javaparser/JavaToken.java
index eb0b3058d..ab774b1af 100644
--- a/javaparser-core/src/main/java/com/github/javaparser/JavaToken.java
+++ b/javaparser-core/src/main/java/com/github/javaparser/JavaToken.java
@@ -24,30 +24,33 @@ package com.github.javaparser;
 import java.util.List;
 import java.util.Optional;
 
-import static com.github.javaparser.Position.pos;
+import static com.github.javaparser.utils.CodeGenerationUtils.f;
+import static com.github.javaparser.utils.Utils.EOL;
 import static com.github.javaparser.utils.Utils.assertNotNull;
 
 /**
  * A token from a parsed source file.
  * (Awkwardly named "Java"Token since JavaCC already generates an internal class Token.)
+ * It is a node in a double linked list called token list.
  */
 public class JavaToken {
     public static final JavaToken INVALID = new JavaToken();
 
-    private final Range range;
+    private Range range;
     private int kind;
-    private final String text;
-    private final Optional<JavaToken> previousToken;
-    private Optional<JavaToken> nextToken = Optional.empty();
+    private String text;
+    private JavaToken previousToken = null;
+    private JavaToken nextToken = null;
 
     private JavaToken() {
-        range = new Range(pos(-1, -1), pos(-1, -1));
-        kind = 0;
-        text = "INVALID";
-        previousToken = Optional.empty();
+        this(null, 0, "INVALID", null, null);
     }
 
-    public JavaToken(Token token, List<JavaToken> tokens) {
+    public JavaToken(int kind, String text) {
+        this(null, kind, text, null, null);
+    }
+
+    JavaToken(Token token, List<JavaToken> tokens) {
         Range range = Range.range(token.beginLine, token.beginColumn, token.endLine, token.endColumn);
         String text = token.image;
 
@@ -95,14 +98,34 @@ public class JavaToken {
         this.text = text;
         if (!tokens.isEmpty()) {
             final JavaToken previousToken = tokens.get(tokens.size() - 1);
-            this.previousToken = Optional.of(previousToken);
-            previousToken.nextToken = Optional.of(this);
+            this.previousToken = previousToken;
+            previousToken.nextToken = this;
         } else {
-            previousToken = Optional.empty();
+            previousToken = null;
+        }
+    }
+
+    /**
+     * Create a token of a certain kind.
+     */
+    public JavaToken(int kind) {
+        String content = GeneratedJavaParserConstants.tokenImage[kind];
+        if (content.startsWith("\"")) {
+            content = content.substring(1, content.length() - 1);
+        }
+        if (TokenTypes.isEndOfLineToken(kind)) {
+            content = EOL;
+        } else if (TokenTypes.isWhitespace(kind)) {
+            content = " ";
         }
+        this.kind = kind;
+        this.text = content;
     }
 
-    public JavaToken(Range range, int kind, String text, Optional<JavaToken> previousToken, Optional<JavaToken> nextToken) {
+
+    public JavaToken(Range range, int kind, String text, JavaToken previousToken, JavaToken nextToken) {
+        assertNotNull(text);
+
         this.range = range;
         this.kind = kind;
         this.text = text;
@@ -110,8 +133,8 @@ public class JavaToken {
         this.nextToken = nextToken;
     }
 
-    public Range getRange() {
-        return range;
+    public Optional<Range> getRange() {
+        return Optional.ofNullable(range);
     }
 
     public int getKind() {
@@ -127,16 +150,31 @@ public class JavaToken {
     }
 
     public Optional<JavaToken> getNextToken() {
-        return nextToken;
+        return Optional.ofNullable(nextToken);
     }
 
     public Optional<JavaToken> getPreviousToken() {
-        return previousToken;
+        return Optional.ofNullable(previousToken);
+    }
+
+    public void setRange(Range range) {
+        this.range = range;
+    }
+
+    public void setText(String text) {
+        this.text = text;
+    }
+
+    public String asString() {
+        return text;
     }
 
     @Override
     public String toString() {
-        return text;
+        return f("\"%s\" <%s> %s",
+                getText(),
+                getKind(),
+                getRange().map(Range::toString).orElse("(?)-(?)"));
     }
 
     /**
@@ -211,4 +249,78 @@ public class JavaToken {
     public JavaToken.Category getCategory() {
         return TokenTypes.getCategory(kind);
     }
+
+    /**
+     * Inserts newToken into the token list just before this token.
+     */
+    public void insert(JavaToken newToken) {
+        assertNotNull(newToken);
+        getPreviousToken().ifPresent(p -> {
+            p.nextToken = newToken;
+            newToken.previousToken = p;
+        });
+        previousToken = newToken;
+        newToken.nextToken = this;
+    }
+
+    /**
+     * Inserts newToken into the token list just after this token.
+     */
+    public void insertAfter(JavaToken newToken) {
+        assertNotNull(newToken);
+        getNextToken().ifPresent(n -> {
+            n.previousToken = newToken;
+            newToken.nextToken = n;
+        });
+        nextToken = newToken;
+        newToken.previousToken = this;
+    }
+
+    /**
+     * Links the tokens around the current token together, making the current token disappear from the list.
+     */
+    public void deleteToken() {
+        final Optional<JavaToken> nextToken = getNextToken();
+        final Optional<JavaToken> previousToken = getPreviousToken();
+
+        previousToken.ifPresent(p -> p.nextToken = nextToken.orElse(null));
+        nextToken.ifPresent(n -> n.previousToken = previousToken.orElse(null));
+    }
+
+    /**
+     * Replaces the current token with newToken.
+     */
+    public void replaceToken(JavaToken newToken) {
+        assertNotNull(newToken);
+        getPreviousToken().ifPresent(p -> {
+            p.nextToken = newToken;
+            newToken.previousToken = p;
+        });
+        getNextToken().ifPresent(n -> {
+            n.previousToken = newToken;
+            newToken.nextToken = n;
+        });
+    }
+
+    /**
+     * @return the last token in the token list.
+     */
+    public JavaToken findLastToken() {
+        JavaToken current = this;
+        while (current.getNextToken().isPresent()) {
+            current = current.getNextToken().get();
+        }
+        return current;
+    }
+
+    /**
+     * @return the first token in the token list.
+     */
+    public JavaToken findFirstToken() {
+        JavaToken current = this;
+        while (current.getPreviousToken().isPresent()) {
+            current = current.getPreviousToken().get();
+        }
+        return current;
+    }
 }
diff --git a/javaparser-core/src/main/java/com/github/javaparser/Problem.java b/javaparser-core/src/main/java/com/github/javaparser/Problem.java
index 156bb2f25..2fc053226 100644
--- a/javaparser-core/src/main/java/com/github/javaparser/Problem.java
+++ b/javaparser-core/src/main/java/com/github/javaparser/Problem.java
@@ -69,7 +69,7 @@ public class Problem {
      * @return the message plus location information.
      */
     public String getVerboseMessage() {
-        return getLocation().map(l -> l.getBegin().getRange().begin + " " + message).orElse(message);
+        return getLocation().map(l -> l.getBegin().getRange().map(r -> r.begin.toString()).orElse("(line ?,col ?)") + " " + message).orElse(message);
     }
 
     /**
@@ -98,8 +98,11 @@ public class Problem {
      * Sorts problems on position.
      */
     public static Comparator<Problem> PROBLEM_BY_BEGIN_POSITION = (a, b) -> {
-        if (a.getLocation().isPresent() && b.getLocation().isPresent()) {
-            return a.getLocation().get().getBegin().getRange().begin.compareTo(b.getLocation().get().getBegin().getRange().begin);
+        final Optional<Position> aBegin= a.getLocation().flatMap(l -> l.getBegin().getRange().map(r -> r.begin));
+        final Optional<Position> bBegin = b.getLocation().flatMap(l -> l.getBegin().getRange().map(r -> r.begin));
+
+        if (aBegin.isPresent() && bBegin.isPresent()) {
+            return aBegin.get().compareTo(bBegin.get());
         }
         if (a.getLocation().isPresent() || b.getLocation().isPresent()) {
             if (a.getLocation().isPresent()) {
diff --git a/javaparser-core/src/main/java/com/github/javaparser/TokenRange.java b/javaparser-core/src/main/java/com/github/javaparser/TokenRange.java
index 7746e856d..b6958eb0d 100644
--- a/javaparser-core/src/main/java/com/github/javaparser/TokenRange.java
+++ b/javaparser-core/src/main/java/com/github/javaparser/TokenRange.java
@@ -26,8 +26,11 @@ public class TokenRange {
         return end;
     }
 
-    public Range toRange() {
-        return new Range(begin.getRange().begin, end.getRange().end);
+    public Optional<Range> toRange() {
+        if (begin.getRange().isPresent() && end.getRange().isPresent()) {
+            return Optional.of(new Range(begin.getRange().get().begin, end.getRange().get().end));
+        }
+        return Optional.empty();
     }
 
     public TokenRange withBegin(JavaToken begin) {
diff --git a/javaparser-core/src/main/java/com/github/javaparser/ast/Node.java b/javaparser-core/src/main/java/com/github/javaparser/ast/Node.java
index d8630d979..43263b16e 100644
--- a/javaparser-core/src/main/java/com/github/javaparser/ast/Node.java
+++ b/javaparser-core/src/main/java/com/github/javaparser/ast/Node.java
@@ -200,10 +200,11 @@ public abstract class Node implements Cloneable, HasParentNode<Node>, Visitable,
 
     public Node setTokenRange(TokenRange tokenRange) {
         this.tokenRange = tokenRange;
-        if (tokenRange == null) {
+        if (tokenRange == null || 
+                !(tokenRange.getBegin().getRange().isPresent() && tokenRange.getBegin().getRange().isPresent())) {
             range = null;
         } else {
-            range = new Range(tokenRange.getBegin().getRange().begin, tokenRange.getEnd().getRange().end);
+            range = new Range(tokenRange.getBegin().getRange().get().begin, tokenRange.getEnd().getRange().get().end);
         }
         return this;
     }
diff --git a/javaparser-core/src/main/java/com/github/javaparser/ast/type/ArrayType.java b/javaparser-core/src/main/java/com/github/javaparser/ast/type/ArrayType.java
index 37133cecc..12e4ab46c 100644
--- a/javaparser-core/src/main/java/com/github/javaparser/ast/type/ArrayType.java
+++ b/javaparser-core/src/main/java/com/github/javaparser/ast/type/ArrayType.java
@@ -113,7 +113,7 @@ public class ArrayType extends ReferenceType implements NodeWithAnnotations<Arra
                     }
                     type = new ArrayType(tokenRange, type, pair.getAnnotations());
                     if (tokenRange != null) {
-                        type.setRange(tokenRange.toRange());
+                        type.setRange(tokenRange.toRange().get());
                     }
                 }
             }
diff --git a/javaparser-core/src/main/java/com/github/javaparser/printer/lexicalpreservation/LexicalPreservingPrinter.java b/javaparser-core/src/main/java/com/github/javaparser/printer/lexicalpreservation/LexicalPreservingPrinter.java
index cb150bca7..8f719fdab 100644
--- a/javaparser-core/src/main/java/com/github/javaparser/printer/lexicalpreservation/LexicalPreservingPrinter.java
+++ b/javaparser-core/src/main/java/com/github/javaparser/printer/lexicalpreservation/LexicalPreservingPrinter.java
@@ -202,11 +202,11 @@ public class LexicalPreservingPrinter {
         // We go over tokens and find to which nodes belong. Note that we start from the most specific nodes
         // and we move up to more general nodes
         for (JavaToken token : documentTokens) {
-            Optional<Node> maybeOwner = nodesDepthFirst.stream().filter(n -> n.getRange().get().contains(token.getRange())).findFirst();
-            if (!maybeOwner.isPresent()) {
-                throw new RuntimeException("Token without node owning it: " + token);
-            }
-            Node owner = maybeOwner.get();
+            Range tokenRange = token.getRange().orElseThrow(() -> new RuntimeException("Token without range: " + token));
+            Node owner = nodesDepthFirst.stream()
+                    .filter(n -> n.getRange().get().contains(tokenRange))
+                    .findFirst()
+                    .orElseThrow(() -> new RuntimeException("Token without node owning it: " + token));
             if (!tokensByNode.containsKey(owner)) {
                 tokensByNode.put(owner, new LinkedList<>());
             }
@@ -238,7 +238,7 @@ public class LexicalPreservingPrinter {
             }
         }
         for (JavaToken token : nodeTokens) {
-            elements.add(new Pair<>(token.getRange(), new TokenTextElement(token)));
+            elements.add(new Pair<>(token.getRange().get(), new TokenTextElement(token)));
         }
         elements.sort(Comparator.comparing(e -> e.a.begin));
         textForNodes.put(node, new NodeText(this, elements.stream().map(p -> p.b).collect(Collectors.toList())));
diff --git a/javaparser-testing/src/test/java/com/github/javaparser/JavaParserTest.java b/javaparser-testing/src/test/java/com/github/javaparser/JavaParserTest.java
index 828e54ddb..1082cd61f 100644
--- a/javaparser-testing/src/test/java/com/github/javaparser/JavaParserTest.java
+++ b/javaparser-testing/src/test/java/com/github/javaparser/JavaParserTest.java
@@ -102,7 +102,7 @@ public class JavaParserTest {
         ParseResult<CompilationUnit> result = new JavaParser().parse(COMPILATION_UNIT, Providers.provider("class X { // blah"));
 
         Problem problem = result.getProblem(0);
-        assertEquals(range(1, 9, 1, 17), problem.getLocation().get().toRange());
+        assertEquals(range(1, 9, 1, 17), problem.getLocation().get().toRange().get());
         assertEquals("Parse error. Found <EOF>, expected one of  \";\" \"<\" \"@\" \"abstract\" \"boolean\" \"byte\" \"char\" \"class\" \"default\" \"double\" \"enum\" \"exports\" \"final\" \"float\" \"int\" \"interface\" \"long\" \"module\" \"native\" \"open\" \"opens\" \"private\" \"protected\" \"provides\" \"public\" \"requires\" \"short\" \"static\" \"strictfp\" \"synchronized\" \"to\" \"transient\" \"transitive\" \"uses\" \"void\" \"volatile\" \"with\" \"{\" \"}\" <IDENTIFIER>", problem.getMessage());
         assertInstanceOf(ParseException.class, problem.getCause().get());
     }
diff --git a/javaparser-testing/src/test/java/com/github/javaparser/JavaTokenTest.java b/javaparser-testing/src/test/java/com/github/javaparser/JavaTokenTest.java
index c197e91a2..0e38f6d3b 100644
--- a/javaparser-testing/src/test/java/com/github/javaparser/JavaTokenTest.java
+++ b/javaparser-testing/src/test/java/com/github/javaparser/JavaTokenTest.java
@@ -53,11 +53,19 @@ public class JavaTokenTest {
 
     private void assertToken(String image, Range range, int kind, JavaToken.Category category, JavaToken token) {
         assertEquals(image, token.getText());
-        assertEquals(range, token.getRange());
+        assertEquals(range, token.getRange().get());
         assertEquals(kind, token.getKind());
         assertEquals(category, token.getCategory());
         token.getNextToken().ifPresent(nt -> assertEquals(token, nt.getPreviousToken().get()));
         token.getPreviousToken().ifPresent(pt -> assertEquals(token, pt.getNextToken().get()));
         assertTrue(token.getNextToken().isPresent() || token.getPreviousToken().isPresent());
     }
+    
+    @Test
+    public void testAFewImagesForTokenKinds() {
+        assertEquals("=", new JavaToken(ASSIGN).getText());
+        // TODO this shouldn't be a space.
+        assertEquals(" ", new JavaToken(EOF).getText());
+        assertEquals("*/", new JavaToken(JAVA_DOC_COMMENT).getText());
+    }
 }
diff --git a/javaparser-testing/src/test/java/com/github/javaparser/ProblemTest.java b/javaparser-testing/src/test/java/com/github/javaparser/ProblemTest.java
index a26be5d5e..0654344ba 100644
--- a/javaparser-testing/src/test/java/com/github/javaparser/ProblemTest.java
+++ b/javaparser-testing/src/test/java/com/github/javaparser/ProblemTest.java
@@ -20,7 +20,7 @@ public class ProblemTest {
     public void testVerboseMessage() {
         Problem problem = new Problem("Parse error", TokenRange.INVALID, null);
 
-        assertEquals("(line -1,col -1) Parse error", problem.getVerboseMessage());
+        assertEquals("(line ?,col ?) Parse error", problem.getVerboseMessage());
     }
 
     @Test
diff --git a/javaparser-testing/src/test/java/com/github/javaparser/ast/expr/LambdaExprTest.java b/javaparser-testing/src/test/java/com/github/javaparser/ast/expr/LambdaExprTest.java
index 11b6dffe0..bfa007867 100644
--- a/javaparser-testing/src/test/java/com/github/javaparser/ast/expr/LambdaExprTest.java
+++ b/javaparser-testing/src/test/java/com/github/javaparser/ast/expr/LambdaExprTest.java
@@ -22,7 +22,7 @@ public class LambdaExprTest {
 
     private void assertRange(String startToken, String endToken, Node node) {
         TokenRange tokenRange = node.getTokenRange().get();
-        assertEquals(startToken, tokenRange.getBegin().toString());
-        assertEquals(endToken, tokenRange.getEnd().toString());
+        assertEquals(startToken, tokenRange.getBegin().asString());
+        assertEquals(endToken, tokenRange.getEnd().asString());
     }
 }
\ No newline at end of file