View Javadoc

    1   /***
    2    * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
    3    */
    4   package net.sourceforge.pmd.cpd;
    5
    6   import net.sourceforge.pmd.cpd.cppast.CPPParserTokenManager;
    7   import net.sourceforge.pmd.cpd.cppast.SimpleCharStream;
    8   import net.sourceforge.pmd.cpd.cppast.Token;
    9   import net.sourceforge.pmd.cpd.cppast.TokenMgrError;
    10
    11  import java.io.StringReader;
    12  import java.util.ArrayList;
    13  import java.util.List;
    14
    15  public class CPPTokenizer implements Tokenizer {
    16      protected String EOL = System.getProperty("line.separator", "\n");
    17
    18      private static SimpleCharStream charStream;
    19
    20      public void tokenize(SourceCode sourceCode, Tokens tokenEntries) {
    21          StringBuffer sb = sourceCode.getCodeBuffer();
    22          try {
    23              if (charStream == null) {
    24                  charStream = new SimpleCharStream(new StringReader(sb.toString()));
    25              } else {
    26                  charStream.ReInit(new StringReader(sb.toString()));
    27              }
    28              CPPParserTokenManager.ReInit(charStream);
    29              CPPParserTokenManager.setFileName(sourceCode.getFileName());
    30              Token currToken = CPPParserTokenManager.getNextToken();
    31              while (currToken.image.length() > 0) {
    32                  tokenEntries.add(new TokenEntry(currToken.image, sourceCode.getFileName(), currToken.beginLine));
    33                  currToken = CPPParserTokenManager.getNextToken();
    34              }
    35              tokenEntries.add(TokenEntry.getEOF());
    36              System.out.println("Added " + sourceCode.getFileName());
    37          } catch (TokenMgrError err) {
    38              err.printStackTrace();
    39              System.out.println("Skipping " + sourceCode.getFileName() + " due to parse error");
    40              List emptyCode = new ArrayList();
    41              emptyCode.add("");
    42              tokenEntries.add(TokenEntry.getEOF());
    43          }
    44      }
    45  }