javacc-wrapper.xml

<!--
  ~ BSD-style license; for more info see http://pmd.sourceforge.net/license.html
  -->

<project name="pmd" default="alljavacc" basedir="/">

    <!-- Wraps a JavaCC/JJTree task and adapts the generated sources to the PMD
         codebase.

    Input parameters:

         - lang-name:       Capitalized name of the language eg Jsp or Java.
                             By convention the grammar file must be named exactly
                             so with a .jj[t] extension.

         - lang-name-camelcase: Name of the language in camelcase. Useful for languages
                                whose name is an acronym, eg PLSQL (in camelcase, "Plsql").
                                Defaults to lang-name.

         - lang-id: The language id, used in the conventional package names

         - no-jjtree: true/false. Use true for CPD-only languages. Then a parser won't
                      be generated. By default, a parser will be generated (and jjtree will be called).

         It also uses the following maven properties:

         - javacc.outputDirectory:  Directory in which to root the generated package tree
         - javacc.jar:              JAR of JavaCC in the local maven repository
         - some properties of project.build

    -->

    <condition property="lang-name-camelcase" value="${lang-name}">
        <!-- Eg for "PLSQL": "Plsql" -->
        <not>
            <isset property="lang-name-camelcase" />
        </not>
    </condition>

    <property name="target-package-dir" value="${javacc.outputDirectory}/net/sourceforge/pmd/lang/${lang-id}/ast" />
    <property name="stamp-file" value="${project.build.directory}/last-generated-timestamp" />

    <property name="javacc-home.path" value="${project.build.directory}/lib/javacc" />

    <property name="lang-ast-package" value="net.sourceforge.pmd.lang.${lang-id}.ast" />
    <property name="ast-api-package" value="net.sourceforge.pmd.lang.ast" />
    <property name="ast-impl-package" value="${ast-api-package}.impl.javacc" />

    <property name="grammar-file" value="${basedir}/etc/grammar/${lang-name}.jjt" />

    <property name="parser-name" value="${lang-name}ParserImpl" />
    <property name="parser-file" value="${target-package-dir}/${parser-name}.java" />
    <property name="ast-prefix" value="AST" />

    <property name="constants-itf-name" value="${parser-name}Constants" />
    <property name="constants-itf-file" value="${target-package-dir}/${constants-itf-name}.java" />

    <property name="token-constants-name" value="${lang-name}TokenKinds" />
    <property name="token-constants-file" value="${target-package-dir}/${token-constants-name}.java" />

    <property name="node-name" value="${lang-name}Node" />
    <property name="base-class-name" value="Abstract${lang-name}Node" />

    <!-- This will be moved to impl package when all language modules have been ported   -->
    <property name="base-tokenmgr" value="${ast-impl-package}.AbstractTokenManager"/>
    <property name="charstream-itf" value="${ast-impl-package}.CharStream"/>


    <property name="tokenmgr-name" value="${parser-name}TokenManager" />
    <property name="tokenmgr-file" value="${target-package-dir}/${tokenmgr-name}.java" />


    <!-- Visitor names -->
    <!-- This is the default name generated by JJTree   -->
    <property name="gen-visitor-name" value="${parser-name}Visitor" />

    <!-- This is the actual name we use   -->
    <property name="generic-visitor-interface-name" value="${lang-name-camelcase}Visitor" />
    <property name="generic-visitor-interface-file"
              value="${target-package-dir}/${generic-visitor-interface-name}.java" />


    <!-- TARGETS -->

    <target name="alljavacc"
            description="Generates JavaCC sources and cleans them up"
            depends="checkUpToDate,init,jjtree,jjtree-ersatz,javacc,adapt-generated,default-visitor,cleanup" />

    <target name="checkUpToDate"
            description="Checks the input files are up to date">

        <uptodate property="javaccBuildNotRequired" targetfile="${stamp-file}">
            <srcfiles dir="etc/grammar" includes="${lang-name}.jj*" />
            <srcfiles file="${ant.file}" />
        </uptodate>
        <echo message="Up-to-date check: javaccBuildNotRequired=${javaccBuildNotRequired}" />
        <condition property="jjtreeBuildNotRequired">
            <or>
                <isset property="no-jjtree" />
                <isset property="javaccBuildNotRequired" />
            </or>
        </condition>
    </target>


    <target name="init" unless="javaccBuildNotRequired" description="Initialize build">
        <mkdir dir="${javacc-home.path}" />
        <copy file="${javacc.jar}" tofile="${javacc-home.path}/javacc.jar" />

        <mkdir dir="${javacc.outputDirectory}" />
        <touch file="${stamp-file}" />
        <delete dir="${target-package-dir}" />
        <mkdir dir="${target-package-dir}" />

        <condition property="option-token-manager-uses-parser">
            <resourcecontains resource="${grammar-file}" substring="TOKEN_MANAGER_USES_PARSER = true"/>
        </condition>
    </target>

    <target name="cleanup" unless="javaccBuildNotRequired">
        <delete dir="${javacc-home.path}" />
    </target>


    <target name="jjtree" unless="jjtreeBuildNotRequired" description="Runs JJTree">

        <!--
             Token-tracking behavior is implemented in the JJTree builder,
             so the grammar should have TRACK_TOKEN=false; (this is the default value)
        -->
        <java fork="true"
              classname="jjtree"
              classpath="${javacc-home.path}/javacc.jar">
            <sysproperty key="file.encoding" value="UTF-8" />
            <arg value="-OUTPUT_DIRECTORY:${target-package-dir}" />
            <arg value="-NODE_USES_PARSER:false" />
            <arg value="-NODE_PACKAGE:${lang-ast-package}" />
            <arg value="${grammar-file}" />
        </java>
    </target>

    <target name="jjtree-ersatz" if="no-jjtree" unless="javaccBuildNotRequired">
        <!--  If no jjtree is run, then we look for a .jj file to place
              in the target dir for the javacc target to pick up on it -->
        <copy file="etc/grammar/${lang-name}.jj" todir="${target-package-dir}" />
    </target>

    <target name="javacc" depends="jjtree" unless="javaccBuildNotRequired">
        <java fork="true"
              classname="javacc"
              classpath="${javacc-home.path}/javacc.jar">
            <sysproperty key="file.encoding" value="UTF-8" />
            <arg value="-STATIC:false" />
            <arg value="-USER_CHAR_STREAM:true" />
            <arg value="-UNICODE_INPUT:true" />
            <arg value="-OUTPUT_DIRECTORY:${target-package-dir}" />
            <arg value="${target-package-dir}/${lang-name}.jj" />
        </java>

    </target>

    <target name="adapt-generated" unless="javaccBuildNotRequired">

        <delete file="${target-package-dir}/Node.java" />
        <delete file="${target-package-dir}/SimpleNode.java" />
        <delete file="${target-package-dir}/JJT${parser-name}State.java" />

        <delete file="${target-package-dir}/CharStream.java" />
        <delete file="${target-package-dir}/ParseException.java" />
        <delete file="${target-package-dir}/Token.java" />
        <delete file="${target-package-dir}/TokenMgrError.java" />
        <!-- Remove all duplicate files       -->
        <delete>
            <fileset dir="${javacc.outputDirectory}">
                <present present="both" targetdir="${project.build.sourceDirectory}" />
            </fileset>
        </delete>

        <antcall target="cleanup-token-manager" />
        <antcall target="cleanup-parser" />
        <antcall target="cleanup-nodes" />

        <!-- Map Javacc names to our names -->

        <replaceregexp flags="g">
            <regexp pattern="\bToken\b" />
            <substitution expression="${ast-impl-package}.JavaccToken" />
            <fileset dir="${target-package-dir}" />
        </replaceregexp>

        <replaceregexp flags="g">
            <regexp pattern="(?&lt;!\.)\bCharStream\b" />
            <substitution expression="${charstream-itf}" />
            <fileset dir="${target-package-dir}" />
        </replaceregexp>


        <antcall target="constants-files" />
        <antcall target="constants-files-token-manager-uses-parser" />

        <!-- For compatibility -->

        <replaceregexp flags="g">
            <regexp pattern="\.image\b(?! =)" />
            <substitution expression=".getImage()" />
            <fileset dir="${target-package-dir}" />
        </replaceregexp>

        <replace token=".beginLine" value=".getReportLocation().getStartLine()">
            <fileset dir="${target-package-dir}" />
        </replace>

        <replace token=".beginColumn" value=".getReportLocation().getStartColumn()">
            <fileset dir="${target-package-dir}" />
        </replace>

        <!-- Used by debug modes -->
        <replace token="TokenMgrError.addEscapes" value="net.sourceforge.pmd.util.StringUtil.escapeJava">
            <fileset dir="${target-package-dir}" />
        </replace>


    </target>

    <target name="cleanup-parser" unless="jjtreeBuildNotRequired">


        <replace token="new Token()" value="token_source.input_stream.getTokenDocument().open()">
            <file name="${parser-file}" />
        </replace>

        <replaceregexp flags="g">
            <regexp pattern='jjtree.openNodeScope\((jjtn.*?)\);' />
            <substitution expression='jjtree.openNodeScope(\1, getToken(1));' />
            <file name="${parser-file}" />
        </replaceregexp>

        <replaceregexp flags="g">
            <regexp pattern='jjtree.closeNodeScope\((jjtn.*?)\);' />
            <substitution expression='jjtree.closeNodeScope(\1, getToken(0));' />
            <file name="${parser-file}" />
        </replaceregexp>

        <!-- The tree builder is generic now -->
        <replaceregexp flags="g">
            <regexp pattern="\bJJT${parser-name}State\b" />
            <substitution expression="${ast-impl-package}.JjtreeBuilder&lt;${base-class-name}&gt;" />
            <file name="${parser-file}" />
        </replaceregexp>

        <replaceregexp flags="g">
            <regexp pattern="new ParseException\(token, exptokseq, tokenImage\);" />
            <substitution expression="new ParseException(token, exptokseq);" />
            <file name="${parser-file}" />
        </replaceregexp>

        <replaceregexp flags="g">
            <regexp pattern="throw new ParseException\(\);" />
            <substitution expression='throw net.sourceforge.pmd.util.AssertionUtil.shouldNotReachHere("consumetoken(-1) should have thrown");' />
            <file name="${parser-file}" />
        </replaceregexp>

        <replaceregexp>
            <regexp pattern="public interface"/>
            <substitution expression="interface"/>
            <file name="${target-package-dir}/${parser-name}TreeConstants.java" />
        </replaceregexp>

    </target>

    <target name="cleanup-token-manager">

        <replaceregexp>
            <regexp pattern='(public )?class ${tokenmgr-name}' />
            <substitution expression='class ${tokenmgr-name} extends ${base-tokenmgr}' />
            <file name="${tokenmgr-file}" />
        </replaceregexp>

        <!-- This is a programming error and should be caught in tests -->
        <replaceregexp>
            <regexp pattern='throw new TokenMgrError\("Error: Ignoring invalid lexical state.*?\);' />
            <substitution expression='throw new IllegalArgumentException("Invalid lexical state " + lexState);' />
            <file name="${tokenmgr-file}" />
        </replaceregexp>

        <!-- Use own LexException instead of JavaCC's TokenMgrError -->
        <replaceregexp>
            <regexp pattern='throw new TokenMgrError\(EOFSeen' />
            <substitution expression='throw net.sourceforge.pmd.lang.ast.InternalApiBridge.newLexException(EOFSeen' />
            <file name="${tokenmgr-file}" />
        </replaceregexp>

        <!-- Useless argument, also replace lex state ID with its name  -->
        <replaceregexp>
            <regexp pattern='curLexState, error_line, error_column, error_after, curChar, TokenMgrError.LEXICAL_ERROR\)' />
            <substitution expression='lexStateNames[curLexState], error_line, error_column, error_after, curChar)' />
            <file name="${tokenmgr-file}" />
        </replaceregexp>

        <!-- Patch token creation routine, delegates to the token document. -->
        <replaceregexp flags="s">
            <file name="${tokenmgr-file}" />
            <regexp pattern="protected Token jjFillToken.*?(?=int curLexState = )" />
            <substitution expression="protected Token jjFillToken() {return input_stream.getTokenDocument().createToken(jjmatchedKind, input_stream, jjstrLiteralImages[jjmatchedKind]);}" />
        </replaceregexp>

        <!-- Renamed CharStream methods       -->
        <replaceregexp flags="g">
            <file name="${tokenmgr-file}" />
            <regexp pattern="image.append\(input_stream.GetSuffix\(" />
            <substitution expression="input_stream.appendSuffix(image,(" />
        </replaceregexp>

        <replaceregexp flags="g">
            <file name="${tokenmgr-file}" />
            <regexp pattern="input_stream.GetImage" />
            <substitution expression="input_stream.getTokenImage" />
        </replaceregexp>

        <replaceregexp flags="g">
            <file name="${tokenmgr-file}" />
            <regexp pattern="input_stream.BeginToken" />
            <substitution expression="input_stream.markTokenStart" />
        </replaceregexp>

        <!-- This is used to allow for tokens to be immutable. The lexical actions
             return the new token instead of mutating it. -->
        <replaceregexp flags="sg">
            <file name="${tokenmgr-file}" />
            <regexp pattern="void ((Token|Skip)LexicalActions\(Token matchedToken\))\R\{(.*?)}\R(?=(private )?void)" />
            <substitution expression="Token \1 { \3
             return matchedToken;
        }" />
        </replaceregexp>

        <!-- Update call sites for the previous routines -->
        <replaceregexp>
            <file name="${tokenmgr-file}" />
            <regexp pattern="(Token|Skip)LexicalActions\(matchedToken\);" />
            <substitution expression="matchedToken = \0" />
        </replaceregexp>


        <!-- Fix end column being now exclusive -->
        <!-- JavaCC assumes it's inclusive, uses it for character positions of errors -->
        <!-- This is also used in debug mode -->
        <replaceregexp flags="sg">
            <file name="${tokenmgr-file}" />
            <regexp pattern="input_stream.getEndColumn\(\)" />
            <substitution expression="(input_stream.getEndColumn() - 1)" />
        </replaceregexp>

    </target>

    <target name="constants-files">

        <replaceregexp flags="g">
            <regexp pattern="implements ${constants-itf-name}," />
            <substitution expression="implements" />
            <fileset dir="${target-package-dir}" />
        </replaceregexp>

        <replaceregexp flags="g">
            <regexp pattern=", ${constants-itf-name}\b" />
            <substitution expression="" />
            <fileset dir="${target-package-dir}" />
        </replaceregexp>

        <replaceregexp flags="g">
            <regexp pattern="implements ${constants-itf-name}\b" />
            <substitution expression="" />
            <fileset dir="${target-package-dir}" />
        </replaceregexp>

        <replaceregexp>
            <regexp pattern="package ${lang-ast-package};" />
            <substitution expression="\0${line.separator}${line.separator}import static ${lang-ast-package}.${token-constants-name}.*;" />
            <file name="${tokenmgr-file}" />
            <file name="${parser-file}" />
        </replaceregexp>

        <!-- Create constants file  -->

        <move file="${constants-itf-file}" tofile="${token-constants-file}" />

        <replaceregexp>
            <regexp pattern="(public )?interface ${constants-itf-name} \{" />
            <substitution expression="/** Token kinds ({@link ${ast-impl-package}.JavaccToken#kind}) for this language. */${line.separator}
@net.sourceforge.pmd.annotation.InternalApi${line.separator}
public final class ${token-constants-name} \{${line.separator}
    private ${token-constants-name}() { /* Utility class */ }${line.separator}
"/>
            <file name="${token-constants-file}" />
        </replaceregexp>

        <replaceregexp flags="g">
            <regexp pattern="/\*\* RegularExpression Id. \*/\R" />
            <substitution expression="public static final " />
            <file name="${token-constants-file}" />
        </replaceregexp>

        <replaceregexp flags="s">
            <regexp pattern="/\*\* End of File. \*/\R" />
            <substitution expression="public static final " />
            <file name="${token-constants-file}" />
        </replaceregexp>

        <!-- Lexical states are kept package-private -->
        <replaceregexp flags="g">
            <regexp pattern="/\*\* Lexical state. \*/\R" />
            <substitution expression="static final " />
            <file name="${token-constants-file}" />
        </replaceregexp>

        <replace>
            <replacetoken>/** Literal token values. */</replacetoken>
            <replacevalue><![CDATA[
  /** Returns a string describing the given token kind.
    * Returns null if the kind is unknown.
    *
    * @param kind Kind of token
    *
    * @return A string describing the given kind
    */
  public static @org.checkerframework.checker.nullness.qual.Nullable
                String describe(int kind) {
    return kind < 0 || kind >= tokenImage.length
        ? null
        : tokenImage[kind];
  }

  /** Returns a new token manager for this language. The CharStream
    * should be configured with the correct language-specific escaping
    * and token document. This is not a published API, it's intended to
    * be used as a basis for a CPD Tokenizer.
    */
  @net.sourceforge.pmd.annotation.InternalApi
  public static net.sourceforge.pmd.lang.TokenManager<%%%API_PACK%%%.impl.javacc.JavaccToken> newTokenManager(%%%API_PACK%%%.impl.javacc.CharStream cs) {
    return new %%%TOKEN_MGR_NAME%%%(cs);
  }

    private static final
]]>            </replacevalue>
            <file name="${token-constants-file}" />
        </replace>

        <replace>
            <replacetoken>};</replacetoken>
            <replacevalue><![CDATA[
  };

  /** Nams of the tokens, each index corresponds to a kind. See also {@link #describe(int)}. */
  public static final java.util.List<String> TOKEN_NAMES = java.util.Collections.unmodifiableList(java.util.Arrays.asList(tokenImage));
]]>            </replacevalue>
            <file name="${token-constants-file}" />
        </replace>

        <replaceregexp>
            <regexp pattern="%%%TOKEN_MGR_NAME%%%" />
            <substitution expression="${tokenmgr-name}" />
            <file name="${token-constants-file}" />
        </replaceregexp>

        <replaceregexp flags="g">
            <regexp pattern="%%%API_PACK%%%" />
            <substitution expression="${ast-api-package}" />
            <file name="${token-constants-file}" />
        </replaceregexp>

    </target>

    <target name="constants-files-token-manager-uses-parser" if="option-token-manager-uses-parser">
        <echo level="info">Option TOKEN_MANAGER_USES_PARSER is enabled</echo>

        <replaceregexp flags="g">
            <regexp pattern="return new ${tokenmgr-name}\(cs\);" />
            <!-- TODO this implementation does not work as the first token is skipped, since it is fetched by the parser itself. -->
            <substitution expression="return new ${parser-name}(cs).token_source;" />
            <file name="${token-constants-file}" />
        </replaceregexp>
    </target>

    <target name="cleanup-nodes">

        <replaceregexp flags="g">
            <regexp pattern="\bParseException\b" />
            <substitution expression="${ast-api-package}.ParseException" />
            <fileset dir="${target-package-dir}" />
        </replaceregexp>

        <replaceregexp flags="g">
            <regexp pattern="extends SimpleNode\b" />
            <substitution expression="extends ${base-class-name}" />
            <fileset dir="${target-package-dir}" />
        </replaceregexp>

        <!-- Remove constructor with parser -->
        <replaceregexp flags="s">
            <regexp pattern="public \w+\(${parser-name}.*?}" />
            <substitution expression="" />
            <fileset dir="${target-package-dir}">
                <include name="${ast-prefix}*" />
            </fileset>
        </replaceregexp>

        <!-- Make constructor package-private -->
        <replaceregexp>
            <regexp pattern="public (\w+\(int)" />
            <substitution expression="\1" />
            <fileset dir="${target-package-dir}">
                <include name="${ast-prefix}*" />
            </fileset>
        </replaceregexp>

        <!-- Make constructor package-private -->
        <replaceregexp>
            <regexp pattern="(?&lt;!public(\R| ))class" />
            <substitution expression="public class" />
            <fileset dir="${target-package-dir}">
                <include name="${ast-prefix}*" />
            </fileset>
        </replaceregexp>

        <replaceregexp flags="g">
            <regexp pattern="/\*\* Accept the visitor. \*\*/" />
            <substitution expression="" />
            <fileset dir="${target-package-dir}">
                <include name="${ast-prefix}*" />
            </fileset>
        </replaceregexp>

    </target>

    <target name="default-visitor" depends="jjtree" unless="jjtreeBuildNotRequired">
        <move file="${target-package-dir}/${gen-visitor-name}.java" tofile="${generic-visitor-interface-file}" />
        <replace file="${generic-visitor-interface-file}">
            <replacefilter token="${gen-visitor-name}"
                           value="${generic-visitor-interface-name}&lt;P, R> extends ${ast-api-package}.AstVisitor&lt;P, R>" />

            <replacefilter token="SimpleNode" value="${node-name}" />
            <!-- Root method, eg visitJavaNode -->
            <replacefilter
                    token="Object visit(${node-name} node, Object data);"
                    value="default R visit${lang-name-camelcase}Node(${node-name} node, P data) { return visitNode(node, data); }">
            </replacefilter>


            <replacefilter token="public Object visit(" value="default R visit(" />
            <replacefilter token="Object data);" value="P data) { return visit${lang-name-camelcase}Node(node, data); }" />
        </replace>

        <replace>
            <replacefilter token="public Object jjtAccept(${gen-visitor-name} visitor, Object data)"
                           value="@Override protected &lt;P, R> R accept${generic-visitor-interface-name}(${generic-visitor-interface-name}&lt;? super P, ? extends R> visitor, P data)" />

            <fileset dir="${target-package-dir}">
                <include name="${ast-prefix}*" />
            </fileset>
        </replace>
    </target>

</project>