eu.cqse.check.framework.util.abap.AbapLanguageFeatureParser Maven / Gradle / Ivy
Show all versions of teamscale-check-api Show documentation
/*
* Copyright (c) CQSE GmbH
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package eu.cqse.check.framework.util.abap;
import static eu.cqse.check.framework.scanner.ETokenType.CHANGING;
import static eu.cqse.check.framework.scanner.ETokenType.DEFAULT;
import static eu.cqse.check.framework.scanner.ETokenType.DOT;
import static eu.cqse.check.framework.scanner.ETokenType.EVENT;
import static eu.cqse.check.framework.scanner.ETokenType.EXPORTING;
import static eu.cqse.check.framework.scanner.ETokenType.FOR;
import static eu.cqse.check.framework.scanner.ETokenType.IDENTIFIER;
import static eu.cqse.check.framework.scanner.ETokenType.IMPORTING;
import static eu.cqse.check.framework.scanner.ETokenType.LIKE;
import static eu.cqse.check.framework.scanner.ETokenType.LPAREN;
import static eu.cqse.check.framework.scanner.ETokenType.OPTIONAL;
import static eu.cqse.check.framework.scanner.ETokenType.RAISING;
import static eu.cqse.check.framework.scanner.ETokenType.RETURNING;
import static eu.cqse.check.framework.scanner.ETokenType.RPAREN;
import static eu.cqse.check.framework.scanner.ETokenType.STRUCTURE;
import static eu.cqse.check.framework.scanner.ETokenType.TABLES;
import static eu.cqse.check.framework.scanner.ETokenType.TESTING;
import static eu.cqse.check.framework.scanner.ETokenType.TYPE;
import static eu.cqse.check.framework.scanner.ETokenType.USING;
import java.util.ArrayList;
import java.util.Collections;
import java.util.EnumSet;
import java.util.List;
import java.util.Optional;
import java.util.regex.Pattern;
import org.conqat.lib.commons.collections.CollectionUtils;
import org.conqat.lib.commons.collections.Pair;
import org.conqat.lib.commons.string.StringUtils;
import eu.cqse.check.framework.core.CheckException;
import eu.cqse.check.framework.scanner.ELanguage;
import eu.cqse.check.framework.scanner.ETokenType;
import eu.cqse.check.framework.scanner.IToken;
import eu.cqse.check.framework.shallowparser.SubTypeNames;
import eu.cqse.check.framework.shallowparser.TokenStreamTextUtils;
import eu.cqse.check.framework.shallowparser.TokenStreamUtils;
import eu.cqse.check.framework.shallowparser.framework.EShallowEntityType;
import eu.cqse.check.framework.shallowparser.framework.ShallowEntity;
import eu.cqse.check.framework.shallowparser.framework.ShallowEntityTraversalUtils;
import eu.cqse.check.framework.typetracker.TypedVariable;
import eu.cqse.check.framework.util.ILanguageFeatureParser;
/**
* Language feature parser for ABAP.
*
* Since ABAP is case-insensitive, all variables and types are lower-cased.
*/
public class AbapLanguageFeatureParser implements ILanguageFeatureParser {
/**
* EnumSet containing all token types which can appear as section headers in a parameter
* declaration.
*/
public static final EnumSet PARAMETER_SECTION_TOKENS = EnumSet.of(USING, CHANGING, IMPORTING, EXPORTING,
RETURNING, TABLES, TESTING, RAISING);
/**
* Pattern to match possible identifier names which were not parsed as identifiers but as keywords
* or operators. Note that this pattern does not comprise all valid identifiers names (which can
* contain some special characters), but only excludes keywords and operators which can not be used
* as identifiers. Thus, the pattern does not match e.g. to the operator symbols like '=' or '<>' or
* keywords containing spaces. See also {@link #isPossiblyIdentifier(IToken)}.
*/
private static final Pattern KEYWORD_OR_OPERATOR_AS_IDENTIFIER_PATTERN = Pattern.compile("(?i)[A-Z][A-Z0-9_]*");
/** {@inheritDoc} */
@Override
public ELanguage getLanguage() {
return ELanguage.ABAP;
}
/**
* Gets the type name from the given list of tokens.
*
* @param tokens
* List of tokens. For example v1 TYPE t value(v2) LIKE c v TYPE t v3.
* @return Returns a tuple of the type name and the corresponding index of the last type token. The
* index is ensured to be bigger or equal to startIndex-1. If no further type is found, ("",
* startIndex-1) is returned.
*/
public Pair getNextTypeName(List tokens, int startIndex) {
int typeKeywordIndex = TokenStreamUtils.firstTokenOfType(tokens, startIndex, TYPE, LIKE, STRUCTURE);
if (typeKeywordIndex == TokenStreamUtils.NOT_FOUND) {
// No type is given
return new Pair<>(StringUtils.EMPTY_STRING, startIndex - 1);
}
// Find the beginning of the next type declaration and try to step
// backwards in order to find the end of the current variable type. This
// is done because you can put so many different expressions behind the
// type keyword, that it is nearly impossible to get an exhaustive list.
int endIndex = TokenStreamUtils.firstTokenOfType(tokens, typeKeywordIndex + 1, TYPE, LIKE, STRUCTURE);
if (endIndex == TokenStreamUtils.NOT_FOUND) {
endIndex = TokenStreamUtils.firstTokenOfType(tokens, typeKeywordIndex, DOT);
if (endIndex == TokenStreamUtils.NOT_FOUND) {
endIndex = tokens.size();
}
} else if (endIndex >= 3 && tokens.get(endIndex - 3).getType() == LPAREN
&& tokens.get(endIndex - 1).getType() == RPAREN) {
// If the next parameter is defined using a value(var2) or similar
// notation we have to skip those tokens in order to get to the
// type we are looking for.
endIndex -= 4;
} else {
// Normally the TYPE keyword is preceded by a parameter name, which
// we have to skip to reach the end of the previous parameter type.
endIndex--;
}
if (endIndex > 2 && endIndex <= tokens.size()) {
if (tokens.get(endIndex - 1).getType() == OPTIONAL) {
endIndex--;
} else if (tokens.get(endIndex - 2).getType() == DEFAULT) {
endIndex -= 2;
}
}
/*
* Type declarations may be ambiguous. Look at the following: I TYPE STRUCTURE TYPE STRUCTURE. This
* can either mean we have 5 variables (I, Type, STRUCTURE, ...) or (I of type STRUCTURE, TYPE and
* STRUCTURE) or (I, TYPE and STRUCTURE of type STRUCTURE) and so on. And we did not yet manage to
* find a reliable pattern on how the compiler parses those declarations. Especially because leaving
* out the type of a parameter is not officially documented in any ABAP book, but it is used
* throughout the whole MR project we used to test it. And the discussion is still open on how to
* deal with it. Currently we agreed on ignoring cases that we are not able to parse correctly and
* providing a workaround for cases where the above method fails. The workaround is what follows.
*/
if (typeKeywordIndex < endIndex) {
return new Pair<>(TokenStreamTextUtils
.concatTokenTexts(tokens.subList(typeKeywordIndex + 1, endIndex), StringUtils.SPACE).toLowerCase(),
endIndex - 1);
}
return new Pair<>(StringUtils.EMPTY_STRING, Math.max(startIndex, endIndex));
}
/**
* Gets type information from a list of tokens holding a method's parameter declaration part, which
* may either use the FOR EVENT ... OF ... IMPORTING or the normal sectioned parameter
* style.
*
* The given {@link ShallowEntity} is stored as declaring entity in the returned
* {@link TypedVariable}s.
*/
public List getTypeInfoForMethodParameters(ShallowEntity entity, List tokens) {
if (TokenStreamUtils.startsWith(tokens, FOR, EVENT)) {
return processMethodEventHandler(entity, tokens);
}
return processParameterList(entity, tokens);
}
/**
* Gets type information from a list of tokens holding a method's parameter declaration part, which
* may contain USING, CHANGING, IMPORTING, EXPORTING and RETURNING sections.
*
* For example METHODS method IMPORTING v TYPE t value(v2) LIKE c v3
* CHANGING v4 TYPE t EXPORTING v5 TYPE t
*/
private List processParameterList(ShallowEntity entity, List tokens) {
List typeInfo = new ArrayList<>();
List positionsOfSectionTokens = TokenStreamUtils.findAll(tokens, PARAMETER_SECTION_TOKENS);
// Tokens before first parameter section token are irrelevant because
// they cannot declare parameters
for (int i = 0; i < positionsOfSectionTokens.size(); i++) {
int positionOfCurrentSectionToken = positionsOfSectionTokens.get(i);
IToken sectionName = tokens.get(positionOfCurrentSectionToken);
int endOfSection;
if (i < positionsOfSectionTokens.size() - 1) {
endOfSection = positionsOfSectionTokens.get(i + 1);
} else {
endOfSection = tokens.size();
}
List section = tokens.subList(positionOfCurrentSectionToken + 1, endOfSection);
processHeaderSection(entity, typeInfo, section, sectionName);
}
return typeInfo;
}
/**
* Processes a list of parameters that have been defined within the same parameter section. The
* detected parameters are added to the given list of typeInfos.
*/
private void processHeaderSection(ShallowEntity containingEntity, List typeInfo,
List section, IToken sectionName) {
List sectionModifiers = new ArrayList<>();
sectionModifiers.add(sectionName);
int dotIndex = TokenStreamUtils.firstTokenOfType(section, DOT);
if (dotIndex != TokenStreamUtils.NOT_FOUND) {
section = section.subList(0, dotIndex);
}
int currentIndex = 0;
while (currentIndex < section.size()) {
List currentParameterModifiers = new ArrayList<>(sectionModifiers);
String variableName;
// If the method parameter is marked as pass-by-value with
// value(varName) TYPE t, skip the parenthesized part
if (TokenStreamUtils.hasTokenTypeSequence(section, currentIndex + 1, LPAREN, IDENTIFIER, RPAREN)) {
variableName = section.get(currentIndex + 2).getText();
currentIndex += 4;
} else {
variableName = section.get(currentIndex).getText();
currentIndex++;
}
Pair type = getNextTypeName(section, currentIndex);
Pair, Integer> afterTypeModifiers = getAfterTypeModifiers(section, type.getSecond() + 1);
currentParameterModifiers.addAll(afterTypeModifiers.getFirst());
typeInfo.add(new TypedVariable(normalizeVariable(variableName), type.getFirst().toLowerCase(),
currentParameterModifiers, containingEntity));
currentIndex = afterTypeModifiers.getSecond() + 1;
}
}
/**
* Returns the modifier tokens which can be used after the type declaration of a parameter (e.g.,
* OPTIONAL or DEFAULT 'x'). Returns the position of the last token of the modifiers as second
* return value. Returns startIndex-1 if no OPTIONAL or DEFAULT is found.
*/
private static Pair, Integer> getAfterTypeModifiers(List tokens, Integer startIndex) {
if (tokens.size() <= startIndex) {
return new Pair<>(Collections.emptyList(), startIndex - 1);
}
if (tokens.get(startIndex).getType() == OPTIONAL) {
return new Pair<>(Collections.singletonList(tokens.get(startIndex)), startIndex);
} else if (tokens.get(startIndex).getType() == DEFAULT) {
// also skip the default initialization value
return new Pair<>(Collections.singletonList(tokens.get(startIndex)), startIndex + 1);
}
return new Pair<>(Collections.emptyList(), startIndex - 1);
}
/**
* Gets type information from a list of tokens holding a method's parameter declaration part in the
* form of ([CLASS-]METHODS handler )FOR EVENT evt OF class|intf IMPORTING e1 e2 ...
* The parameters that are returned are e1, e2, ...
*/
private static List processMethodEventHandler(ShallowEntity entity, List tokens)
throws AssertionError {
int importingIndex = TokenStreamUtils.firstTokenOfType(tokens, IMPORTING);
if (importingIndex == TokenStreamUtils.NOT_FOUND) {
// Only happens in non standard conform code.
return CollectionUtils.emptyList();
}
List parameterTokens = tokens.subList(importingIndex + 1, tokens.size());
return CollectionUtils.filterAndMap(parameterTokens, token -> token.getType() != DOT,
token -> new TypedVariable(normalizeVariable(token.getText()), StringUtils.EMPTY_STRING,
CollectionUtils.emptyList(), entity));
}
/**
* Returns the method declaration which belongs to the given method implementation entity. May
* return null if the declaration was not found, because the rootEntities given are
* incomplete.
*/
public List getDeclarationTokensForMethod(List rootEntities, ShallowEntity methodEntity) {
return Optional.ofNullable(getMethodDeclaration(rootEntities, methodEntity)).map(ShallowEntity::includedTokens)
.orElse(null);
}
/**
* Gets the corresponding class declaration for the given class implementation. This may return
* null if the rootEntities are incomplete.
*/
public static ShallowEntity getClassDeclaration(List rootEntities, ShallowEntity classEntity) {
if (classEntity == null || classEntity.getName() == null) {
return null;
}
if (isClassOrInterfaceDefinition(classEntity)) {
// In case we already have a definition entity, return this immediately.
return classEntity;
}
List result = new ArrayList<>();
ShallowEntity.traverse(rootEntities,
entity -> visitToFindClassDeclaration(entity, classEntity.getName(), result));
return CollectionUtils.getAny(result);
}
/**
* {@link eu.cqse.check.framework.shallowparser.framework.IShallowEntityVisitor} implementation to
* search for class declarations. Optimized to early return on the first match, and to not traverse
* into entities that we know cannot contain class declarations.
*/
private static boolean visitToFindClassDeclaration(ShallowEntity entity, String name, List result) {
if (isClassOrInterfaceDefinition(entity) && name.equals(entity.getName())) {
result.add(entity);
}
// Do not traverse into methods, classes, or interfaces, as these cannot contain
// class definitions (reports, however, can!)
return result.isEmpty()
&& !(entity.getType() == EShallowEntityType.METHOD || isClassOrInterfaceDefinition(entity)
|| entity.getSubtype().equals(SubTypeNames.CLASS_IMPLEMENTATION));
}
/** Whether the given entity is a class or interface declaration. */
private static boolean isClassOrInterfaceDefinition(ShallowEntity entity) {
return entity.getType() == EShallowEntityType.TYPE && StringUtils.equalsOneOf(entity.getSubtype(),
SubTypeNames.CLASS_DEFINITION, SubTypeNames.INTERFACE_DEFINITION);
}
/**
* Gets the corresponding method declaration for the given method implementation. This may return
* null if the rootEntities are incomplete or the declaration is only available in a
* base class or an interface.
*/
public static ShallowEntity getMethodDeclaration(List rootEntities,
ShallowEntity methodImplementation) {
ShallowEntity classDeclaration = null;
if (methodImplementation.getParent() != null) {
classDeclaration = getClassDeclaration(rootEntities, methodImplementation.getParent());
}
if (classDeclaration == null) {
return null;
}
List methodEntities = ShallowEntityTraversalUtils
.listEntitiesOfType(classDeclaration.getChildren(), EShallowEntityType.METHOD);
for (ShallowEntity methodEntity : methodEntities) {
if (methodEntity.getSubtype().equals(SubTypeNames.METHOD_DECLARATION)
&& methodImplementation.getName() != null
&& methodImplementation.getName().equals(methodEntity.getName())) {
return methodEntity;
}
}
return null;
}
/**
* Parses a function call string.
*
* @return {@link Optional}, empty if the given entity is not a CALL FUNCTION statement otherwise
* containing a {@link FunctionCallInfo}
* @throws CheckException
* if the given statement can not be parsed
*/
public Optional getFunctionCallInfo(ShallowEntity callFunctionStatement) throws CheckException {
if (callFunctionStatement.getType() != EShallowEntityType.STATEMENT) {
return Optional.empty();
}
List tokens = callFunctionStatement.ownStartTokens();
if (tokens.size() < 3 || !TokenStreamUtils.startsWith(tokens, ETokenType.CALL, ETokenType.FUNCTION)) {
return Optional.empty();
}
return Optional.of(new FunctionCallInfo(filterIllegalCharacterTokens(tokens)));
}
/**
* Filters illegal character tokens from the given token list.
*/
private static List filterIllegalCharacterTokens(List tokens) {
return CollectionUtils.filter(tokens, token -> token.getType() != ETokenType.ILLEGAL_CHARACTER);
}
/**
* Lower-cases and removes leading '!', since this does not belong to the variable name.
*/
public static String normalizeVariable(String name) {
return StringUtils.stripPrefix(name, "!").toLowerCase();
}
/**
* Checks if a token is possibly an identifier. This also considers tokens which are wrongly parsed
* as keyword or operator.
*
* @return true if 1) the type of the given token is either an identifier (regardless
* if the text is a valid ABAP identifier name) 2) the type is a keyword or an operator and
* the token text is a possible variable name. false otherwise.
*/
public static boolean isPossiblyIdentifier(IToken token) {
ETokenType tokenType = token.getType();
if (tokenType.isIdentifier()) {
return true;
}
if (tokenType.isKeyword() || tokenType.isOperator()) {
return KEYWORD_OR_OPERATOR_AS_IDENTIFIER_PATTERN.matcher(token.getText()).matches();
}
return false;
}
/**
* Checks whether the given class name is the name of a generic exception class, i.e., one that
* should usually not be caught in application code.
*/
public boolean isGenericExceptionClass(String className) {
return Optional.ofNullable(className).map(s -> s.equalsIgnoreCase("cx_root")).orElse(false);
}
}