{

QProcess* parser = new QProcess();

// we call a python script to parse the code for us. It returns an XML string with the AST

kDebug() << "+++ Starting parser for file " << filename.path();

parser->start("/usr/bin/env", QStringList() << "python" << QString(INSTALL_PATH) + QString("/pythonpythonparser.py"));

qint64 length = contents.length();

qint64 written = parser->write(contents.toAscii().data(), length);

kDebug() << "Content length: " << length << ", Bytes written: " << written;

parser->closeWriteChannel();

if ( written != length ) {

Q_ASSERT(false);

}

parser->waitForFinished();

kDebug() << " ** Reading results...";

// TODO this is not clean

if ( parser->exitStatus() != QProcess::NormalExit ) {

kError() << "Error parsing file: " << parser->errorString();

return "0";

}

QString result = parser->readAllStandardOutput();

kDebug() << " ** XML for " << filename << ": length" << result.length();

if ( ! result.length() ) {

result = parser->readAllStandardError();

QStringList position = result.split(":::");

QString additionalExplanation = "";

if ( position.length() < 4 ) {

kError() << "Could not parse error message! This should not happen.";

kError() << "Raw data was: " << result;

return "0";

}

qint64 lineno = position.at(0).toInt() - 1;

qint64 colno = position.at(1).toInt() - 1;

kDebug() << lineno << colno;

if ( position.at(2) == "SyntaxError" ) {

additionalExplanation = "Something's wrong with your syntax. Check for missing brackets, commas, and colons.";

}

else if ( position.at(2) == "IndentationError" ) {

additionalExplanation = "You indented your code incorrectly. Also check that you didn't mix tabs and spaces in an incorrect way!";

}

KDevelop::ProblemPointer p(new KDevelop::Problem());

p->setFinalLocation(KDevelop::DocumentRange(KDevelop::IndexedString(filename), KDevelop::SimpleRange(lineno, colno - 5 < 0 ? 0 : colno - 5, lineno, colno + 5)));

p->setSource(KDevelop::ProblemData::Parser);

p->setDescription(position.at(2));

p->setExplanation(position.at(3) + "<br><br>" + additionalExplanation);

p->setSeverity(KDevelop::ProblemData::Error);

{

DUChainWriteLocker lock(DUChain::lock());

m_problems.clear();

m_problems.append(p);

}

kWarning() << "Parse Error: " << result;

return "0";

}

delete parser;

return result;

bool nodeAdded = false;

while ( ! xmlast->atEnd() && ! xmlast->hasError() ) {

// Advance to the next (first) token

QXmlStreamReader::TokenType token = xmlast->readNext();

// Store everything we need later into local variables

QString currentElementName = xmlast->name().toString();

QString currentElementText = xmlast->text().toString();

QList<QXmlStreamAttribute> currentElementAttributes = xmlast->attributes().toList();

// We ignore startDocument and EndDocument

if ( token == QXmlStreamReader::StartDocument || token == QXmlStreamReader::EndDocument ) {

continue;

}

// We recursively continue parsing if we find another element

else if ( token == QXmlStreamReader::StartElement ) {

// Here we can now assemble an actual node with the attributes extracted above

// Skip the document root element

if ( currentElementName == "pythonast" ) {

parseXmlAstNode(xmlast, token);

continue;

}

// this will push a parent onto the stack

nodeAdded = parseAstNode(currentElementName, currentElementText, currentElementAttributes);

if ( ! nodeAdded ) {

m_isRealNodeMap.append(false);

continue;

}

m_isRealNodeMap.append(true);

m_currentNode = m_nodeStack.last();

parseXmlAstNode(xmlast, token);

}

else if ( token == QXmlStreamReader::EndElement ) {

if ( currentElementName == "pythonast" ) continue;

// now we pop the parent off

bool isreal = m_isRealNodeMap.last();

m_isRealNodeMap.removeLast();

if ( isreal ) {

m_currentNode = m_nodeStack.last();

m_nodeStack.removeLast();

}

}

// Everything else (stuff between tags, comments...) is ignored

else continue;

}

if ( xmlast->hasError() ) {

kWarning() << "Invalid XML file: " << xmlast->errorString();

kWarning() << "Aborting!";

Q_ASSERT(false);

}

{

Ast* ast;

QMap<QString, QString> attributeDict;

for ( int i=0; i<attributes.length(); i++ ) {

attributeDict.insert(attributes.at(i).name().toString(), attributes.at(i).value().toString());

}

int node_id = attributeDict["nodecnt"].toInt();

// TODO implent this using QMap/QBinaryFind/enum/parser table

// things in the comments are definitely found like this in the XML file

name = name.toLower();

if ( name == "aliasast" ) ast = new AliasAst(m_nodeStack.last()); // aliasAst

else if ( name == "argumentsast" ) ast = new ArgumentsAst(m_nodeStack.last()); // argumentsAst

else if ( name == "assertast" ) ast = new AssertionAst(m_nodeStack.last()); // assertAst

else if ( name == "assignast" ) ast = new AssignmentAst(m_nodeStack.last()); // assignAst

else if ( name == "attributeast" ) ast = new AttributeAst(m_nodeStack.last()); // attributeAst

else if ( name == "augassignast" ) ast = new AugmentedAssignmentAst(m_nodeStack.last()); // augassignast

else if ( name == "binopast" ) ast = new BinaryOperationAst(m_nodeStack.last()); // binOpAst

else if ( name == "boolopast" ) ast = new BooleanOperationAst(m_nodeStack.last()); // boolOpAst

else if ( name == "breakast" ) ast = new BreakAst(m_nodeStack.last()); // breakAst

else if ( name == "callast" ) ast = new CallAst(m_nodeStack.last()); // callAst

else if ( name == "classdefast" ) ast = new ClassDefinitionAst(m_nodeStack.last()); // classDefAst

else if ( name == "moduleast" ) ast = new CodeAst(); // moduleAst

else if ( name == "compareast" ) ast = new CompareAst(m_nodeStack.last()); // compareAst

else if ( name == "comprehensionast" ) ast = new ComprehensionAst(m_nodeStack.last()); // comprehensionAst

else if ( name == "continueast" ) ast = new ContinueAst(m_nodeStack.last()); // continueAst

else if ( name == "deleteast" ) ast = new DeleteAst(m_nodeStack.last()); // deleteAst

else if ( name == "dictast" ) ast = new DictAst(m_nodeStack.last()); // dictAst

else if ( name == "ellipsisast" ) ast = new EllipsisAst(m_nodeStack.last()); // ellipsisAst

else if ( name == "excepthandlerast" ) ast = new ExceptionHandlerAst(m_nodeStack.last()); // exceptHandlerAst

else if ( name == "execast" ) ast = new ExecAst(m_nodeStack.last()); // execAst

else if ( name == "exprast" ) ast = new ExpressionAst(m_nodeStack.last()); // exprast

else if ( name == "extsliceast" ) ast = new ExtendedSliceAst(m_nodeStack.last()); // extsliceast

else if ( name == "forast" ) ast = new ForAst(m_nodeStack.last()); // forAst

else if ( name == "functiondefast" ) ast = new FunctionDefinitionAst(m_nodeStack.last()); // functionDefAst

else if ( name == "globalast" ) ast = new GlobalAst(m_nodeStack.last()); // globalAst

else if ( name == "ifast" ) ast = new IfAst(m_nodeStack.last()); // ifAst

else if ( name == "ifexpast" ) ast = new IfExpressionAst(m_nodeStack.last()); // ifExpAst

else if ( name == "importast" ) ast = new ImportAst(m_nodeStack.last()); // importAst

else if ( name == "importfromast" ) ast = new ImportFromAst(m_nodeStack.last()); // importFromAst

else if ( name == "indexast" ) ast = new IndexAst(m_nodeStack.last()); // indexAst

else if ( name == "keywordast" ) ast = new KeywordAst(m_nodeStack.last()); // keywordAst

else if ( name == "lambdaast" ) ast = new LambdaAst(m_nodeStack.last()); // lambdaAst

else if ( name == "listast" ) ast = new ListAst(m_nodeStack.last()); // listAst

else if ( name == "listcompast" ) ast = new ListComprehensionAst(m_nodeStack.last()); // listCompAst

else if ( name == "nameast" ) ast = new NameAst(m_nodeStack.last()); // nameAst

else if ( name == "numast" ) ast = new NumberAst(m_nodeStack.last()); // numAst

else if ( name == "passast" ) ast = new PassAst(m_nodeStack.last()); // passAst

else if ( name == "printast" ) ast = new PrintAst(m_nodeStack.last()); // printAst

else if ( name == "raiseast" ) ast = new RaiseAst(m_nodeStack.last()); // raiseAst

else if ( name == "returnast" ) ast = new ReturnAst(m_nodeStack.last()); // returnAst

else if ( name == "sliceast" ) ast = new SliceAst(m_nodeStack.last()); // sliceAst

else if ( name == "strast" ) ast = new StringAst(m_nodeStack.last()); // strAst

else if ( name == "subscriptast" ) ast = new SubscriptAst(m_nodeStack.last()); // subscriptAst

else if ( name == "tryexceptast" ) ast = new TryExceptAst(m_nodeStack.last()); // tryExceptAst

else if ( name == "tryfinallyast" ) ast = new TryFinallyAst(m_nodeStack.last()); // tryFinallyAst

else if ( name == "tupleast" ) ast = new TupleAst(m_nodeStack.last()); // tupleAst

else if ( name == "unaryopast" ) ast = new UnaryOperationAst(m_nodeStack.last()); // unaryOpAst

else if ( name == "whileast" ) ast = new WhileAst(m_nodeStack.last()); // whileAst

else if ( name == "withast" ) ast = new WithAst(m_nodeStack.last()); // withAst

else if ( name == "yieldast" ) ast = new YieldAst(m_nodeStack.last()); // yieldAst

else {

// assemble AST primitives

if ( name == "loadast" ) m_contextNodeMap.insert(node_id, ExpressionAst::Load);

else if ( name == "storeast") m_contextNodeMap.insert(node_id, ExpressionAst::Store);

else if ( name == "deleteast" ) m_contextNodeMap.insert(node_id, ExpressionAst::Delete);

else if ( name == "augassignast" ) m_contextNodeMap.insert(node_id, ExpressionAst::AugStore);

else if ( name == "paramast" ) m_contextNodeMap.insert(node_id, ExpressionAst::Parameter);

else if ( name == "addast" ) m_opNodeMap.insert(node_id, Ast::OperatorAdd);

else if ( name == "subast" ) m_opNodeMap.insert(node_id, Ast::OperatorSub);

else if ( name == "multast" ) m_opNodeMap.insert(node_id, Ast::OperatorMult);

else if ( name == "divast" ) m_opNodeMap.insert(node_id, Ast::OperatorDiv);

else if ( name == "modast" ) m_opNodeMap.insert(node_id, Ast::OperatorMod);

else if ( name == "bitxorast" ) m_opNodeMap.insert(node_id, Ast::OperatorBitwiseXor);

else if ( name == "bitandast" ) m_opNodeMap.insert(node_id, Ast::OperatorBitwiseAnd);

else if ( name == "bitorast" ) m_opNodeMap.insert(node_id, Ast::OperatorBitwiseOr);

else if ( name == "powast" ) m_opNodeMap.insert(node_id, Ast::OperatorPow);

else if ( name == "rshiftast" ) m_opNodeMap.insert(node_id, Ast::OperatorRightShift);

else if ( name == "lshiftast" ) m_opNodeMap.insert(node_id, Ast::OperatorLeftShift);

else if ( name == "notast" ) m_unaryOpNodeMap.insert(node_id, Ast::UnaryOperatorNot);

else if ( name == "uaddast" ) m_unaryOpNodeMap.insert(node_id, Ast::UnaryOperatorAdd);

else if ( name == "usubast" ) m_unaryOpNodeMap.insert(node_id, Ast::UnaryOperatorSub);

else if ( name == "invertast" ) m_unaryOpNodeMap.insert(node_id, Ast::UnaryOperatorInvert);

else if ( name == "eqast" ) m_compOpNodeMap.insert(node_id, Ast::ComparisonOperatorEquals);

else if ( name == "noteqast" ) m_compOpNodeMap.insert(node_id, Ast::ComparisonOperatorNotEquals);

else if ( name == "ltast" ) m_compOpNodeMap.insert(node_id, Ast::ComparisonOperatorLessThan);

else if ( name == "lteast" ) m_compOpNodeMap.insert(node_id, Ast::ComparisonOperatorLessThanEqual);

else if ( name == "gtast" ) m_compOpNodeMap.insert(node_id, Ast::ComparisonOperatorGreaterThan);

else if ( name == "gteast" ) m_compOpNodeMap.insert(node_id, Ast::ComparisonOperatorGreaterThanEqual);

else if ( name == "isast" ) m_compOpNodeMap.insert(node_id, Ast::ComparisonOperatorIs);

else if ( name == "isnotast" ) m_compOpNodeMap.insert(node_id, Ast::ComparisonOperatorIsNot);

else if ( name == "inast" ) m_compOpNodeMap.insert(node_id, Ast::ComparisonOperatorIn);

else if ( name == "notinast" ) m_compOpNodeMap.insert(node_id, Ast::ComparisonOperatorNotIn);

else if ( name == "andast" ) m_boolOpNodeMap.insert(node_id, Ast::BooleanAnd);

else if ( name == "orast" ) m_boolOpNodeMap.insert(node_id, Ast::BooleanOr);

else {

kWarning() << "Unknown AST type" << name;

}

// we did not push a node onto the stack, so we return false

return false;

}

ast->startLine = -5;

m_nodeMap.insert(node_id, ast);

m_attributeStore.insert(node_id, attributeDict);

m_nodeStack.append(ast);

return true;

{

QList<T*> items;

items.clear();

QStringList identifiers = commaSeperatedIdentifiers.split(",");

T* found;

for ( int i=0; i<identifiers.length(); i++ ) {

// make sure we have no null pointers in our lists

found = 0;

if ( identifiers.at(i).length() ) found = resolveNode<T>(identifiers.at(i));

if ( found ) items << found;

}

return items;

{

int id = identifier.toInt();

if ( ! id ) return ExpressionAst::Invalid;

return m_contextNodeMap.value(id);

{

int id = identifier.toInt();

if ( ! id ) return Ast::BooleanInvalidOperation;

return m_boolOpNodeMap.value(id);

{

int id = identifier.toInt();

if ( ! id ) return Ast::OperatorInvalid;

return m_opNodeMap.value(id);

{

int id = identifier.toInt();

if ( ! id ) return Ast::UnaryOperatorInvalid;

return m_unaryOpNodeMap.value(id);

{

int id = identifier.toInt();

if ( ! id ) return Ast::ComparisonOperatorInvalid;

return m_compOpNodeMap.value(id);

{

QList<Ast::ComparisonOperatorTypes> items;

QList<QString> ids = identifiers.split(",");

for ( int i=0; i < ids.length(); i++ ) {

items << resolveComparisonOperator(ids.at(i));

}

return items;

{

Ast* currentAbstractNode;

stringDictionary currentAttributes;

QMapIterator<int, Ast*> i(m_nodeMap);

while ( i.hasNext() ) {

i.next();

currentAbstractNode = i.value();

currentAttributes = m_attributeStore.value(i.key());

stringDictionary::const_iterator i = currentAttributes.begin();

if ( currentAttributes.value("lineno").length() > 0 && currentAttributes.value("col_offset").length() > 0 )

currentAbstractNode->hasUsefulRangeInformation = true;

else currentAbstractNode->hasUsefulRangeInformation = false;

int startLine = currentAttributes.value("lineno").toInt() - 1; // start = 0 <> start = 1

currentAbstractNode->startLine = startLine;

currentAbstractNode->endLine = startLine;

int startCol = currentAttributes.value("col_offset").toInt();

currentAbstractNode->startCol = startCol;

currentAbstractNode->endCol = startCol; // this is justified if necessary (only an AST with an actual value or identifier will know the true range)

switch ( currentAbstractNode->astType ) {

case Ast::CodeAstType: currentAbstractNode = populateCodeAst(currentAbstractNode, currentAttributes); break;

case Ast::FunctionDefinitionAstType: currentAbstractNode = populateFunctionDefinitionAst(currentAbstractNode, currentAttributes); break;

case Ast::ClassDefinitionAstType: currentAbstractNode = populateClassDefinitonAst(currentAbstractNode, currentAttributes); break;

case Ast::ReturnAstType: currentAbstractNode = populateReturnAst(currentAbstractNode, currentAttributes); break;

case Ast::DeleteAstType: currentAbstractNode = populateDeleteAst(currentAbstractNode, currentAttributes); break;

case Ast::AssignmentAstType: currentAbstractNode = populateAssignmentAst(currentAbstractNode, currentAttributes); break;

case Ast::AugmentedAssignmentAstType: currentAbstractNode = populateAugmentedAssignmentAst(currentAbstractNode, currentAttributes); break;

case Ast::ForAstType: currentAbstractNode = populateForAst(currentAbstractNode, currentAttributes); break;

case Ast::WhileAstType: currentAbstractNode = populateWhileAst(currentAbstractNode, currentAttributes); break;

case Ast::IfAstType: currentAbstractNode = populateIfAst(currentAbstractNode, currentAttributes); break;

case Ast::WithAstType: currentAbstractNode = populateWithAst(currentAbstractNode, currentAttributes); break;

case Ast::RaiseAstType: currentAbstractNode = populateRaiseAst(currentAbstractNode, currentAttributes); break;

case Ast::TryExceptAstType: currentAbstractNode = populateTryExceptAst(currentAbstractNode, currentAttributes); break;

case Ast::TryFinallyAstType: currentAbstractNode = populateTryFinallyAst(currentAbstractNode, currentAttributes); break;

case Ast::AssertionAstType: currentAbstractNode = populateAssertionAst(currentAbstractNode, currentAttributes); break;

case Ast::ImportAstType: currentAbstractNode = populateImportAst(currentAbstractNode, currentAttributes); break;

case Ast::ImportFromAstType: currentAbstractNode = populateImportFromAst(currentAbstractNode, currentAttributes); break;

case Ast::GlobalAstType: currentAbstractNode = populateGlobalAst(currentAbstractNode, currentAttributes); break;

case Ast::BreakAstType: break; // ok

case Ast::ContinueAstType: break; // ok

case Ast::PrintAstType: currentAbstractNode = populatePrintAst(currentAbstractNode, currentAttributes); break;

case Ast::PassAstType: break; // ok

case Ast::BooleanOperationAstType: currentAbstractNode = populateBooleanOperationAst(currentAbstractNode, currentAttributes); break;

case Ast::BinaryOperationAstType: currentAbstractNode = populateBinaryOperationAst(currentAbstractNode, currentAttributes); break;

case Ast::UnaryOperationAstType: currentAbstractNode = populateUnaryOperationAst(currentAbstractNode, currentAttributes); break;

case Ast::LambdaAstType: currentAbstractNode = populateLambdaAst(currentAbstractNode, currentAttributes); break;

case Ast::IfExpressionAstType: currentAbstractNode = populateIfExpressionAst(currentAbstractNode, currentAttributes); break;

case Ast::DictAstType: currentAbstractNode = populateDictAst(currentAbstractNode, currentAttributes); break;

case Ast::ListComprehensionAstType: currentAbstractNode = populateListComprehensionAst(currentAbstractNode, currentAttributes); break;

case Ast::CompareAstType: currentAbstractNode = populateCompareAst(currentAbstractNode, currentAttributes); break;

case Ast::NumberAstType: currentAbstractNode = populateNumberAst(currentAbstractNode, currentAttributes); break;

case Ast::StringAstType: currentAbstractNode = populateStringAst(currentAbstractNode, currentAttributes); break;

case Ast::NameAstType: currentAbstractNode = populateNameAst(currentAbstractNode, currentAttributes); break;

case Ast::CallAstType: currentAbstractNode = populateCallAst(currentAbstractNode, currentAttributes); break;

case Ast::AttributeAstType: currentAbstractNode = populateAttributeAst(currentAbstractNode, currentAttributes); break;

case Ast::SubscriptAstType: currentAbstractNode = populateSubscriptAst(currentAbstractNode, currentAttributes); break;

case Ast::ListAstType: currentAbstractNode = populateListAst(currentAbstractNode, currentAttributes); break;

case Ast::TupleAstType: currentAbstractNode = populateTupleAst(currentAbstractNode, currentAttributes); break;

case Ast::SliceAstType: currentAbstractNode = populateSliceAst(currentAbstractNode, currentAttributes); break;

case Ast::IndexAstType: currentAbstractNode = populateIndexAst(currentAbstractNode, currentAttributes); break;

case Ast::ArgumentsAstType: currentAbstractNode = populateArgumentsAst(currentAbstractNode, currentAttributes); break;

case Ast::KeywordAstType: currentAbstractNode = populateKeywordAst(currentAbstractNode, currentAttributes); break;

case Ast::ComprehensionAstType: currentAbstractNode = populateComprehensionAst(currentAbstractNode, currentAttributes); break;

case Ast::ExceptionHandlerAstType: currentAbstractNode = populateExceptionHandlerAst(currentAbstractNode, currentAttributes); break;

case Ast::AliasAstType: currentAbstractNode = populateAliasAst(currentAbstractNode, currentAttributes); break;

case Ast::ExpressionAstType: currentAbstractNode = populateExpressionAst(currentAbstractNode, currentAttributes); break;

case Ast::StatementAstType: break; // ok

default: kWarning() << "Unsupported AST type: " << currentAbstractNode->astType; break;

}

// Walk through the tree and set proper end columns and lines, as the python parser sadly does not do this for us

if ( currentAbstractNode->hasUsefulRangeInformation ) {

Ast* parent = currentAbstractNode->parent;

while ( parent ) {

if ( parent->endLine < currentAbstractNode->endLine ) {

parent->endLine = currentAbstractNode->endLine;

parent->endCol = currentAbstractNode->endCol;

}

if ( ! parent->hasUsefulRangeInformation && parent->startLine == -5 ) {

parent->startLine = currentAbstractNode->startLine;

parent->startCol = currentAbstractNode->startCol;

}

parent = parent->parent;

}

}

}