Fix release scripts and pocodoc (pocoproject#5194)

* fix(release): modified mkrelease to use CMake insteadk of make

* fix(release): release scripts fail on error.

* fix(CppParser): incorrect handling of "default" keyword caused parser to crash at dereferencing null pointer.

* fix(CppParser): Fix hangs while parsing, parse anonymous namespaces and improved parsing of string literals.

* fix(CppParser): add unit tests and address PR review comments

Author: matejk

.github/workflows/packages-qa.yml

@@ -16,13 +16,12 @@ jobs:
         uses: actions/checkout@v2
       -
         name: Install packages
-        run: sudo apt -y update && sudo apt -y install libssl-dev unixodbc-dev redis-server libmysqlclient-dev libpq-dev
+        run: sudo apt -y update && sudo apt -y install cmake ninja-build libssl-dev unixodbc-dev redis-server libmysqlclient-dev libpq-dev
       -
         name: Build documentation
         run: |
           export POCO_BASE=`pwd`
           export PATH=$POCO_BASE/release/script:$PATH
-          export LD_LIBRARY_PATH=$POCO_BASE/stage/tools/lib/Linux/x86_64
           mkdoc all
 
   mkrelease_win:

.github/workflows/publish_release.yml

@@ -20,7 +20,7 @@ jobs:
         uses: actions/checkout@v2
       -
         name: Install packages
-        run: sudo apt -y update && sudo apt -y install libssl-dev unixodbc-dev redis-server libmysqlclient-dev libpq-dev
+        run: sudo apt -y update && sudo apt -y install cmake ninja-build libssl-dev unixodbc-dev redis-server libmysqlclient-dev libpq-dev
       -
         name: Install SSH key
         run: |
@@ -33,7 +33,6 @@ jobs:
         run: |
           export POCO_BASE=`pwd`
           export PATH=$POCO_BASE/release/script:$PATH
-          export LD_LIBRARY_PATH=$POCO_BASE/stage/tools/lib/Linux/x86_64
           mkdoc all
       -
         name: Copy to web server

CppParser/include/Poco/CppParser/CppToken.h

@@ -209,6 +209,8 @@ class CppParser_API IdentifierToken: public CppToken
 	int asInteger() const;
 
 private:
+	void finishRawString(std::istream& istr);
+
 	using KWMap = std::map<std::string, int>;
 
 	KWMap _kwMap;

CppParser/include/Poco/CppParser/Parser.h

@@ -81,8 +81,10 @@ class CppParser_API Parser
 	const Poco::Token* parseBaseClassList(const Poco::Token* pNext, Struct* pClass);
 	const Poco::Token* parseClassMembers(const Poco::Token* pNext, Struct* pClass);
 	const Poco::Token* parseAccess(const Poco::Token* pNext);
+	const Poco::Token* parseStaticAssert(const Poco::Token* pNext);
 	const Poco::Token* parseIdentifier(const Poco::Token* pNext, std::string& id);
 	const Poco::Token* parseAttributes(const Poco::Token* pNext, std::string& attrs);
+	const Poco::Token* parseParenthesized(const Poco::Token* pNext, std::string& decl);
 
 	void addSymbol(Symbol* pSymbol, int lineNumber, bool addGST = true);
 	void pushNameSpace(NameSpace* pNameSpace, int lineNumber, bool addGST = true);

CppParser/src/CppToken.cpp

@@ -369,6 +369,54 @@ void IdentifierToken::finish(std::istream& istr)
 		_value += (char) istr.get();
 		next = istr.peek();
 	}
+
+	// C++11 raw string literals: R"delim(...)delim", also u8R"...", uR"...", UR"...", LR"..."
+	// Without this handling, the tokenizer would split R"(")" into separate tokens:
+	//   identifier 'R', string literal '("', operator ')', and an unterminated '"',
+	// causing "Unterminated character literal" errors downstream.
+	if ((_value == "R" || _value == "u8R" || _value == "uR" || _value == "UR" || _value == "LR")
+		&& istr.peek() == '"')
+	{
+		finishRawString(istr);
+	}
+}
+
+
+void IdentifierToken::finishRawString(std::istream& istr)
+{
+	// Consume the entire raw string literal as part of this identifier token
+	// so that parseBlock() and other consumers can safely skip over it.
+	// The delimiter between " and ( can be any character sequence,
+	// e.g. R"---(content)---" has delimiter "---".
+	_value += (char) istr.get(); // consume opening "
+
+	// Read optional delimiter characters until '('
+	std::string delimiter;
+	int next = istr.peek();
+	while (next != -1 && next != '(' && next != '\n')
+	{
+		delimiter += (char) istr.get();
+		_value += delimiter.back();
+		next = istr.peek();
+	}
+	if (next == '(')
+	{
+		_value += (char) istr.get(); // consume '('
+		// Scan for closing sequence: )delimiter"
+		// Use a sliding window to match the end pattern correctly.
+		std::string endPattern = ")" + delimiter + "\"";
+		std::string window;
+		while (!istr.eof())
+		{
+			char c = (char) istr.get();
+			_value += c;
+			window += c;
+			if (window.size() > endPattern.size())
+				window.erase(0, 1);
+			if (window == endPattern)
+				break;
+		}
+	}
 }
 
 
@@ -420,6 +468,23 @@ void StringLiteralToken::finish(std::istream& istr)
 		_value += (char) next;
 	}
 	else throw SyntaxException("Unterminated string literal");
+
+	// C++14 string literal suffixes: "str"s (std::string), "str"sv (std::string_view),
+	// and user-defined literal suffixes starting with _ (e.g. "str"_mylit).
+	// Consume the suffix as part of this token so the parser doesn't see a
+	// stray identifier immediately following a string literal.
+	next = istr.peek();
+	if (next == 's' || next == '_')
+	{
+		while ((next >= 'A' && next <= 'Z') ||
+			   (next >= 'a' && next <= 'z') ||
+			   (next >= '0' && next <= '9') ||
+			   next == '_')
+		{
+			_value += (char) istr.get();
+			next = istr.peek();
+		}
+	}
 }
 
 

CppParser/src/Parser.cpp

@@ -46,6 +46,12 @@ namespace Poco {
 namespace CppParser {
 
 
+// Context-sensitive identifiers (not reserved keywords in C++,
+// but have special meaning in certain contexts)
+static const std::string ID_OVERRIDE("override");
+static const std::string ID_FINAL("final");
+
+
 Parser::Parser(NameSpace::SymbolTable& gst, const std::string& file, std::istream& istr):
 	_gst(gst),
 	_istr(istr),
@@ -198,6 +204,9 @@ const Token* Parser::parseFile(const Token* pNext)
 		case IdentifierToken::KW_ENUM:
 			pNext = parseEnum(pNext);
 			break;
+		case IdentifierToken::KW_STATIC_ASSERT:
+			pNext = parseStaticAssert(pNext);
+			break;
 		default:
 			pNext = parseVarFunc(pNext);
 		}
@@ -263,13 +272,23 @@ const Token* Parser::parseNameSpace(const Token* pNext)
 			case IdentifierToken::KW_ENUM:
 				pNext = parseEnum(pNext);
 				break;
+			case IdentifierToken::KW_STATIC_ASSERT:
+				pNext = parseStaticAssert(pNext);
+				break;
 			default:
 				pNext = parseVarFunc(pNext);
 			}
 		}
 		expectOperator(pNext, OperatorToken::OP_CLOSBRACE, "}");
 		pNext = next();
 	}
+	else if (isOperator(pNext, OperatorToken::OP_OPENBRACE))
+	{
+		// Anonymous namespace (namespace { ... }) contains implementation
+		// details not relevant for API documentation; skip the entire block.
+		pNext = parseBlock(pNext);
+		return pNext;
+	}
 	else syntaxError("namespace name");
 	popNameSpace();
 	return pNext;
@@ -305,7 +324,7 @@ const Token* Parser::parseClass(const Token* pNext, std::string& decl)
 	pNext = next();
 
 	bool isFinal = false;
-	if (isIdentifier(pNext) && pNext->asString() == "final")
+	if (isIdentifier(pNext) && pNext->asString() == ID_FINAL)
 	{
 		pNext = next();
 		isFinal = true;
@@ -424,6 +443,9 @@ const Token* Parser::parseClassMembers(const Token* pNext, Struct* /*pClass*/)
 		case IdentifierToken::KW_FRIEND:
 			pNext = parseFriend(pNext);
 			break;
+		case IdentifierToken::KW_STATIC_ASSERT:
+			pNext = parseStaticAssert(pNext);
+			break;
 		case OperatorToken::OP_COMPL:
 		default:
 			pNext = parseVarFunc(pNext);
@@ -569,12 +591,19 @@ const Token* Parser::parseFriend(const Token* pNext)
 {
 	poco_assert (isKeyword(pNext, IdentifierToken::KW_FRIEND));
 
+	// Friend declarations come in two forms:
+	//   friend class Foo;                          — ends with ;
+	//   friend bool operator==(A a, B b) { ... }  — inline definition ends with }
+	// Skip the entire declaration in both cases.
 	pNext = next();
-
-	while (!isOperator(pNext, OperatorToken::OP_SEMICOLON) && !isEOF(pNext))
+	while (!isOperator(pNext, OperatorToken::OP_SEMICOLON) &&
+	       !isOperator(pNext, OperatorToken::OP_OPENBRACE) &&
+	       !isEOF(pNext))
 		pNext = next();
 
-	if (isOperator(pNext, OperatorToken::OP_SEMICOLON))
+	if (isOperator(pNext, OperatorToken::OP_OPENBRACE))
+		pNext = parseBlock(pNext);
+	else if (isOperator(pNext, OperatorToken::OP_SEMICOLON))
 		pNext = next();
 	return pNext;
 }
@@ -601,6 +630,18 @@ const Token* Parser::parseVarFunc(const Token* pNext, std::string& decl)
 	}
 	else
 	{
+		while (isKeyword(pNext, IdentifierToken::KW_ALIGNAS) ||
+		       isKeyword(pNext, IdentifierToken::KW_DECLTYPE))
+		{
+			// These keywords take parenthesized arguments and are part
+			// of the declaration, not function names
+			append(decl, pNext);
+			pNext = next();
+			if (isOperator(pNext, OperatorToken::OP_OPENPARENT))
+			{
+				pNext = parseParenthesized(pNext, decl);
+			}
+		}
 		append(decl, pNext);
 		pNext = next();
 		bool isOperatorKeyword = false;
@@ -691,6 +732,7 @@ const Token* Parser::parseFunc(const Token* pNext, const std::string& attrs, std
 		{
 			if (pFunc) pFunc->makeConst();
 			pNext = next();
+			continue;
 		}
 		if (isKeyword(pNext, IdentifierToken::KW_THROW))
 		{
@@ -702,13 +744,18 @@ const Token* Parser::parseFunc(const Token* pNext, const std::string& attrs, std
 		{
 			if (pFunc) pFunc->makeNoexcept();
 			pNext = next();
+			if (isOperator(pNext, OperatorToken::OP_OPENPARENT))
+			{
+				std::string tmp;
+				pNext = parseParenthesized(pNext, tmp);
+			}
 		}
-		else if (isIdentifier(pNext) && pNext->asString() == "override")
+		else if (isIdentifier(pNext) && pNext->asString() == ID_OVERRIDE)
 		{
 			if (pFunc) pFunc->makeOverride();
 			pNext = next();
 		}
-		else if (isIdentifier(pNext) && pNext->asString() == "final")
+		else if (isIdentifier(pNext) && pNext->asString() == ID_FINAL)
 		{
 			if (pFunc) pFunc->makeFinal();
 			pNext = next();
@@ -717,18 +764,26 @@ const Token* Parser::parseFunc(const Token* pNext, const std::string& attrs, std
 		{
 			break; // handled below
 		}
+		else break;
 	}
 	if (isOperator(pNext, OperatorToken::OP_ASSIGN))
 	{
 		pNext = next();
 		if (!pNext->is(Token::INTEGER_LITERAL_TOKEN) && !isKeyword(pNext, IdentifierToken::KW_DEFAULT) && !isKeyword(pNext, IdentifierToken::KW_DELETE))
 			syntaxError("0, default or delete");
 		if (isKeyword(pNext, IdentifierToken::KW_DEFAULT))
-			pFunc->makeDefault();
+		{
+			if (pFunc) pFunc->makeDefault();
+		}
 		else if (isKeyword(pNext, IdentifierToken::KW_DELETE))
-			pFunc->makeDelete();
+		{
+			if (pFunc) pFunc->makeDelete();
+		}
+		else
+		{
+			if (pFunc) pFunc->makePureVirtual();
+		}
 		pNext = next();
-		if (pFunc) pFunc->makePureVirtual();
 		expectOperator(pNext, OperatorToken::OP_SEMICOLON, ";");
 	}
 	else if (isOperator(pNext, OperatorToken::OP_OPENBRACE) || isOperator(pNext, OperatorToken::OP_COLON))
@@ -957,6 +1012,44 @@ const Poco::Token* Parser::parseAttributes(const Poco::Token* pNext, std::string
 }
 
 
+const Token* Parser::parseParenthesized(const Token* pNext, std::string& decl)
+{
+	poco_assert (isOperator(pNext, OperatorToken::OP_OPENPARENT));
+
+	append(decl, pNext);
+	pNext = next();
+	int depth = 1;
+	while (depth > 0 && !isEOF(pNext))
+	{
+		if (isOperator(pNext, OperatorToken::OP_OPENPARENT))
+			++depth;
+		else if (isOperator(pNext, OperatorToken::OP_CLOSPARENT))
+			--depth;
+		if (depth > 0)
+		{
+			append(decl, pNext);
+			pNext = next();
+		}
+	}
+	append(decl, pNext); // closing ')'
+	pNext = next();
+	return pNext;
+}
+
+
+const Token* Parser::parseStaticAssert(const Token* pNext)
+{
+	poco_assert (isKeyword(pNext, IdentifierToken::KW_STATIC_ASSERT));
+
+	// skip static_assert(...); — not a declaration
+	while (!isOperator(pNext, OperatorToken::OP_SEMICOLON) && !isEOF(pNext))
+		pNext = next();
+	if (isOperator(pNext, OperatorToken::OP_SEMICOLON))
+		pNext = next();
+	return pNext;
+}
+
+
 void Parser::addSymbol(Symbol* pSymbol, int lineNumber, bool addGST)
 {
 	pSymbol->setLineNumber(lineNumber);