[yul] Add support for parsing debug data attributes.

.circleci/parallel_bytecode_report.sh

@@ -45,7 +45,7 @@ echo "Preparing input files"
 python3 ../scripts/isolate_tests.py ../test/
 
 # FIXME: These cases crash because of https://github.com/ethereum/solidity/issues/13583
-rm ./*_bytecode_too_large_*.sol ./*_combined_too_large_*.sol
+rm ./*_bytecode_too_large_*.sol ./*_combined_too_large_*.sol ./*_initcode_too_large_*.sol
 
 if [[ $binary_type == native || $binary_type == "osx_intel" ]]; then
     interface=$(echo -e "standard-json\ncli" | circleci tests split)

liblangutil/DebugData.h

@@ -19,6 +19,7 @@
 #pragma once
 
 #include <liblangutil/SourceLocation.h>
+#include <libsolutil/JSON.h>
 #include <optional>
 #include <memory>
 
@@ -32,23 +33,27 @@ struct DebugData
 	explicit DebugData(
 		langutil::SourceLocation _nativeLocation = {},
 		langutil::SourceLocation _originLocation = {},
-		std::optional<int64_t> _astID = {}
+		std::optional<int64_t> _astID = {},
+		Json _attributes = {}
 	):
 		nativeLocation(std::move(_nativeLocation)),
 		originLocation(std::move(_originLocation)),
-		astID(_astID)
+		astID(_astID),
+		attributes(std::move(_attributes))
 	{}
 
 	static DebugData::ConstPtr create(
 		langutil::SourceLocation _nativeLocation,
 		langutil::SourceLocation _originLocation = {},
-		std::optional<int64_t> _astID = {}
+		std::optional<int64_t> _astID = {},
+		Json _attributes = {}
 	)
 	{
 		return std::make_shared<DebugData>(
 			std::move(_nativeLocation),
 			std::move(_originLocation),
-			_astID
+			_astID,
+			std::move(_attributes)
 		);
 	}
 
@@ -65,6 +70,8 @@ struct DebugData
 	langutil::SourceLocation originLocation;
 	/// ID in the (Solidity) source AST.
 	std::optional<int64_t> astID;
+	/// Additional debug data attributes.
+	Json attributes;
 };
 
 } // namespace solidity::langutil

libyul/AsmParser.cpp

@@ -64,11 +64,11 @@ langutil::DebugData::ConstPtr Parser::createDebugData() const
 	switch (m_useSourceLocationFrom)
 	{
 		case UseSourceLocationFrom::Scanner:
-			return DebugData::create(ParserBase::currentLocation(), ParserBase::currentLocation());
+			return DebugData::create(ParserBase::currentLocation(), ParserBase::currentLocation(), {}, m_currentDebugDataAttributes);
 		case UseSourceLocationFrom::LocationOverride:
-			return DebugData::create(m_locationOverride, m_locationOverride);
+			return DebugData::create(m_locationOverride, m_locationOverride, {}, m_currentDebugDataAttributes);
 		case UseSourceLocationFrom::Comments:
-			return DebugData::create(ParserBase::currentLocation(), m_locationFromComment, m_astIDFromComment);
+			return DebugData::create(ParserBase::currentLocation(), m_locationFromComment, m_astIDFromComment, m_currentDebugDataAttributes);
 	}
 	solAssert(false, "");
 }
@@ -122,8 +122,7 @@ std::unique_ptr<Block> Parser::parseInline(std::shared_ptr<Scanner> const& _scan
 	try
 	{
 		m_scanner = _scanner;
-		if (m_useSourceLocationFrom == UseSourceLocationFrom::Comments)
-			fetchDebugDataFromComment();
+		fetchDebugDataFromComment();
 		return std::make_unique<Block>(parseBlock());
 	}
 	catch (FatalError const&)
@@ -137,24 +136,20 @@ std::unique_ptr<Block> Parser::parseInline(std::shared_ptr<Scanner> const& _scan
 langutil::Token Parser::advance()
 {
 	auto const token = ParserBase::advance();
-	if (m_useSourceLocationFrom == UseSourceLocationFrom::Comments)
-		fetchDebugDataFromComment();
+	fetchDebugDataFromComment();
 	return token;
 }
 
 void Parser::fetchDebugDataFromComment()
 {
-	solAssert(m_sourceNames.has_value(), "");
-
 	static std::regex const tagRegex = std::regex(
-		R"~~((?:^|\s+)(@[a-zA-Z0-9\-_]+)(?:\s+|$))~~", // tag, e.g: @src
+		R"~~((?:^|\s+)(@[a-zA-Z0-9\-\._]+)(?:\s+|$))~~", // tag, e.g: @src
 		std::regex_constants::ECMAScript | std::regex_constants::optimize
 	);
 
 	std::string_view commentLiteral = m_scanner->currentCommentLiteral();
 	std::match_results<std::string_view::const_iterator> match;
 
-	langutil::SourceLocation originLocation = m_locationFromComment;
 	// Empty for each new node.
 	std::optional<int> astID;
 
@@ -165,10 +160,14 @@ void Parser::fetchDebugDataFromComment()
 
 		if (match[1] == "@src")
 		{
-			if (auto parseResult = parseSrcComment(commentLiteral, m_scanner->currentCommentLocation()))
-				tie(commentLiteral, originLocation) = *parseResult;
-			else
-				break;
+			if (m_useSourceLocationFrom == UseSourceLocationFrom::Comments)
+			{
+				solAssert(m_sourceNames.has_value(), "");
+				if (auto parseResult = parseSrcComment(commentLiteral, m_scanner->currentCommentLocation()))
+					tie(commentLiteral, m_locationFromComment) = *parseResult;
+				else
+					break;
+			}
 		}
 		else if (match[1] == "@ast-id")
 		{
@@ -177,15 +176,101 @@ void Parser::fetchDebugDataFromComment()
 			else
 				break;
 		}
+		else if (match[1] == "@debug.set")
+		{
+			if (auto parseResult = parseDebugDataAttributeOperationComment(match[1], commentLiteral, m_scanner->currentCommentLocation()))
+			{
+				commentLiteral = parseResult->first;
+				if (parseResult->second.has_value())
+					m_currentDebugDataAttributes = parseResult->second.value();
+			}
+			else
+				break;
+		}
+		else if (match[1] == "@debug.merge")
+		{
+			if (auto parseResult = parseDebugDataAttributeOperationComment(match[1], commentLiteral, m_scanner->currentCommentLocation()))
+			{
+				commentLiteral = parseResult->first;
+				if (parseResult->second.has_value())
+					m_currentDebugDataAttributes.merge_patch(parseResult->second.value());
+			}
+			else
+				break;
+		}
+		else if (match[1] == "@debug.patch")
+		{
+			if (auto parseResult = parseDebugDataAttributeOperationComment(match[1], commentLiteral, m_scanner->currentCommentLocation()))
+			{
+				commentLiteral = parseResult->first;
+				if (parseResult->second.has_value())
+					applyDebugDataAttributePatch(parseResult->second.value(), m_scanner->currentCommentLocation());
+			}
+			else
+				break;
+		}
 		else
 			// Ignore unrecognized tags.
 			continue;
 	}
 
-	m_locationFromComment = originLocation;
 	m_astIDFromComment = astID;
 }
 
+std::optional<std::pair<std::string_view, std::optional<Json>>> Parser::parseDebugDataAttributeOperationComment(
+	std::string const& _command,
+	std::string_view _arguments,
+	langutil::SourceLocation const& _location
+)
+{
+	std::optional<Json> jsonData;
+	try
+	{
+		jsonData = Json::parse(_arguments.begin(), _arguments.end(), nullptr, true);
+	}
+	catch (nlohmann::json::parse_error& e)
+	{
+		try
+		{
+			jsonData = Json::parse(_arguments.substr(0, e.byte - 1), nullptr, true);
+		}
+		catch(nlohmann::json::parse_error& ee)
+		{
+			m_errorReporter.syntaxError(
+				5721_error,
+				_location,
+			_command + ": Could not parse debug data: " + removeNlohmannInternalErrorIdentifier(ee.what())
+			);
+			jsonData.reset();
+		}
+		_arguments = _arguments.substr(e.byte - 1);
+	}
+	return {{_arguments, jsonData}};
+}
+
+void Parser::applyDebugDataAttributePatch(Json const& _jsonPatch, langutil::SourceLocation const& _location)
+{
+	try
+	{
+		if (_jsonPatch.is_object())
+		{
+			Json array = Json::array();
+			array.push_back(_jsonPatch);
+			m_currentDebugDataAttributes = m_currentDebugDataAttributes.patch(array);
+		}
+		else
+			m_currentDebugDataAttributes = m_currentDebugDataAttributes.patch(_jsonPatch);
+	}
+	catch(nlohmann::json::parse_error& ee)
+	{
+		m_errorReporter.syntaxError(
+			9426_error,
+			_location,
+			"@debug.patch: Could not patch debug data: " + removeNlohmannInternalErrorIdentifier(ee.what())
+		);
+	}
+}
+
 std::optional<std::pair<std::string_view, SourceLocation>> Parser::parseSrcComment(
 	std::string_view const _arguments,
 	langutil::SourceLocation const& _commentLocation

libyul/AsmParser.h

@@ -117,6 +117,14 @@ class Parser: public langutil::ParserBase
 		langutil::SourceLocation const& _commentLocation
 	);
 
+	std::optional<std::pair<std::string_view, std::optional<Json>>> parseDebugDataAttributeOperationComment(
+		std::string const& _command,
+		std::string_view _arguments,
+		langutil::SourceLocation const& _commentLocation
+	);
+
+	void applyDebugDataAttributePatch(Json const& _jsonPatch, langutil::SourceLocation const& _location);
+
 	/// Creates a DebugData object with the correct source location set.
 	langutil::DebugData::ConstPtr createDebugData() const;
 
@@ -163,6 +171,7 @@ class Parser: public langutil::ParserBase
 	UseSourceLocationFrom m_useSourceLocationFrom = UseSourceLocationFrom::Scanner;
 	ForLoopComponent m_currentForLoopComponent = ForLoopComponent::None;
 	bool m_insideFunction = false;
+	Json m_currentDebugDataAttributes = Json::object();
 };
 
 }

test/libyul/yulSyntaxTests/invalid/invalid_debug_merge.yul

@@ -0,0 +1,7 @@
+object "object" {
+    code {
+        /// @debug.merge {"HELLO": 2
+    }
+}
+// ----
+// SyntaxError 5721: (37-65): @debug.merge: Could not parse debug data: parse error at line 1, column 12: syntax error while parsing object - unexpected end of input; expected '}'

test/libyul/yulSyntaxTests/invalid/invalid_debug_patch.yul

@@ -0,0 +1,7 @@
+object "object" {
+    code {
+        /// @debug.patch {"HELLO": invalid
+    }
+}
+// ----
+// SyntaxError 5721: (37-71): @debug.patch: Could not parse debug data: parse error at line 1, column 11: syntax error while parsing value - unexpected end of input; expected '[', '{', or a literal

test/libyul/yulSyntaxTests/invalid/invalid_debug_patch_patch_object.yul

@@ -0,0 +1,7 @@
+object "object" {
+    code {
+        /// @debug.patch { "op": "unknown_operation", "path": "/variable_a", "value": ["test"] }
+    }
+}
+// ----
+// SyntaxError 9426: (37-125): @debug.patch: Could not patch debug data: parse error: operation value 'unknown_operation' is invalid

test/libyul/yulSyntaxTests/invalid/invalid_debug_set.yul

@@ -0,0 +1,7 @@
+object "object" {
+    code {
+        /// @debug.set {"HELLO": "WORLD
+    }
+}
+// ----
+// SyntaxError 5721: (37-68): @debug.set: Could not parse debug data: parse error at line 1, column 17: syntax error while parsing value - invalid string: missing closing quote; last read: '"WORLD'