Use returndatacopy for retrieving dynamically sized outputs.

Changelog.md

@@ -1,6 +1,7 @@
 ### 0.4.22 (unreleased)
 
 Features:
+ * General: Support accessing dynamic return data in post-byzantium EVMs.
 
 Bugfixes:
  * Code Generator: Allow ``block.blockhash`` without being called.

libsolidity/analysis/TypeChecker.cpp

@@ -1551,16 +1551,22 @@ bool TypeChecker::visit(FunctionCall const& _functionCall)
 			_functionCall.expression().annotation().isPure &&
 			functionType->isPure();
 
+	bool allowDynamicTypes = m_evmVersion.supportsReturndata();
 	if (!functionType)
 	{
 		m_errorReporter.typeError(_functionCall.location(), "Type is not callable");
 		_functionCall.annotation().type = make_shared<TupleType>();
 		return false;
 	}
-	else if (functionType->returnParameterTypes().size() == 1)
-		_functionCall.annotation().type = functionType->returnParameterTypes().front();
+
+	auto returnTypes =
+		allowDynamicTypes ?
+		functionType->returnParameterTypes() :
+		functionType->returnParameterTypesWithoutDynamicTypes();
+	if (returnTypes.size() == 1)
+		_functionCall.annotation().type = returnTypes.front();
 	else
-		_functionCall.annotation().type = make_shared<TupleType>(functionType->returnParameterTypes());
+		_functionCall.annotation().type = make_shared<TupleType>(returnTypes);
 
 	if (auto functionName = dynamic_cast<Identifier const*>(&_functionCall.expression()))
 	{

libsolidity/ast/Types.cpp

@@ -2311,6 +2311,18 @@ vector<string> FunctionType::parameterNames() const
 	return vector<string>(m_parameterNames.cbegin() + 1, m_parameterNames.cend());
 }
 
+TypePointers FunctionType::returnParameterTypesWithoutDynamicTypes() const
+{
+	TypePointers returnParameterTypes = m_returnParameterTypes;
+
+	if (m_kind == Kind::External || m_kind == Kind::CallCode || m_kind == Kind::DelegateCall)
+		for (auto& param: returnParameterTypes)
+			if (param->isDynamicallySized() && !param->dataStoredIn(DataLocation::Storage))
+				param = make_shared<InaccessibleDynamicType>();
+
+	return returnParameterTypes;
+}
+
 TypePointers FunctionType::parameterTypes() const
 {
 	if (!bound())
@@ -2772,18 +2784,9 @@ FunctionTypePointer FunctionType::asMemberFunction(bool _inLibrary, bool _bound)
 			kind = Kind::DelegateCall;
 	}
 
-	TypePointers returnParameterTypes = m_returnParameterTypes;
-	if (kind != Kind::Internal)
-	{
-		// Alter dynamic types to be non-accessible.
-		for (auto& param: returnParameterTypes)
-			if (param->isDynamicallySized())
-				param = make_shared<InaccessibleDynamicType>();
-	}
-
 	return make_shared<FunctionType>(
 		parameterTypes,
-		returnParameterTypes,
+		m_returnParameterTypes,
 		m_parameterNames,
 		m_returnParameterNames,
 		kind,

libsolidity/ast/Types.h

@@ -973,6 +973,9 @@ class FunctionType: public Type
 	TypePointers parameterTypes() const;
 	std::vector<std::string> parameterNames() const;
 	TypePointers const& returnParameterTypes() const { return m_returnParameterTypes; }
+	/// @returns the list of return parameter types. All dynamically-sized types (this excludes
+	/// storage pointers) are replaced by InaccessibleDynamicType instances.
+	TypePointers returnParameterTypesWithoutDynamicTypes() const;
 	std::vector<std::string> const& returnParameterNames() const { return m_returnParameterNames; }
 	/// @returns the "self" parameter type for a bound function
 	TypePointer const& selfType() const;

libsolidity/codegen/ABIFunctions.cpp

@@ -253,6 +253,9 @@ string ABIFunctions::cleanupFunction(Type const& _type, bool _revertOnFailure)
 			templ("body", w.render());
 			break;
 		}
+		case Type::Category::InaccessibleDynamic:
+			templ("body", "cleaned := 0");
+			break;
 		default:
 			solAssert(false, "Cleanup of type " + _type.identifier() + " requested.");
 		}

libsolidity/codegen/CompilerUtils.cpp

@@ -22,11 +22,14 @@
 
 #include <libsolidity/codegen/CompilerUtils.h>
 #include <libsolidity/ast/AST.h>
-#include <libevmasm/Instruction.h>
 #include <libsolidity/codegen/ArrayUtils.h>
 #include <libsolidity/codegen/LValue.h>
 #include <libsolidity/codegen/ABIFunctions.h>
 
+#include <libevmasm/Instruction.h>
+
+#include <libdevcore/Whiskers.h>
+
 using namespace std;
 
 namespace dev
@@ -159,6 +162,163 @@ void CompilerUtils::storeInMemoryDynamic(Type const& _type, bool _padToWordBound
 	}
 }
 
+void CompilerUtils::abiDecode(TypePointers const& _typeParameters, bool _fromMemory, bool _revertOnOutOfBounds)
+{
+	/// Stack: <source_offset> <length>
+	if (m_context.experimentalFeatureActive(ExperimentalFeature::ABIEncoderV2))
+	{
+		// Use the new JULIA-based decoding function
+		auto stackHeightBefore = m_context.stackHeight();
+		abiDecodeV2(_typeParameters, _fromMemory);
+		solAssert(m_context.stackHeight() - stackHeightBefore == sizeOnStack(_typeParameters) - 2, "");
+		return;
+	}
+
+	//@todo this does not yet support nested dynamic arrays
+
+	if (_revertOnOutOfBounds)
+	{
+		size_t encodedSize = 0;
+		for (auto const& t: _typeParameters)
+			encodedSize += t->decodingType()->calldataEncodedSize(true);
+		m_context.appendInlineAssembly("{ if lt(len, " + to_string(encodedSize) + ") { revert(0, 0) } }", {"len"});
+	}
+
+	m_context << Instruction::DUP2 << Instruction::ADD;
+	m_context << Instruction::SWAP1;
+	/// Stack: <input_end> <source_offset>
+
+	// Retain the offset pointer as base_offset, the point from which the data offsets are computed.
+	m_context << Instruction::DUP1;
+	for (TypePointer const& parameterType: _typeParameters)
+	{
+		// stack: v1 v2 ... v(k-1) input_end base_offset current_offset
+		TypePointer type = parameterType->decodingType();
+		solUnimplementedAssert(type, "No decoding type found.");
+		if (type->category() == Type::Category::Array)
+		{
+			auto const& arrayType = dynamic_cast<ArrayType const&>(*type);
+			solUnimplementedAssert(!arrayType.baseType()->isDynamicallyEncoded(), "Nested arrays not yet implemented.");
+			if (_fromMemory)
+			{
+				solUnimplementedAssert(
+					arrayType.baseType()->isValueType(),
+					"Nested memory arrays not yet implemented here."
+				);
+				// @todo If base type is an array or struct, it is still calldata-style encoded, so
+				// we would have to convert it like below.
+				solAssert(arrayType.location() == DataLocation::Memory, "");
+				if (arrayType.isDynamicallySized())
+				{
+					// compute data pointer
+					m_context << Instruction::DUP1 << Instruction::MLOAD;
+					if (_revertOnOutOfBounds)
+					{
+						// Check that the data pointer is valid and that length times
+						// item size is still inside the range.
+						Whiskers templ(R"({
+							if gt(ptr, 0x100000000) { revert(0, 0) }
+							ptr := add(ptr, base_offset)
+							let array_data_start := add(ptr, 0x20)
+							if gt(array_data_start, input_end) { revert(0, 0) }
+							let array_length := mload(ptr)
+							if or(
+								gt(array_length, 0x100000000),
+								gt(add(array_data_start, mul(array_length, <item_size>)), input_end)
+							) { revert(0, 0) }
+						})");
+						templ("item_size", to_string(arrayType.isByteArray() ? 1 : arrayType.baseType()->calldataEncodedSize(true)));
+						m_context.appendInlineAssembly(templ.render(), {"input_end", "base_offset", "offset", "ptr"});
+					}
+					else
+						m_context << Instruction::DUP3 << Instruction::ADD;
+					// stack: v1 v2 ... v(k-1) input_end base_offset current_offset v(k)
+					moveIntoStack(3);
+					m_context << u256(0x20) << Instruction::ADD;
+				}
+				else
+				{
+					// Size has already been checked for this one.
+					moveIntoStack(2);
+					m_context << Instruction::DUP3;
+					m_context << u256(arrayType.calldataEncodedSize(true)) << Instruction::ADD;
+				}
+			}
+			else
+			{
+				// first load from calldata and potentially convert to memory if arrayType is memory
+				TypePointer calldataType = arrayType.copyForLocation(DataLocation::CallData, false);
+				if (calldataType->isDynamicallySized())
+				{
+					// put on stack: data_pointer length
+					loadFromMemoryDynamic(IntegerType(256), !_fromMemory);
+					m_context << Instruction::SWAP1;
+					// stack: input_end base_offset next_pointer data_offset
+					if (_revertOnOutOfBounds)
+						m_context.appendInlineAssembly("{ if gt(data_offset, 0x100000000) { revert(0, 0) } }", {"data_offset"});
+					m_context << Instruction::DUP3 << Instruction::ADD;
+					// stack: input_end base_offset next_pointer array_head_ptr
+					if (_revertOnOutOfBounds)
+						m_context.appendInlineAssembly(
+							"{ if gt(add(array_head_ptr, 0x20), input_end) { revert(0, 0) } }",
+							{"input_end", "base_offset", "next_ptr", "array_head_ptr"}
+						);
+					// retrieve length
+					loadFromMemoryDynamic(IntegerType(256), !_fromMemory, true);
+					// stack: input_end base_offset next_pointer array_length data_pointer
+					m_context << Instruction::SWAP2;
+					// stack: input_end base_offset data_pointer array_length next_pointer
+					if (_revertOnOutOfBounds)
+					{
+						unsigned itemSize = arrayType.isByteArray() ? 1 : arrayType.baseType()->calldataEncodedSize(true);
+						m_context.appendInlineAssembly(R"({
+							if or(
+								gt(array_length, 0x100000000),
+								gt(add(data_ptr, mul(array_length, )" + to_string(itemSize) + R"()), input_end)
+							) { revert(0, 0) }
+						})", {"input_end", "base_offset", "data_ptr", "array_length", "next_ptr"});
+					}
+				}
+				else
+				{
+					// size has already been checked
+					// stack: input_end base_offset data_offset
+					m_context << Instruction::DUP1;
+					m_context << u256(calldataType->calldataEncodedSize()) << Instruction::ADD;
+				}
+				if (arrayType.location() == DataLocation::Memory)
+				{
+					// stack: input_end base_offset calldata_ref [length] next_calldata
+					// copy to memory
+					// move calldata type up again
+					moveIntoStack(calldataType->sizeOnStack());
+					convertType(*calldataType, arrayType, false, false, true);
+					// fetch next pointer again
+					moveToStackTop(arrayType.sizeOnStack());
+				}
+				// move input_end up
+				// stack: input_end base_offset calldata_ref [length] next_calldata
+				moveToStackTop(2 + arrayType.sizeOnStack());
+				m_context << Instruction::SWAP1;
+				// stack: base_offset calldata_ref [length] input_end next_calldata
+				moveToStackTop(2 + arrayType.sizeOnStack());
+				m_context << Instruction::SWAP1;
+				// stack: calldata_ref [length] input_end base_offset next_calldata
+			}
+		}
+		else
+		{
+			solAssert(!type->isDynamicallyEncoded(), "Unknown dynamically sized type: " + type->toString());
+			loadFromMemoryDynamic(*type, !_fromMemory, true);
+			// stack: v1 v2 ... v(k-1) input_end base_offset v(k) mem_offset
+			moveToStackTop(1, type->sizeOnStack());
+			moveIntoStack(3, type->sizeOnStack());
+		}
+		// stack: v1 v2 ... v(k-1) v(k) input_end base_offset next_offset
+	}
+	popStackSlots(3);
+}
+
 void CompilerUtils::encodeToMemory(
 	TypePointers const& _givenTypes,
 	TypePointers const& _targetTypes,
@@ -321,15 +481,13 @@ void CompilerUtils::abiEncodeV2(
 
 void CompilerUtils::abiDecodeV2(TypePointers const& _parameterTypes, bool _fromMemory)
 {
-	// stack: <source_offset>
+	// stack: <source_offset> <length> [stack top]
 	auto ret = m_context.pushNewTag();
+	moveIntoStack(2);
+	// stack: <return tag> <source_offset> <length> [stack top]
+	m_context << Instruction::DUP2 << Instruction::ADD;
 	m_context << Instruction::SWAP1;
-	if (_fromMemory)
-		// TODO pass correct size for the memory case
-		m_context << (u256(1) << 63);
-	else
-		m_context << Instruction::CALLDATASIZE;
-	m_context << Instruction::SWAP1;
+	// stack: <return tag> <end> <start>
 	string decoderName = m_context.abiFunctions().tupleDecoder(_parameterTypes, _fromMemory);
 	m_context.appendJumpTo(m_context.namedTag(decoderName));
 	m_context.adjustStackOffset(int(sizeOnStack(_parameterTypes)) - 3);

libsolidity/codegen/CompilerUtils.h

@@ -88,6 +88,15 @@ class CompilerUtils
 	/// Stack post: (memory_offset+length)
 	void storeInMemoryDynamic(Type const& _type, bool _padToWords = true);
 
+	/// Creates code that unpacks the arguments according to their types specified by a vector of TypePointers.
+	/// From memory if @a _fromMemory is true, otherwise from call data.
+	/// Calls revert if @a _revertOnOutOfBounds is true and the supplied size is shorter
+	/// than the static data requirements or if dynamic data pointers reach outside of the
+	/// area. Also has a hard cap of 0x100000000 for any given length/offset field.
+	/// Stack pre: <source_offset> <length>
+	/// Stack post: <value0> <value1> ... <valuen>
+	void abiDecode(TypePointers const& _typeParameters, bool _fromMemory = false, bool _revertOnOutOfBounds = false);
+
 	/// Copies values (of types @a _givenTypes) given on the stack to a location in memory given
 	/// at the stack top, encoding them according to the ABI as the given types @a _targetTypes.
 	/// Removes the values from the stack and leaves the updated memory pointer.
@@ -149,7 +158,7 @@ class CompilerUtils
 	/// Decodes data from ABI encoding into internal encoding. If @a _fromMemory is set to true,
 	/// the data is taken from memory instead of from calldata.
 	/// Can allocate memory.
-	/// Stack pre: <source_offset>
+	/// Stack pre: <source_offset> <length>
 	/// Stack post: <value0> <value1> ... <valuen>
 	void abiDecodeV2(TypePointers const& _parameterTypes, bool _fromMemory = false);