From 235af2b4d1245f59525d9fd148d7892ea89b46a0 Mon Sep 17 00:00:00 2001 From: Alexander Arlt Date: Thu, 4 Nov 2021 16:53:32 -0500 Subject: [PATCH 01/19] [solc] Add --import-asm-json input mode. --- solc/CommandLineInterface.cpp | 277 ++++++++++++++++------------- solc/CommandLineParser.cpp | 48 +++-- solc/CommandLineParser.h | 1 + test/solc/CommandLineInterface.cpp | 2 +- 4 files changed, 188 insertions(+), 140 deletions(-) diff --git a/solc/CommandLineInterface.cpp b/solc/CommandLineInterface.cpp index 1d069f925152..4a6598fbc08e 100644 --- a/solc/CommandLineInterface.cpp +++ b/solc/CommandLineInterface.cpp @@ -634,6 +634,7 @@ bool CommandLineInterface::processInput() break; case InputMode::Compiler: case InputMode::CompilerWithASTImport: + case InputMode::CompilerWithEvmAssemblyJsonImport: if (!compile()) return false; outputCompilationResults(); @@ -657,105 +658,112 @@ void CommandLineInterface::printLicense() bool CommandLineInterface::compile() { - solAssert(m_options.input.mode == InputMode::Compiler || m_options.input.mode == InputMode::CompilerWithASTImport, ""); + solAssert( + m_options.input.mode == InputMode::Compiler || + m_options.input.mode == InputMode::CompilerWithASTImport || + m_options.input.mode == InputMode::CompilerWithEvmAssemblyJsonImport, "" + ); m_compiler = make_unique(m_fileReader.reader()); - SourceReferenceFormatter formatter(serr(false), *m_compiler, coloredOutput(m_options), m_options.formatting.withErrorIds); - - try + if (m_options.input.mode == InputMode::CompilerWithEvmAssemblyJsonImport) { - if (m_options.metadata.literalSources) - m_compiler->useMetadataLiteralSources(true); - m_compiler->setMetadataHash(m_options.metadata.hash); - if (m_options.modelChecker.initialize) - m_compiler->setModelCheckerSettings(m_options.modelChecker.settings); - m_compiler->setRemappings(m_options.input.remappings); - m_compiler->setLibraries(m_options.linker.libraries); - m_compiler->setViaIR(m_options.output.experimentalViaIR); - m_compiler->setEVMVersion(m_options.output.evmVersion); - m_compiler->setRevertStringBehaviour(m_options.output.revertStrings); - if (m_options.output.debugInfoSelection.has_value()) - m_compiler->selectDebugInfo(m_options.output.debugInfoSelection.value()); - // TODO: Perhaps we should not compile unless requested - - m_compiler->enableIRGeneration(m_options.compiler.outputs.ir || m_options.compiler.outputs.irOptimized); - m_compiler->enableEwasmGeneration(m_options.compiler.outputs.ewasm); - m_compiler->enableEvmBytecodeGeneration( - m_options.compiler.estimateGas || - m_options.compiler.outputs.asm_ || - m_options.compiler.outputs.asmJson || - m_options.compiler.outputs.opcodes || - m_options.compiler.outputs.binary || - m_options.compiler.outputs.binaryRuntime || - (m_options.compiler.combinedJsonRequests && ( - m_options.compiler.combinedJsonRequests->binary || - m_options.compiler.combinedJsonRequests->binaryRuntime || - m_options.compiler.combinedJsonRequests->opcodes || - m_options.compiler.combinedJsonRequests->asm_ || - m_options.compiler.combinedJsonRequests->generatedSources || - m_options.compiler.combinedJsonRequests->generatedSourcesRuntime || - m_options.compiler.combinedJsonRequests->srcMap || - m_options.compiler.combinedJsonRequests->srcMapRuntime || - m_options.compiler.combinedJsonRequests->funDebug || - m_options.compiler.combinedJsonRequests->funDebugRuntime - )) - ); - m_compiler->setOptimiserSettings(m_options.optimiserSettings()); + } + else + { + SourceReferenceFormatter + formatter(serr(false), *m_compiler, coloredOutput(m_options), m_options.formatting.withErrorIds); - if (m_options.input.mode == InputMode::CompilerWithASTImport) + try { - try + if (m_options.metadata.literalSources) + m_compiler->useMetadataLiteralSources(true); + m_compiler->setMetadataHash(m_options.metadata.hash); + if (m_options.modelChecker.initialize) + m_compiler->setModelCheckerSettings(m_options.modelChecker.settings); + m_compiler->setRemappings(m_options.input.remappings); + m_compiler->setLibraries(m_options.linker.libraries); + m_compiler->setViaIR(m_options.output.experimentalViaIR); + m_compiler->setEVMVersion(m_options.output.evmVersion); + m_compiler->setRevertStringBehaviour(m_options.output.revertStrings); + if (m_options.output.debugInfoSelection.has_value()) + m_compiler->selectDebugInfo(m_options.output.debugInfoSelection.value()); + // TODO: Perhaps we should not compile unless requested + + m_compiler->enableIRGeneration(m_options.compiler.outputs.ir || m_options.compiler.outputs.irOptimized); + m_compiler->enableEwasmGeneration(m_options.compiler.outputs.ewasm); + m_compiler->enableEvmBytecodeGeneration( + m_options.compiler.estimateGas || m_options.compiler.outputs.asm_ || m_options.compiler.outputs.asmJson + || m_options.compiler.outputs.opcodes || m_options.compiler.outputs.binary + || m_options.compiler.outputs.binaryRuntime + || (m_options.compiler.combinedJsonRequests + && (m_options.compiler.combinedJsonRequests->binary + || m_options.compiler.combinedJsonRequests->binaryRuntime + || m_options.compiler.combinedJsonRequests->opcodes + || m_options.compiler.combinedJsonRequests->asm_ + || m_options.compiler.combinedJsonRequests->generatedSources + || m_options.compiler.combinedJsonRequests->generatedSourcesRuntime + || m_options.compiler.combinedJsonRequests->srcMap + || m_options.compiler.combinedJsonRequests->srcMapRuntime + || m_options.compiler.combinedJsonRequests->funDebug + || m_options.compiler.combinedJsonRequests->funDebugRuntime))); + + m_compiler->setOptimiserSettings(m_options.optimiserSettings()); + + if (m_options.input.mode == InputMode::CompilerWithASTImport) { - m_compiler->importASTs(parseAstFromInput()); + try + { + m_compiler->importASTs(parseAstFromInput()); - if (!m_compiler->analyze()) + if (!m_compiler->analyze()) + { + formatter.printErrorInformation(m_compiler->errors()); + astAssert(false, "Analysis of the AST failed"); + } + } + catch (Exception const& _exc) { - formatter.printErrorInformation(m_compiler->errors()); - astAssert(false, "Analysis of the AST failed"); + serr() << string("Failed to import AST: ") << _exc.what() << endl; + return false; } } - catch (Exception const& _exc) + else { - serr() << string("Failed to import AST: ") << _exc.what() << endl; - return false; + m_compiler->setSources(m_fileReader.sourceCodes()); + m_compiler->setParserErrorRecovery(m_options.input.errorRecovery); } - } - else - { - m_compiler->setSources(m_fileReader.sourceCodes()); - m_compiler->setParserErrorRecovery(m_options.input.errorRecovery); - } - bool successful = m_compiler->compile(m_options.output.stopAfter); + bool successful = m_compiler->compile(m_options.output.stopAfter); - for (auto const& error: m_compiler->errors()) - { - m_hasOutput = true; - formatter.printErrorInformation(*error); - } + for (auto const& error: m_compiler->errors()) + { + m_hasOutput = true; + formatter.printErrorInformation(*error); + } - if (!successful) - return m_options.input.errorRecovery; - } - catch (CompilerError const& _exception) - { - m_hasOutput = true; - formatter.printExceptionInformation(_exception, "Compiler error"); - return false; - } - catch (Error const& _error) - { - if (_error.type() == Error::Type::DocstringParsingError) - serr() << "Documentation parsing error: " << *boost::get_error_info(_error) << endl; - else + if (!successful) + return m_options.input.errorRecovery; + } + catch (CompilerError const& _exception) { m_hasOutput = true; - formatter.printExceptionInformation(_error, _error.typeName()); + formatter.printExceptionInformation(_exception, "Compiler error"); + return false; } + catch (Error const& _error) + { + if (_error.type() == Error::Type::DocstringParsingError) + serr() << "Documentation parsing error: " << *boost::get_error_info(_error) << endl; + else + { + m_hasOutput = true; + formatter.printExceptionInformation(_error, _error.typeName()); + } - return false; + return false; + } } return true; @@ -763,7 +771,11 @@ bool CommandLineInterface::compile() void CommandLineInterface::handleCombinedJSON() { - solAssert(m_options.input.mode == InputMode::Compiler || m_options.input.mode == InputMode::CompilerWithASTImport, ""); + solAssert( + m_options.input.mode == InputMode::Compiler || + m_options.input.mode == InputMode::CompilerWithEvmAssemblyJsonImport || + m_options.input.mode == InputMode::CompilerWithASTImport, "" + ); if (!m_options.compiler.combinedJsonRequests.has_value()) return; @@ -1095,68 +1107,77 @@ bool CommandLineInterface::assemble(yul::AssemblyStack::Language _language, yul: void CommandLineInterface::outputCompilationResults() { - solAssert(m_options.input.mode == InputMode::Compiler || m_options.input.mode == InputMode::CompilerWithASTImport, ""); + solAssert( + m_options.input.mode == InputMode::Compiler || + m_options.input.mode == InputMode::CompilerWithEvmAssemblyJsonImport || + m_options.input.mode == InputMode::CompilerWithASTImport, "" + ); handleCombinedJSON(); - // do we need AST output? - handleAst(); - - if ( - !m_compiler->compilationSuccessful() && - m_options.output.stopAfter == CompilerStack::State::CompilationSuccessful - ) + if (m_options.input.mode == InputMode::CompilerWithEvmAssemblyJsonImport) { - serr() << endl << "Compilation halted after AST generation due to errors." << endl; - return; + handleBytecode(""); } - - vector contracts = m_compiler->contractNames(); - for (string const& contract: contracts) + else { - if (needsHumanTargetedStdout(m_options)) - sout() << endl << "======= " << contract << " =======" << endl; + // do we need AST output? + handleAst(); - // do we need EVM assembly? - if (m_options.compiler.outputs.asm_ || m_options.compiler.outputs.asmJson) + if (!m_compiler->compilationSuccessful() + && m_options.output.stopAfter == CompilerStack::State::CompilationSuccessful) { - string ret; - if (m_options.compiler.outputs.asmJson) - ret = jsonPrettyPrint(removeNullMembers(m_compiler->assemblyJSON(contract))); - else - ret = m_compiler->assemblyString(contract, m_fileReader.sourceCodes()); + serr() << endl << "Compilation halted after AST generation due to errors." << endl; + return; + } - if (!m_options.output.dir.empty()) + vector contracts = m_compiler->contractNames(); + for (string const& contract: contracts) + { + if (needsHumanTargetedStdout(m_options)) + sout() << endl << "======= " << contract << " =======" << endl; + + // do we need EVM assembly? + if (m_options.compiler.outputs.asm_ || m_options.compiler.outputs.asmJson) { - createFile(m_compiler->filesystemFriendlyName(contract) + (m_options.compiler.outputs.asmJson ? "_evm.json" : ".evm"), ret); + string ret; + if (m_options.compiler.outputs.asmJson) + ret = jsonPrettyPrint(removeNullMembers(m_compiler->assemblyJSON(contract))); + else + ret = m_compiler->assemblyString(contract, m_fileReader.sourceCodes()); + + if (!m_options.output.dir.empty()) + { + createFile(m_compiler->filesystemFriendlyName(contract) + (m_options.compiler.outputs.asmJson ? "_evm.json" : ".evm"), ret); + } + else + { + sout() << "EVM assembly:" << endl << ret << endl; + } } + + if (m_options.compiler.estimateGas) + handleGasEstimation(contract); + + handleBytecode(contract); + handleIR(contract); + handleIROptimized(contract); + handleEwasm(contract); + handleSignatureHashes(contract); + handleMetadata(contract); + handleABI(contract); + handleStorageLayout(contract); + handleNatspec(true, contract); + handleNatspec(false, contract); + } // end of contracts iteration + + if (!m_hasOutput) + { + if (!m_options.output.dir.empty()) + sout() << "Compiler run successful. Artifact(s) can be found in directory " << m_options.output.dir << "." << endl; else - { - sout() << "EVM assembly:" << endl << ret << endl; - } + serr() << "Compiler run successful, no output requested." << endl; } - - if (m_options.compiler.estimateGas) - handleGasEstimation(contract); - - handleBytecode(contract); - handleIR(contract); - handleIROptimized(contract); - handleEwasm(contract); - handleSignatureHashes(contract); - handleMetadata(contract); - handleABI(contract); - handleStorageLayout(contract); - handleNatspec(true, contract); - handleNatspec(false, contract); - } // end of contracts iteration - - if (!m_hasOutput) - { - if (!m_options.output.dir.empty()) - sout() << "Compiler run successful. Artifact(s) can be found in directory " << m_options.output.dir << "." << endl; - else - serr() << "Compiler run successful, no output requested." << endl; } } diff --git a/solc/CommandLineParser.cpp b/solc/CommandLineParser.cpp index 9356c96625d0..acfc09c8765f 100644 --- a/solc/CommandLineParser.cpp +++ b/solc/CommandLineParser.cpp @@ -46,6 +46,7 @@ static string const g_strAllowPaths = "allow-paths"; static string const g_strBasePath = "base-path"; static string const g_strIncludePath = "include-path"; static string const g_strAssemble = "assemble"; +static string const g_strImportEvmAssemblerJson = "import-asm-json"; static string const g_strCombinedJson = "combined-json"; static string const g_strErrorRecovery = "error-recovery"; static string const g_strEVM = "evm"; @@ -134,6 +135,7 @@ static map const g_inputModeName = { {InputMode::Compiler, "compiler"}, {InputMode::CompilerWithASTImport, "compiler (AST import)"}, {InputMode::Assembler, "assembler"}, + {InputMode::CompilerWithEvmAssemblyJsonImport, "assembler (EVM ASM JSON import)"}, {InputMode::StandardJson, "standard JSON"}, {InputMode::Linker, "linker"}, }; @@ -321,7 +323,20 @@ bool CommandLineParser::parseInputPathsAndRemappings() m_options.input.paths.insert(positionalArg); } - if (m_options.input.mode == InputMode::StandardJson) + if (m_options.input.mode == InputMode::CompilerWithEvmAssemblyJsonImport) + { + if (m_options.input.paths.size() > 1 || (m_options.input.paths.size() == 1 && m_options.input.addStdin)) + { + serr() << "Too many input files for --" << g_strImportEvmAssemblerJson << "." << endl; + serr() << "Please either specify a single file name or provide its content on standard input." << endl; + return false; + } + else if (m_options.input.paths.size() == 0) + // Standard JSON mode input used to be handled separately and zero files meant "read from stdin". + // Keep it working that way for backwards-compatibility. + m_options.input.addStdin = true; + } + else if (m_options.input.mode == InputMode::StandardJson) { if (m_options.input.paths.size() > 1 || (m_options.input.paths.size() == 1 && m_options.input.addStdin)) { @@ -466,6 +481,7 @@ bool CommandLineParser::parseOutputSelection() case InputMode::Version: solAssert(false); case InputMode::Compiler: + case InputMode::CompilerWithEvmAssemblyJsonImport: case InputMode::CompilerWithASTImport: return contains(compilerModeOutputs, _outputName); case InputMode::Assembler: @@ -641,8 +657,12 @@ General Information)").c_str(), ( g_strImportAst.c_str(), ("Import ASTs to be compiled, assumes input holds the AST in compact JSON format. " - "Supported Inputs is the output of the --" + g_strStandardJSON + " or the one produced by " - "--" + g_strCombinedJson + " " + CombinedJsonRequests::componentName(&CombinedJsonRequests::ast)).c_str() + "Supported Inputs is the output of the --" + g_strStandardJSON + " or the one produced by " + "--" + g_strCombinedJson + " " + CombinedJsonRequests::componentName(&CombinedJsonRequests::ast)).c_str() + ) + ( + g_strImportEvmAssemblerJson.c_str(), + "Import evm assembler json to be compiled, assumes input holds the evm assembly in JSON format." ) ; desc.add(alternativeInputModes); @@ -879,6 +899,7 @@ bool CommandLineParser::processArgs() g_strStrictAssembly, g_strYul, g_strImportAst, + g_strImportEvmAssemblerJson, })) return false; @@ -892,6 +913,8 @@ bool CommandLineParser::processArgs() m_options.input.mode = InputMode::StandardJson; else if (m_args.count(g_strAssemble) > 0 || m_args.count(g_strStrictAssembly) > 0 || m_args.count(g_strYul) > 0) m_options.input.mode = InputMode::Assembler; + else if (m_args.count(g_strImportEvmAssemblerJson) > 0) + m_options.input.mode = InputMode::CompilerWithEvmAssemblyJsonImport; else if (m_args.count(g_strLink) > 0) m_options.input.mode = InputMode::Linker; else if (m_args.count(g_strImportAst) > 0) @@ -946,8 +969,8 @@ bool CommandLineParser::processArgs() if ( m_options.input.mode != InputMode::Compiler && m_options.input.mode != InputMode::CompilerWithASTImport && - m_options.input.mode != InputMode::Assembler - ) + m_options.input.mode != InputMode::Assembler && + m_options.input.mode != InputMode::CompilerWithEvmAssemblyJsonImport) { if (!m_args[g_strOptimizeRuns].defaulted()) { @@ -1127,16 +1150,18 @@ bool CommandLineParser::processArgs() m_options.optimizer.yulSteps = m_args[g_strYulOptimizations].as(); } - if (m_options.input.mode == InputMode::Assembler) + if (m_options.input.mode == InputMode::Assembler || + m_options.input.mode == InputMode::CompilerWithEvmAssemblyJsonImport) { - vector const nonAssemblyModeOptions = { + vector nonAssemblyModeOptions = { // TODO: The list is not complete. Add more. g_strOutputDir, g_strGas, - g_strCombinedJson, g_strOptimizeYul, g_strNoOptimizeYul, }; + if (m_options.input.mode == InputMode::Assembler) + nonAssemblyModeOptions.emplace_back(g_strCombinedJson); if (countEnabledOptions(nonAssemblyModeOptions) >= 1) { auto optionEnabled = [&](string const& name){ return m_args.count(name) > 0; }; @@ -1206,9 +1231,10 @@ bool CommandLineParser::processArgs() serr() << "and automatic translation is not available." << endl; return false; } - serr() << - "Warning: Yul is still experimental. Please use the output with care." << - endl; + if (m_options.input.mode == InputMode::Assembler) + serr() << + "Warning: Yul is still experimental. Please use the output with care." << + endl; return true; } diff --git a/solc/CommandLineParser.h b/solc/CommandLineParser.h index a1d13c690756..c946f120400c 100644 --- a/solc/CommandLineParser.h +++ b/solc/CommandLineParser.h @@ -56,6 +56,7 @@ enum class InputMode StandardJson, Linker, Assembler, + CompilerWithEvmAssemblyJsonImport, }; struct CompilerOutputs diff --git a/test/solc/CommandLineInterface.cpp b/test/solc/CommandLineInterface.cpp index 0269ec462fc6..fcb3d6d67c48 100644 --- a/test/solc/CommandLineInterface.cpp +++ b/test/solc/CommandLineInterface.cpp @@ -157,7 +157,7 @@ BOOST_AUTO_TEST_CASE(multiple_input_modes) }; string expectedMessage = "The following options are mutually exclusive: " - "--help, --license, --version, --standard-json, --link, --assemble, --strict-assembly, --yul, --import-ast. " + "--help, --license, --version, --standard-json, --link, --assemble, --strict-assembly, --yul, --import-ast, --import-asm-json. " "Select at most one.\n"; for (string const& mode1: inputModeOptions) From 9a3e248d51c5e0776029fc308bbf9bbd96e032dc Mon Sep 17 00:00:00 2001 From: Alexander Arlt Date: Mon, 8 Nov 2021 14:30:11 -0500 Subject: [PATCH 02/19] [libsolidity] Prepare evm assembly json import. --- libsolidity/interface/CompilerStack.cpp | 11 + libsolidity/interface/CompilerStack.h | 3 + solc/CommandLineInterface.cpp | 314 +++++++++++++----------- solc/CommandLineInterface.h | 2 + 4 files changed, 191 insertions(+), 139 deletions(-) diff --git a/libsolidity/interface/CompilerStack.cpp b/libsolidity/interface/CompilerStack.cpp index 4b7faf4966e2..03f03bf7d2da 100644 --- a/libsolidity/interface/CompilerStack.cpp +++ b/libsolidity/interface/CompilerStack.cpp @@ -388,6 +388,17 @@ bool CompilerStack::parse() return !m_hasError; } +void CompilerStack::importEvmAssemblyJson(map const& _sources) +{ + solAssert(_sources.size() == 1, ""); + if (m_stackState != Empty) + solThrow(CompilerError, "Must call importEvmAssemblyJson only before the SourcesSet state."); + + m_evmAssemblyJson = std::make_unique(_sources.begin()->second); + m_importedSources = true; + m_stackState = SourcesSet; +} + void CompilerStack::importASTs(map const& _sources) { if (m_stackState != Empty) diff --git a/libsolidity/interface/CompilerStack.h b/libsolidity/interface/CompilerStack.h index 3609662e02de..85f527862fc4 100644 --- a/libsolidity/interface/CompilerStack.h +++ b/libsolidity/interface/CompilerStack.h @@ -222,6 +222,8 @@ class CompilerStack: public langutil::CharStreamProvider /// Will throw errors if the import fails void importASTs(std::map const& _sources); + void importEvmAssemblyJson(std::map const& _sources); + /// Performs the analysis steps (imports, scopesetting, syntaxCheck, referenceResolving, /// typechecking, staticAnalysis) on previously parsed sources. /// @returns false on error. @@ -499,6 +501,7 @@ class CompilerStack: public langutil::CharStreamProvider std::map m_sources; // if imported, store AST-JSONS for each filename std::map m_sourceJsons; + std::unique_ptr m_evmAssemblyJson; std::vector m_unhandledSMTLib2Queries; std::map m_smtlib2Responses; std::shared_ptr m_globalContext; diff --git a/solc/CommandLineInterface.cpp b/solc/CommandLineInterface.cpp index 4a6598fbc08e..5cbfec9849a7 100644 --- a/solc/CommandLineInterface.cpp +++ b/solc/CommandLineInterface.cpp @@ -155,7 +155,11 @@ static bool coloredOutput(CommandLineOptions const& _options) void CommandLineInterface::handleBinary(string const& _contract) { - solAssert(m_options.input.mode == InputMode::Compiler || m_options.input.mode == InputMode::CompilerWithASTImport, ""); + solAssert( + m_options.input.mode == InputMode::Compiler || + m_options.input.mode == InputMode::CompilerWithASTImport || + m_options.input.mode == InputMode::CompilerWithEvmAssemblyJsonImport , "" + ); if (m_options.compiler.outputs.binary) { @@ -250,7 +254,11 @@ void CommandLineInterface::handleEwasm(string const& _contractName) void CommandLineInterface::handleBytecode(string const& _contract) { - solAssert(m_options.input.mode == InputMode::Compiler || m_options.input.mode == InputMode::CompilerWithASTImport, ""); + solAssert( + m_options.input.mode == InputMode::Compiler || + m_options.input.mode == InputMode::CompilerWithASTImport || + m_options.input.mode == InputMode::CompilerWithEvmAssemblyJsonImport , "" + ); if (m_options.compiler.outputs.opcodes) handleOpcode(_contract); @@ -546,6 +554,25 @@ map CommandLineInterface::parseAstFromInput() return sourceJsons; } +map CommandLineInterface::parseEvmAssemblyJsonFromInput() +{ + solAssert(m_options.input.mode == InputMode::CompilerWithEvmAssemblyJsonImport, ""); + solAssert(m_fileReader.sourceCodes().size() == 1, ""); + + map sourceJsons; + + for (auto const& iter: m_fileReader.sourceCodes()) + { + Json::Value evmAsmJson; + astAssert(jsonParseStrict(iter.second, evmAsmJson), "Input file could not be parsed to JSON"); + astAssert(evmAsmJson.isMember(".code"), "Invalid Format for assembly-JSON: Must have '.code'-object"); + astAssert(evmAsmJson.isMember(".data"), "Invalid Format for assembly-JSON: Must have '.data'-object"); + sourceJsons[iter.first] = evmAsmJson; + } + + return sourceJsons; +} + void CommandLineInterface::createFile(string const& _fileName, string const& _data) { namespace fs = boost::filesystem; @@ -666,104 +693,114 @@ bool CommandLineInterface::compile() m_compiler = make_unique(m_fileReader.reader()); - if (m_options.input.mode == InputMode::CompilerWithEvmAssemblyJsonImport) - { + SourceReferenceFormatter formatter(serr(false), *m_compiler, coloredOutput(m_options), m_options.formatting.withErrorIds); - } - else + try { - SourceReferenceFormatter - formatter(serr(false), *m_compiler, coloredOutput(m_options), m_options.formatting.withErrorIds); + if (m_options.metadata.literalSources) + m_compiler->useMetadataLiteralSources(true); + m_compiler->setMetadataHash(m_options.metadata.hash); + if (m_options.modelChecker.initialize) + m_compiler->setModelCheckerSettings(m_options.modelChecker.settings); + m_compiler->setRemappings(m_options.input.remappings); + m_compiler->setLibraries(m_options.linker.libraries); + m_compiler->setViaIR(m_options.output.experimentalViaIR); + m_compiler->setEVMVersion(m_options.output.evmVersion); + m_compiler->setRevertStringBehaviour(m_options.output.revertStrings); + if (m_options.output.debugInfoSelection.has_value()) + m_compiler->selectDebugInfo(m_options.output.debugInfoSelection.value()); + // TODO: Perhaps we should not compile unless requested + + m_compiler->enableIRGeneration(m_options.compiler.outputs.ir || m_options.compiler.outputs.irOptimized); + m_compiler->enableEwasmGeneration(m_options.compiler.outputs.ewasm); + m_compiler->enableEvmBytecodeGeneration( + m_options.compiler.estimateGas || + m_options.compiler.outputs.asm_ || + m_options.compiler.outputs.asmJson || + m_options.compiler.outputs.opcodes || + m_options.compiler.outputs.binary || + m_options.compiler.outputs.binaryRuntime || + (m_options.compiler.combinedJsonRequests && ( + m_options.compiler.combinedJsonRequests->binary || + m_options.compiler.combinedJsonRequests->binaryRuntime || + m_options.compiler.combinedJsonRequests->opcodes || + m_options.compiler.combinedJsonRequests->asm_ || + m_options.compiler.combinedJsonRequests->generatedSources || + m_options.compiler.combinedJsonRequests->generatedSourcesRuntime || + m_options.compiler.combinedJsonRequests->srcMap || + m_options.compiler.combinedJsonRequests->srcMapRuntime || + m_options.compiler.combinedJsonRequests->funDebug || + m_options.compiler.combinedJsonRequests->funDebugRuntime + )) + ); - try + m_compiler->setOptimiserSettings(m_options.optimiserSettings()); + + if (m_options.input.mode == InputMode::CompilerWithASTImport) { - if (m_options.metadata.literalSources) - m_compiler->useMetadataLiteralSources(true); - m_compiler->setMetadataHash(m_options.metadata.hash); - if (m_options.modelChecker.initialize) - m_compiler->setModelCheckerSettings(m_options.modelChecker.settings); - m_compiler->setRemappings(m_options.input.remappings); - m_compiler->setLibraries(m_options.linker.libraries); - m_compiler->setViaIR(m_options.output.experimentalViaIR); - m_compiler->setEVMVersion(m_options.output.evmVersion); - m_compiler->setRevertStringBehaviour(m_options.output.revertStrings); - if (m_options.output.debugInfoSelection.has_value()) - m_compiler->selectDebugInfo(m_options.output.debugInfoSelection.value()); - // TODO: Perhaps we should not compile unless requested - - m_compiler->enableIRGeneration(m_options.compiler.outputs.ir || m_options.compiler.outputs.irOptimized); - m_compiler->enableEwasmGeneration(m_options.compiler.outputs.ewasm); - m_compiler->enableEvmBytecodeGeneration( - m_options.compiler.estimateGas || m_options.compiler.outputs.asm_ || m_options.compiler.outputs.asmJson - || m_options.compiler.outputs.opcodes || m_options.compiler.outputs.binary - || m_options.compiler.outputs.binaryRuntime - || (m_options.compiler.combinedJsonRequests - && (m_options.compiler.combinedJsonRequests->binary - || m_options.compiler.combinedJsonRequests->binaryRuntime - || m_options.compiler.combinedJsonRequests->opcodes - || m_options.compiler.combinedJsonRequests->asm_ - || m_options.compiler.combinedJsonRequests->generatedSources - || m_options.compiler.combinedJsonRequests->generatedSourcesRuntime - || m_options.compiler.combinedJsonRequests->srcMap - || m_options.compiler.combinedJsonRequests->srcMapRuntime - || m_options.compiler.combinedJsonRequests->funDebug - || m_options.compiler.combinedJsonRequests->funDebugRuntime))); - - m_compiler->setOptimiserSettings(m_options.optimiserSettings()); - - if (m_options.input.mode == InputMode::CompilerWithASTImport) + try { - try - { - m_compiler->importASTs(parseAstFromInput()); + m_compiler->importASTs(parseAstFromInput()); - if (!m_compiler->analyze()) - { - formatter.printErrorInformation(m_compiler->errors()); - astAssert(false, "Analysis of the AST failed"); - } - } - catch (Exception const& _exc) + if (!m_compiler->analyze()) { - serr() << string("Failed to import AST: ") << _exc.what() << endl; - return false; + formatter.printErrorInformation(m_compiler->errors()); + astAssert(false, "Analysis of the AST failed"); } } - else + catch (Exception const& _exc) { - m_compiler->setSources(m_fileReader.sourceCodes()); - m_compiler->setParserErrorRecovery(m_options.input.errorRecovery); + serr() << string("Failed to import AST: ") << _exc.what() << endl; + return false; } - - bool successful = m_compiler->compile(m_options.output.stopAfter); - - for (auto const& error: m_compiler->errors()) + } + if (m_options.input.mode == InputMode::CompilerWithEvmAssemblyJsonImport) + { + solAssert(m_fileReader.sourceCodes().size() == 1, ""); + try { - m_hasOutput = true; - formatter.printErrorInformation(*error); + m_compiler->importEvmAssemblyJson(parseEvmAssemblyJsonFromInput()); + } + catch (Exception const& _exc) + { + serr() << string("Failed to import Assembly JSON: ") << _exc.what() << endl; + return false; } - - if (!successful) - return m_options.input.errorRecovery; } - catch (CompilerError const& _exception) + else { - m_hasOutput = true; - formatter.printExceptionInformation(_exception, "Compiler error"); - return false; + m_compiler->setSources(m_fileReader.sourceCodes()); + m_compiler->setParserErrorRecovery(m_options.input.errorRecovery); } - catch (Error const& _error) + + bool successful = m_compiler->compile(m_options.output.stopAfter); + + for (auto const& error: m_compiler->errors()) { - if (_error.type() == Error::Type::DocstringParsingError) - serr() << "Documentation parsing error: " << *boost::get_error_info(_error) << endl; - else - { - m_hasOutput = true; - formatter.printExceptionInformation(_error, _error.typeName()); - } + m_hasOutput = true; + formatter.printErrorInformation(*error); + } - return false; + if (!successful) + return m_options.input.errorRecovery; + } + catch (CompilerError const& _exception) + { + m_hasOutput = true; + formatter.printExceptionInformation(_exception, "Compiler error"); + return false; + } + catch (Error const& _error) + { + if (_error.type() == Error::Type::DocstringParsingError) + serr() << "Documentation parsing error: " << *boost::get_error_info(_error) << endl; + else + { + m_hasOutput = true; + formatter.printExceptionInformation(_error, _error.typeName()); } + + return false; } return true; @@ -773,8 +810,8 @@ void CommandLineInterface::handleCombinedJSON() { solAssert( m_options.input.mode == InputMode::Compiler || - m_options.input.mode == InputMode::CompilerWithEvmAssemblyJsonImport || - m_options.input.mode == InputMode::CompilerWithASTImport, "" + m_options.input.mode == InputMode::CompilerWithASTImport || + m_options.input.mode == InputMode::CompilerWithEvmAssemblyJsonImport, "" ); if (!m_options.compiler.combinedJsonRequests.has_value()) @@ -867,7 +904,11 @@ void CommandLineInterface::handleCombinedJSON() void CommandLineInterface::handleAst() { - solAssert(m_options.input.mode == InputMode::Compiler || m_options.input.mode == InputMode::CompilerWithASTImport, ""); + solAssert( + m_options.input.mode == InputMode::Compiler || + m_options.input.mode == InputMode::CompilerWithASTImport || + m_options.input.mode == InputMode::CompilerWithEvmAssemblyJsonImport, "" + ); if (!m_options.compiler.outputs.astCompactJson) return; @@ -1109,75 +1150,70 @@ void CommandLineInterface::outputCompilationResults() { solAssert( m_options.input.mode == InputMode::Compiler || - m_options.input.mode == InputMode::CompilerWithEvmAssemblyJsonImport || - m_options.input.mode == InputMode::CompilerWithASTImport, "" + m_options.input.mode == InputMode::CompilerWithASTImport || + m_options.input.mode == InputMode::CompilerWithEvmAssemblyJsonImport, "" ); handleCombinedJSON(); - if (m_options.input.mode == InputMode::CompilerWithEvmAssemblyJsonImport) + // do we need AST output? + handleAst(); + + if ( + !m_compiler->compilationSuccessful() && + m_options.output.stopAfter == CompilerStack::State::CompilationSuccessful + ) { - handleBytecode(""); + serr() << endl << "Compilation halted after AST generation due to errors." << endl; + return; } - else - { - // do we need AST output? - handleAst(); - if (!m_compiler->compilationSuccessful() - && m_options.output.stopAfter == CompilerStack::State::CompilationSuccessful) - { - serr() << endl << "Compilation halted after AST generation due to errors." << endl; - return; - } + vector contracts = m_compiler->contractNames(); + for (string const& contract: contracts) + { + if (needsHumanTargetedStdout(m_options)) + sout() << endl << "======= " << contract << " =======" << endl; - vector contracts = m_compiler->contractNames(); - for (string const& contract: contracts) + // do we need EVM assembly? + if (m_options.compiler.outputs.asm_ || m_options.compiler.outputs.asmJson) { - if (needsHumanTargetedStdout(m_options)) - sout() << endl << "======= " << contract << " =======" << endl; + string ret; + if (m_options.compiler.outputs.asmJson) + ret = jsonPrettyPrint(removeNullMembers(m_compiler->assemblyJSON(contract))); + else + ret = m_compiler->assemblyString(contract, m_fileReader.sourceCodes()); - // do we need EVM assembly? - if (m_options.compiler.outputs.asm_ || m_options.compiler.outputs.asmJson) + if (!m_options.output.dir.empty()) { - string ret; - if (m_options.compiler.outputs.asmJson) - ret = jsonPrettyPrint(removeNullMembers(m_compiler->assemblyJSON(contract))); - else - ret = m_compiler->assemblyString(contract, m_fileReader.sourceCodes()); - - if (!m_options.output.dir.empty()) - { - createFile(m_compiler->filesystemFriendlyName(contract) + (m_options.compiler.outputs.asmJson ? "_evm.json" : ".evm"), ret); - } - else - { - sout() << "EVM assembly:" << endl << ret << endl; - } + createFile(m_compiler->filesystemFriendlyName(contract) + (m_options.compiler.outputs.asmJson ? "_evm.json" : ".evm"), ret); } - - if (m_options.compiler.estimateGas) - handleGasEstimation(contract); - - handleBytecode(contract); - handleIR(contract); - handleIROptimized(contract); - handleEwasm(contract); - handleSignatureHashes(contract); - handleMetadata(contract); - handleABI(contract); - handleStorageLayout(contract); - handleNatspec(true, contract); - handleNatspec(false, contract); - } // end of contracts iteration - - if (!m_hasOutput) - { - if (!m_options.output.dir.empty()) - sout() << "Compiler run successful. Artifact(s) can be found in directory " << m_options.output.dir << "." << endl; else - serr() << "Compiler run successful, no output requested." << endl; + { + sout() << "EVM assembly:" << endl << ret << endl; + } } + + if (m_options.compiler.estimateGas) + handleGasEstimation(contract); + + handleBytecode(contract); + handleIR(contract); + handleIROptimized(contract); + handleEwasm(contract); + handleSignatureHashes(contract); + handleMetadata(contract); + handleABI(contract); + handleStorageLayout(contract); + handleNatspec(true, contract); + handleNatspec(false, contract); + } // end of contracts iteration + + if (!m_hasOutput) + { + if (!m_options.output.dir.empty()) + sout() << "Compiler run successful. Artifact(s) can be found in directory " << m_options.output.dir << "." << endl; + else + serr() << "Compiler run successful, no output requested." << endl; } } diff --git a/solc/CommandLineInterface.h b/solc/CommandLineInterface.h index ee5057468273..250d30de902e 100644 --- a/solc/CommandLineInterface.h +++ b/solc/CommandLineInterface.h @@ -98,6 +98,8 @@ class CommandLineInterface /// or standard-json output std::map parseAstFromInput(); + std::map parseEvmAssemblyJsonFromInput(); + /// Create a file in the given directory /// @arg _fileName the name of the file /// @arg _data to be written From 34492b3fe386d3dda1853970c5a2604f72291794 Mon Sep 17 00:00:00 2001 From: Alexander Arlt Date: Mon, 8 Nov 2021 19:59:57 -0500 Subject: [PATCH 03/19] [solc] Enable handling for InputMode::CompilerWithEvmAssemblyJsonImport. --- solc/CommandLineInterface.cpp | 54 +++++++++++++++++++++++++++++------ 1 file changed, 45 insertions(+), 9 deletions(-) diff --git a/solc/CommandLineInterface.cpp b/solc/CommandLineInterface.cpp index 5cbfec9849a7..ed694ab740d9 100644 --- a/solc/CommandLineInterface.cpp +++ b/solc/CommandLineInterface.cpp @@ -185,7 +185,11 @@ void CommandLineInterface::handleBinary(string const& _contract) void CommandLineInterface::handleOpcode(string const& _contract) { - solAssert(m_options.input.mode == InputMode::Compiler || m_options.input.mode == InputMode::CompilerWithASTImport, ""); + solAssert( + m_options.input.mode == InputMode::Compiler || + m_options.input.mode == InputMode::CompilerWithASTImport || + m_options.input.mode == InputMode::CompilerWithEvmAssemblyJsonImport , "" + ); if (!m_options.output.dir.empty()) createFile(m_compiler->filesystemFriendlyName(_contract) + ".opcode", evmasm::disassemble(m_compiler->object(_contract).bytecode)); @@ -199,7 +203,11 @@ void CommandLineInterface::handleOpcode(string const& _contract) void CommandLineInterface::handleIR(string const& _contractName) { - solAssert(m_options.input.mode == InputMode::Compiler || m_options.input.mode == InputMode::CompilerWithASTImport, ""); + solAssert( + m_options.input.mode == InputMode::Compiler || + m_options.input.mode == InputMode::CompilerWithASTImport || + m_options.input.mode == InputMode::CompilerWithEvmAssemblyJsonImport , "" + ); if (!m_options.compiler.outputs.ir) return; @@ -215,7 +223,11 @@ void CommandLineInterface::handleIR(string const& _contractName) void CommandLineInterface::handleIROptimized(string const& _contractName) { - solAssert(m_options.input.mode == InputMode::Compiler || m_options.input.mode == InputMode::CompilerWithASTImport, ""); + solAssert( + m_options.input.mode == InputMode::Compiler || + m_options.input.mode == InputMode::CompilerWithASTImport || + m_options.input.mode == InputMode::CompilerWithEvmAssemblyJsonImport , "" + ); if (!m_options.compiler.outputs.irOptimized) return; @@ -231,7 +243,11 @@ void CommandLineInterface::handleIROptimized(string const& _contractName) void CommandLineInterface::handleEwasm(string const& _contractName) { - solAssert(m_options.input.mode == InputMode::Compiler || m_options.input.mode == InputMode::CompilerWithASTImport, ""); + solAssert( + m_options.input.mode == InputMode::Compiler || + m_options.input.mode == InputMode::CompilerWithASTImport || + m_options.input.mode == InputMode::CompilerWithEvmAssemblyJsonImport , "" + ); if (!m_options.compiler.outputs.ewasm) return; @@ -268,7 +284,11 @@ void CommandLineInterface::handleBytecode(string const& _contract) void CommandLineInterface::handleSignatureHashes(string const& _contract) { - solAssert(m_options.input.mode == InputMode::Compiler || m_options.input.mode == InputMode::CompilerWithASTImport, ""); + solAssert( + m_options.input.mode == InputMode::Compiler || + m_options.input.mode == InputMode::CompilerWithASTImport || + m_options.input.mode == InputMode::CompilerWithEvmAssemblyJsonImport , "" + ); if (!m_options.compiler.outputs.signatureHashes) return; @@ -286,7 +306,11 @@ void CommandLineInterface::handleSignatureHashes(string const& _contract) void CommandLineInterface::handleMetadata(string const& _contract) { - solAssert(m_options.input.mode == InputMode::Compiler || m_options.input.mode == InputMode::CompilerWithASTImport, ""); + solAssert( + m_options.input.mode == InputMode::Compiler || + m_options.input.mode == InputMode::CompilerWithASTImport || + m_options.input.mode == InputMode::CompilerWithEvmAssemblyJsonImport , "" + ); if (!m_options.compiler.outputs.metadata) return; @@ -300,7 +324,11 @@ void CommandLineInterface::handleMetadata(string const& _contract) void CommandLineInterface::handleABI(string const& _contract) { - solAssert(m_options.input.mode == InputMode::Compiler || m_options.input.mode == InputMode::CompilerWithASTImport, ""); + solAssert( + m_options.input.mode == InputMode::Compiler || + m_options.input.mode == InputMode::CompilerWithASTImport || + m_options.input.mode == InputMode::CompilerWithEvmAssemblyJsonImport , "" + ); if (!m_options.compiler.outputs.abi) return; @@ -314,7 +342,11 @@ void CommandLineInterface::handleABI(string const& _contract) void CommandLineInterface::handleStorageLayout(string const& _contract) { - solAssert(m_options.input.mode == InputMode::Compiler || m_options.input.mode == InputMode::CompilerWithASTImport, ""); + solAssert( + m_options.input.mode == InputMode::Compiler || + m_options.input.mode == InputMode::CompilerWithASTImport || + m_options.input.mode == InputMode::CompilerWithEvmAssemblyJsonImport , "" + ); if (!m_options.compiler.outputs.storageLayout) return; @@ -328,7 +360,11 @@ void CommandLineInterface::handleStorageLayout(string const& _contract) void CommandLineInterface::handleNatspec(bool _natspecDev, string const& _contract) { - solAssert(m_options.input.mode == InputMode::Compiler || m_options.input.mode == InputMode::CompilerWithASTImport, ""); + solAssert( + m_options.input.mode == InputMode::Compiler || + m_options.input.mode == InputMode::CompilerWithASTImport || + m_options.input.mode == InputMode::CompilerWithEvmAssemblyJsonImport , "" + ); bool enabled = false; std::string suffix; From f11044d9dd64e01c999bb3769fb648d12fe327bc Mon Sep 17 00:00:00 2001 From: Alexander Arlt Date: Mon, 8 Nov 2021 20:01:12 -0500 Subject: [PATCH 04/19] [libevmasm] Add support for assembly json import. --- libevmasm/Assembly.cpp | 362 ++++++++++++++++++++++++++++--------- libevmasm/Assembly.h | 21 ++- libevmasm/AssemblyItem.cpp | 12 ++ libevmasm/AssemblyItem.h | 1 + 4 files changed, 309 insertions(+), 87 deletions(-) diff --git a/libevmasm/Assembly.cpp b/libevmasm/Assembly.cpp index 12c7be23038d..15819805e242 100644 --- a/libevmasm/Assembly.cpp +++ b/libevmasm/Assembly.cpp @@ -34,7 +34,7 @@ #include #include -#include +#include #include #include @@ -243,12 +243,223 @@ string Assembly::toStringInHex(u256 _value) return hexStr.str(); } -Json::Value Assembly::assemblyJSON(map const& _sourceIndices) const +AssemblyItem Assembly::loadItemFromJSON(Json::Value const& _json) +{ + std::string name = _json["name"].isString() ? _json["name"].asString() : ""; + int begin = _json["begin"].isInt() ? _json["begin"].asInt() : -1; + int end = _json["end"].isInt() ? _json["end"].asInt() : -1; + int srcIndex = _json["source"].isInt() ? _json["source"].asInt() : -1; + std::string value = _json["value"].isString() ? _json["value"].asString() : ""; + std::string jumpType = _json["jumpType"].isString() ? _json["jumpType"].asString() : ""; + solAssert(!name.empty(), ""); + + auto updateUsedTags = [&](u256 const& data) { + auto tag = static_cast(data); + if (this->m_usedTags <= tag) + this->m_usedTags = tag + 1; + }; + + auto updateImmutables = [&](string const& _immutableName) -> h256 { + h256 hash(util::keccak256(value)); + this->m_immutables[hash] = _immutableName; + return hash; + }; + + auto updateLibraries = [&](string const& _libraryName) -> h256 { + h256 hash(util::keccak256(_libraryName)); + this->m_libraries[hash] = _libraryName; + return hash; + }; + + SourceLocation location; + location.start = begin; + location.end = end; + if (srcIndex > -1 && srcIndex < (int)sources().size()) + location.sourceName = sources()[static_cast(srcIndex)]; + + if (c_instructions.find(name) != c_instructions.end()) + { + AssemblyItem item{c_instructions.at(name), location}; + if (!value.empty()) + item.setJumpType(value); + return item; + } + else + { + u256 data; + if (name == "PUSH") + { + if (!value.empty()) + data = u256("0x" + value); + AssemblyItem item{AssemblyItemType::Push, data, location}; + if (!jumpType.empty()) + item.setJumpType(jumpType); + return item; + } + else if (name == "PUSH [ErrorTag]") + return {AssemblyItemType::PushTag, data, location}; + else if (name == "PUSH [tag]") + { + if (!value.empty()) + data = u256(value); + updateUsedTags(data); + return {AssemblyItemType::PushTag, data, location}; + } + else if (name == "PUSH [$]") + { + if (!value.empty()) + data = u256("0x" + value); + return {AssemblyItemType::PushSub, data, location}; + } + else if (name == "PUSH #[$]") + { + if (!value.empty()) + data = u256("0x" + value); + return {AssemblyItemType::PushSubSize, data, location}; + } + else if (name == "PUSHSIZE") + return {AssemblyItemType::PushProgramSize, data, location}; + else if (name == "PUSHLIB") + { + h256 hash = updateLibraries(value); + return {AssemblyItemType::PushLibraryAddress, hash, location}; + } + else if (name == "PUSHDEPLOYADDRESS") + return {AssemblyItemType::PushDeployTimeAddress, data, location}; + else if (name == "PUSHIMMUTABLE") + { + h256 hash = updateImmutables(value); + return {AssemblyItemType::PushImmutable, hash, location}; + } + else if (name == "ASSIGNIMMUTABLE") + { + h256 hash = updateImmutables(value); + return {AssemblyItemType::AssignImmutable, hash, location}; + } + else if (name == "tag") + { + if (!value.empty()) + data = u256(value); + return {AssemblyItemType::Tag, data, location}; + } + else if (name == "PUSH data") + { + if (!value.empty()) + data = u256("0x" + value); + return {AssemblyItemType::PushData, data, location}; + } + else if (name == "VERBATIM") + { + AssemblyItem item(fromHex(value), 0, 0); + item.setLocation(location); + return item; + } + else + assertThrow(false, InvalidOpcode, ""); + } +} + +vector Assembly::assemblyItemAsJSON(AssemblyItem const& _item, int _sourceIndex) const +{ + vector result; + + switch (_item.type()) + { + case Operation: + result.emplace_back(createJsonValue( + instructionInfo(_item.instruction()).name, + _sourceIndex, + _item.location().start, + _item.location().end, + _item.getJumpTypeAsString())); + break; + case Push: + result.emplace_back(createJsonValue( + "PUSH", + _sourceIndex, + _item.location().start, + _item.location().end, + toStringInHex(_item.data()), + _item.getJumpTypeAsString())); + break; + case PushTag: + if (_item.data() == 0) + result.emplace_back( + createJsonValue("PUSH [ErrorTag]", _sourceIndex, _item.location().start, _item.location().end, "")); + else + result.emplace_back(createJsonValue( + "PUSH [tag]", _sourceIndex, _item.location().start, _item.location().end, toString(_item.data()))); + break; + case PushSub: + result.emplace_back(createJsonValue( + "PUSH [$]", _sourceIndex, _item.location().start, _item.location().end, toString(h256(_item.data())))); + break; + case PushSubSize: + result.emplace_back(createJsonValue( + "PUSH #[$]", _sourceIndex, _item.location().start, _item.location().end, toString(h256(_item.data())))); + break; + case PushProgramSize: + result.emplace_back(createJsonValue("PUSHSIZE", _sourceIndex, _item.location().start, _item.location().end)); + break; + case PushLibraryAddress: + result.emplace_back(createJsonValue( + "PUSHLIB", _sourceIndex, _item.location().start, _item.location().end, m_libraries.at(h256(_item.data())))); + break; + case PushDeployTimeAddress: + result.emplace_back( + createJsonValue("PUSHDEPLOYADDRESS", _sourceIndex, _item.location().start, _item.location().end)); + break; + case PushImmutable: + result.emplace_back(createJsonValue( + "PUSHIMMUTABLE", + _sourceIndex, + _item.location().start, + _item.location().end, + m_immutables.at(h256(_item.data())))); + break; + case AssignImmutable: + result.emplace_back(createJsonValue( + "ASSIGNIMMUTABLE", + _sourceIndex, + _item.location().start, + _item.location().end, + m_immutables.at(h256(_item.data())))); + break; + case Tag: + result.emplace_back( + createJsonValue("tag", _sourceIndex, _item.location().start, _item.location().end, toString(_item.data()))); + result.emplace_back(createJsonValue("JUMPDEST", _sourceIndex, _item.location().start, _item.location().end)); + break; + case PushData: + result.emplace_back(createJsonValue( + "PUSH data", _sourceIndex, _item.location().start, _item.location().end, toStringInHex(_item.data()))); + break; + case VerbatimBytecode: + result.emplace_back(createJsonValue( + "VERBATIM", _sourceIndex, _item.location().start, _item.location().end, util::toHex(_item.verbatimData()))); + break; + default: + assertThrow(false, InvalidOpcode, ""); + } + return result; +} + +Json::Value Assembly::assemblyJSON(map const& _sourceIndices, bool _includeSourceList) const { Json::Value root; - root[".code"] = Json::arrayValue; + if (_includeSourceList) + { + root["sourceList"] = Json::arrayValue; + Json::Value& sourceList = root["sourceList"]; + std::vector sources(_sourceIndices.size()); + for (auto const& item: _sourceIndices) + sources[item.second] = item.first; + for (auto const& item: sources) + sourceList.append(item); + } - Json::Value& collection = root[".code"]; + root[".code"] = Json::arrayValue; + Json::Value& code = root[".code"]; for (AssemblyItem const& i: m_items) { int sourceIndex = -1; @@ -259,85 +470,8 @@ Json::Value Assembly::assemblyJSON(map const& _sourceIndices) sourceIndex = static_cast(iter->second); } - switch (i.type()) - { - case Operation: - collection.append( - createJsonValue( - instructionInfo(i.instruction()).name, - sourceIndex, - i.location().start, - i.location().end, - i.getJumpTypeAsString()) - ); - break; - case Push: - collection.append( - createJsonValue("PUSH", sourceIndex, i.location().start, i.location().end, toStringInHex(i.data()), i.getJumpTypeAsString())); - break; - case PushTag: - if (i.data() == 0) - collection.append( - createJsonValue("PUSH [ErrorTag]", sourceIndex, i.location().start, i.location().end, "")); - else - collection.append( - createJsonValue("PUSH [tag]", sourceIndex, i.location().start, i.location().end, toString(i.data()))); - break; - case PushSub: - collection.append( - createJsonValue("PUSH [$]", sourceIndex, i.location().start, i.location().end, toString(h256(i.data())))); - break; - case PushSubSize: - collection.append( - createJsonValue("PUSH #[$]", sourceIndex, i.location().start, i.location().end, toString(h256(i.data())))); - break; - case PushProgramSize: - collection.append( - createJsonValue("PUSHSIZE", sourceIndex, i.location().start, i.location().end)); - break; - case PushLibraryAddress: - collection.append( - createJsonValue("PUSHLIB", sourceIndex, i.location().start, i.location().end, m_libraries.at(h256(i.data()))) - ); - break; - case PushDeployTimeAddress: - collection.append( - createJsonValue("PUSHDEPLOYADDRESS", sourceIndex, i.location().start, i.location().end) - ); - break; - case PushImmutable: - collection.append(createJsonValue( - "PUSHIMMUTABLE", - sourceIndex, - i.location().start, - i.location().end, - m_immutables.at(h256(i.data())) - )); - break; - case AssignImmutable: - collection.append(createJsonValue( - "ASSIGNIMMUTABLE", - sourceIndex, - i.location().start, - i.location().end, - m_immutables.at(h256(i.data())) - )); - break; - case Tag: - collection.append( - createJsonValue("tag", sourceIndex, i.location().start, i.location().end, toString(i.data()))); - collection.append( - createJsonValue("JUMPDEST", sourceIndex, i.location().start, i.location().end)); - break; - case PushData: - collection.append(createJsonValue("PUSH data", sourceIndex, i.location().start, i.location().end, toStringInHex(i.data()))); - break; - case VerbatimBytecode: - collection.append(createJsonValue("VERBATIM", sourceIndex, i.location().start, i.location().end, util::toHex(i.verbatimData()))); - break; - default: - assertThrow(false, InvalidOpcode, ""); - } + for (Json::Value const& item: assemblyItemAsJSON(i, sourceIndex)) + code.append(item); } if (!m_data.empty() || !m_subs.empty()) @@ -352,16 +486,73 @@ Json::Value Assembly::assemblyJSON(map const& _sourceIndices) { std::stringstream hexStr; hexStr << hex << i; - data[hexStr.str()] = m_subs[i]->assemblyJSON(_sourceIndices); + data[hexStr.str()] = m_subs[i]->assemblyJSON(_sourceIndices, false); } } - if (m_auxiliaryData.size() > 0) + if (!m_auxiliaryData.empty()) root[".auxdata"] = util::toHex(m_auxiliaryData); return root; } +bool Assembly::addAssemblyItemsFromJSON(Json::Value const& _code) +{ + solAssert(_code.isArray(), ""); + for (auto const& it: _code) + this->m_items.emplace_back(loadItemFromJSON(it)); + + for (auto current = this->m_items.begin(); current != this->m_items.end(); ++current) + { + // During the assembly json export a `JUMPDEST` is always generated after a `tag`. + // So we just ignore exactly these `JUMPDEST`'s. + auto const next = std::next(current); + if ( + current->type() == AssemblyItemType::Tag && + next->type() == AssemblyItemType::Operation && + next->instruction() == Instruction::JUMPDEST + ) + this->m_items.erase(next); + } + + return true; +} + +bool Assembly::loadFromAssemblyJSON(Json::Value const& _json) +{ + if (!_json[".code"].isArray()) + return false; + bool result{true}; + if (_json.isMember("sourceList")) + { + vector sourceList; + for (auto const& it: _json["sourceList"]) + sourceList.emplace_back(it.asString()); + this->setSources(sourceList); + } + + addAssemblyItemsFromJSON(_json[".code"]); + if (_json[".auxdata"].isString()) + this->m_auxiliaryData = fromHex(_json[".auxdata"].asString()); + Json::Value const& data = _json[".data"]; + for (Json::ValueConstIterator itr = data.begin(); itr != data.end(); itr++) + { + solAssert(itr.key().isString(), ""); + std::string key = itr.key().asString(); + Json::Value const& code = data[key]; + if (code.isString()) + this->m_data[h256(fromHex(key))] = fromHex(code.asString()); + else + { + shared_ptr subassembly = make_shared(); + subassembly->setSources(this->sources()); + result &= subassembly->loadFromAssemblyJSON(code); + this->m_subs.emplace_back(subassembly); + } + } + return result; +} + AssemblyItem Assembly::namedTag(string const& _name, size_t _params, size_t _returns, optional _sourceID) { assertThrow(!_name.empty(), AssemblyException, "Empty named tag."); @@ -416,7 +607,6 @@ Assembly& Assembly::optimise(bool _enable, EVMVersion _evmVersion, bool _isCreat return *this; } - Assembly& Assembly::optimise(OptimiserSettings const& _settings) { optimiseInternal(_settings, {}); diff --git a/libevmasm/Assembly.h b/libevmasm/Assembly.h index f768ffe31414..14fdc98c4bb5 100644 --- a/libevmasm/Assembly.h +++ b/libevmasm/Assembly.h @@ -155,15 +155,28 @@ class Assembly /// Create a JSON representation of the assembly. Json::Value assemblyJSON( - std::map const& _sourceIndices = std::map() + std::map const& _sourceIndices = std::map(), + bool _includeSourceList = true ) const; + bool loadFromAssemblyJSON(Json::Value const& _json); + /// Mark this assembly as invalid. Calling ``assemble`` on it will throw. void markAsInvalid() { m_invalid = true; } std::vector decodeSubPath(size_t _subObjectId) const; size_t encodeSubPath(std::vector const& _subPath); + void setSources(std::vector> _sources) { + m_sources = _sources; + } + + void setSources(std::vector const& _sources) { + for (auto const& item: _sources) + m_sources.emplace_back(std::make_shared(item)); + } + std::vector> sources() const& { return m_sources; } + protected: /// Does the same operations as @a optimise, but should only be applied to a sub and /// returns the replaced tags. Also takes an argument containing the tags of this assembly @@ -172,7 +185,11 @@ class Assembly unsigned codeSize(unsigned subTagSize) const; + AssemblyItem loadItemFromJSON(Json::Value const& _json); + std::vector assemblyItemAsJSON(AssemblyItem const& _item, int _sourceIndex) const; + private: + bool addAssemblyItemsFromJSON(Json::Value const& _code); static Json::Value createJsonValue( std::string _name, int _source, @@ -226,6 +243,8 @@ class Assembly std::string m_name; langutil::SourceLocation m_currentSourceLocation; + std::vector> m_sources; + public: size_t m_currentModifierDepth = 0; }; diff --git a/libevmasm/AssemblyItem.cpp b/libevmasm/AssemblyItem.cpp index eeeadb3ef3ea..2cb4eaa1eaf5 100644 --- a/libevmasm/AssemblyItem.cpp +++ b/libevmasm/AssemblyItem.cpp @@ -192,6 +192,18 @@ string AssemblyItem::getJumpTypeAsString() const } } +void AssemblyItem::setJumpType(std::string const& _jumpType) +{ + if (_jumpType == "[in]") + m_jumpType = JumpType::IntoFunction; + else if (_jumpType == "[out]") + m_jumpType = JumpType::OutOfFunction; + else if (_jumpType.empty()) + m_jumpType = JumpType::Ordinary; + else + assertThrow(false, AssemblyException, "Invalid jump type."); +} + string AssemblyItem::toAssemblyText(Assembly const& _assembly) const { string text; diff --git a/libevmasm/AssemblyItem.h b/libevmasm/AssemblyItem.h index 1cbfe768d6ed..652a83208501 100644 --- a/libevmasm/AssemblyItem.h +++ b/libevmasm/AssemblyItem.h @@ -165,6 +165,7 @@ class AssemblyItem langutil::SourceLocation const& location() const { return m_location; } void setJumpType(JumpType _jumpType) { m_jumpType = _jumpType; } + void setJumpType(std::string const& _jumpType); JumpType getJumpType() const { return m_jumpType; } std::string getJumpTypeAsString() const; From 4473b3ca2d7b6c6ef9ce4897434a4eb1637b20fc Mon Sep 17 00:00:00 2001 From: Alexander Arlt Date: Mon, 8 Nov 2021 20:02:05 -0500 Subject: [PATCH 05/19] [libsolidity] Basic output generation of assembly json import. --- libsolidity/interface/CompilerStack.cpp | 105 ++++++++++++++---------- libsolidity/interface/CompilerStack.h | 2 +- 2 files changed, 61 insertions(+), 46 deletions(-) diff --git a/libsolidity/interface/CompilerStack.cpp b/libsolidity/interface/CompilerStack.cpp index 03f03bf7d2da..c890dcf2f707 100644 --- a/libsolidity/interface/CompilerStack.cpp +++ b/libsolidity/interface/CompilerStack.cpp @@ -394,7 +394,7 @@ void CompilerStack::importEvmAssemblyJson(map const& _sourc if (m_stackState != Empty) solThrow(CompilerError, "Must call importEvmAssemblyJson only before the SourcesSet state."); - m_evmAssemblyJson = std::make_unique(_sources.begin()->second); + m_evmAssemblyJson[_sources.begin()->first] = _sources.begin()->second; m_importedSources = true; m_stackState = SourcesSet; } @@ -654,58 +654,73 @@ bool CompilerStack::compile(State _stopAfter) if (m_hasError) solThrow(CompilerError, "Called compile with errors."); - // Only compile contracts individually which have been requested. - map> otherCompilers; + if (!m_evmAssemblyJson.empty()) + { + solAssert(m_importedSources, ""); + solAssert(m_evmAssemblyJson.size() == 1, ""); - for (Source const* source: m_sourceOrder) - for (ASTPointer const& node: source->ast->nodes()) - if (auto contract = dynamic_cast(node.get())) - if (isRequestedContract(*contract)) - { - try + string const evmAssemblyJsonSource = m_evmAssemblyJson.begin()->first; + + m_contracts[evmAssemblyJsonSource].evmAssembly = make_shared(evmAssemblyJsonSource); + m_contracts[evmAssemblyJsonSource].evmAssembly->loadFromAssemblyJSON(m_evmAssemblyJson[evmAssemblyJsonSource]); + m_contracts[evmAssemblyJsonSource].object = m_contracts[evmAssemblyJsonSource].evmAssembly->assemble(); + + m_contracts[evmAssemblyJsonSource].evmRuntimeAssembly = make_shared(evmAssemblyJsonSource); + m_contracts[evmAssemblyJsonSource].evmRuntimeAssembly->loadFromAssemblyJSON(m_evmAssemblyJson[evmAssemblyJsonSource][".data"]["0"]); + m_contracts[evmAssemblyJsonSource].runtimeObject = m_contracts[evmAssemblyJsonSource].evmRuntimeAssembly->assemble(); + } + else + { + // Only compile contracts individually which have been requested. + map> otherCompilers; + + for (Source const* source: m_sourceOrder) + for (ASTPointer const& node: source->ast->nodes()) + if (auto contract = dynamic_cast(node.get())) + if (isRequestedContract(*contract)) { - if (m_viaIR || m_generateIR || m_generateEwasm) - generateIR(*contract); - if (m_generateEvmBytecode) + try { - if (m_viaIR) - generateEVMFromIR(*contract); - else - compileContract(*contract, otherCompilers); + if (m_viaIR || m_generateIR || m_generateEwasm) + generateIR(*contract); + if (m_generateEvmBytecode) + { + if (m_viaIR) + generateEVMFromIR(*contract); + else + compileContract(*contract, otherCompilers); + } + if (m_generateEwasm) + generateEwasm(*contract); } - if (m_generateEwasm) - generateEwasm(*contract); - } - catch (Error const& _error) - { - if (_error.type() != Error::Type::CodeGenerationError) - throw; - m_errorReporter.error(_error.errorId(), _error.type(), SourceLocation(), _error.what()); - return false; - } - catch (UnimplementedFeatureError const& _unimplementedError) - { - if ( - SourceLocation const* sourceLocation = - boost::get_error_info(_unimplementedError) - ) + catch (Error const& _error) { - string const* comment = _unimplementedError.comment(); - m_errorReporter.error( - 1834_error, - Error::Type::CodeGenerationError, - *sourceLocation, - "Unimplemented feature error" + - ((comment && !comment->empty()) ? ": " + *comment : string{}) + - " in " + - _unimplementedError.lineInfo() - ); + if (_error.type() != Error::Type::CodeGenerationError) + throw; + m_errorReporter.error(_error.errorId(), _error.type(), SourceLocation(), _error.what()); return false; } - else - throw; + catch (UnimplementedFeatureError const& _unimplementedError) + { + if (SourceLocation const* sourceLocation + = boost::get_error_info(_unimplementedError)) + { + string const* comment = _unimplementedError.comment(); + m_errorReporter.error( + 1834_error, + Error::Type::CodeGenerationError, + *sourceLocation, + "Unimplemented feature error" + + ((comment && !comment->empty()) ? ": " + *comment : string{}) + " in " + + _unimplementedError.lineInfo()); + return false; + } + else + throw; + } } - } + } + m_stackState = CompilationSuccessful; this->link(); return true; diff --git a/libsolidity/interface/CompilerStack.h b/libsolidity/interface/CompilerStack.h index 85f527862fc4..9e25797e503c 100644 --- a/libsolidity/interface/CompilerStack.h +++ b/libsolidity/interface/CompilerStack.h @@ -501,7 +501,7 @@ class CompilerStack: public langutil::CharStreamProvider std::map m_sources; // if imported, store AST-JSONS for each filename std::map m_sourceJsons; - std::unique_ptr m_evmAssemblyJson; + std::map m_evmAssemblyJson; std::vector m_unhandledSMTLib2Queries; std::map m_smtlib2Responses; std::shared_ptr m_globalContext; From 4f2056ff8fbb20201fd70c40cf68e4e43b856d8a Mon Sep 17 00:00:00 2001 From: Alexander Arlt Date: Mon, 8 Nov 2021 20:05:41 -0500 Subject: [PATCH 06/19] [libsolidity] Add basic optimizer support for assembly json import. --- libsolidity/interface/CompilerStack.cpp | 12 ++++++++++++ solc/CommandLineParser.cpp | 15 +++++++++------ 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/libsolidity/interface/CompilerStack.cpp b/libsolidity/interface/CompilerStack.cpp index c890dcf2f707..0e3017d675e6 100644 --- a/libsolidity/interface/CompilerStack.cpp +++ b/libsolidity/interface/CompilerStack.cpp @@ -661,12 +661,24 @@ bool CompilerStack::compile(State _stopAfter) string const evmAssemblyJsonSource = m_evmAssemblyJson.begin()->first; + evmasm::Assembly::OptimiserSettings optimiserSettings; + optimiserSettings.evmVersion = m_evmVersion; + optimiserSettings.expectedExecutionsPerDeployment = m_optimiserSettings.expectedExecutionsPerDeployment; + optimiserSettings.runCSE = m_optimiserSettings.runCSE; + optimiserSettings.runConstantOptimiser = m_optimiserSettings.runConstantOptimiser; + optimiserSettings.runDeduplicate = m_optimiserSettings.runDeduplicate; + optimiserSettings.runInliner = m_optimiserSettings.runInliner; + optimiserSettings.runJumpdestRemover = m_optimiserSettings.runJumpdestRemover; + optimiserSettings.runPeephole = m_optimiserSettings.runPeephole; + m_contracts[evmAssemblyJsonSource].evmAssembly = make_shared(evmAssemblyJsonSource); m_contracts[evmAssemblyJsonSource].evmAssembly->loadFromAssemblyJSON(m_evmAssemblyJson[evmAssemblyJsonSource]); + m_contracts[evmAssemblyJsonSource].evmAssembly->optimise(optimiserSettings); m_contracts[evmAssemblyJsonSource].object = m_contracts[evmAssemblyJsonSource].evmAssembly->assemble(); m_contracts[evmAssemblyJsonSource].evmRuntimeAssembly = make_shared(evmAssemblyJsonSource); m_contracts[evmAssemblyJsonSource].evmRuntimeAssembly->loadFromAssemblyJSON(m_evmAssemblyJson[evmAssemblyJsonSource][".data"]["0"]); + m_contracts[evmAssemblyJsonSource].evmRuntimeAssembly->optimise(optimiserSettings); m_contracts[evmAssemblyJsonSource].runtimeObject = m_contracts[evmAssemblyJsonSource].evmRuntimeAssembly->assemble(); } else diff --git a/solc/CommandLineParser.cpp b/solc/CommandLineParser.cpp index acfc09c8765f..6cf664c891c3 100644 --- a/solc/CommandLineParser.cpp +++ b/solc/CommandLineParser.cpp @@ -1218,12 +1218,15 @@ bool CommandLineParser::processArgs() } if (m_options.optimizer.enabled && (m_options.assembly.inputLanguage != Input::StrictAssembly && m_options.assembly.inputLanguage != Input::Ewasm)) { - serr() << - "Optimizer can only be used for strict assembly. Use --" << - g_strStrictAssembly << - "." << - endl; - return false; + if (m_options.input.mode != InputMode::CompilerWithEvmAssemblyJsonImport) + { + serr() << + "Optimizer can only be used for strict assembly or with assembly import. Use --" << + g_strStrictAssembly << + " or --" << g_strImportEvmAssemblerJson << "." << + endl; + return false; + } } if (m_options.assembly.targetMachine == Machine::Ewasm && m_options.assembly.inputLanguage != Input::StrictAssembly && m_options.assembly.inputLanguage != Input::Ewasm) { From 200684a3b9356ea40da3dc330537e7ad7331c1fd Mon Sep 17 00:00:00 2001 From: Alexander Arlt Date: Tue, 9 Nov 2021 13:35:02 -0500 Subject: [PATCH 07/19] [libsolidity] Preserve sourceList order, if assembly json import is used. --- libsolidity/interface/CompilerStack.cpp | 39 ++++++++++++++++++++----- 1 file changed, 31 insertions(+), 8 deletions(-) diff --git a/libsolidity/interface/CompilerStack.cpp b/libsolidity/interface/CompilerStack.cpp index 0e3017d675e6..dfed2bb75bea 100644 --- a/libsolidity/interface/CompilerStack.cpp +++ b/libsolidity/interface/CompilerStack.cpp @@ -391,10 +391,23 @@ bool CompilerStack::parse() void CompilerStack::importEvmAssemblyJson(map const& _sources) { solAssert(_sources.size() == 1, ""); + solAssert(m_sources.empty(), ""); + solAssert(m_sourceOrder.empty(), ""); if (m_stackState != Empty) solThrow(CompilerError, "Must call importEvmAssemblyJson only before the SourcesSet state."); - m_evmAssemblyJson[_sources.begin()->first] = _sources.begin()->second; + Json::Value jsonValue = _sources.begin()->second; + if (jsonValue.isMember("sourceList")) + { + for (auto const& item: jsonValue["sourceList"]) + { + Source source; + source.charStream = std::make_shared(item.asString(), ""); + m_sources.emplace(std::make_pair(item.asString(), source)); + m_sourceOrder.push_back(&m_sources[item.asString()]); + } + } + m_evmAssemblyJson[_sources.begin()->first] = jsonValue; m_importedSources = true; m_stackState = SourcesSet; } @@ -608,6 +621,9 @@ bool CompilerStack::parseAndAnalyze(State _stopAfter) { m_stopAfter = _stopAfter; + if (!m_evmAssemblyJson.empty()) + return true; + bool success = parse(); if (m_stackState >= m_stopAfter) return success; @@ -962,9 +978,10 @@ Json::Value CompilerStack::assemblyJSON(string const& _contractName) const vector CompilerStack::sourceNames() const { - vector names; - for (auto const& s: m_sources) - names.push_back(s.first); + map indices = sourceIndices(); + vector names(indices.size()); + for (auto const& s: indices) + names[s.second] = s.first; return names; } @@ -972,10 +989,16 @@ map CompilerStack::sourceIndices() const { map indices; unsigned index = 0; - for (auto const& s: m_sources) - indices[s.first] = index++; - solAssert(!indices.count(CompilerContext::yulUtilityFileName()), ""); - indices[CompilerContext::yulUtilityFileName()] = index++; + if (m_evmAssemblyJson.empty()) + { + for (auto const& s: m_sources) + indices[s.first] = index++; + solAssert(!indices.count(CompilerContext::yulUtilityFileName()), ""); + indices[CompilerContext::yulUtilityFileName()] = index++; + } + else + for (auto const& s: m_sourceOrder) + indices[s->charStream->source()] = index++; return indices; } From ef150b7fa683e8b639ce4f6509e8208998640331 Mon Sep 17 00:00:00 2001 From: Alexander Arlt Date: Tue, 9 Nov 2021 16:04:36 -0500 Subject: [PATCH 08/19] [libevmasm] Add more tests. --- test/libevmasm/Assembler.cpp | 59 ++++++++++++++++++++++++++++++------ 1 file changed, 50 insertions(+), 9 deletions(-) diff --git a/test/libevmasm/Assembler.cpp b/test/libevmasm/Assembler.cpp index c7f28f654cd9..4a974ac27b54 100644 --- a/test/libevmasm/Assembler.cpp +++ b/test/libevmasm/Assembler.cpp @@ -55,7 +55,8 @@ BOOST_AUTO_TEST_CASE(all_assembly_items) { map indices = { { "root.asm", 0 }, - { "sub.asm", 1 } + { "sub.asm", 1 }, + { "verbatim.asm", 2 } }; Assembly _assembly; auto root_asm = make_shared("root.asm"); @@ -64,11 +65,22 @@ BOOST_AUTO_TEST_CASE(all_assembly_items) Assembly _subAsm; auto sub_asm = make_shared("sub.asm"); _subAsm.setSourceLocation({6, 8, sub_asm}); + + Assembly _verbatimAsm; + auto verbatim_asm = make_shared("verbatim.asm"); + _verbatimAsm.setSourceLocation({8, 18, verbatim_asm}); + // PushImmutable _subAsm.appendImmutable("someImmutable"); + _subAsm.append(AssemblyItem(PushTag, 0)); _subAsm.append(Instruction::INVALID); shared_ptr _subAsmPtr = make_shared(_subAsm); + _verbatimAsm.appendVerbatim({0xff,0xff}, 0, 0); + _verbatimAsm.appendVerbatim({0x74, 0x65, 0x73, 0x74}, 0, 1); + _verbatimAsm.append(Instruction::MSTORE); + shared_ptr _verbatimAsmPtr = make_shared(_verbatimAsm); + // Tag auto tag = _assembly.newTag(); _assembly.append(tag); @@ -89,6 +101,10 @@ BOOST_AUTO_TEST_CASE(all_assembly_items) auto sub = _assembly.appendSubroutine(_subAsmPtr); // PushSub _assembly.pushSubroutineOffset(static_cast(sub.data())); + // PushSubSize + auto verbatim_sub = _assembly.appendSubroutine(_verbatimAsmPtr); + // PushSub + _assembly.pushSubroutineOffset(static_cast(verbatim_sub.data())); // PushDeployTimeAddress _assembly.append(PushDeployTimeAddress); // AssignImmutable. @@ -105,12 +121,12 @@ BOOST_AUTO_TEST_CASE(all_assembly_items) BOOST_CHECK_EQUAL( _assembly.assemble().toHex(), - "5b6001600220606f73__$bf005014d9d0f534b8fcb268bd84c491a2$__" - "6000566067602260457300000000000000000000000000000000000000005050" + "5b6001600220607c73__$bf005014d9d0f534b8fcb268bd84c491a2$__" + "6000566074602460496007606d7300000000000000000000000000000000000000005050" "600260010152" "00fe" "7f0000000000000000000000000000000000000000000000000000000000000000" - "fe010203044266eeaa" + "6000feffff7465737452010203044266eeaa" ); BOOST_CHECK_EQUAL( _assembly.assemblyString(), @@ -123,6 +139,8 @@ BOOST_AUTO_TEST_CASE(all_assembly_items) " data_a6885b3731702da62e8e4a8f584ac46a7f6822f4e2ba50fba902f67b1588d23b\n" " dataSize(sub_0)\n" " dataOffset(sub_0)\n" + " dataSize(sub_1)\n" + " dataOffset(sub_1)\n" " deployTimeAddress()\n" " assignImmutable(\"0xc3978657661c4d8e32e3d5f42597c009f0d3859e9f9d0d94325268f9799e2bfb\")\n" " 0x02\n" @@ -134,13 +152,20 @@ BOOST_AUTO_TEST_CASE(all_assembly_items) "sub_0: assembly {\n" " /* \"sub.asm\":6:8 */\n" " immutable(\"0x26f2c0195e9d408feff3abd77d83f2971f3c9a18d1e8a9437c7835ae4211fc9f\")\n" + " tag_0\n" " invalid\n" "}\n" "\n" + "sub_1: assembly {\n" + " /* \"verbatim.asm\":8:18 */\n" + " verbatimbytecode_ffff\n" + " verbatimbytecode_74657374\n" + " mstore\n" + "}\n" + "\n" "auxdata: 0x4266eeaa\n" ); - BOOST_CHECK_EQUAL( - util::jsonCompactPrint(_assembly.assemblyJSON(indices)), + string json{ "{\".auxdata\":\"4266eeaa\",\".code\":[" "{\"begin\":1,\"end\":3,\"name\":\"tag\",\"source\":0,\"value\":\"1\"}," "{\"begin\":1,\"end\":3,\"name\":\"JUMPDEST\",\"source\":0}," @@ -154,6 +179,8 @@ BOOST_AUTO_TEST_CASE(all_assembly_items) "{\"begin\":1,\"end\":3,\"name\":\"PUSH data\",\"source\":0,\"value\":\"A6885B3731702DA62E8E4A8F584AC46A7F6822F4E2BA50FBA902F67B1588D23B\"}," "{\"begin\":1,\"end\":3,\"name\":\"PUSH #[$]\",\"source\":0,\"value\":\"0000000000000000000000000000000000000000000000000000000000000000\"}," "{\"begin\":1,\"end\":3,\"name\":\"PUSH [$]\",\"source\":0,\"value\":\"0000000000000000000000000000000000000000000000000000000000000000\"}," + "{\"begin\":1,\"end\":3,\"name\":\"PUSH #[$]\",\"source\":0,\"value\":\"0000000000000000000000000000000000000000000000000000000000000001\"}," + "{\"begin\":1,\"end\":3,\"name\":\"PUSH [$]\",\"source\":0,\"value\":\"0000000000000000000000000000000000000000000000000000000000000001\"}," "{\"begin\":1,\"end\":3,\"name\":\"PUSHDEPLOYADDRESS\",\"source\":0}," "{\"begin\":1,\"end\":3,\"name\":\"ASSIGNIMMUTABLE\",\"source\":0,\"value\":\"someOtherImmutable\"}," "{\"begin\":1,\"end\":3,\"name\":\"PUSH\",\"source\":0,\"value\":\"2\"}," @@ -161,9 +188,23 @@ BOOST_AUTO_TEST_CASE(all_assembly_items) "{\"begin\":1,\"end\":3,\"name\":\"STOP\",\"source\":0}" "],\".data\":{\"0\":{\".code\":[" "{\"begin\":6,\"end\":8,\"name\":\"PUSHIMMUTABLE\",\"source\":1,\"value\":\"someImmutable\"}," + "{\"begin\":6,\"end\":8,\"name\":\"PUSH [ErrorTag]\",\"source\":1}," "{\"begin\":6,\"end\":8,\"name\":\"INVALID\",\"source\":1}" - "]},\"A6885B3731702DA62E8E4A8F584AC46A7F6822F4E2BA50FBA902F67B1588D23B\":\"01020304\"}}" - ); + "]}," + "\"1\":{\".code\":[" + "{\"begin\":8,\"end\":18,\"name\":\"VERBATIM\",\"source\":2,\"value\":\"ffff\"}," + "{\"begin\":8,\"end\":18,\"name\":\"VERBATIM\",\"source\":2,\"value\":\"74657374\"}," + "{\"begin\":8,\"end\":18,\"name\":\"MSTORE\",\"source\":2}" + "]},\"A6885B3731702DA62E8E4A8F584AC46A7F6822F4E2BA50FBA902F67B1588D23B\":\"01020304\"},\"sourceList\":[\"root.asm\",\"sub.asm\",\"verbatim.asm\"]}" + }; + Json::Value jsonValue; + BOOST_CHECK(util::jsonParseStrict(json, jsonValue)); + BOOST_CHECK_EQUAL(util::jsonCompactPrint(_assembly.assemblyJSON(indices)), util::jsonCompactPrint(jsonValue)); + + Assembly _assemblyFromJson; + _assemblyFromJson.loadFromAssemblyJSON(_assembly.assemblyJSON(indices)); + BOOST_CHECK_EQUAL(util::jsonCompactPrint(_assemblyFromJson.assemblyJSON(indices)), util::jsonCompactPrint(jsonValue)); + BOOST_CHECK_EQUAL(_assembly.assemble().toHex(), _assemblyFromJson.assemble().toHex()); } BOOST_AUTO_TEST_CASE(immutables_and_its_source_maps) @@ -342,7 +383,7 @@ BOOST_AUTO_TEST_CASE(immutable) "{\"begin\":6,\"end\":8,\"name\":\"PUSHIMMUTABLE\",\"source\":1,\"value\":\"someImmutable\"}," "{\"begin\":6,\"end\":8,\"name\":\"PUSHIMMUTABLE\",\"source\":1,\"value\":\"someOtherImmutable\"}," "{\"begin\":6,\"end\":8,\"name\":\"PUSHIMMUTABLE\",\"source\":1,\"value\":\"someImmutable\"}" - "]}}}" + "]}},\"sourceList\":[\"root.asm\",\"sub.asm\"]}" ); } From 8608bc3b8c71cadf49669d120d92ae6b180cc935 Mon Sep 17 00:00:00 2001 From: Alexander Arlt Date: Tue, 9 Nov 2021 17:06:48 -0500 Subject: [PATCH 09/19] [libsolidity] sourceIndices: selectable internal sources. --- libsolidity/interface/CompilerStack.cpp | 7 ++++--- libsolidity/interface/CompilerStack.h | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/libsolidity/interface/CompilerStack.cpp b/libsolidity/interface/CompilerStack.cpp index dfed2bb75bea..c77a99e26f27 100644 --- a/libsolidity/interface/CompilerStack.cpp +++ b/libsolidity/interface/CompilerStack.cpp @@ -978,14 +978,14 @@ Json::Value CompilerStack::assemblyJSON(string const& _contractName) const vector CompilerStack::sourceNames() const { - map indices = sourceIndices(); + map indices = sourceIndices(false); vector names(indices.size()); for (auto const& s: indices) names[s.second] = s.first; return names; } -map CompilerStack::sourceIndices() const +map CompilerStack::sourceIndices(bool _includeInternalSources /* = true */) const { map indices; unsigned index = 0; @@ -994,7 +994,8 @@ map CompilerStack::sourceIndices() const for (auto const& s: m_sources) indices[s.first] = index++; solAssert(!indices.count(CompilerContext::yulUtilityFileName()), ""); - indices[CompilerContext::yulUtilityFileName()] = index++; + if (_includeInternalSources) + indices[CompilerContext::yulUtilityFileName()] = index++; } else for (auto const& s: m_sourceOrder) diff --git a/libsolidity/interface/CompilerStack.h b/libsolidity/interface/CompilerStack.h index 9e25797e503c..26be9da83aaa 100644 --- a/libsolidity/interface/CompilerStack.h +++ b/libsolidity/interface/CompilerStack.h @@ -242,7 +242,7 @@ class CompilerStack: public langutil::CharStreamProvider /// @returns a mapping assigning each source name its index inside the vector returned /// by sourceNames(). - std::map sourceIndices() const; + std::map sourceIndices(bool _includeInternalSources = true) const; /// @returns the previously used character stream, useful for counting lines during error reporting. langutil::CharStream const& charStream(std::string const& _sourceName) const override; From c8222636469a626e37fc2c9b603ceb34e2c7adf2 Mon Sep 17 00:00:00 2001 From: Alexander Arlt Date: Tue, 9 Nov 2021 18:02:51 -0500 Subject: [PATCH 10/19] [libsolidity] srcmap sourceList handling corrections. --- libevmasm/Assembly.cpp | 8 -------- libsolidity/interface/CompilerStack.cpp | 9 +++++++-- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/libevmasm/Assembly.cpp b/libevmasm/Assembly.cpp index 15819805e242..6b0c73fcbbdc 100644 --- a/libevmasm/Assembly.cpp +++ b/libevmasm/Assembly.cpp @@ -523,14 +523,6 @@ bool Assembly::loadFromAssemblyJSON(Json::Value const& _json) if (!_json[".code"].isArray()) return false; bool result{true}; - if (_json.isMember("sourceList")) - { - vector sourceList; - for (auto const& it: _json["sourceList"]) - sourceList.emplace_back(it.asString()); - this->setSources(sourceList); - } - addAssemblyItemsFromJSON(_json[".code"]); if (_json[".auxdata"].isString()) this->m_auxiliaryData = fromHex(_json[".auxdata"].asString()); diff --git a/libsolidity/interface/CompilerStack.cpp b/libsolidity/interface/CompilerStack.cpp index c77a99e26f27..cd8a91472003 100644 --- a/libsolidity/interface/CompilerStack.cpp +++ b/libsolidity/interface/CompilerStack.cpp @@ -398,7 +398,6 @@ void CompilerStack::importEvmAssemblyJson(map const& _sourc Json::Value jsonValue = _sources.begin()->second; if (jsonValue.isMember("sourceList")) - { for (auto const& item: jsonValue["sourceList"]) { Source source; @@ -406,7 +405,6 @@ void CompilerStack::importEvmAssemblyJson(map const& _sourc m_sources.emplace(std::make_pair(item.asString(), source)); m_sourceOrder.push_back(&m_sources[item.asString()]); } - } m_evmAssemblyJson[_sources.begin()->first] = jsonValue; m_importedSources = true; m_stackState = SourcesSet; @@ -687,12 +685,19 @@ bool CompilerStack::compile(State _stopAfter) optimiserSettings.runJumpdestRemover = m_optimiserSettings.runJumpdestRemover; optimiserSettings.runPeephole = m_optimiserSettings.runPeephole; + vector sourceList; + if (m_evmAssemblyJson[evmAssemblyJsonSource].isMember("sourceList")) + for (auto const& it: m_evmAssemblyJson[evmAssemblyJsonSource]["sourceList"]) + sourceList.emplace_back(it.asString()); + m_contracts[evmAssemblyJsonSource].evmAssembly = make_shared(evmAssemblyJsonSource); + m_contracts[evmAssemblyJsonSource].evmAssembly->setSources(sourceList); m_contracts[evmAssemblyJsonSource].evmAssembly->loadFromAssemblyJSON(m_evmAssemblyJson[evmAssemblyJsonSource]); m_contracts[evmAssemblyJsonSource].evmAssembly->optimise(optimiserSettings); m_contracts[evmAssemblyJsonSource].object = m_contracts[evmAssemblyJsonSource].evmAssembly->assemble(); m_contracts[evmAssemblyJsonSource].evmRuntimeAssembly = make_shared(evmAssemblyJsonSource); + m_contracts[evmAssemblyJsonSource].evmRuntimeAssembly->setSources(sourceList); m_contracts[evmAssemblyJsonSource].evmRuntimeAssembly->loadFromAssemblyJSON(m_evmAssemblyJson[evmAssemblyJsonSource][".data"]["0"]); m_contracts[evmAssemblyJsonSource].evmRuntimeAssembly->optimise(optimiserSettings); m_contracts[evmAssemblyJsonSource].runtimeObject = m_contracts[evmAssemblyJsonSource].evmRuntimeAssembly->assemble(); From dcb697f6d4f0c61078a3ffaea159ecccea63a519 Mon Sep 17 00:00:00 2001 From: Alexander Arlt Date: Tue, 9 Nov 2021 18:54:53 -0500 Subject: [PATCH 11/19] [libevmasm] loadFromAssemblyJSON: optionally load from sourceList. --- libevmasm/Assembly.cpp | 14 ++++++++++++-- libevmasm/Assembly.h | 5 +++-- libsolidity/interface/CompilerStack.cpp | 10 ++-------- 3 files changed, 17 insertions(+), 12 deletions(-) diff --git a/libevmasm/Assembly.cpp b/libevmasm/Assembly.cpp index 6b0c73fcbbdc..dbb56b0ad6fd 100644 --- a/libevmasm/Assembly.cpp +++ b/libevmasm/Assembly.cpp @@ -518,11 +518,21 @@ bool Assembly::addAssemblyItemsFromJSON(Json::Value const& _code) return true; } -bool Assembly::loadFromAssemblyJSON(Json::Value const& _json) +bool Assembly::loadFromAssemblyJSON(Json::Value const& _json, bool _loadSources /* = true */) { if (!_json[".code"].isArray()) return false; bool result{true}; + + if (_loadSources) + { + vector sourceList; + if (_json.isMember("sourceList")) + for (auto const& it: _json["sourceList"]) + sourceList.emplace_back(it.asString()); + setSources(sourceList); + } + addAssemblyItemsFromJSON(_json[".code"]); if (_json[".auxdata"].isString()) this->m_auxiliaryData = fromHex(_json[".auxdata"].asString()); @@ -538,7 +548,7 @@ bool Assembly::loadFromAssemblyJSON(Json::Value const& _json) { shared_ptr subassembly = make_shared(); subassembly->setSources(this->sources()); - result &= subassembly->loadFromAssemblyJSON(code); + result &= subassembly->loadFromAssemblyJSON(code, false); this->m_subs.emplace_back(subassembly); } } diff --git a/libevmasm/Assembly.h b/libevmasm/Assembly.h index 14fdc98c4bb5..dcf76d4f3d9b 100644 --- a/libevmasm/Assembly.h +++ b/libevmasm/Assembly.h @@ -39,6 +39,7 @@ #include #include #include +#include namespace solidity::evmasm { @@ -159,7 +160,7 @@ class Assembly bool _includeSourceList = true ) const; - bool loadFromAssemblyJSON(Json::Value const& _json); + bool loadFromAssemblyJSON(Json::Value const& _json, bool _loadSources = true); /// Mark this assembly as invalid. Calling ``assemble`` on it will throw. void markAsInvalid() { m_invalid = true; } @@ -168,7 +169,7 @@ class Assembly size_t encodeSubPath(std::vector const& _subPath); void setSources(std::vector> _sources) { - m_sources = _sources; + m_sources = std::move(_sources); } void setSources(std::vector const& _sources) { diff --git a/libsolidity/interface/CompilerStack.cpp b/libsolidity/interface/CompilerStack.cpp index cd8a91472003..58c55459e863 100644 --- a/libsolidity/interface/CompilerStack.cpp +++ b/libsolidity/interface/CompilerStack.cpp @@ -685,20 +685,14 @@ bool CompilerStack::compile(State _stopAfter) optimiserSettings.runJumpdestRemover = m_optimiserSettings.runJumpdestRemover; optimiserSettings.runPeephole = m_optimiserSettings.runPeephole; - vector sourceList; - if (m_evmAssemblyJson[evmAssemblyJsonSource].isMember("sourceList")) - for (auto const& it: m_evmAssemblyJson[evmAssemblyJsonSource]["sourceList"]) - sourceList.emplace_back(it.asString()); - m_contracts[evmAssemblyJsonSource].evmAssembly = make_shared(evmAssemblyJsonSource); - m_contracts[evmAssemblyJsonSource].evmAssembly->setSources(sourceList); m_contracts[evmAssemblyJsonSource].evmAssembly->loadFromAssemblyJSON(m_evmAssemblyJson[evmAssemblyJsonSource]); m_contracts[evmAssemblyJsonSource].evmAssembly->optimise(optimiserSettings); m_contracts[evmAssemblyJsonSource].object = m_contracts[evmAssemblyJsonSource].evmAssembly->assemble(); m_contracts[evmAssemblyJsonSource].evmRuntimeAssembly = make_shared(evmAssemblyJsonSource); - m_contracts[evmAssemblyJsonSource].evmRuntimeAssembly->setSources(sourceList); - m_contracts[evmAssemblyJsonSource].evmRuntimeAssembly->loadFromAssemblyJSON(m_evmAssemblyJson[evmAssemblyJsonSource][".data"]["0"]); + m_contracts[evmAssemblyJsonSource].evmRuntimeAssembly->setSources(m_contracts[evmAssemblyJsonSource].evmAssembly->sources()); + m_contracts[evmAssemblyJsonSource].evmRuntimeAssembly->loadFromAssemblyJSON(m_evmAssemblyJson[evmAssemblyJsonSource][".data"]["0"], false); m_contracts[evmAssemblyJsonSource].evmRuntimeAssembly->optimise(optimiserSettings); m_contracts[evmAssemblyJsonSource].runtimeObject = m_contracts[evmAssemblyJsonSource].evmRuntimeAssembly->assemble(); } From 2855921d54aed932ac716191a73a194a186b57ac Mon Sep 17 00:00:00 2001 From: Alexander Arlt Date: Tue, 9 Nov 2021 22:30:26 -0500 Subject: [PATCH 12/19] [test] Add some assembly json import tests. --- scripts/AsmJsonImportTest.sh | 99 +++++++++++++++++++++++++++++++ scripts/common_import.sh | 79 ++++++++++++++++++++++++ solc/CommandLineInterface.cpp | 2 +- test/cmdlineTests.sh | 16 ++++- test/cmdlineTests/asm_json/output | 7 ++- 5 files changed, 199 insertions(+), 4 deletions(-) create mode 100755 scripts/AsmJsonImportTest.sh create mode 100644 scripts/common_import.sh diff --git a/scripts/AsmJsonImportTest.sh b/scripts/AsmJsonImportTest.sh new file mode 100755 index 000000000000..7709197f0e54 --- /dev/null +++ b/scripts/AsmJsonImportTest.sh @@ -0,0 +1,99 @@ +#!/usr/bin/env bash + +# Bash script to test the asm-json-import input mode of the compiler by +# first exporting a .sol file to JSON that containing assembly json and corresponding bytecode, +# then the compiler is invoked in assembly json import mode `--import-asm-json` and uses the previously +# generated assembly json as input, where the corresponding bytecode output will be stored. +# Finally, the originally generated bytecode will be compared with the one that was generated by using the +# assembly json file as input. + +set -eo pipefail +READLINK=readlink +if [[ "$OSTYPE" == "darwin"* ]]; then + READLINK=greadlink +fi +REPO_ROOT=$(${READLINK} -f "$(dirname "$0")"/..) +# shellcheck source=scripts/common_import.sh +source "${REPO_ROOT}/scripts/common_import.sh" + +SEMANTICTESTS_DIR="${REPO_ROOT}/test/libsolidity/semanticTests" +NSOURCES="$(find "$SEMANTICTESTS_DIR" -type f | wc -l)" + +init_import_tests + +# function tests whether importing an assembly json file creates identical bytecode. +# Results are recorded by adding to FAILED or UNCOMPILABLE. +# Also, in case of a mismatch a diff and the respective ASTs are printed +# Expected parameters: +# $1 name of the file to be exported and imported +# $2 any files needed to do so that might be in parent directories +function testImportExportEquivalence { + local nth_input_file="$1" + IFS=" " read -r -a all_input_files <<< "$2" + + if $SOLC "$nth_input_file" "${all_input_files[@]}" --combined-json asm,bin > /dev/null 2>&1 + then + local types=( "bin" "bin-runtime" "opcodes" "asm" "srcmap" "srcmap-runtime" ) + local test_types=( "bin" "bin-runtime" "opcodes" "asm" ) + + # save exported json as expected result (silently) + $SOLC --combined-json asm,opcodes,bin,srcmap,srcmap-runtime,bin-runtime --pretty-json "$nth_input_file" "${all_input_files[@]}" > expected.json 2> /dev/null + for contract in $(jq '.contracts | keys | .[]' expected.json 2> /dev/null) + do + for type in "${types[@]}" + do + jq --raw-output ".contracts.${contract}.\"${type}\"" expected.json > "expected.${type}" + done + expected_bin=$(cat expected.bin) + if [[ $expected_bin == "" ]] + then + continue + fi + + if ! "$SOLC" --import-asm-json expected.asm --combined-json asm,opcodes,bin,srcmap,srcmap-runtime,bin-runtime > imported.json 2> /dev/null + then + # For investigating, use exit 1 here so the script stops at the + # first failing test + # exit 1 + echo "" + echo "Failed with contract ${contract}!?" + echo "" + FAILED=$((FAILED + 1)) + return 1 + fi + + for type in "${test_types[@]}" + do + jq --raw-output ".contracts.\"expected.asm\".\"${type}\"" imported.json > "imported.${type}" + if ! diff "expected.${type}" "imported.${type}" + then + echo "" + echo "Failed with contract ${contract} (${type})." + echo "" + if [ "$DIFFVIEW" == "" ] + then + echo "Expected:" + cat "./expected.${type}" + echo "Obtained:" + cat "./imported.${type}" + else + # Use user supplied diff view binary + $DIFFVIEW "expected.${type}" "imported.${type}" + fi + FAILED=$((FAILED + 1)) + return 2 + fi + done + done + TESTED=$((TESTED + 1)) + rm -f expected.json asm.json expected.bin imported.bin + else + # echo "contract $solfile could not be compiled " + UNCOMPILABLE=$((UNCOMPILABLE + 1)) + fi + # return 0 +} +echo "Looking at $NSOURCES .sol files..." + +TEST_FILES=$(find "$SEMANTICTESTS_DIR" -name "*.sol") +run_import_tests "$TEST_FILES" "$SPLITSOURCES" "$NSOURCES" "$PWD" diff --git a/scripts/common_import.sh b/scripts/common_import.sh new file mode 100644 index 000000000000..37cbbe30703d --- /dev/null +++ b/scripts/common_import.sh @@ -0,0 +1,79 @@ +#!/usr/bin/env bash + +set -eo pipefail + +function init_import_tests() +{ + export SOLIDITY_BUILD_DIR=${SOLIDITY_BUILD_DIR:-${REPO_ROOT}/build} + export SOLC=${SOLIDITY_BUILD_DIR}/solc/solc + export SPLITSOURCES=${REPO_ROOT}/scripts/splitSources.py + export FAILED=0 + export UNCOMPILABLE=0 + export TESTED=0 + + if [[ "$(find . -maxdepth 0 -type d -empty)" == "" ]]; then + echo "Test directory not empty. Skipping!" + exit 1 + fi +} + +function run_import_tests() +{ + local TEST_FILES=$1 + local SPLITSOURCES=$2 + local NSOURCES=$3 + local WORKINGDIR=$4 + + for solfile in $TEST_FILES; do + echo -n "." + # create a temporary sub-directory + local FILETMP + FILETMP=$(mktemp -d) + cd "$FILETMP" + + set +e + local OUTPUT + OUTPUT=$("$SPLITSOURCES" "$solfile") + local SPLITSOURCES_RC=$? + set -e + if [ ${SPLITSOURCES_RC} == 0 ]; then + # echo $OUTPUT + NSOURCES=$((NSOURCES - 1)) + for i in $OUTPUT; do + testImportExportEquivalence "$i" "$OUTPUT" + NSOURCES=$((NSOURCES + 1)) + done + elif [ ${SPLITSOURCES_RC} == 1 ]; then + testImportExportEquivalence "$solfile" + elif [ ${SPLITSOURCES_RC} == 2 ]; then + # The script will exit with return code 2, if an UnicodeDecodeError occurred. + # This is the case if e.g. some tests are using invalid utf-8 sequences. We will ignore + # these errors, but print the actual output of the script. + echo -e "\n${OUTPUT}\n" + testImportExportEquivalence "$solfile" + else + # All other return codes will be treated as critical errors. The script will exit. + echo -e "\nGot unexpected return code ${SPLITSOURCES_RC} from ${SPLITSOURCES}. Aborting." + echo -e "\n${OUTPUT}\n" + + cd "$WORKINGDIR" + # Delete temporary files + rm -rf "$FILETMP" + + exit 1 + fi + + cd "$WORKINGDIR" + # Delete temporary files + rm -rf "$FILETMP" + done + + echo "" + + if [ "$FAILED" = 0 ]; then + echo "SUCCESS: $TESTED tests passed, $FAILED failed, $UNCOMPILABLE could not be compiled ($NSOURCES sources total)." + else + echo "FAILURE: Out of $NSOURCES sources, $FAILED failed, ($UNCOMPILABLE could not be compiled)." + exit 1 + fi +} diff --git a/solc/CommandLineInterface.cpp b/solc/CommandLineInterface.cpp index ed694ab740d9..dcefb0e678d3 100644 --- a/solc/CommandLineInterface.cpp +++ b/solc/CommandLineInterface.cpp @@ -790,7 +790,7 @@ bool CommandLineInterface::compile() return false; } } - if (m_options.input.mode == InputMode::CompilerWithEvmAssemblyJsonImport) + else if (m_options.input.mode == InputMode::CompilerWithEvmAssemblyJsonImport) { solAssert(m_fileReader.sourceCodes().size() == 1, ""); try diff --git a/test/cmdlineTests.sh b/test/cmdlineTests.sh index 2076eb993e5a..87820bf64e6b 100755 --- a/test/cmdlineTests.sh +++ b/test/cmdlineTests.sh @@ -154,7 +154,7 @@ function ask_expectation_update # General helper function for testing SOLC behaviour, based on file name, compile opts, exit code, stdout and stderr. # An failure is expected. -function test_solc_behaviour +function test_solc_behaviour() { local filename="${1}" local solc_args @@ -288,7 +288,7 @@ EOF } -function test_solc_assembly_output +function test_solc_assembly_output() { local input="${1}" local expected="${2}" @@ -572,6 +572,18 @@ SOLTMPDIR=$(mktemp -d) ) rm -r "$SOLTMPDIR" +printTask "Testing ASM-JSON import..." +SOLTMPDIR=$(mktemp -d) +( + cd "$SOLTMPDIR" + if ! "$REPO_ROOT/scripts/AsmJsonImportTest.sh" + then + rm -rf "$SOLTMPDIR" + exit 1 + fi +) +rm -rf "$SOLTMPDIR" + printTask "Testing AST export with stop-after=parsing..." "$REPO_ROOT/test/stopAfterParseTests.sh" diff --git a/test/cmdlineTests/asm_json/output b/test/cmdlineTests/asm_json/output index 26ab87147a30..0128c541c474 100644 --- a/test/cmdlineTests/asm_json/output +++ b/test/cmdlineTests/asm_json/output @@ -1582,5 +1582,10 @@ EVM assembly: } ] } - } + }, + "sourceList": + [ + "asm_json/input.sol", + "#utility.yul" + ] } From 66368177ed5ddcb9d35674463d66ea3f567f47e4 Mon Sep 17 00:00:00 2001 From: Alexander Arlt Date: Tue, 9 Nov 2021 22:38:09 -0500 Subject: [PATCH 13/19] [libsolidity] temporary disable optimization for assembly json import. --- libsolidity/interface/CompilerStack.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libsolidity/interface/CompilerStack.cpp b/libsolidity/interface/CompilerStack.cpp index 58c55459e863..da2e222b59e6 100644 --- a/libsolidity/interface/CompilerStack.cpp +++ b/libsolidity/interface/CompilerStack.cpp @@ -687,13 +687,13 @@ bool CompilerStack::compile(State _stopAfter) m_contracts[evmAssemblyJsonSource].evmAssembly = make_shared(evmAssemblyJsonSource); m_contracts[evmAssemblyJsonSource].evmAssembly->loadFromAssemblyJSON(m_evmAssemblyJson[evmAssemblyJsonSource]); - m_contracts[evmAssemblyJsonSource].evmAssembly->optimise(optimiserSettings); +// m_contracts[evmAssemblyJsonSource].evmAssembly->optimise(optimiserSettings); m_contracts[evmAssemblyJsonSource].object = m_contracts[evmAssemblyJsonSource].evmAssembly->assemble(); m_contracts[evmAssemblyJsonSource].evmRuntimeAssembly = make_shared(evmAssemblyJsonSource); m_contracts[evmAssemblyJsonSource].evmRuntimeAssembly->setSources(m_contracts[evmAssemblyJsonSource].evmAssembly->sources()); m_contracts[evmAssemblyJsonSource].evmRuntimeAssembly->loadFromAssemblyJSON(m_evmAssemblyJson[evmAssemblyJsonSource][".data"]["0"], false); - m_contracts[evmAssemblyJsonSource].evmRuntimeAssembly->optimise(optimiserSettings); +// m_contracts[evmAssemblyJsonSource].evmRuntimeAssembly->optimise(optimiserSettings); m_contracts[evmAssemblyJsonSource].runtimeObject = m_contracts[evmAssemblyJsonSource].evmRuntimeAssembly->assemble(); } else From 3de13c4d21785cd237ebab57e2dd35dcbfab09f2 Mon Sep 17 00:00:00 2001 From: Alexander Arlt Date: Tue, 9 Nov 2021 23:38:03 -0500 Subject: [PATCH 14/19] [solc] CommandLineParser: Fix Coding style error. --- solc/CommandLineParser.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/solc/CommandLineParser.cpp b/solc/CommandLineParser.cpp index 6cf664c891c3..2116baba74ba 100644 --- a/solc/CommandLineParser.cpp +++ b/solc/CommandLineParser.cpp @@ -657,8 +657,8 @@ General Information)").c_str(), ( g_strImportAst.c_str(), ("Import ASTs to be compiled, assumes input holds the AST in compact JSON format. " - "Supported Inputs is the output of the --" + g_strStandardJSON + " or the one produced by " - "--" + g_strCombinedJson + " " + CombinedJsonRequests::componentName(&CombinedJsonRequests::ast)).c_str() + "Supported Inputs is the output of the --" + g_strStandardJSON + " or the one produced by " + "--" + g_strCombinedJson + " " + CombinedJsonRequests::componentName(&CombinedJsonRequests::ast)).c_str() ) ( g_strImportEvmAssemblerJson.c_str(), From 5d748613b7fb5b10fb5631fc2df5fdab02f3d823 Mon Sep 17 00:00:00 2001 From: Alexander Arlt Date: Fri, 12 Nov 2021 10:29:46 -0500 Subject: [PATCH 15/19] [libevmasm] Add 'modifierDepth' to assembly json. --- libevmasm/Assembly.cpp | 119 ++++++++++++++++++++++++++++++----------- libevmasm/Assembly.h | 3 +- 2 files changed, 89 insertions(+), 33 deletions(-) diff --git a/libevmasm/Assembly.cpp b/libevmasm/Assembly.cpp index dbb56b0ad6fd..3dc19dab67c7 100644 --- a/libevmasm/Assembly.cpp +++ b/libevmasm/Assembly.cpp @@ -222,13 +222,16 @@ string Assembly::assemblyString( return tmp.str(); } -Json::Value Assembly::createJsonValue(string _name, int _source, int _begin, int _end, string _value, string _jumpType) +Json::Value Assembly::createJsonValue( + string _name, int _sourceIndex, size_t _modifierDepth, int _begin, int _end, string _value, string _jumpType) { Json::Value value{Json::objectValue}; value["name"] = _name; - value["source"] = _source; + value["source"] = _sourceIndex; value["begin"] = _begin; value["end"] = _end; + if (_modifierDepth != 0) + value["modifierDepth"] = static_cast(_modifierDepth); if (!_value.empty()) value["value"] = _value; if (!_jumpType.empty()) @@ -248,7 +251,8 @@ AssemblyItem Assembly::loadItemFromJSON(Json::Value const& _json) std::string name = _json["name"].isString() ? _json["name"].asString() : ""; int begin = _json["begin"].isInt() ? _json["begin"].asInt() : -1; int end = _json["end"].isInt() ? _json["end"].asInt() : -1; - int srcIndex = _json["source"].isInt() ? _json["source"].asInt() : -1; + int srcIndex = _json["source"].isInt() ? _json["source"].asInt() : -1; + size_t modifierDepth = _json["modifierDepth"].isInt() ? static_cast(_json["modifierDepth"].asInt()) : 0; std::string value = _json["value"].isString() ? _json["value"].asString() : ""; std::string jumpType = _json["jumpType"].isString() ? _json["jumpType"].asString() : ""; solAssert(!name.empty(), ""); @@ -274,15 +278,18 @@ AssemblyItem Assembly::loadItemFromJSON(Json::Value const& _json) SourceLocation location; location.start = begin; location.end = end; - if (srcIndex > -1 && srcIndex < (int)sources().size()) + if (srcIndex > -1 && srcIndex < (int) sources().size()) location.sourceName = sources()[static_cast(srcIndex)]; + AssemblyItem result(0); + if (c_instructions.find(name) != c_instructions.end()) { AssemblyItem item{c_instructions.at(name), location}; + item.m_modifierDepth = modifierDepth; if (!value.empty()) item.setJumpType(value); - return item; + result = item; } else { @@ -294,69 +301,71 @@ AssemblyItem Assembly::loadItemFromJSON(Json::Value const& _json) AssemblyItem item{AssemblyItemType::Push, data, location}; if (!jumpType.empty()) item.setJumpType(jumpType); - return item; + result = item; } else if (name == "PUSH [ErrorTag]") - return {AssemblyItemType::PushTag, data, location}; + result = {AssemblyItemType::PushTag, data, location}; else if (name == "PUSH [tag]") { if (!value.empty()) data = u256(value); updateUsedTags(data); - return {AssemblyItemType::PushTag, data, location}; + result = {AssemblyItemType::PushTag, data, location}; } else if (name == "PUSH [$]") { if (!value.empty()) data = u256("0x" + value); - return {AssemblyItemType::PushSub, data, location}; + result = {AssemblyItemType::PushSub, data, location}; } else if (name == "PUSH #[$]") { if (!value.empty()) data = u256("0x" + value); - return {AssemblyItemType::PushSubSize, data, location}; + result = {AssemblyItemType::PushSubSize, data, location}; } else if (name == "PUSHSIZE") - return {AssemblyItemType::PushProgramSize, data, location}; + result = {AssemblyItemType::PushProgramSize, data, location}; else if (name == "PUSHLIB") { h256 hash = updateLibraries(value); - return {AssemblyItemType::PushLibraryAddress, hash, location}; + result = {AssemblyItemType::PushLibraryAddress, hash, location}; } else if (name == "PUSHDEPLOYADDRESS") - return {AssemblyItemType::PushDeployTimeAddress, data, location}; + result = {AssemblyItemType::PushDeployTimeAddress, data, location}; else if (name == "PUSHIMMUTABLE") { h256 hash = updateImmutables(value); - return {AssemblyItemType::PushImmutable, hash, location}; + result = {AssemblyItemType::PushImmutable, hash, location}; } else if (name == "ASSIGNIMMUTABLE") { h256 hash = updateImmutables(value); - return {AssemblyItemType::AssignImmutable, hash, location}; + result = {AssemblyItemType::AssignImmutable, hash, location}; } else if (name == "tag") { if (!value.empty()) data = u256(value); - return {AssemblyItemType::Tag, data, location}; + result = {AssemblyItemType::Tag, data, location}; } else if (name == "PUSH data") { if (!value.empty()) data = u256("0x" + value); - return {AssemblyItemType::PushData, data, location}; + result = {AssemblyItemType::PushData, data, location}; } else if (name == "VERBATIM") { AssemblyItem item(fromHex(value), 0, 0); item.setLocation(location); - return item; + result = item; } else assertThrow(false, InvalidOpcode, ""); } + result.m_modifierDepth = modifierDepth; + return result; } vector Assembly::assemblyItemAsJSON(AssemblyItem const& _item, int _sourceIndex) const @@ -369,6 +378,7 @@ vector Assembly::assemblyItemAsJSON(AssemblyItem const& _item, int result.emplace_back(createJsonValue( instructionInfo(_item.instruction()).name, _sourceIndex, + _item.m_modifierDepth, _item.location().start, _item.location().end, _item.getJumpTypeAsString())); @@ -377,6 +387,7 @@ vector Assembly::assemblyItemAsJSON(AssemblyItem const& _item, int result.emplace_back(createJsonValue( "PUSH", _sourceIndex, + _item.m_modifierDepth, _item.location().start, _item.location().end, toStringInHex(_item.data()), @@ -384,35 +395,62 @@ vector Assembly::assemblyItemAsJSON(AssemblyItem const& _item, int break; case PushTag: if (_item.data() == 0) - result.emplace_back( - createJsonValue("PUSH [ErrorTag]", _sourceIndex, _item.location().start, _item.location().end, "")); + result.emplace_back(createJsonValue( + "PUSH [ErrorTag]", + _sourceIndex, + _item.m_modifierDepth, + _item.location().start, + _item.location().end, + "")); else result.emplace_back(createJsonValue( - "PUSH [tag]", _sourceIndex, _item.location().start, _item.location().end, toString(_item.data()))); + "PUSH [tag]", + _sourceIndex, + _item.m_modifierDepth, + _item.location().start, + _item.location().end, + toString(_item.data()))); break; case PushSub: result.emplace_back(createJsonValue( - "PUSH [$]", _sourceIndex, _item.location().start, _item.location().end, toString(h256(_item.data())))); + "PUSH [$]", + _sourceIndex, + _item.m_modifierDepth, + _item.location().start, + _item.location().end, + toString(h256(_item.data())))); break; case PushSubSize: result.emplace_back(createJsonValue( - "PUSH #[$]", _sourceIndex, _item.location().start, _item.location().end, toString(h256(_item.data())))); + "PUSH #[$]", + _sourceIndex, + _item.m_modifierDepth, + _item.location().start, + _item.location().end, + toString(h256(_item.data())))); break; case PushProgramSize: - result.emplace_back(createJsonValue("PUSHSIZE", _sourceIndex, _item.location().start, _item.location().end)); + result.emplace_back(createJsonValue( + "PUSHSIZE", _sourceIndex, _item.m_modifierDepth, _item.location().start, _item.location().end)); break; case PushLibraryAddress: result.emplace_back(createJsonValue( - "PUSHLIB", _sourceIndex, _item.location().start, _item.location().end, m_libraries.at(h256(_item.data())))); + "PUSHLIB", + _sourceIndex, + _item.m_modifierDepth, + _item.location().start, + _item.location().end, + m_libraries.at(h256(_item.data())))); break; case PushDeployTimeAddress: - result.emplace_back( - createJsonValue("PUSHDEPLOYADDRESS", _sourceIndex, _item.location().start, _item.location().end)); + result.emplace_back(createJsonValue( + "PUSHDEPLOYADDRESS", _sourceIndex, _item.m_modifierDepth, _item.location().start, _item.location().end)); break; case PushImmutable: result.emplace_back(createJsonValue( "PUSHIMMUTABLE", _sourceIndex, + _item.m_modifierDepth, _item.location().start, _item.location().end, m_immutables.at(h256(_item.data())))); @@ -421,22 +459,39 @@ vector Assembly::assemblyItemAsJSON(AssemblyItem const& _item, int result.emplace_back(createJsonValue( "ASSIGNIMMUTABLE", _sourceIndex, + _item.m_modifierDepth, _item.location().start, _item.location().end, m_immutables.at(h256(_item.data())))); break; case Tag: - result.emplace_back( - createJsonValue("tag", _sourceIndex, _item.location().start, _item.location().end, toString(_item.data()))); - result.emplace_back(createJsonValue("JUMPDEST", _sourceIndex, _item.location().start, _item.location().end)); + result.emplace_back(createJsonValue( + "tag", + _sourceIndex, + _item.m_modifierDepth, + _item.location().start, + _item.location().end, + toString(_item.data()))); + result.emplace_back(createJsonValue( + "JUMPDEST", _sourceIndex, _item.m_modifierDepth, _item.location().start, _item.location().end)); break; case PushData: result.emplace_back(createJsonValue( - "PUSH data", _sourceIndex, _item.location().start, _item.location().end, toStringInHex(_item.data()))); + "PUSH data", + _sourceIndex, + _item.m_modifierDepth, + _item.location().start, + _item.location().end, + toStringInHex(_item.data()))); break; case VerbatimBytecode: result.emplace_back(createJsonValue( - "VERBATIM", _sourceIndex, _item.location().start, _item.location().end, util::toHex(_item.verbatimData()))); + "VERBATIM", + _sourceIndex, + _item.m_modifierDepth, + _item.location().start, + _item.location().end, + util::toHex(_item.verbatimData()))); break; default: assertThrow(false, InvalidOpcode, ""); diff --git a/libevmasm/Assembly.h b/libevmasm/Assembly.h index dcf76d4f3d9b..f1d3edcccecd 100644 --- a/libevmasm/Assembly.h +++ b/libevmasm/Assembly.h @@ -193,7 +193,8 @@ class Assembly bool addAssemblyItemsFromJSON(Json::Value const& _code); static Json::Value createJsonValue( std::string _name, - int _source, + int _sourceIndex, + size_t _modifierDepth, int _begin, int _end, std::string _value = std::string(), From ba62b1a401629d804886698b131250cdc7656bbc Mon Sep 17 00:00:00 2001 From: Alexander Arlt Date: Fri, 12 Nov 2021 10:32:04 -0500 Subject: [PATCH 16/19] [libsolidity] OptimiserSettings: add enabled flag. --- libsolidity/interface/CompilerStack.cpp | 6 ++++-- libsolidity/interface/OptimiserSettings.h | 2 ++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/libsolidity/interface/CompilerStack.cpp b/libsolidity/interface/CompilerStack.cpp index da2e222b59e6..c1adf5e78c9f 100644 --- a/libsolidity/interface/CompilerStack.cpp +++ b/libsolidity/interface/CompilerStack.cpp @@ -687,13 +687,15 @@ bool CompilerStack::compile(State _stopAfter) m_contracts[evmAssemblyJsonSource].evmAssembly = make_shared(evmAssemblyJsonSource); m_contracts[evmAssemblyJsonSource].evmAssembly->loadFromAssemblyJSON(m_evmAssemblyJson[evmAssemblyJsonSource]); -// m_contracts[evmAssemblyJsonSource].evmAssembly->optimise(optimiserSettings); + if (m_optimiserSettings.enabled) + m_contracts[evmAssemblyJsonSource].evmAssembly->optimise(optimiserSettings); m_contracts[evmAssemblyJsonSource].object = m_contracts[evmAssemblyJsonSource].evmAssembly->assemble(); m_contracts[evmAssemblyJsonSource].evmRuntimeAssembly = make_shared(evmAssemblyJsonSource); m_contracts[evmAssemblyJsonSource].evmRuntimeAssembly->setSources(m_contracts[evmAssemblyJsonSource].evmAssembly->sources()); m_contracts[evmAssemblyJsonSource].evmRuntimeAssembly->loadFromAssemblyJSON(m_evmAssemblyJson[evmAssemblyJsonSource][".data"]["0"], false); -// m_contracts[evmAssemblyJsonSource].evmRuntimeAssembly->optimise(optimiserSettings); + if (m_optimiserSettings.enabled) + m_contracts[evmAssemblyJsonSource].evmRuntimeAssembly->optimise(optimiserSettings); m_contracts[evmAssemblyJsonSource].runtimeObject = m_contracts[evmAssemblyJsonSource].evmRuntimeAssembly->assemble(); } else diff --git a/libsolidity/interface/OptimiserSettings.h b/libsolidity/interface/OptimiserSettings.h index 5317acee4bd4..528730eca74b 100644 --- a/libsolidity/interface/OptimiserSettings.h +++ b/libsolidity/interface/OptimiserSettings.h @@ -121,6 +121,8 @@ struct OptimiserSettings expectedExecutionsPerDeployment == _other.expectedExecutionsPerDeployment; } + /// Optimizer enabled. + bool enabled = false; /// Move literals to the right of commutative binary operators during code generation. /// This helps exploiting associativity. bool runOrderLiterals = false; From 5c3a618b6cd1b837e4d04ae2248d23849d044326 Mon Sep 17 00:00:00 2001 From: Alexander Arlt Date: Fri, 12 Nov 2021 10:33:03 -0500 Subject: [PATCH 17/19] [solc] Set flag to indicate whether optimiser was enabled. --- solc/CommandLineParser.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/solc/CommandLineParser.cpp b/solc/CommandLineParser.cpp index 2116baba74ba..06539aa89f13 100644 --- a/solc/CommandLineParser.cpp +++ b/solc/CommandLineParser.cpp @@ -267,6 +267,8 @@ OptimiserSettings CommandLineOptions::optimiserSettings() const if (optimizer.yulSteps.has_value()) settings.yulOptimiserSteps = optimizer.yulSteps.value(); + settings.enabled = optimizer.enabled; + return settings; } From ed9c09b2e257eb024fdb1f13f520169641bdd46c Mon Sep 17 00:00:00 2001 From: Alexander Arlt Date: Fri, 12 Nov 2021 11:02:01 -0500 Subject: [PATCH 18/19] [scripts/AsmJsonImportTest.sh] Enable checks of source maps & minor improvements. --- scripts/AsmJsonImportTest.sh | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/scripts/AsmJsonImportTest.sh b/scripts/AsmJsonImportTest.sh index 7709197f0e54..fe4b345d656a 100755 --- a/scripts/AsmJsonImportTest.sh +++ b/scripts/AsmJsonImportTest.sh @@ -1,11 +1,11 @@ #!/usr/bin/env bash # Bash script to test the asm-json-import input mode of the compiler by -# first exporting a .sol file to JSON that containing assembly json and corresponding bytecode, +# first exporting a .sol file to JSON that containing assembly json, deploy & runtime bytecode, opcodes and source mappings, # then the compiler is invoked in assembly json import mode `--import-asm-json` and uses the previously -# generated assembly json as input, where the corresponding bytecode output will be stored. -# Finally, the originally generated bytecode will be compared with the one that was generated by using the -# assembly json file as input. +# generated assembly. This output will be stored. +# Finally, the originally generated outputs (bin, bin-runtime, opcodes, asm, srcmap and srcmap-runtime) +# will be compared with the outputs that where generated by using the assembly json file as input. set -eo pipefail READLINK=readlink @@ -34,7 +34,6 @@ function testImportExportEquivalence { if $SOLC "$nth_input_file" "${all_input_files[@]}" --combined-json asm,bin > /dev/null 2>&1 then local types=( "bin" "bin-runtime" "opcodes" "asm" "srcmap" "srcmap-runtime" ) - local test_types=( "bin" "bin-runtime" "opcodes" "asm" ) # save exported json as expected result (silently) $SOLC --combined-json asm,opcodes,bin,srcmap,srcmap-runtime,bin-runtime --pretty-json "$nth_input_file" "${all_input_files[@]}" > expected.json 2> /dev/null @@ -62,7 +61,7 @@ function testImportExportEquivalence { return 1 fi - for type in "${test_types[@]}" + for type in "${types[@]}" do jq --raw-output ".contracts.\"expected.asm\".\"${type}\"" imported.json > "imported.${type}" if ! diff "expected.${type}" "imported.${type}" @@ -77,7 +76,7 @@ function testImportExportEquivalence { echo "Obtained:" cat "./imported.${type}" else - # Use user supplied diff view binary + # Use user supplied diff view mismatched output $DIFFVIEW "expected.${type}" "imported.${type}" fi FAILED=$((FAILED + 1)) @@ -86,12 +85,17 @@ function testImportExportEquivalence { done done TESTED=$((TESTED + 1)) - rm -f expected.json asm.json expected.bin imported.bin + rm -f expected.json + rm -f imported.json + for type in "${types[@]}" + do + rm -f "expected.${type}" + rm -f "imported.${type}" + done else # echo "contract $solfile could not be compiled " UNCOMPILABLE=$((UNCOMPILABLE + 1)) fi - # return 0 } echo "Looking at $NSOURCES .sol files..." From 9721c82569ed756f00211835aa997fadd0ec5051 Mon Sep 17 00:00:00 2001 From: Alexander Arlt Date: Mon, 15 Nov 2021 21:39:53 -0500 Subject: [PATCH 19/19] [solc] Improve parameter checks for --import-asm-json. --- solc/CommandLineInterface.cpp | 49 ++++++++++++++++------------------- solc/CommandLineParser.cpp | 17 ++++++++++-- solc/CommandLineParser.h | 12 +++++++++ 3 files changed, 50 insertions(+), 28 deletions(-) diff --git a/solc/CommandLineInterface.cpp b/solc/CommandLineInterface.cpp index dcefb0e678d3..490a0996c287 100644 --- a/solc/CommandLineInterface.cpp +++ b/solc/CommandLineInterface.cpp @@ -205,8 +205,7 @@ void CommandLineInterface::handleIR(string const& _contractName) { solAssert( m_options.input.mode == InputMode::Compiler || - m_options.input.mode == InputMode::CompilerWithASTImport || - m_options.input.mode == InputMode::CompilerWithEvmAssemblyJsonImport , "" + m_options.input.mode == InputMode::CompilerWithASTImport ); if (!m_options.compiler.outputs.ir) @@ -225,8 +224,7 @@ void CommandLineInterface::handleIROptimized(string const& _contractName) { solAssert( m_options.input.mode == InputMode::Compiler || - m_options.input.mode == InputMode::CompilerWithASTImport || - m_options.input.mode == InputMode::CompilerWithEvmAssemblyJsonImport , "" + m_options.input.mode == InputMode::CompilerWithASTImport ); if (!m_options.compiler.outputs.irOptimized) @@ -245,8 +243,7 @@ void CommandLineInterface::handleEwasm(string const& _contractName) { solAssert( m_options.input.mode == InputMode::Compiler || - m_options.input.mode == InputMode::CompilerWithASTImport || - m_options.input.mode == InputMode::CompilerWithEvmAssemblyJsonImport , "" + m_options.input.mode == InputMode::CompilerWithASTImport ); if (!m_options.compiler.outputs.ewasm) @@ -286,8 +283,7 @@ void CommandLineInterface::handleSignatureHashes(string const& _contract) { solAssert( m_options.input.mode == InputMode::Compiler || - m_options.input.mode == InputMode::CompilerWithASTImport || - m_options.input.mode == InputMode::CompilerWithEvmAssemblyJsonImport , "" + m_options.input.mode == InputMode::CompilerWithASTImport ); if (!m_options.compiler.outputs.signatureHashes) @@ -326,8 +322,7 @@ void CommandLineInterface::handleABI(string const& _contract) { solAssert( m_options.input.mode == InputMode::Compiler || - m_options.input.mode == InputMode::CompilerWithASTImport || - m_options.input.mode == InputMode::CompilerWithEvmAssemblyJsonImport , "" + m_options.input.mode == InputMode::CompilerWithASTImport ); if (!m_options.compiler.outputs.abi) @@ -344,8 +339,7 @@ void CommandLineInterface::handleStorageLayout(string const& _contract) { solAssert( m_options.input.mode == InputMode::Compiler || - m_options.input.mode == InputMode::CompilerWithASTImport || - m_options.input.mode == InputMode::CompilerWithEvmAssemblyJsonImport , "" + m_options.input.mode == InputMode::CompilerWithASTImport ); if (!m_options.compiler.outputs.storageLayout) @@ -362,8 +356,7 @@ void CommandLineInterface::handleNatspec(bool _natspecDev, string const& _contra { solAssert( m_options.input.mode == InputMode::Compiler || - m_options.input.mode == InputMode::CompilerWithASTImport || - m_options.input.mode == InputMode::CompilerWithEvmAssemblyJsonImport , "" + m_options.input.mode == InputMode::CompilerWithASTImport ); bool enabled = false; @@ -942,8 +935,7 @@ void CommandLineInterface::handleAst() { solAssert( m_options.input.mode == InputMode::Compiler || - m_options.input.mode == InputMode::CompilerWithASTImport || - m_options.input.mode == InputMode::CompilerWithEvmAssemblyJsonImport, "" + m_options.input.mode == InputMode::CompilerWithASTImport ); if (!m_options.compiler.outputs.astCompactJson) @@ -1193,7 +1185,8 @@ void CommandLineInterface::outputCompilationResults() handleCombinedJSON(); // do we need AST output? - handleAst(); + if (m_options.input.mode != InputMode::CompilerWithEvmAssemblyJsonImport) + handleAst(); if ( !m_compiler->compilationSuccessful() && @@ -1233,15 +1226,19 @@ void CommandLineInterface::outputCompilationResults() handleGasEstimation(contract); handleBytecode(contract); - handleIR(contract); - handleIROptimized(contract); - handleEwasm(contract); - handleSignatureHashes(contract); - handleMetadata(contract); - handleABI(contract); - handleStorageLayout(contract); - handleNatspec(true, contract); - handleNatspec(false, contract); + + if (m_options.input.mode != InputMode::CompilerWithEvmAssemblyJsonImport) + { + handleIR(contract); + handleIROptimized(contract); + handleEwasm(contract); + handleSignatureHashes(contract); + handleMetadata(contract); + handleABI(contract); + handleStorageLayout(contract); + handleNatspec(true, contract); + handleNatspec(false, contract); + } } // end of contracts iteration if (!m_hasOutput) diff --git a/solc/CommandLineParser.cpp b/solc/CommandLineParser.cpp index 06539aa89f13..11e86f2e0146 100644 --- a/solc/CommandLineParser.cpp +++ b/solc/CommandLineParser.cpp @@ -475,6 +475,12 @@ bool CommandLineParser::parseOutputSelection() CompilerOutputs::componentName(&CompilerOutputs::ewasm), CompilerOutputs::componentName(&CompilerOutputs::ewasmIR), }; + static set const assemblyJsonImportModeOutputs = { + CompilerOutputs::componentName(&CompilerOutputs::asm_), + CompilerOutputs::componentName(&CompilerOutputs::binary), + CompilerOutputs::componentName(&CompilerOutputs::binaryRuntime), + CompilerOutputs::componentName(&CompilerOutputs::opcodes), + }; switch (_mode) { @@ -483,9 +489,10 @@ bool CommandLineParser::parseOutputSelection() case InputMode::Version: solAssert(false); case InputMode::Compiler: - case InputMode::CompilerWithEvmAssemblyJsonImport: case InputMode::CompilerWithASTImport: return contains(compilerModeOutputs, _outputName); + case InputMode::CompilerWithEvmAssemblyJsonImport: + return contains(assemblyJsonImportModeOutputs, _outputName); case InputMode::Assembler: return contains(assemblerModeOutputs, _outputName); case InputMode::StandardJson: @@ -1360,7 +1367,13 @@ bool CommandLineParser::parseCombinedJsonOption() set requests; for (string const& item: boost::split(requests, m_args[g_strCombinedJson].as(), boost::is_any_of(","))) - if (CombinedJsonRequests::componentMap().count(item) == 0) + if (m_options.input.mode == InputMode::CompilerWithEvmAssemblyJsonImport && + CombinedJsonRequests::componentMapAssemblyJsonImport().count(item) == 0) + { + serr() << "Invalid option to --" << g_strCombinedJson << ": " << item << ", for --" << g_strImportEvmAssemblerJson << endl; + return false; + } + else if (CombinedJsonRequests::componentMap().count(item) == 0) { serr() << "Invalid option to --" << g_strCombinedJson << ": " << item << endl; return false; diff --git a/solc/CommandLineParser.h b/solc/CommandLineParser.h index c946f120400c..0545023e1983 100644 --- a/solc/CommandLineParser.h +++ b/solc/CommandLineParser.h @@ -114,6 +114,18 @@ struct CombinedJsonRequests friend std::ostream& operator<<(std::ostream& _out, CombinedJsonRequests const& _requests); static std::string const& componentName(bool CombinedJsonRequests::* _component); + static auto const& componentMapAssemblyJsonImport() + { + static std::map const components = { + {"bin", &CombinedJsonRequests::binary}, + {"bin-runtime", &CombinedJsonRequests::binaryRuntime}, + {"opcodes", &CombinedJsonRequests::opcodes}, + {"asm", &CombinedJsonRequests::asm_}, + {"srcmap", &CombinedJsonRequests::srcMap}, + {"srcmap-runtime", &CombinedJsonRequests::srcMapRuntime}, + }; + return components; + } static auto const& componentMap() { static std::map const components = {