From 5ea5f3b45d980be692f28f494e25a02e4a6679b1 Mon Sep 17 00:00:00 2001 From: Doug Gregor Date: Wed, 31 Aug 2022 16:39:10 -0700 Subject: [PATCH 1/4] Optional support for round-trip testing the new Swift parser. When enabled, compile in support for round-trip testing the new SwiftSyntax-provided Swift parser alongside the existing parser. Right now, this means parsing every source file with the new parser and ensuring that the resulting syntax tree can reproduce the input source precisely. Over time, this is expected to grow. Opt in to this behavior by passing the following to build-script: build-script --early-swiftsyntax --extra-cmake-options=-DSWIFT_SWIFT_PARSER_MODE:STRING=ROUNDTRIP --- CMakeLists.txt | 17 +++++++++++ cmake/modules/AddSwift.cmake | 10 +++++++ lib/Parse/CMakeLists.txt | 28 +++++++++++++++++++ lib/Parse/ParseRequests.cpp | 20 +++++++++++++ .../swift_build_support/products/swift.py | 25 +++++++++++++++-- 5 files changed, 98 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index a925b40f48941..df0e6419b9945 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -868,6 +868,23 @@ if(XCODE) set(SWIFT_SDKS "OSX") endif() +# When we have the early SwiftSyntax build, we can include its parser. +if(SWIFT_PATH_TO_EARLYSWIFTSYNTAX_BUILD_DIR) + # An option that specifies how the SwiftSyntax-based Swift parser is + # integrated. + option(SWIFT_SWIFT_PARSER_MODE [=[ +How to use the SwiftSyntax-based Swift parser. Possible values are + OFF: do not use this parser at all + ROUNDTRIP: ensure that the SwiftSyntax-based Swift parser round-trips +]=] OFF) + + if(SWIFT_SWIFT_PARSER_MODE) + set(SWIFT_SWIFT_PARSER TRUE) + include(${SWIFT_PATH_TO_EARLYSWIFTSYNTAX_BUILD_DIR}/cmake/SwiftSyntaxTargets.cmake) + endif() +endif() + + # FIXME: the parameters we specify in SWIFT_SDKS are lacking architecture specifics, # so we need to hard-code it. For example, the SDK for Android is just 'ANDROID', # and we have to specify SWIFT_SDK_ANDROID_ARCHITECTURES separately. diff --git a/cmake/modules/AddSwift.cmake b/cmake/modules/AddSwift.cmake index d52f780bc0e2c..4bf1d691bd326 100644 --- a/cmake/modules/AddSwift.cmake +++ b/cmake/modules/AddSwift.cmake @@ -608,6 +608,11 @@ function(add_swift_host_library name) translate_flags(ASHL "${options}") + # Once the new Swift parser is linked, everything has Swift modules. + if (SWIFT_SWIFT_PARSER AND ASHL_SHARED) + set(ASHL_HAS_SWIFT_MODULES ON) + endif() + if(NOT ASHL_SHARED AND NOT ASHL_STATIC AND NOT ASHL_OBJECT) message(FATAL_ERROR "One of SHARED/STATIC/OBJECT must be specified") endif() @@ -834,6 +839,11 @@ function(add_swift_host_tool executable) JOB_POOL_LINK swift_link_job_pool) endif() + # Once the new Swift parser is linked in, every host tool has Swift modules. + if (SWIFT_SWIFT_PARSER) + set(ASHT_HAS_SWIFT_MODULES ON) + endif() + if (ASHT_HAS_SWIFT_MODULES) _add_swift_runtime_link_flags(${executable} "../lib" "${ASHT_BOOTSTRAPPING}") endif() diff --git a/lib/Parse/CMakeLists.txt b/lib/Parse/CMakeLists.txt index b7d694b6eeb12..c483aafdb04db 100644 --- a/lib/Parse/CMakeLists.txt +++ b/lib/Parse/CMakeLists.txt @@ -35,6 +35,34 @@ target_link_libraries(swiftParse PRIVATE swiftSyntaxParse ) +if (SWIFT_SWIFT_PARSER) + # Link against the SwiftSyntax parser and libraries it depends on. The actual + # formulation of this is a hack to work around a CMake bug in Ninja file + # generation that results in multiple Ninja targets producing the same file in + # a downstream SourceKit target. This should be expressed as: + # + # target_link_libraries(swiftParse + # PRIVATE + # SwiftSyntax::SwiftParser + # ) + target_link_libraries(swiftParse + PRIVATE + $ + $ + $ + ) + + target_include_directories(swiftParse + PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/../../../swift-syntax/Sources/SwiftParser + ) + + target_compile_definitions(swiftParse + PRIVATE + SWIFT_SWIFT_PARSER_ROUNDTRIP + ) +endif() + add_dependencies(swiftParse swift-parse-syntax-generated-headers) set_swift_llvm_is_available(swiftParse) diff --git a/lib/Parse/ParseRequests.cpp b/lib/Parse/ParseRequests.cpp index 3cef99a457ab0..65e01e5787291 100644 --- a/lib/Parse/ParseRequests.cpp +++ b/lib/Parse/ParseRequests.cpp @@ -26,6 +26,10 @@ #include "swift/Syntax/SyntaxNodes.h" #include "swift/SyntaxParse/SyntaxTreeCreator.h" +#ifdef SWIFT_SWIFT_PARSER_ROUNDTRIP +#include "SwiftParserCompilerSupport.h" +#endif + using namespace swift; namespace swift { @@ -189,6 +193,22 @@ SourceFileParsingResult ParseSourceFileRequest::evaluate(Evaluator &evaluator, if (auto tokens = parser.takeTokenReceiver()->finalize()) tokensRef = ctx.AllocateCopy(*tokens); +#ifdef SWIFT_SWIFT_PARSER_ROUNDTRIP + if (ctx.SourceMgr.getCodeCompletionBufferID() != bufferID) { + auto bufferRange = ctx.SourceMgr.getRangeForBuffer(*bufferID); + unsigned int flags = SPCC_RoundTrip; + + int roundTripResult = + swift_parser_consistencyCheck( + bufferRange.str().data(), bufferRange.getByteLength(), + SF->getFilename().str().c_str(), flags); + + // FIXME: Produce an error on round-trip failure. + if (roundTripResult) + abort(); + } +#endif + return SourceFileParsingResult{ctx.AllocateCopy(decls), tokensRef, parser.CurrentTokenHash, syntaxRoot}; } diff --git a/utils/swift_build_support/swift_build_support/products/swift.py b/utils/swift_build_support/swift_build_support/products/swift.py index 16e3ff6b55cbe..c61b1a48774ff 100644 --- a/utils/swift_build_support/swift_build_support/products/swift.py +++ b/utils/swift_build_support/swift_build_support/products/swift.py @@ -10,14 +10,16 @@ # # ---------------------------------------------------------------------------- +import os + from . import cmark from . import earlyswiftdriver +from . import earlyswiftsyntax from . import libcxx from . import llvm from . import product from ..cmake import CMakeOptions - class Swift(product.Product): def __init__(self, args, toolchain, source_dir, build_dir): @@ -55,6 +57,9 @@ def __init__(self, args, toolchain, source_dir, build_dir): # Add experimental distributed flag. self.cmake_options.extend(self._enable_experimental_distributed) + # Add path for the early SwiftSyntax build. + self.cmake_options.extend(self._early_swiftsyntax_flags) + # Add static vprintf flag self.cmake_options.extend(self._enable_stdlib_static_vprintf) @@ -167,6 +172,18 @@ def _enable_experimental_distributed(self): return [('SWIFT_ENABLE_EXPERIMENTAL_DISTRIBUTED:BOOL', self.args.enable_experimental_distributed)] + @property + def _early_swiftsyntax_flags(self): + result=[] + if self.args.build_early_swiftsyntax: + build_root = os.path.dirname(self.build_dir) + early_swiftsyntax_build_dir = os.path.join( + '..', build_root, '%s-%s' % ('earlyswiftsyntax', + self.args.host_target)) + result.append(('SWIFT_PATH_TO_EARLYSWIFTSYNTAX_BUILD_DIR:PATH', + early_swiftsyntax_build_dir)) + return result + @property def _enable_stdlib_static_vprintf(self): return [('SWIFT_STDLIB_STATIC_PRINT', @@ -194,7 +211,11 @@ def _swift_tools_ld64_lto_codegen_only_for_supporting_targets(self): @classmethod def get_dependencies(cls): - return [cmark.CMark, + deps = [cmark.CMark, earlyswiftdriver.EarlySwiftDriver, llvm.LLVM, libcxx.LibCXX] + if self.args.build_early_swiftsyntax: + deps.append(earlyswiftsyntax.EarlySwiftSyntax) + + return deps From 6854355b08508c0d00e5fde712e85ddf8bbd4b25 Mon Sep 17 00:00:00 2001 From: Doug Gregor Date: Thu, 1 Sep 2022 08:53:46 -0700 Subject: [PATCH 2/4] Address Python lint and testing issues --- .../swift_build_support/products/swift.py | 10 ++++------ utils/swift_build_support/tests/products/test_swift.py | 1 + 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/utils/swift_build_support/swift_build_support/products/swift.py b/utils/swift_build_support/swift_build_support/products/swift.py index c61b1a48774ff..dcf356a7e435c 100644 --- a/utils/swift_build_support/swift_build_support/products/swift.py +++ b/utils/swift_build_support/swift_build_support/products/swift.py @@ -20,6 +20,7 @@ from . import product from ..cmake import CMakeOptions + class Swift(product.Product): def __init__(self, args, toolchain, source_dir, build_dir): @@ -174,7 +175,7 @@ def _enable_experimental_distributed(self): @property def _early_swiftsyntax_flags(self): - result=[] + result = [] if self.args.build_early_swiftsyntax: build_root = os.path.dirname(self.build_dir) early_swiftsyntax_build_dir = os.path.join( @@ -211,11 +212,8 @@ def _swift_tools_ld64_lto_codegen_only_for_supporting_targets(self): @classmethod def get_dependencies(cls): - deps = [cmark.CMark, + return [cmark.CMark, earlyswiftdriver.EarlySwiftDriver, + earlyswiftsyntax.EarlySwiftSyntax, llvm.LLVM, libcxx.LibCXX] - if self.args.build_early_swiftsyntax: - deps.append(earlyswiftsyntax.EarlySwiftSyntax) - - return deps diff --git a/utils/swift_build_support/tests/products/test_swift.py b/utils/swift_build_support/tests/products/test_swift.py index bcd413641dbc0..1225d27409640 100644 --- a/utils/swift_build_support/tests/products/test_swift.py +++ b/utils/swift_build_support/tests/products/test_swift.py @@ -56,6 +56,7 @@ def setUp(self): enable_experimental_differentiable_programming=False, enable_experimental_concurrency=False, enable_experimental_distributed=False, + build_early_swiftsyntax=False, build_swift_stdlib_static_print=False, build_swift_stdlib_unicode_data=True, swift_freestanding_is_darwin=False, From 3f9f16f4d2ff23f294cc0a18882a7bd7442ccdd8 Mon Sep 17 00:00:00 2001 From: Doug Gregor Date: Thu, 1 Sep 2022 11:45:16 -0700 Subject: [PATCH 3/4] [Build] Work around build-scripts single-root requirement. Replace the correct Swift -> EarlySwiftSyntax dependency with an EarlySwiftDriver -> EarlySwiftSyntax dependency, to keep a linear dependency order in this part of the build graph. --- .../swift_build_support/products/earlyswiftdriver.py | 7 ++++++- .../swift_build_support/products/swift.py | 2 -- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/utils/swift_build_support/swift_build_support/products/earlyswiftdriver.py b/utils/swift_build_support/swift_build_support/products/earlyswiftdriver.py index 34438b93bc1b1..eb5348a284da2 100644 --- a/utils/swift_build_support/swift_build_support/products/earlyswiftdriver.py +++ b/utils/swift_build_support/swift_build_support/products/earlyswiftdriver.py @@ -12,6 +12,7 @@ import os +from . import earlyswiftsyntax from . import product from .. import shell from .. import toolchain @@ -57,7 +58,11 @@ def should_build(self, host_target): @classmethod def get_dependencies(cls): - return [] + # FIXME: This isn't a real dependency, but is necessary to linearize the + # dependency graph from Swift to EarlySwiftSyntax. If we properly + # express the dependency from Swift -> EarlySwiftSyntax, build_graph.py + # asserts that there are multiple roots to the graph. + return [earlyswiftsyntax.EarlySwiftSyntax] def should_clean(self, host_target): return self.args.clean_early_swift_driver diff --git a/utils/swift_build_support/swift_build_support/products/swift.py b/utils/swift_build_support/swift_build_support/products/swift.py index dcf356a7e435c..ad6665fefc80a 100644 --- a/utils/swift_build_support/swift_build_support/products/swift.py +++ b/utils/swift_build_support/swift_build_support/products/swift.py @@ -14,7 +14,6 @@ from . import cmark from . import earlyswiftdriver -from . import earlyswiftsyntax from . import libcxx from . import llvm from . import product @@ -214,6 +213,5 @@ def _swift_tools_ld64_lto_codegen_only_for_supporting_targets(self): def get_dependencies(cls): return [cmark.CMark, earlyswiftdriver.EarlySwiftDriver, - earlyswiftsyntax.EarlySwiftSyntax, llvm.LLVM, libcxx.LibCXX] From 9e387148d1cb9c9b5395163d9a98e8754bfa6a3c Mon Sep 17 00:00:00 2001 From: Doug Gregor Date: Thu, 1 Sep 2022 11:57:35 -0700 Subject: [PATCH 4/4] Ensure to link unit tests against the Swift libraries when using the new parser --- cmake/modules/AddSwift.cmake | 2 +- cmake/modules/AddSwiftUnittests.cmake | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/cmake/modules/AddSwift.cmake b/cmake/modules/AddSwift.cmake index 4bf1d691bd326..71f08914d9e2a 100644 --- a/cmake/modules/AddSwift.cmake +++ b/cmake/modules/AddSwift.cmake @@ -504,7 +504,7 @@ function(_add_swift_runtime_link_flags target relpath_to_lib_dir bootstrapping) # Workaround to make lldb happy: we have to explicitly add all swift compiler modules # to the linker command line. - set(swift_ast_path_flags "-Wl") + set(swift_ast_path_flags " -Wl") get_property(modules GLOBAL PROPERTY swift_compiler_modules) foreach(module ${modules}) get_target_property(module_file "SwiftModule${module}" "module_file") diff --git a/cmake/modules/AddSwiftUnittests.cmake b/cmake/modules/AddSwiftUnittests.cmake index 277347bb7a3f7..cdb522e9fc464 100644 --- a/cmake/modules/AddSwiftUnittests.cmake +++ b/cmake/modules/AddSwiftUnittests.cmake @@ -78,5 +78,9 @@ function(add_swift_unittest test_dirname) LINK_FLAGS " -fsanitize=thread") endif() endif() + + if (SWIFT_SWIFT_PARSER) + _add_swift_runtime_link_flags(${test_dirname} "../../lib" "") + endif() endfunction()