diff --git a/Dockerfile b/Dockerfile index 11303462b..f6af34a11 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,6 +2,7 @@ FROM openjdk:11.0.16-jdk as build ENV KOTLIN_LIB=1.8.20-Beta ENV KOTLIN_LIB_JS=1.8.20-Beta-js +ENV KOTLIN_CACHES_JS=1.8.20-Beta-js-caches RUN mkdir -p /kotlin-compiler-server WORKDIR /kotlin-compiler-server @@ -20,6 +21,7 @@ COPY --from=build /build/libs/META-INF /kotlin-compiler-server/META-INF COPY --from=build /build/libs/BOOT-INF/classes /kotlin-compiler-server COPY --from=build /kotlin-compiler-server/${KOTLIN_LIB} /kotlin-compiler-server/${KOTLIN_LIB} COPY --from=build /kotlin-compiler-server/${KOTLIN_LIB_JS} /kotlin-compiler-server/${KOTLIN_LIB_JS} +COPY --from=build /kotlin-compiler-server/${KOTLIN_CACHES_JS} /kotlin-compiler-server/${KOTLIN_CACHES_JS} COPY --from=build /kotlin-compiler-server/executor.policy /kotlin-compiler-server/ COPY --from=build /kotlin-compiler-server/indexes.json /kotlin-compiler-server/ diff --git a/build.gradle.kts b/build.gradle.kts index 319b2a98f..1152ad74a 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -25,11 +25,31 @@ val kotlinJsDependency: Configuration by configurations.creating { ) attribute( KotlinJsCompilerAttribute.jsCompilerAttribute, - KotlinJsCompilerAttribute.legacy + KotlinJsCompilerAttribute.ir ) } } + +val kotlinJsIcCache: Configuration by configurations.creating { + isTransitive = false + attributes { + attribute( + KotlinPlatformType.attribute, + KotlinPlatformType.js + ) + attribute( + KotlinJsCompilerAttribute.jsCompilerAttribute, + KotlinJsCompilerAttribute.ir + ) + attribute( + LibraryElements.LIBRARY_ELEMENTS_ATTRIBUTE, + objects.named(LibraryElements::class.java, "js-ir-cache") + ) + } +} + val libJSFolder = "$kotlinVersion-js" +val libJSCachesFolder = "$kotlinVersion-js-caches" val libJVMFolder = kotlinVersion val propertyFile = "application.properties" val jacksonVersionKotlinDependencyJar = "2.14.0" // don't forget to update version in `executor.policy` file. @@ -43,6 +63,11 @@ val copyJSDependencies by tasks.creating(Copy::class) { into(libJSFolder) } +val copyJSCaches by tasks.creating(Copy::class) { + from(kotlinJsIcCache) + into(libJSCachesFolder) +} + plugins { id("org.springframework.boot") version "2.7.8" id("io.spring.dependency-management") version "1.1.0" @@ -87,6 +112,8 @@ dependencies { kotlinDependency("org.jetbrains.kotlinx:kotlinx-coroutines-core-jvm:1.6.4") kotlinJsDependency("org.jetbrains.kotlin:kotlin-stdlib-js:$kotlinVersion") + kotlinJsIcCache(project(":cache-preparator")) + annotationProcessor("org.springframework:spring-context-indexer") implementation("org.springframework.boot:spring-boot-starter-web") implementation("com.amazonaws.serverless:aws-serverless-java-container-springboot2:1.9.1") @@ -131,7 +158,10 @@ fun generateProperties(prefix: String = "") = """ indexesJs.file=${prefix + indexesJs} libraries.folder.jvm=${prefix + libJVMFolder} libraries.folder.js=${prefix + libJSFolder} + caches.folder.js=${prefix + libJSCachesFolder} spring.mvc.pathmatch.matching-strategy=ant_path_matcher + server.compression.enabled=true + server.compression.mime-types=application/json """.trimIndent() tasks.withType { @@ -141,6 +171,7 @@ tasks.withType { } dependsOn(copyDependencies) dependsOn(copyJSDependencies) + dependsOn(copyJSCaches) dependsOn(":executors:jar") dependsOn(":indexation:run") buildPropertyFile() diff --git a/cache-preparator/build.gradle.kts b/cache-preparator/build.gradle.kts new file mode 100644 index 000000000..d173271a0 --- /dev/null +++ b/cache-preparator/build.gradle.kts @@ -0,0 +1,55 @@ +import org.jetbrains.kotlin.gradle.plugin.KotlinPlatformType +import org.jetbrains.kotlin.gradle.targets.js.KotlinJsCompilerAttribute +import org.jetbrains.kotlin.gradle.targets.js.dsl.KotlinJsBinaryMode +import org.jetbrains.kotlin.gradle.targets.js.ir.JsIrBinary + +plugins { + kotlin("multiplatform") +} + +val kotlinJsDependency by rootProject.configurations + +kotlin { + js(IR) { + nodejs() + val executables = binaries.executable() + val main by compilations.getting + main.configurations.apiConfiguration.extendsFrom(kotlinJsDependency) + + val jsCaches by configurations.creating { + isVisible = false + isCanBeResolved = false + isCanBeConsumed = true + + attributes { + attribute( + KotlinPlatformType.attribute, + KotlinPlatformType.js + ) + attribute( + KotlinJsCompilerAttribute.jsCompilerAttribute, + KotlinJsCompilerAttribute.ir + ) + attribute( + LibraryElements.LIBRARY_ELEMENTS_ATTRIBUTE, + objects.named(LibraryElements::class.java, "js-ir-cache") + ) + } + } + + val linkTask = executables + .filterIsInstance() + .single { it.mode == KotlinJsBinaryMode.DEVELOPMENT } + .linkTask + + val rootCacheDir = linkTask + .map { it.rootCacheDirectory } + + artifacts.add(jsCaches.name, rootCacheDir) { + builtBy(linkTask) + } + } +} + +// just to calm down root project's allprojects block +configurations.register("implementation") \ No newline at end of file diff --git a/cache-preparator/src/jsMain/kotlin/File.kt b/cache-preparator/src/jsMain/kotlin/File.kt new file mode 100644 index 000000000..222d270e8 --- /dev/null +++ b/cache-preparator/src/jsMain/kotlin/File.kt @@ -0,0 +1,3 @@ +fun main() { + println("Hello, world") +} \ No newline at end of file diff --git a/common/src/main/kotlin/component/KotlinEnvironment.kt b/common/src/main/kotlin/component/KotlinEnvironment.kt index 5dddf648f..2989eaa86 100644 --- a/common/src/main/kotlin/component/KotlinEnvironment.kt +++ b/common/src/main/kotlin/component/KotlinEnvironment.kt @@ -28,7 +28,8 @@ import java.io.File class KotlinEnvironment( val classpath: List, - additionalJsClasspath: List + additionalJsClasspath: List, + val cachesJsDir: File ) { companion object { /** diff --git a/indexation/src/main/kotlin/KotlinEnvironmentConfiguration.kt b/indexation/src/main/kotlin/KotlinEnvironmentConfiguration.kt index cff2374ad..d355434ef 100644 --- a/indexation/src/main/kotlin/KotlinEnvironmentConfiguration.kt +++ b/indexation/src/main/kotlin/KotlinEnvironmentConfiguration.kt @@ -7,6 +7,7 @@ class KotlinEnvironmentConfiguration(fileName: String) { val kotlinEnvironment = run { val jvmFile = File(fileName) val jsFile = File("$fileName-js") + val cachesJsDir = File("$fileName-js-caches") val classPath = listOfNotNull(jvmFile) .flatMap { @@ -15,6 +16,6 @@ class KotlinEnvironmentConfiguration(fileName: String) { } val additionalJsClasspath = listOfNotNull(jsFile) - KotlinEnvironment(classPath, additionalJsClasspath) + KotlinEnvironment(classPath, additionalJsClasspath, cachesJsDir) } } diff --git a/settings.gradle.kts b/settings.gradle.kts index 506057a4c..ca96936db 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -1,4 +1,5 @@ rootProject.name = "kotlin-compiler-server" include(":executors") include(":indexation") -include(":common") \ No newline at end of file +include(":common") +include(":cache-preparator") \ No newline at end of file diff --git a/src/main/kotlin/com/compiler/server/compiler/components/JsCompiler.kt b/src/main/kotlin/com/compiler/server/compiler/components/JsCompiler.kt new file mode 100644 index 000000000..46abbee7d --- /dev/null +++ b/src/main/kotlin/com/compiler/server/compiler/components/JsCompiler.kt @@ -0,0 +1,104 @@ +package com.compiler.server.compiler.components + +import com.intellij.openapi.project.Project +import org.jetbrains.kotlin.cli.common.messages.AnalyzerWithCompilerReport +import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity +import org.jetbrains.kotlin.cli.common.messages.MessageCollector +import org.jetbrains.kotlin.cli.js.klib.generateIrForKlibSerialization +import org.jetbrains.kotlin.config.CompilerConfiguration +import org.jetbrains.kotlin.descriptors.DeclarationDescriptor +import org.jetbrains.kotlin.ir.backend.js.* +import org.jetbrains.kotlin.ir.backend.js.codegen.JsGenerationGranularity +import org.jetbrains.kotlin.ir.backend.js.ic.CacheUpdater +import org.jetbrains.kotlin.ir.backend.js.ic.DirtyFileState +import org.jetbrains.kotlin.ir.backend.js.ic.ModuleArtifact +import org.jetbrains.kotlin.ir.declarations.impl.IrFactoryImpl +import org.jetbrains.kotlin.ir.declarations.impl.IrFactoryImplForJsIC +import org.jetbrains.kotlin.ir.symbols.IrSymbol +import org.jetbrains.kotlin.psi.KtFile +import java.io.File + +// copy from compiler private funs + +fun processSourceModule( + project: Project, + files: List, + libraries: List, + friendLibraries: List, + configuration: CompilerConfiguration, + outputKlibPath: String +): ModulesStructure { + val sourceModule: ModulesStructure = prepareAnalyzedSourceModule( + project, + files, + configuration, + libraries, + friendLibraries, + AnalyzerWithCompilerReport(configuration) + ) + + val moduleSourceFiles = (sourceModule.mainModule as MainModule.SourceFiles).files + val expectDescriptorToSymbol = mutableMapOf() + + val (moduleFragment, _) = generateIrForKlibSerialization( + project, + moduleSourceFiles, + configuration, + sourceModule.jsFrontEndResult.jsAnalysisResult, + sourceModule.allDependencies.map { it.library }, + emptyList(), + expectDescriptorToSymbol, + IrFactoryImpl, + verifySignatures = true + ) { + sourceModule.getModuleDescriptor(it) + } + + val metadataSerializer = + KlibMetadataIncrementalSerializer( + configuration, + sourceModule.project, + sourceModule.jsFrontEndResult.hasErrors + ) + + generateKLib( + sourceModule, + outputKlibPath, + nopack = true, + jsOutputName = null, + icData = emptyList(), + expectDescriptorToSymbol = expectDescriptorToSymbol, + moduleFragment = moduleFragment + ) { file -> + metadataSerializer.serializeScope(file, sourceModule.jsFrontEndResult.bindingContext, moduleFragment.descriptor) + } + return sourceModule +} + +fun prepareIcCaches( + includes: String, + cacheDirectory: String, + libraries: List, + friendLibraries: List, + configurationJs: CompilerConfiguration, +): List { + val cacheUpdater = CacheUpdater( + mainModule = includes, + allModules = libraries, + mainModuleFriends = friendLibraries, + cacheDir = cacheDirectory, + compilerConfiguration = configurationJs, + irFactory = { IrFactoryImplForJsIC(WholeWorldStageController()) }, + mainArguments = emptyList(), + compilerInterfaceFactory = { mainModule, cfg -> + JsIrCompilerWithIC( + mainModule, + cfg, + JsGenerationGranularity.WHOLE_PROGRAM, + es6mode = false + ) + } + ) + + return cacheUpdater.actualizeCaches() +} \ No newline at end of file diff --git a/src/main/kotlin/com/compiler/server/compiler/components/KotlinEnvironment.kt b/src/main/kotlin/com/compiler/server/compiler/components/KotlinEnvironment.kt index e450e73a9..55723f3e7 100644 --- a/src/main/kotlin/com/compiler/server/compiler/components/KotlinEnvironment.kt +++ b/src/main/kotlin/com/compiler/server/compiler/components/KotlinEnvironment.kt @@ -1,12 +1,16 @@ package com.compiler.server.compiler.components +import com.compiler.server.model.bean.CachesFile import com.compiler.server.model.bean.LibrariesFile import component.KotlinEnvironment import org.springframework.context.annotation.Bean import org.springframework.context.annotation.Configuration @Configuration -class KotlinEnvironmentConfiguration(val librariesFile: LibrariesFile) { +class KotlinEnvironmentConfiguration( + val librariesFile: LibrariesFile, + val cachesFiles: CachesFile +) { @Bean fun kotlinEnvironment(): KotlinEnvironment { val classPath = @@ -17,6 +21,6 @@ class KotlinEnvironmentConfiguration(val librariesFile: LibrariesFile) { } val additionalJsClasspath = listOfNotNull(librariesFile.js) - return KotlinEnvironment(classPath, additionalJsClasspath) + return KotlinEnvironment(classPath, additionalJsClasspath, cachesFiles.js) } } diff --git a/src/main/kotlin/com/compiler/server/compiler/components/KotlinToJSTranslator.kt b/src/main/kotlin/com/compiler/server/compiler/components/KotlinToJSTranslator.kt index a9e5d6b44..66493cd10 100644 --- a/src/main/kotlin/com/compiler/server/compiler/components/KotlinToJSTranslator.kt +++ b/src/main/kotlin/com/compiler/server/compiler/components/KotlinToJSTranslator.kt @@ -4,15 +4,11 @@ import com.compiler.server.model.ErrorDescriptor import com.compiler.server.model.TranslationJSResult import com.compiler.server.model.toExceptionDescriptor import component.KotlinEnvironment -import org.jetbrains.kotlin.cli.common.messages.AnalyzerWithCompilerReport import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment -import org.jetbrains.kotlin.ir.backend.js.CompilerResult -import org.jetbrains.kotlin.ir.backend.js.WholeWorldStageController -import org.jetbrains.kotlin.ir.backend.js.compile -import org.jetbrains.kotlin.ir.backend.js.prepareAnalyzedSourceModule -import org.jetbrains.kotlin.ir.backend.js.transformers.irToJs.IrModuleToJsTransformer -import org.jetbrains.kotlin.ir.backend.js.transformers.irToJs.TranslationMode -import org.jetbrains.kotlin.ir.declarations.impl.IrFactoryImplForJsIC +import org.jetbrains.kotlin.ir.backend.js.* +import org.jetbrains.kotlin.ir.backend.js.ic.JsExecutableProducer +import org.jetbrains.kotlin.ir.backend.js.transformers.irToJs.CompilationOutputs +import org.jetbrains.kotlin.ir.backend.js.transformers.irToJs.CompilationOutputsBuilt import org.jetbrains.kotlin.js.config.JsConfig import org.jetbrains.kotlin.js.facade.K2JSTranslator import org.jetbrains.kotlin.js.facade.MainCallParameters @@ -20,7 +16,11 @@ import org.jetbrains.kotlin.js.facade.TranslationResult import org.jetbrains.kotlin.js.facade.exceptions.TranslationException import org.jetbrains.kotlin.psi.KtFile import org.jetbrains.kotlin.resolve.CompilerEnvironment +import org.jetbrains.kotlin.serialization.js.ModuleKind import org.springframework.stereotype.Service +import kotlin.io.path.absolutePathString +import kotlin.io.path.createTempDirectory +import kotlin.io.path.deleteRecursively @Service class KotlinToJSTranslator( @@ -104,33 +104,47 @@ class KotlinToJSTranslator( arguments: List, coreEnvironment: KotlinCoreEnvironment ): TranslationJSResult { - val currentProject = coreEnvironment.project - - val sourceModule = prepareAnalyzedSourceModule( - currentProject, - files, - kotlinEnvironment.jsConfiguration, - kotlinEnvironment.JS_LIBRARIES, - friendDependencies = emptyList(), - analyzer = AnalyzerWithCompilerReport(kotlinEnvironment.jsConfiguration), - ) - val ir = compile( - sourceModule, - kotlinEnvironment.jsIrPhaseConfig, - irFactory = IrFactoryImplForJsIC(WholeWorldStageController()) - ) - val transformer = IrModuleToJsTransformer( - ir.context, - arguments - ) - - val compiledModule: CompilerResult = transformer.generateModule( - modules = ir.allModules, - modes = setOf(TranslationMode.FULL_DEV), - relativeRequirePath = false - ) + val tmpDir = createTempDirectory() + val mainKlib = tmpDir.resolve("main").normalize().absolutePathString() + + val cachesDir = tmpDir.resolve("caches").normalize() + val caches = cachesDir.absolutePathString() + + kotlinEnvironment.cachesJsDir.copyRecursively(cachesDir.toFile()) + + val outputs = try { + processSourceModule( + coreEnvironment.project, + files, + kotlinEnvironment.JS_LIBRARIES, + friendLibraries = emptyList(), + kotlinEnvironment.jsConfiguration, + mainKlib + ) + + val icCaches = prepareIcCaches( + includes = mainKlib, + cacheDirectory = caches, + libraries = kotlinEnvironment.JS_LIBRARIES + mainKlib, + friendLibraries = emptyList(), + configurationJs = kotlinEnvironment.jsConfiguration, + ) + + val jsExecutableProducer = JsExecutableProducer( + mainModuleName = "moduleId", + moduleKind = ModuleKind.PLAIN, + sourceMapsInfo = null, + caches = icCaches, + relativeRequirePath = true + ) + + val (outputs, _) = jsExecutableProducer.buildExecutable(multiModule = false, outJsProgram = false) + outputs + } finally { + tmpDir.normalize().toAbsolutePath().toFile().deleteRecursively() + } - val jsCode = getJsCodeFromModule(compiledModule) + val jsCode = getJsCodeFromOutputs(outputs) val listLines = jsCode .lineSequence() @@ -143,8 +157,8 @@ class KotlinToJSTranslator( return TranslationJSResult(listLines.joinToString("\n")) } - private fun getJsCodeFromModule(compiledModule: CompilerResult): String { - val jsCodeObject = compiledModule.outputs.values.single() + private fun getJsCodeFromOutputs(outputs: CompilationOutputs): String { + val jsCodeObject = (outputs as CompilationOutputsBuilt) val jsCodeClass = jsCodeObject.javaClass val jsCode = jsCodeClass.getDeclaredField("rawJsCode").let { diff --git a/src/main/kotlin/com/compiler/server/configuration/ApplicationConfiguration.kt b/src/main/kotlin/com/compiler/server/configuration/ApplicationConfiguration.kt index d0b87ffce..4b2c0e1b3 100644 --- a/src/main/kotlin/com/compiler/server/configuration/ApplicationConfiguration.kt +++ b/src/main/kotlin/com/compiler/server/configuration/ApplicationConfiguration.kt @@ -1,5 +1,6 @@ package com.compiler.server.configuration +import com.compiler.server.model.bean.CachesFile import com.compiler.server.model.bean.LibrariesFile import com.compiler.server.model.bean.VersionInfo import org.springframework.beans.factory.annotation.Value @@ -12,10 +13,11 @@ import org.springframework.web.servlet.config.annotation.WebMvcConfigurer import java.io.File @Configuration -@EnableConfigurationProperties(value = [LibrariesFolderProperties::class]) +@EnableConfigurationProperties(value = [LibrariesFolderProperties::class, CachesFolderProperties::class]) class ApplicationConfiguration( @Value("\${kotlin.version}") private val version: String, - private val librariesFolderProperties: LibrariesFolderProperties + private val librariesFolderProperties: LibrariesFolderProperties, + private val cachesFolderProperties: LibrariesFolderProperties ) : WebMvcConfigurer { override fun addFormatters(registry: FormatterRegistry) { registry.addConverter(ProjectConverter()) @@ -32,10 +34,20 @@ class ApplicationConfiguration( File(librariesFolderProperties.jvm), File(librariesFolderProperties.js) ) + + @Bean + fun cachesFiles() = CachesFile( + File(cachesFolderProperties.js) + ) } @ConfigurationProperties(prefix = "libraries.folder") class LibrariesFolderProperties { lateinit var jvm: String lateinit var js: String +} + +@ConfigurationProperties(prefix = "caches.folder") +class CachesFolderProperties { + lateinit var js: String } \ No newline at end of file diff --git a/src/main/kotlin/com/compiler/server/model/bean/CachesFile.kt b/src/main/kotlin/com/compiler/server/model/bean/CachesFile.kt new file mode 100644 index 000000000..a64b8ba00 --- /dev/null +++ b/src/main/kotlin/com/compiler/server/model/bean/CachesFile.kt @@ -0,0 +1,5 @@ +package com.compiler.server.model.bean + +import java.io.File + +class CachesFile(val js: File) \ No newline at end of file diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index fb0a07768..f2df66764 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -5,4 +5,7 @@ indexes.file=indexes.json indexesJs.file=indexesJs.json libraries.folder.jvm=1.8.20-Beta libraries.folder.js=1.8.20-Beta-js -spring.mvc.pathmatch.matching-strategy=ant_path_matcher \ No newline at end of file +caches.folder.js=1.8.20-Beta-js-caches +spring.mvc.pathmatch.matching-strategy=ant_path_matcher +server.compression.enabled=true +server.compression.mime-types=application/json \ No newline at end of file