diff --git a/.changeset/dirty-dolphins-suffer.md b/.changeset/dirty-dolphins-suffer.md
new file mode 100644
index 000000000000..e93cfad16158
--- /dev/null
+++ b/.changeset/dirty-dolphins-suffer.md
@@ -0,0 +1,5 @@
+---
+'@sveltejs/adapter-node': minor
+---
+
+chore: remove build step for adapter node
diff --git a/.changeset/three-hotels-share.md b/.changeset/three-hotels-share.md
new file mode 100644
index 000000000000..60cd79ab45c7
--- /dev/null
+++ b/.changeset/three-hotels-share.md
@@ -0,0 +1,5 @@
+---
+'@sveltejs/adapter-node': minor
+---
+
+feat: load sourcemaps so that they can be merged to point to original sourcefiles
diff --git a/packages/adapter-node/ambient.d.ts b/packages/adapter-node/ambient.d.ts
index 7d45ea6dc64a..6843a8aa5ec7 100644
--- a/packages/adapter-node/ambient.d.ts
+++ b/packages/adapter-node/ambient.d.ts
@@ -1,7 +1,3 @@
-declare module 'ENV' {
-	export function env(key: string, fallback?: any): string;
-}
-
 declare module 'HANDLER' {
 	export const handler: import('polka').Middleware;
 }
@@ -17,6 +13,11 @@ declare module 'SERVER' {
 	export { Server } from '@sveltejs/kit';
 }
 
+interface ImportMeta {
+	SERVER_DIR: string;
+	ENV_PREFIX?: string;
+}
+
 declare namespace App {
 	export interface Platform {
 		/**
diff --git a/packages/adapter-node/index.d.ts b/packages/adapter-node/index.d.ts
index 12ea6273dd66..c3b224198ddb 100644
--- a/packages/adapter-node/index.d.ts
+++ b/packages/adapter-node/index.d.ts
@@ -1,10 +1,6 @@
 import { Adapter } from '@sveltejs/kit';
 import './ambient.js';
 
-declare global {
-	const ENV_PREFIX: string;
-}
-
 interface AdapterOptions {
 	out?: string;
 	precompress?: boolean;
diff --git a/packages/adapter-node/index.js b/packages/adapter-node/index.js
index 8d5c1629a4cf..69eaee9e3e91 100644
--- a/packages/adapter-node/index.js
+++ b/packages/adapter-node/index.js
@@ -1,11 +1,15 @@
-import { readFileSync, writeFileSync } from 'node:fs';
-import { fileURLToPath } from 'node:url';
-import { rollup } from 'rollup';
-import { nodeResolve } from '@rollup/plugin-node-resolve';
 import commonjs from '@rollup/plugin-commonjs';
 import json from '@rollup/plugin-json';
+import { nodeResolve } from '@rollup/plugin-node-resolve';
+import { createFilter, normalizePath } from '@rollup/pluginutils';
+import { existsSync, readFileSync, writeFileSync } from 'node:fs';
+import { readFile } from 'node:fs/promises';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { rollup } from 'rollup';
 
-const files = fileURLToPath(new URL('./files', import.meta.url).href);
+/** @param {string} path */
+const resolve = (path) => fileURLToPath(new URL(path, import.meta.url));
 
 /** @type {import('.').default} */
 export default function (opts = {}) {
@@ -15,7 +19,8 @@ export default function (opts = {}) {
 		name: '@sveltejs/adapter-node',
 
 		async adapt(builder) {
-			const tmp = builder.getBuildDirectory('adapter-node');
+			// use an adjacent temporary directory so that any relative paths in eg. sourcemaps don't break
+			const tmp = path.join(path.dirname(builder.getServerDirectory()), 'adapter-node');
 
 			builder.rimraf(out);
 			builder.rimraf(tmp);
@@ -50,45 +55,85 @@ export default function (opts = {}) {
 			// will get included in the bundled code
 			const bundle = await rollup({
 				input: {
-					index: `${tmp}/index.js`,
-					manifest: `${tmp}/manifest.js`
+					handler: resolve('./src/handler.js'),
+					index: resolve('./src/index.js')
 				},
 				external: [
 					// dependencies could have deep exports, so we need a regex
 					...Object.keys(pkg.dependencies || {}).map((d) => new RegExp(`^${d}(\\/.*)?$`))
 				],
 				plugins: [
+					{
+						name: 'adapter-node-resolve',
+						resolveId(id) {
+							switch (id) {
+								case 'MANIFEST':
+									return `${tmp}/manifest.js`;
+								case 'SERVER':
+									return `${tmp}/index.js`;
+								case 'SHIMS':
+									return '\0virtual:SHIMS';
+							}
+						},
+						load(id) {
+							if (id === '\0virtual:SHIMS') {
+								return polyfill
+									? "import { installPolyfills } from '@sveltejs/kit/node/polyfills'; installPolyfills();"
+									: '';
+							}
+						},
+						resolveImportMeta(property, { chunkId, moduleId }) {
+							if (property === 'SERVER_DIR' && moduleId === resolve('./src/handler.js')) {
+								const segments = chunkId.split('/').length - 1;
+
+								return `new URL("${'../'.repeat(segments) || '.'}", import.meta.url)`;
+							} else if (property === 'ENV_PREFIX' && moduleId === resolve('./src/env.js')) {
+								return JSON.stringify(envPrefix);
+							}
+						}
+					},
 					nodeResolve({
 						preferBuiltins: true,
 						exportConditions: ['node']
 					}),
 					commonjs({ strictRequires: true }),
-					json()
+					json(),
+					merge_sourcemap_plugin(tmp)
 				]
 			});
 
 			await bundle.write({
-				dir: `${out}/server`,
+				dir: out,
 				format: 'esm',
 				sourcemap: true,
-				chunkFileNames: 'chunks/[name]-[hash].js'
+				chunkFileNames: 'server/chunks/[name]-[hash].js',
+				// without this rollup will insert some imports to try speed up
+				// module loading but it doesn't really affect anything on the server side
+				hoistTransitiveImports: false
 			});
+		}
+	};
+}
 
-			builder.copy(files, out, {
-				replace: {
-					ENV: './env.js',
-					HANDLER: './handler.js',
-					MANIFEST: './server/manifest.js',
-					SERVER: './server/index.js',
-					SHIMS: './shims.js',
-					ENV_PREFIX: JSON.stringify(envPrefix)
-				}
-			});
+/**
+ * Load sourcemaps for files in the tmp directory so that the final ones
+ * point to the original source files, instead of the generated files in outDir.
+ * @param {string} tmp
+ * @returns {import('rollup').Plugin}
+ */
+function merge_sourcemap_plugin(tmp) {
+	const should_process_sourcemaps = createFilter(`${normalizePath(tmp)}/**/*.js`);
 
-			// If polyfills aren't wanted then clear the file
-			if (!polyfill) {
-				writeFileSync(`${out}/shims.js`, '', 'utf-8');
-			}
+	return {
+		name: 'adapter-node-sourcemap-loader',
+		async load(id) {
+			if (!should_process_sourcemaps(id)) return;
+			if (!existsSync(`${id}.map`)) return;
+			const [code, map] = await Promise.all([
+				readFile(id, 'utf-8'),
+				readFile(`${id}.map`, 'utf-8')
+			]);
+			return { code, map };
 		}
 	};
 }
diff --git a/packages/adapter-node/package.json b/packages/adapter-node/package.json
index e1a7fc3ac5ff..dca126df7ebd 100644
--- a/packages/adapter-node/package.json
+++ b/packages/adapter-node/package.json
@@ -19,35 +19,32 @@
 	},
 	"types": "index.d.ts",
 	"files": [
-		"files",
+		"src",
 		"index.js",
 		"index.d.ts"
 	],
 	"scripts": {
-		"dev": "rimraf files && rollup -cw",
-		"build": "rimraf files && rollup -c",
 		"test": "echo \"tests temporarily disabled\" # c8 vitest run",
 		"check": "tsc",
 		"lint": "prettier --check . --config ../../.prettierrc --ignore-path .gitignore",
-		"format": "pnpm lint --write",
-		"prepublishOnly": "pnpm build"
+		"format": "pnpm lint --write"
 	},
 	"devDependencies": {
-		"@polka/url": "^1.0.0-next.21",
 		"@sveltejs/kit": "workspace:^",
 		"@types/node": "^16.18.6",
 		"c8": "^8.0.0",
-		"polka": "^1.0.0-next.22",
-		"rimraf": "^5.0.0",
-		"sirv": "^2.0.3",
 		"typescript": "^4.9.4",
 		"vitest": "^0.32.2"
 	},
 	"dependencies": {
+		"@polka/url": "^1.0.0-next.21",
 		"@rollup/plugin-commonjs": "^25.0.0",
 		"@rollup/plugin-json": "^6.0.0",
 		"@rollup/plugin-node-resolve": "^15.0.1",
-		"rollup": "^3.7.0"
+		"@rollup/pluginutils": "^5.0.2",
+		"polka": "^1.0.0-next.22",
+		"rollup": "^3.7.0",
+		"sirv": "^2.0.2"
 	},
 	"peerDependencies": {
 		"@sveltejs/kit": "^1.0.0"
diff --git a/packages/adapter-node/rollup.config.js b/packages/adapter-node/rollup.config.js
deleted file mode 100644
index 16c4a20c37a7..000000000000
--- a/packages/adapter-node/rollup.config.js
+++ /dev/null
@@ -1,44 +0,0 @@
-import { nodeResolve } from '@rollup/plugin-node-resolve';
-import commonjs from '@rollup/plugin-commonjs';
-import json from '@rollup/plugin-json';
-import { builtinModules } from 'node:module';
-
-export default [
-	{
-		input: 'src/index.js',
-		output: {
-			file: 'files/index.js',
-			format: 'esm'
-		},
-		plugins: [nodeResolve({ preferBuiltins: true }), commonjs(), json()],
-		external: ['ENV', 'HANDLER', ...builtinModules]
-	},
-	{
-		input: 'src/env.js',
-		output: {
-			file: 'files/env.js',
-			format: 'esm'
-		},
-		plugins: [nodeResolve(), commonjs(), json()],
-		external: ['HANDLER', ...builtinModules]
-	},
-	{
-		input: 'src/handler.js',
-		output: {
-			file: 'files/handler.js',
-			format: 'esm',
-			inlineDynamicImports: true
-		},
-		plugins: [nodeResolve(), commonjs(), json()],
-		external: ['ENV', 'MANIFEST', 'SERVER', 'SHIMS', ...builtinModules]
-	},
-	{
-		input: 'src/shims.js',
-		output: {
-			file: 'files/shims.js',
-			format: 'esm'
-		},
-		plugins: [nodeResolve(), commonjs()],
-		external: builtinModules
-	}
-];
diff --git a/packages/adapter-node/src/env.js b/packages/adapter-node/src/env.js
index 841bddf2c237..75a7dcc93b28 100644
--- a/packages/adapter-node/src/env.js
+++ b/packages/adapter-node/src/env.js
@@ -1,4 +1,4 @@
-/* global ENV_PREFIX */
+export const ENV_PREFIX = import.meta.ENV_PREFIX;
 
 const expected = new Set([
 	'SOCKET_PATH',
diff --git a/packages/adapter-node/src/handler.js b/packages/adapter-node/src/handler.js
index bae1854bdd7e..495f35db5236 100644
--- a/packages/adapter-node/src/handler.js
+++ b/packages/adapter-node/src/handler.js
@@ -1,15 +1,16 @@
 import 'SHIMS';
+
 import fs from 'node:fs';
 import path from 'node:path';
-import sirv from 'sirv';
 import { fileURLToPath } from 'node:url';
+
 import { parse as polka_url_parser } from '@polka/url';
 import { getRequest, setResponse } from '@sveltejs/kit/node';
-import { Server } from 'SERVER';
-import { manifest, prerendered } from 'MANIFEST';
-import { env } from 'ENV';
+import sirv from 'sirv';
 
-/* global ENV_PREFIX */
+import { env, ENV_PREFIX } from './env.js';
+import { manifest, prerendered } from 'MANIFEST';
+import { Server } from 'SERVER';
 
 const server = new Server(manifest);
 await server.init({ env: process.env });
@@ -20,7 +21,7 @@ const protocol_header = env('PROTOCOL_HEADER', '').toLowerCase();
 const host_header = env('HOST_HEADER', 'host').toLowerCase();
 const body_size_limit = parseInt(env('BODY_SIZE_LIMIT', '524288'));
 
-const dir = path.dirname(fileURLToPath(import.meta.url));
+const dir = fileURLToPath(import.meta.SERVER_DIR);
 
 /**
  * @param {string} path
diff --git a/packages/adapter-node/src/index.js b/packages/adapter-node/src/index.js
index ea3feee05860..5832b244a51e 100644
--- a/packages/adapter-node/src/index.js
+++ b/packages/adapter-node/src/index.js
@@ -1,5 +1,5 @@
-import { handler } from 'HANDLER';
-import { env } from 'ENV';
+import { handler } from './handler.js';
+import { env } from './env.js';
 import polka from 'polka';
 
 export const path = env('SOCKET_PATH', false);
diff --git a/packages/adapter-node/src/shims.js b/packages/adapter-node/src/shims.js
deleted file mode 100644
index 2490311daa1e..000000000000
--- a/packages/adapter-node/src/shims.js
+++ /dev/null
@@ -1,2 +0,0 @@
-import { installPolyfills } from '@sveltejs/kit/node/polyfills';
-installPolyfills();
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index c7a31204d1e6..c7adacd17d61 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -171,6 +171,9 @@ importers:
 
   packages/adapter-node:
     dependencies:
+      '@polka/url':
+        specifier: ^1.0.0-next.21
+        version: 1.0.0-next.21
       '@rollup/plugin-commonjs':
         specifier: ^25.0.0
         version: 25.0.0(rollup@3.7.0)
@@ -180,13 +183,19 @@ importers:
       '@rollup/plugin-node-resolve':
         specifier: ^15.0.1
         version: 15.0.1(rollup@3.7.0)
+      '@rollup/pluginutils':
+        specifier: ^5.0.2
+        version: 5.0.2(rollup@3.7.0)
+      polka:
+        specifier: ^1.0.0-next.22
+        version: 1.0.0-next.22
       rollup:
         specifier: ^3.7.0
         version: 3.7.0
+      sirv:
+        specifier: ^2.0.2
+        version: 2.0.3
     devDependencies:
-      '@polka/url':
-        specifier: ^1.0.0-next.21
-        version: 1.0.0-next.21
       '@sveltejs/kit':
         specifier: workspace:^
         version: link:../kit
@@ -196,15 +205,6 @@ importers:
       c8:
         specifier: ^8.0.0
         version: 8.0.0
-      polka:
-        specifier: ^1.0.0-next.22
-        version: 1.0.0-next.22
-      rimraf:
-        specifier: ^5.0.0
-        version: 5.0.0
-      sirv:
-        specifier: ^2.0.3
-        version: 2.0.3
       typescript:
         specifier: ^4.9.4
         version: 4.9.4
@@ -4814,7 +4814,7 @@ packages:
     dependencies:
       '@polka/url': 1.0.0-next.21
       trouter: 3.2.0
-    dev: true
+    dev: false
 
   /postcss-load-config@3.1.4(postcss@8.4.23):
     resolution: {integrity: sha512-6DiM4E7v4coTE4uzA8U//WhtPwyhiim3eyjEMFCnUpzbrkK9wJHgKDT2mR+HbtSrd/NubVaYTOpSpjUl8NQeRg==}
@@ -5090,7 +5090,7 @@ packages:
   /regexparam@1.3.0:
     resolution: {integrity: sha512-6IQpFBv6e5vz1QAqI+V4k8P2e/3gRrqfCJ9FI+O1FLQTO+Uz6RXZEZOPmTJ6hlGj7gkERzY5BRCv09whKP96/g==}
     engines: {node: '>=6'}
-    dev: true
+    dev: false
 
   /regexparam@2.0.1:
     resolution: {integrity: sha512-zRgSaYemnNYxUv+/5SeoHI0eJIgTL/A2pUtXUPLHQxUldagouJ9p+K6IbIZ/JiQuCEv2E2B1O11SjVQy3aMCkw==}
@@ -5414,7 +5414,6 @@ packages:
       '@polka/url': 1.0.0-next.21
       mrmime: 1.0.1
       totalist: 3.0.0
-    dev: true
 
   /sisteransi@1.0.5:
     resolution: {integrity: sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg==}
@@ -5973,7 +5972,7 @@ packages:
     engines: {node: '>=6'}
     dependencies:
       regexparam: 1.3.0
-    dev: true
+    dev: false
 
   /ts-interface-checker@0.1.13:
     resolution: {integrity: sha512-Y/arvbn+rrz3JCKl9C4kVNfTfSm2/mEp5FSz5EsZSANGPSlQrpRI5M4PKF+mJnE52jOO90PnPSc3Ur3bTQw0gA==}