From f19cd303a50ab49f619384eaf35a6e169e12341e Mon Sep 17 00:00:00 2001 From: pmp-p Date: Wed, 23 Apr 2025 13:46:20 +0200 Subject: [PATCH 01/21] 2 pipeline fixes ( only for socket gw ) --- pglite-REL_17_4_WASM/interactive_one.c | 17 ++++++++++------- pglite-REL_17_4_WASM/pg_proto.c | 2 ++ 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/pglite-REL_17_4_WASM/interactive_one.c b/pglite-REL_17_4_WASM/interactive_one.c index 07e86475ad7ab..650aab9a1b881 100644 --- a/pglite-REL_17_4_WASM/interactive_one.c +++ b/pglite-REL_17_4_WASM/interactive_one.c @@ -285,7 +285,7 @@ startup_auth() { PDEBUG("# 285: sending auth request"); //ClientAuthentication(MyProcPort); discard_input(); - + ClientAuthInProgress = true; md5Salt[0]=0x01; md5Salt[1]=0x23; @@ -592,14 +592,17 @@ PDEBUG("# 367: notified ?"); } #endif - if (!ignore_till_sync) + if (!ignore_till_sync) { + /* initially, or after error */ send_ready_for_query = true; - - if (ignore_till_sync && firstchar != EOF) { - puts("@@@@@@@@@@@@@ 562 TODO: postgres.c 4684 : continue"); - } else { /* process notifications (ASYNC) */ - if (notifyInterruptPending) { PDEBUG("# 565: @@@ has notification @@@@\n"); + if (notifyInterruptPending) ProcessClientReadInterrupt(true); + } else { + /* ignoring till sync will skip all pipeline */ + if (firstchar != EOF) { + if (firstchar != 'S') { + continue; + } } } diff --git a/pglite-REL_17_4_WASM/pg_proto.c b/pglite-REL_17_4_WASM/pg_proto.c index 1abf7024a1a0b..09a014a60a2af 100644 --- a/pglite-REL_17_4_WASM/pg_proto.c +++ b/pglite-REL_17_4_WASM/pg_proto.c @@ -250,6 +250,8 @@ */ PDEBUG("# 251:proc_exit/skip and repl stop"); //proc_exit(0); is_repl = false; + send_ready_for_query = false; + ignore_till_sync = false; break; case 'd': /* copy data */ From dbfa83086673aa196f6038ebf76c1f40f5322d95 Mon Sep 17 00:00:00 2001 From: pmp-p Date: Tue, 29 Apr 2025 11:06:40 +0200 Subject: [PATCH 02/21] sockfixes --- .../src-backend-commands-async.c.diff | 2 +- .../src-bin-initdb-initdb.c.diff | 46 +- pglite-REL_17_4_WASM/build.sh | 222 ++++++-- pglite-REL_17_4_WASM/interactive_one.c | 195 ++++--- pglite-REL_17_4_WASM/pg_main.c | 490 +++++++----------- pglite-REL_17_4_WASM/pg_proto.c | 4 + pglite-REL_17_4_WASM/pgl_initdb.c | 6 +- pglite-REL_17_4_WASM/pgl_mains.c | 42 +- pglite-REL_17_4_WASM/pgl_stubs.h | 52 +- pglite-REL_17_4_WASM/pgl_tools.h | 11 + pglite-REL_17_4_WASM/repl.html | 212 ++++---- wasm-build/build-with-docker.sh | 6 + 12 files changed, 708 insertions(+), 580 deletions(-) diff --git a/patches-REL_17_4_WASM/postgresql-emscripten/src-backend-commands-async.c.diff b/patches-REL_17_4_WASM/postgresql-emscripten/src-backend-commands-async.c.diff index 17646ac8cf3f2..0299b57028d65 100644 --- a/patches-REL_17_4_WASM/postgresql-emscripten/src-backend-commands-async.c.diff +++ b/patches-REL_17_4_WASM/postgresql-emscripten/src-backend-commands-async.c.diff @@ -5,7 +5,7 @@ * just log a low-level debug message if it happens. */ +#if defined(__EMSCRIPTEN__) || defined(__wasi__) -+ HandleNotifyInterrupt(); ++ HandleNotifyInterrupt(); +#else if (SendProcSignal(pid, PROCSIG_NOTIFY_INTERRUPT, procnos[i]) < 0) elog(DEBUG3, "could not signal backend with PID %d: %m", pid); diff --git a/patches-REL_17_4_WASM/postgresql-pglite/src-bin-initdb-initdb.c.diff b/patches-REL_17_4_WASM/postgresql-pglite/src-bin-initdb-initdb.c.diff index cbb044acc6b00..3e110b9c43ff3 100644 --- a/patches-REL_17_4_WASM/postgresql-pglite/src-bin-initdb-initdb.c.diff +++ b/patches-REL_17_4_WASM/postgresql-pglite/src-bin-initdb-initdb.c.diff @@ -20,17 +20,18 @@ const char *username; #ifndef WIN32 -@@ -825,6 +830,9 @@ +@@ -825,6 +830,10 @@ username = get_user_name_or_exit(progname); return pg_strdup(username); +#else ++ setenv("PGUSER", WASM_USERNAME, 0); + return pg_strdup(getenv("PGUSER")); +#endif /* wasm */ } static char * -@@ -1070,6 +1078,9 @@ +@@ -1070,6 +1079,9 @@ static const char * choose_dsm_implementation(void) { @@ -40,7 +41,7 @@ #if defined(HAVE_SHM_OPEN) && !defined(__sun__) int ntries = 10; pg_prng_state prng_state; -@@ -1608,10 +1619,9 @@ +@@ -1608,10 +1620,9 @@ } PG_CMD_CLOSE(); @@ -53,7 +54,7 @@ check_ok(); } -@@ -1711,16 +1721,16 @@ +@@ -1711,16 +1722,16 @@ setup_run_file(FILE *cmdfd, const char *filename) { char **lines; @@ -73,7 +74,7 @@ free(lines); } -@@ -2636,8 +2646,13 @@ +@@ -2636,8 +2647,13 @@ strlcpy(full_path, progname, sizeof(full_path)); if (ret == -1) @@ -87,7 +88,7 @@ else pg_fatal("program \"%s\" was found by \"%s\" but was not the same version as %s", "postgres", full_path, progname); -@@ -3096,7 +3111,7 @@ +@@ -3096,7 +3112,7 @@ initPQExpBuffer(&cmd); printfPQExpBuffer(&cmd, "\"%s\" %s %s template1 >%s", backend_exec, backend_options, extra_options, DEVNULL); @@ -96,7 +97,7 @@ PG_CMD_OPEN(cmd.data); setup_auth(cmdfd); -@@ -3134,14 +3149,53 @@ +@@ -3134,14 +3150,53 @@ PG_CMD_CLOSE(); termPQExpBuffer(&cmd); @@ -133,11 +134,11 @@ + strcat_alloc(PREFIX,"/bin/initdb"), + // "--no-clean", + "--wal-segsize=1", -+ "-g", -+ "-E", "UTF8", "--locale=C.UTF-8", "--locale-provider=libc", -+// "-E", "UTF8", "--locale", "C.UTF-8", "--locale-provider=builtin", -+// "--locale", "en_US.UTF-8", -+// "--icu-locale=en-US", "--locale-provider=icu", ++ "--allow-group-access", "--no-sync", ++ "-E", "UTF8", ++ "--locale=C.UTF-8", "--locale-provider=libc", ++// "--builtin-locale=en_US.UTF-8", "--locale-provider=builtin", ++// "--locale-provider=icu", "--icu-locale=en-US", "--locale-provider=icu", + "-U", WASM_USERNAME, pwfile, //"--pwfile=" WASM_PREFIX "/password", + pgdata, // "--pgdata=" WASM_PREFIX "/base", + NULL @@ -153,7 +154,7 @@ static struct option long_options[] = { {"pgdata", required_argument, NULL, 'D'}, {"encoding", required_argument, NULL, 'E'}, -@@ -3201,9 +3255,15 @@ +@@ -3201,9 +3256,15 @@ * POSIX says we must do this before any other usage of these files. */ setvbuf(stdout, NULL, PG_IOLBF, 0); @@ -170,28 +171,33 @@ set_pglocale_pgservice(argv[0], PG_TEXTDOMAIN("initdb")); if (argc > 1) -@@ -3409,7 +3469,7 @@ +@@ -3409,7 +3470,7 @@ if (icu_rules && locale_provider != COLLPROVIDER_ICU) pg_fatal("%s cannot be specified unless locale provider \"%s\" is chosen", "--icu-rules", "icu"); - -+PDEBUG("# 3463:"__FILE__ " TODO: atexit(cleanup_directories_atexit"); ++PDEBUG("# 3463:"__FILE__ " TODO: atexit(cleanup_directories_atexit)"); atexit(cleanup_directories_atexit); /* If we only need to sync, just do it and exit */ -@@ -3445,9 +3505,9 @@ +@@ -3445,13 +3506,13 @@ get_restricted_token(); setup_pgdata(); - -+puts("# 3493:pgl_initdb_main " __FILE__); ++PDEBUG("# 3493:pgl_initdb_main " __FILE__); setup_bin_paths(argv[0]); - -+puts("# 3495:pgl_initdb_main " __FILE__); ++PDEBUG("# 3495:pgl_initdb_main " __FILE__); effective_user = get_id(); if (!username) username = effective_user; -@@ -3479,7 +3539,7 @@ +- ++PDEBUG("# 3514:pgl_initdb_main " __FILE__); + if (strncmp(username, "pg_", 3) == 0) + pg_fatal("superuser name \"%s\" is disallowed; role names cannot begin with \"pg_\"", username); + +@@ -3479,7 +3540,7 @@ get_su_pwd(); printf("\n"); @@ -200,7 +206,7 @@ initialize_data_directory(); if (do_sync) -@@ -3535,7 +3595,7 @@ +@@ -3535,7 +3596,7 @@ destroyPQExpBuffer(start_db_cmd); } diff --git a/pglite-REL_17_4_WASM/build.sh b/pglite-REL_17_4_WASM/build.sh index 66fb6366e6f6f..bc8f709f84eed 100755 --- a/pglite-REL_17_4_WASM/build.sh +++ b/pglite-REL_17_4_WASM/build.sh @@ -1,26 +1,145 @@ -#!/bin/bash -echo "pglite/build: begin" +export WASI=${WASI:-false} + +if ${WASI} +then + BUILD=wasi +else + BUILD=emscripten + WEBROOT=${PG_DIST}/web +fi + +echo "pglite/build-$BUILD: begin target BUILD_PATH=$BUILD_PATH" WORKSPACE=$(pwd) PGROOT=/tmp/pglite -PGSRC=${WORKSPACE} -PGBUILD=${WORKSPACE}/build/postgres +if [ -d ${WORKSPACE}/src/fe_utils ] +then + PGSRC=${WORKSPACE} +else + PGSRC=${WORKSPACE}/postgresql-${PG_BRANCH} +fi -LIBPGCORE=${PGBUILD}/libpgcore.a +LIBPGCORE=${BUILD_PATH}/libpgcore.a -WEBROOT=${PGBUILD}/web -PGINC="-I/tmp/pglite/include \ - -I${PGSRC}/src/include -I${PGSRC}/src/interfaces/libpq \ - -I${PGBUILD}/src/include" + +PGINC=" -I${BUILD_PATH}/src/include \ +-I${PGROOT}/include -I${PGROOT}/include/postgresql/server \ +-I${PGSRC}/src/include -I${PGSRC}/src/interfaces/libpq -I${PGSRC}/src" + + +GLOBAL_BASE_B=$(python3 -c "print(${CMA_MB}*1024*1024)") if $WASI then - echo TODO +echo " +_______________________ PG_BRANCH=${PG_BRANCH} _____________________ + +wasi : $(which wasi-c) $(wasi-c -v) +python : $(which python3) $(python3 -V) +wasmtime : $(which wasmtime) + +CC=${CC:-undefined} + +Linking to libpgcore static from $LIBPGCORE + +Folders : + source : $PGSRC + build : $BUILD_PATH + target : $PGROOT + retarget : ${PGL_DIST_C} + native : ${PGL_DIST_NATIVE} + + + CPOPTS : $COPTS + DEBUG : $DEBUG + LOPTS : $LOPTS + + CMA_MB : $CMA_MB +GLOBAL_BASE : $GLOBAL_BASE_B + + CC_PGLITE : $CC_PGLITE + + ICU i18n : $USE_ICU + +INCLUDES: $PGINC +________________________________________________________ + + +" + + + if ${CC} -ferror-limit=1 ${CC_PGLITE} \ + ${PGINC} \ + -DPOSTGRES_C=\"../postgresql/src/backend/tcop/postgres.c\" \ + -DPQEXPBUFFER_H=\"../postgresql/src/interfaces/libpq/pqexpbuffer.h\" \ + -DOPTION_UTILS_C=\"../postgresql/src/fe_utils/option_utils.c\" \ + -o ${BUILD_PATH}/pglite.o -c ${WORKSPACE}/pglite-wasm/pg_main.c \ + -Wno-incompatible-pointer-types-discards-qualifiers + then + if ${CC} -fpic -ferror-limit=1 ${CC_PGLITE} ${PGINC} \ + -o ${BUILD_PATH}/sdk_port-wasi.o \ + -c wasm-build/sdk_port-wasi/sdk_port-wasi-dlfcn.c \ + -Wno-incompatible-pointer-types + then + COPTS="$LOPTS" ${CC} ${CC_PGLITE} -ferror-limit=1 -Wl,--global-base=${GLOBAL_BASE_B} -o ${PG_DIST}/pglite.wasi \ + -nostartfiles ${PGINC} ${BUILD_PATH}/pglite.o \ + ${BUILD_PATH}/sdk_port-wasi.o \ + $LINKER $LIBPGCORE \ + $LINK_ICU \ + ${PG_BUILD}/${BUILD}/src/backend/snowball/libdict_snowball.a \ + ${PG_BUILD}/${BUILD}/src/pl/plpgsql/src/libplpgsql.a \ + -lxml2 -lz + else + echo "compilation of libpglite ${BUILD} support failed" + fi + + if [ -f ${PG_DIST}/pglite.wasi ] + then + echo "building minimal wasi FS" + cp ${PG_DIST}/pglite.wasi ${PGROOT}/bin/ + touch ${PGROOT}/bin/initdb ${PGROOT}/bin/postgres + tar -cvJ --files-from=${WORKSPACE}/wasmfs.txt > ${PG_DIST}/pglite-wasi.tar.xz + mkdir -p ${PGL_BUILD_NATIVE} + cat > ${PGL_BUILD_NATIVE}/pglite-native.sh < ${PREFIX}/${cleanup} - done - - mkdir -p ${PG_DIST:-/tmp/sdk/dist}/pglite - COPTS="$LOPTS" ${CC} ${CC_PGLITE} -sGLOBAL_BASE=${CMA_MB}MB -o ${PG_DIST:-/tmp/sdk/dist}/pglite/pglite.html -ferror-limit=1 --shell-file ${WORKSPACE}/pglite-wasm/repl.html \ - $PGPRELOAD \ - -sFORCE_FILESYSTEM=1 -sNO_EXIT_RUNTIME=1 -sENVIRONMENT=node,web \ - -sMODULARIZE=1 -sEXPORT_ES6=1 -sEXPORT_NAME=Module \ - -sALLOW_TABLE_GROWTH -sALLOW_MEMORY_GROWTH -sERROR_ON_UNDEFINED_SYMBOLS \ - -sEXPORTED_RUNTIME_METHODS=${EXPORTED_RUNTIME_METHODS} \ - ${PGINC} ${PGBUILD}/pglite.o \ - $LINKER $LIBPGCORE \ - $LINK_ICU \ - -lnodefs.js -lidbfs.js -lxml2 -lz - + else + echo "compilation of libpglite ${PG_BRANCH} failed" + exit 220 + fi fi -du -hs ${PG_DIST:-/tmp/sdk/dist}/pglite/pglite.* -echo "pglite/build: end" +du -hs ${PG_DIST}/* + +echo "pglite/build($BUILD): end" + diff --git a/pglite-REL_17_4_WASM/interactive_one.c b/pglite-REL_17_4_WASM/interactive_one.c index 650aab9a1b881..195b0647f9361 100644 --- a/pglite-REL_17_4_WASM/interactive_one.c +++ b/pglite-REL_17_4_WASM/interactive_one.c @@ -1,9 +1,9 @@ #include // access, unlink -#if !defined(__wasi__) -volatile sigjmp_buf local_sigjmp_buf; +#if defined(__wasi__) +// volatile sigjmp_buf void*; #else -volatile sigjmp_buf void*; +volatile sigjmp_buf local_sigjmp_buf; #endif /* TODO : prevent multiple write and write while reading ? */ @@ -152,7 +152,7 @@ static void io_init(bool in_auth, bool out_auth) { whereToSendOutput = DestRemote; /* now safe to ereport to client */ if (!MyProcPort) { - PDEBUG("# 137: io_init --------- NO CLIENT (oom) ---------"); + PDEBUG("# 155: io_init --------- NO CLIENT (oom) ---------"); abort(); } #ifdef PG16 @@ -162,49 +162,39 @@ static void io_init(bool in_auth, bool out_auth) { SOCKET_FILE = NULL; SOCKET_DATA = 0; - PDEBUG("\n\n\n\n# 147: io_init --------- Ready for CLIENT ---------"); + PDEBUG("\n\n\n# 165: io_init --------- Ready for CLIENT ---------"); } -volatile int sf_connected = 0; + volatile bool sockfiles = false; volatile bool is_wire = true; extern char * cma_port; +extern void pq_startmsgread(void); - -__attribute__((export_name("interactive_write"))) // EMSCRIPTEN_KEEPALIVE +__attribute__((export_name("interactive_write"))) void interactive_write(int size) { cma_rsize = size; cma_wsize = 0; } -__attribute__((export_name("ping"))) -void -ping(void) { -#if PGDEBUG - puts("pong"); -#endif -} - __attribute__((export_name("use_wire"))) void use_wire(int state) { #if PGDEBUG force_echo=true; -#else - force_echo=false; #endif if (state>0) { #if PGDEBUG - printf("176: wire mode, repl off, echo %d\n", force_echo); + printf("# 199: wire mode, repl off, echo %d\n", force_echo); #endif is_wire = true; is_repl = false; } else { #if PGDEBUG - printf("181: repl mode, no wire, echo %d\n", force_echo); + printf("# 205: repl mode, no wire, echo %d\n", force_echo); #endif is_wire = false; is_repl = true; @@ -278,13 +268,16 @@ void discard_input(){ void startup_auth() { /* code is in handshake/auth domain so read whole msg now */ + send_ready_for_query = false; if (ProcessStartupPacket(MyProcPort, true, true) != STATUS_OK) { - PDEBUG("# 283: ProcessStartupPacket !OK"); + PDEBUG("# 271: ProcessStartupPacket !OK"); } else { - PDEBUG("# 285: sending auth request"); + + sf_connected++; + PDEBUG("# 273: sending auth request"); //ClientAuthentication(MyProcPort); - discard_input(); + discard_input(); ClientAuthInProgress = true; md5Salt[0]=0x01; @@ -309,7 +302,7 @@ startup_pass(bool check) { // auth 'p' if (check) { char *passwd = recv_password_packet(MyProcPort); - printf("# 312: auth recv password: %s\n", "md5***" ); + PDEBUG("# 223: auth recv password: md5***"); /* // TODO: CheckMD5Auth if (passwd == NULL) @@ -321,8 +314,8 @@ startup_pass(bool check) { */ pfree(passwd); } else { - PDEBUG("# 324: auth skip"); - discard_input(); + PDEBUG("# 310: auth skip"); + discard_input(); } ClientAuthInProgress = false; @@ -342,15 +335,15 @@ startup_pass(bool check) { pq_sendint32(&buf, (int32) MyCancelKey); pq_endmessage(&buf); } -PDEBUG("# 305: TODO: set a pg_main started flag"); - sf_connected++; +PDEBUG("# 330: TODO: set a pgl started flag"); send_ready_for_query = true; - + ignore_till_sync = false; + volatile int sf_connected = 0; } extern void pg_startcma(); -EMSCRIPTEN_KEEPALIVE void +__attribute__((export_name("interactive_one"))) void interactive_one() { int peek = -1; /* preview of firstchar with no pos change */ int firstchar = 0; /* character read from getc() */ @@ -358,34 +351,40 @@ interactive_one() { StringInfoData input_message; StringInfoData *inBuf; FILE *stream ; - FILE *fp; + FILE *fp = NULL; int packetlen; bool had_notification = notifyInterruptPending; bool notified = false; -if (notifyInterruptPending) - PDEBUG("# 327: has notification !"); + send_ready_for_query = false; if (!MyProcPort) { + PDEBUG("# 353: client created"); io_init(is_wire, false); } +#if PGDEBUG + puts("\n\n# 377: interactive_one"); + if (notifyInterruptPending) + PDEBUG("# 359: has notification !"); +#endif + // this could be pg_flush in sync mode. // but in fact we are writing socket data that was piled up previous frame async. if (SOCKET_DATA>0) { - puts("331: ERROR flush after frame"); + puts("# 361: ERROR flush after frame"); goto wire_flush; } if (!cma_rsize) { - // prepare reply queue + puts("# 388: socketfiles"); // prepare reply queue if (!SOCKET_FILE) { SOCKET_FILE = fopen(PGS_OLOCK, "w") ; MyProcPort->sock = fileno(SOCKET_FILE); } } - doing_extended_query_message = false; + MemoryContextSwitchTo(MessageContext); MemoryContextResetAndDeleteChildren(MessageContext); @@ -398,32 +397,28 @@ if (notifyInterruptPending) if (send_ready_for_query) { if (IsAbortedTransactionBlockState()) { - puts("@@@@ TODO 356: idle in transaction (aborted)"); + puts("@@@@ TODO 390: idle in transaction (aborted)"); } else if (IsTransactionOrTransactionBlock()) { - puts("@@@@ TODO 359: idle in transaction"); + puts("@@@@ TODO 393: idle in transaction"); } else { if (notifyInterruptPending) { ProcessNotifyInterrupt(false); -PDEBUG("# 367: notified ?"); notified = true; } } send_ready_for_query = false; } - - // postgres.c 4627 DoingCommandRead = true; #if defined(EMUL_CMA) - #define IO ((char *)(1+(int)cma_port)) // temp fix for -O0 but less efficient than literal - #error "inefficient" + // temp fix for -O0 but less efficient than literal + #define IO ((char *)(1+(int)cma_port)) #else #define IO ((char *)(1)) #endif - /* * in cma mode (cma_rsize>0), client call the wire loop itself waiting synchronously for the results * in socketfiles mode, the wire loop polls a pseudo socket made from incoming and outgoing files. @@ -438,30 +433,40 @@ PDEBUG("# 367: notified ?"); if (!is_repl) { whereToSendOutput = DestRemote; if (!is_wire) - PDEBUG("repl message in cma buffer !"); + PDEBUG("# 426: repl message in cma buffer !"); } else { if (is_wire) - PDEBUG("wire message in cma buffer for REPL !"); + PDEBUG("# 429: wire message in cma buffer for REPL !"); whereToSendOutput = DestDebug; } } else { fp = fopen(PGS_IN, "r"); - - // read as a socket. +PDEBUG("# 451:" PGS_IN); + // read file in socket buffer for SocketBackend to consumme. if (fp) { fseek(fp, 0L, SEEK_END); packetlen = ftell(fp); if (packetlen) { - // always. - is_wire = true; - sockfiles = true; - whereToSendOutput = DestRemote; + // set to always true if no REPL. +// is_wire = true; resetStringInfo(inBuf); rewind(fp); /* peek on first char */ peek = getc(fp); rewind(fp); - pq_recvbuf_fill(fp, packetlen); + if (is_repl && !is_wire) { + // sql in buffer + for (int i=0; i true : %c\n", peek); - force_echo = true; + if (is_wire) { + printf("# 499: is_wire -> true : %c\n", peek); + force_echo = true; + } + #endif firstchar = peek; goto incoming; } // wire msg - +PDEBUG("# 500: NO DATA:" PGS_IN ); } // fp data read // is it REPL in cma ? @@ -511,8 +519,8 @@ PDEBUG("# 367: notified ?"); #if PGDEBUG if (packetlen) - IO[packetlen]=0; - printf("# 479: fd=%d is_embed=%d is_repl=%d is_wire=%d fd %s,len=%d peek=%d [%s]\n", MyProcPort->sock, is_embed, is_repl, is_wire, PGS_OLOCK, packetlen, peek, IO); + IO[packetlen]=0; // wire blocks are not zero terminated + printf("\n# 500: fd=%d is_embed=%d is_repl=%d is_wire=%d fd %s,len=%d cma=%d peek=%d [%s]\n", MyProcPort->sock, is_embed, is_repl, is_wire, PGS_OLOCK, packetlen,cma_rsize, peek, IO); #endif // buffer query TODO: direct access ? @@ -524,7 +532,7 @@ PDEBUG("# 367: notified ?"); } if (packetlen<2) { - puts("# 491: WARNING: empty packet"); + puts("# 512: WARNING: empty packet"); cma_rsize= 0; if (is_repl) pg_prompt(); @@ -540,26 +548,32 @@ PDEBUG("# 367: notified ?"); #error "sigsetjmp unsupported" #endif // wasi + while (pipelining) { + if (is_repl) { - // TODO: are we sure repl could not pipeline ? + // are we sure repl could not pipeline ? pipelining = false; /* stdio node repl */ - printf("# 512: enforcing REPL mode, wire off, echo %d\n", force_echo); +#if PGDEBUG + printf("\n# 533: enforcing REPL mode, wire off, echo %d\n", force_echo); +#endif whereToSendOutput = DestDebug; } + DoingCommandRead = true; if (is_wire) { - /* wire on a socket or cma may auth, not handled by pg_proto block */ + /* wire on a socket or cma may auth */ + /* would be handled as error by pg_proto block */ if (peek==0) { - PDEBUG("# 519: handshake/auth"); + PDEBUG("# 540: handshake/auth"); startup_auth(); - PDEBUG("# 521: auth request"); + PDEBUG("# 542: auth request"); break; } if (peek==112) { - PDEBUG("# 525: password"); + PDEBUG("# 547: password"); startup_pass(true); break; } @@ -567,14 +581,16 @@ PDEBUG("# 367: notified ?"); firstchar = SocketBackend(inBuf); pipelining = pq_buffer_remaining_data()>0; +#if PGDEBUG if (!pipelining) { - printf("# 535: end of wire, rfq=%d\n", send_ready_for_query); + printf("# 556: end of wire, rfq=%d\n", send_ready_for_query); } else { - printf("# 537: no end of wire -> pipelining, rfq=%d\n", send_ready_for_query); + printf("# 558: no end of wire -> pipelining, rfq=%d\n", send_ready_for_query); } +#endif } else { /* nowire */ - pipelining = false; + // pipelining = false; if (firstchar == EOF && inBuf->len == 0) { firstchar = EOF; } else { @@ -582,15 +598,16 @@ PDEBUG("# 367: notified ?"); firstchar = 'Q'; } } + DoingCommandRead = false; - #if PGDEBUG +#if PGDEBUG if (!pipelining) { - printf("# 552: wire=%d 1stchar=%c Q: %s\n", is_wire, firstchar, inBuf->data); + printf("# 573: wire=%d 1stchar=%c Q: %s\n", is_wire, firstchar, inBuf->data); force_echo = false; } else { - printf("# 555: PIPELINING [%c]!\n", firstchar); + printf("# 576: PIPELINING [%c]!\n", firstchar); } - #endif +#endif if (!ignore_till_sync) { /* initially, or after error */ @@ -607,6 +624,11 @@ PDEBUG("# 367: notified ?"); } #include "pg_proto.c" + + if (send_ready_for_query) { + ReadyForQuery(whereToSendOutput); + send_ready_for_query = false; + } } if (!is_repl) { @@ -617,14 +639,14 @@ PDEBUG("# 367: notified ?"); ProcessNotifyInterrupt(false); if (send_ready_for_query) { - PDEBUG("# 581: end packet - sending rfq"); + PDEBUG("# 602: end packet - sending rfq\n"); ReadyForQuery(DestRemote); //done at postgres.c 4623 send_ready_for_query = false; } else { - PDEBUG("# 585: end packet - with no rfq"); + PDEBUG("# 606: end packet - with no rfq\n"); } } else { - PDEBUG("# 588: end packet (ClientAuthInProgress - no rfq) "); + PDEBUG("# 609: end packet (ClientAuthInProgress - no rfq)\n"); } if (SOCKET_DATA>0) { @@ -641,29 +663,38 @@ PDEBUG("# 367: notified ?"); SOCKET_FILE = NULL; SOCKET_DATA = 0; if (cma_wsize) - PDEBUG("# 605: cma and sockfile ???"); + PDEBUG("# 626: cma and sockfile ???\n"); if (sockfiles) { #if PGDEBUG - printf("# 608: client:ready -> read(%d) " PGS_OLOCK "->" PGS_OUT"\n", outb); + printf("# 629: client:ready -> read(%d) " PGS_OLOCK "->" PGS_OUT"\n", outb); #endif rename(PGS_OLOCK, PGS_OUT); } } else { #if PGDEBUG - printf("# 614: out queue : %d flushed\n", cma_wsize); + printf("\n# 635: in[%d] out[%d] flushed\n", cma_rsize, cma_wsize); #endif SOCKET_DATA = 0; } } else { - cma_wsize = 0; + cma_wsize = 0; PDEBUG("# 680: no socket data"); } } else { pg_prompt(); +#if PGDEBUG + puts("# 683: repl output"); + if (SOCKET_DATA>0) { + puts("# 686: socket has data"); + if (sockfiles) + printf("# 688: socket file not flushed -> read(%d) " PGS_OLOCK "->" PGS_OUT"\n", SOCKET_DATA); + } + if (cma_wsize) + puts("ERROR: cma was not flushed before socketfile interface"); +#endif } - // always free kernel buffer !!! cma_rsize = 0; IO[0] = 0; diff --git a/pglite-REL_17_4_WASM/pg_main.c b/pglite-REL_17_4_WASM/pg_main.c index 2d18f91f5685f..05533e1b86297 100644 --- a/pglite-REL_17_4_WASM/pg_main.c +++ b/pglite-REL_17_4_WASM/pg_main.c @@ -1,5 +1,8 @@ +#define FIXME 1 + #define PGL_MAIN #define PGL_INITDB_MAIN +#define REPL 0 // #define PGDEBUG_STARTUP // MEMFS files for os pipe simulation @@ -14,8 +17,8 @@ #include "utils/pg_locale.h" #include "tcop/tcopprot.h" -#include /* chdir */ -#include /* mkdir */ +#include /* chdir */ +#include /* mkdir */ // globals @@ -26,20 +29,20 @@ int g_argc; char **g_argv; -extern char ** environ; +extern char **environ; volatile char *PREFIX; volatile char *PGDATA; volatile char *PGUSER; -const char * progname; +const char *progname; volatile bool is_repl = true; volatile bool is_node = true; volatile bool is_embed = false; volatile int pgl_idb_status; -// will backend restart after initdb. defaut is yes. +// now backend start manually after initdb. // TODO: log sync start failures and ask to repair/clean up db. volatile int async_restart = 1; @@ -79,18 +82,13 @@ extern int BootstrapModeMain(int, char **, int); #include "libpq/pqformat.h" - - volatile bool send_ready_for_query = true; volatile bool idle_in_transaction_timeout_enabled = false; volatile bool idle_session_timeout_enabled = false; -/* -bool quote_all_identifiers = false; -FILE* SOCKET_FILE = NULL; -int SOCKET_DATA = 0; -*/ -void pg_free(void *ptr) { + +void +pg_free(void *ptr) { free(ptr); } @@ -112,7 +110,7 @@ static bool force_echo = false; #include "pgl_initdb.c" -// interactive_one +// interactive_one, heart of the async loop. #include "./interactive_one.c" @@ -122,38 +120,38 @@ main_pre(int argc, char *argv[]) { char key[256]; - int i=0; + int i = 0; // extra env is always after normal args - PDEBUG("# ============= extra argv dump =================="); + PDEBUG("# ============= extra argv dump ==================\n"); { - for (;inextOid < ((Oid) FirstNormalObjectId)) { /* IsPostmasterEnvironment is now true - these will be executed when required in varsup.c/GetNewObjectId - TransamVariables->nextOid = FirstNormalObjectId; - TransamVariables->oidCount = 0; - */ + these will be executed when required in varsup.c/GetNewObjectId + TransamVariables->nextOid = FirstNormalObjectId; + TransamVariables->oidCount = 0; + */ #if PGDEBUG - puts("# 382: initdb done, oid base too low but OID range will be set because IsPostmasterEnvironment"); + puts("# 403: initdb done, oid base too low but OID range will be set because IsPostmasterEnvironment"); #endif } -} + } #if defined(__EMSCRIPTEN__) -EMSCRIPTEN_KEEPALIVE + EMSCRIPTEN_KEEPALIVE #else -__attribute__((export_name("pgl_initdb"))) + __attribute__ ((export_name("pgl_initdb"))) #endif -int -pgl_initdb() { - PDEBUG("# 433: pg_initdb()"); + int pgl_initdb() { + PDEBUG("# 412: pg_initdb()"); optind = 1; pgl_idb_status |= IDB_FAILED; - if (!chdir(PGDATA)){ + if (!chdir(PGDATA)) { if (access("PG_VERSION", F_OK) == 0) { - chdir("/"); + chdir("/"); pgl_idb_status |= IDB_HASDB; /* assume auth success for now */ pgl_idb_status |= IDB_HASUSER; #if PGDEBUG - fprintf(stdout, "# 446: pg_initdb: db exists at : %s TODO: test for db name : %s \n", PGDATA, getenv("PGDATABASE")); + fprintf(stdout, "# 427: pg_initdb: db exists at : %s TODO: test for db name : %s \n", PGDATA, getenv("PGDATABASE")); #endif // PGDEBUG async_restart = 0; goto initdb_done; } - chdir("/"); + chdir("/"); #if PGDEBUG - fprintf(stderr, "# 454: pg_initdb no db found at : %s\n", PGDATA ); + fprintf(stderr, "# 435: pg_initdb no db found at : %s\n", PGDATA); #endif // PGDEBUG } else { #if PGDEBUG - fprintf(stderr, "# 458: pg_initdb db folder not found at : %s\n", PGDATA ); + fprintf(stderr, "# 439: pg_initdb db folder not found at : %s\n", PGDATA); #endif // PGDEBUG } int initdb_rc = pgl_initdb_main(); #if PGDEBUG - fprintf(stderr, "\n\n# 465: " __FILE__ "pgl_initdb_main = %d\n", initdb_rc ); + fprintf(stderr, "\n\n# 444: " __FILE__ "pgl_initdb_main = %d\n", initdb_rc); #endif // PGDEBUG - PDEBUG("# 467:" __FILE__); + PDEBUG("# 448:" __FILE__); /* save stdin and use previous initdb output to feed boot mode */ int saved_stdin = dup(STDIN_FILENO); { - PDEBUG("# 471: restarting in boot mode for initdb"); + PDEBUG("# 450: restarting in boot mode for initdb"); freopen(IDB_PIPE_BOOT, "r", stdin); char *boot_argv[] = { g_argv[0], "--boot", "-D", PGDATA, - "-d","3", + "-d", "3", WASM_PGOPTS, "-X", "1048576", NULL }; - int boot_argc = sizeof(boot_argv) / sizeof(char*) - 1; + int boot_argc = sizeof(boot_argv) / sizeof(char *) - 1; - set_pglocale_pgservice(boot_argv[0], PG_TEXTDOMAIN("initdb")); + set_pglocale_pgservice(boot_argv[0], PG_TEXTDOMAIN("initdb")); optind = 1; BootstrapModeMain(boot_argc, boot_argv, false); fclose(stdin); +#if PGDEBUG + puts("BOOT FILE:"); + puts(IDB_PIPE_BOOT); +#else remove(IDB_PIPE_BOOT); +#endif stdin = fdopen(saved_stdin, "r"); - PDEBUG("# 493: initdb faking shutdown to complete WAL/OID states"); + PDEBUG("# 479: initdb faking shutdown to complete WAL/OID states"); pg_proc_exit(66); } @@ -481,7 +488,7 @@ pgl_initdb() { //IsPostmasterEnvironment = true; if (TransamVariables->nextOid < ((Oid) FirstNormalObjectId)) { #if PGDEBUG - puts("# 503: warning oid base too low, will need to set OID range after initdb(bootstrap/single)"); + puts("# 482: warning oid base too low, will need to set OID range after initdb(bootstrap/single)"); #endif } /* @@ -506,110 +513,21 @@ PDEBUG("# 498: initdb faking shutdown to complete WAL/OID states in single mode" async_restart = 1; } */ - async_restart = 1; -initdb_done:; + async_restart = 1; + initdb_done:; pgl_idb_status |= IDB_CALLED; - if (optind>0) { + if (optind > 0) { /* RESET getopt */ optind = 1; /* we did not fail, clear the default failed state */ pgl_idb_status &= IDB_OK; } else { - PDEBUG("# 511: exiting on initdb-single error"); + PDEBUG("# 524: exiting on initdb-single error"); // TODO raise js exception } return pgl_idb_status; -} // pg_initdb - - - - - -int -main_repl() { - bool hadloop_error = false; - - whereToSendOutput = DestNone; - - if (!mkdir(PGDATA, 0700)) { - /* no db : run initdb now. */ -#if defined(PGDEBUG_STARTUP) - fprintf(stderr, "PGDATA=%s not found, running initdb with default=%s\n",PGDATA, WASM_PGDATA ); -#endif - #if defined(PG_INITDB_MAIN) - #warning "web build" - hadloop_error = pg_initdb() & IDB_FAILED; - #else - #warning "node build" - #if defined(__wasi__) - hadloop_error = pg_initdb() & IDB_FAILED; - #endif - #endif - - } else { - // download a db case ? - mkdirp(WASM_PGDATA); - - // db fixup because empty dirs may not be packaged in git case - mksub_dir(WASM_PGDATA, "/pg_wal"); - mksub_dir(WASM_PGDATA, "/pg_wal/archive_status"); - mksub_dir(WASM_PGDATA, "/pg_wal/summaries"); - - mksub_dir(WASM_PGDATA, "/pg_tblspc"); - mksub_dir(WASM_PGDATA, "/pg_snapshots"); - mksub_dir(WASM_PGDATA, "/pg_commit_ts"); - mksub_dir(WASM_PGDATA, "/pg_notify"); - mksub_dir(WASM_PGDATA, "/pg_replslot"); - mksub_dir(WASM_PGDATA, "/pg_twophase"); - - mksub_dir(WASM_PGDATA, "/pg_logical"); - mksub_dir(WASM_PGDATA, "/pg_logical/snapshots"); - mksub_dir(WASM_PGDATA, "/pg_logical/mappings"); - - } - - if (!hadloop_error) { - main_post(); - - /* - * Catch standard options before doing much else, in particular before we - * insist on not being root. - */ - if (g_argc > 1) { - if (strcmp(g_argv[1], "--help") == 0 || strcmp(g_argv[1], "-?") == 0) - { - help(progname); - exit(0); - } - if (strcmp(g_argv[1], "--version") == 0 || strcmp(g_argv[1], "-V") == 0) - { - fputs(PG_BACKEND_VERSIONSTR, stdout); - exit(0); - } - - } - - if (g_argc > 1 && strcmp(g_argv[1], "--check") == 0) { - BootstrapModeMain(g_argc, g_argv, true); - return 0; - } - - if (g_argc > 1 && strcmp(g_argv[1], "--boot") == 0) { - PDEBUG("# 1410: boot: " __FILE__ ); - BootstrapModeMain(g_argc, g_argv, false); - return 0; - } - - PDEBUG("# 570: single: " __FILE__ ); - AsyncPostgresSingleUserMain(g_argc, g_argv, PGUSER, 0); - } - return 0; -} - - - - +} // pgl_initdb @@ -626,59 +544,43 @@ main_repl() { PGOPTIONS */ - - - -// ???? __attribute__((export_name("_main"))) -int -main(int argc, char **argv) -{ - int exit_code = 0; - main_pre(argc,argv); +// __attribute__((export_name("main"))) + int main(int argc, char **argv) { + int exit_code = 0; + main_pre(argc, argv); #if PGDEBUG - printf("# 616: argv0 (%s) PGUSER=%s PGDATA=%s\n PGDATABASE=%s REPL=%s\n", - argv[0], PGUSER, PGDATA, getenv("PGDATABASE"), getenv("REPL") ); + printf("# 550: argv0 (%s) PGUSER=%s PGDATA=%s\n PGDATABASE=%s REPL=%s\n", + argv[0], PGUSER, PGDATA, getenv("PGDATABASE"), getenv("REPL")); #endif - progname = get_progname(argv[0]); - startup_hacks(progname); - g_argv = argv; - g_argc = argc; + progname = get_progname(argv[0]); + startup_hacks(progname); + g_argv = argv; + g_argc = argc; - is_repl = strlen(getenv("REPL")) && getenv("REPL")[0]!='N'; - is_embed = true; + is_repl = strlen(getenv("REPL")) && getenv("REPL")[0] != 'N'; + is_embed = true; - if (!is_repl) { - PDEBUG("# 628: exit with live runtime (nodb)"); - return 0; - } -/* + if (!is_repl) { + PDEBUG("# 562: exit with live runtime (nodb)"); + return 0; + } +#if defined(__wasi__) + + +#else + /* main_post(); - PDEBUG("# 634: repl"); + PDEBUG("# 565: repl"); // so it is repl main_repl(); if (is_node) { - PDEBUG("# 639: node repl"); + PDEBUG("# 570: node repl"); pg_repl_raf(); } -*/ - emscripten_force_exit(exit_code); - return exit_code; + */ + emscripten_force_exit(exit_code); +#endif + return exit_code; } - - - - - - - - - - - - - - - - diff --git a/pglite-REL_17_4_WASM/pg_proto.c b/pglite-REL_17_4_WASM/pg_proto.c index 09a014a60a2af..30201c8e49b75 100644 --- a/pglite-REL_17_4_WASM/pg_proto.c +++ b/pglite-REL_17_4_WASM/pg_proto.c @@ -248,6 +248,10 @@ * it will fail to be called during other backend-shutdown * scenarios. */ +if (sf_connected) + sf_connected--; +else + puts("ERROR: more exits than connections"); PDEBUG("# 251:proc_exit/skip and repl stop"); //proc_exit(0); is_repl = false; send_ready_for_query = false; diff --git a/pglite-REL_17_4_WASM/pgl_initdb.c b/pglite-REL_17_4_WASM/pgl_initdb.c index 3fc19685ca549..19bf7fd405457 100644 --- a/pglite-REL_17_4_WASM/pgl_initdb.c +++ b/pglite-REL_17_4_WASM/pgl_initdb.c @@ -36,18 +36,18 @@ pg_chmod(const char * path, int mode_t) { #endif #define FRONTEND -# include "../postgresql/src/common/logging.c" +# include "common/logging.c" #undef FRONTEND -#include "../postgresql/src/interfaces/libpq/pqexpbuffer.c" +#include "interfaces/libpq/pqexpbuffer.c" #define sync_pgdata(...) #define icu_language_tag(loc_str) icu_language_tag_idb(loc_str) #define icu_validate_locale(loc_str) icu_validate_locale_idb(loc_str) -#include "../postgresql/src/bin/initdb/initdb.c" +#include "bin/initdb/initdb.c" void use_socketfile(void) { is_repl = true; diff --git a/pglite-REL_17_4_WASM/pgl_mains.c b/pglite-REL_17_4_WASM/pgl_mains.c index 0f8305b48a214..e09f3e79738d0 100644 --- a/pglite-REL_17_4_WASM/pgl_mains.c +++ b/pglite-REL_17_4_WASM/pgl_mains.c @@ -1,5 +1,6 @@ #include +volatile int sf_connected = 0; FILE * single_mode_feed = NULL; volatile bool inloop = false; volatile sigjmp_buf local_sigjmp_buf; @@ -12,6 +13,17 @@ pg_shutdown() { proc_exit(66); } +__attribute__((export_name("pgl_closed"))) +int +pgl_closed() { + if (sf_connected>0) + return 1; + return 0; +} + +#if FIXME +extern bool startswith(const char *str, const char *prefix); +#endif void interactive_file() { @@ -20,7 +32,8 @@ interactive_file() { StringInfoData input_message; StringInfoData *inBuf; FILE *stream ; - + int sql_line=1; + bool sql_skip = false; /* * At top of loop, reset extended-query-message flag, so that any * errors encountered in "idle" state don't provoke skip. @@ -45,6 +58,7 @@ interactive_file() { while ((c = getc(stream)) != EOF) { if (c == '\n') { + sql_line++; if (UseSemiNewlineNewline) { /* @@ -93,14 +107,26 @@ interactive_file() { /* Add '\0' to make it look the same as message case. */ appendStringInfoChar(inBuf, (char) '\0'); firstchar = 'Q'; -PDEBUG(inBuf->data); - +#if FIXME +#warning "FIXME: REVOKE ALL ON pg_largeobject FROM PUBLIC;" +#warning "FIXME: REVOKE CREATE,TEMPORARY ON DATABASE template1 FROM public;" + sql_skip |= startswith(inBuf->data , "REVOKE ALL ON pg_largeobject FROM PUBLIC;"); + sql_skip |= startswith(inBuf->data , "REVOKE CREATE,TEMPORARY ON DATABASE template1 FROM public;"); + if (sql_skip) { + fprintf(stdout, "# 106: SKIPPED: %d: %s\n", sql_line, inBuf->data); + sql_skip = false; + continue; + } else { + // fprintf(stderr, "%d: %s\n", sql_line, inBuf->data); + } +#endif // ??? if (ignore_till_sync && firstchar != EOF) continue; #include "pg_proto.c" } + PDEBUG("# 115: interactive_file: end"); } void @@ -220,7 +246,7 @@ PDEBUG("# 164:" __FILE__); /* while (repl) { interactive_file(); } - PDEBUG("# 232: REPL:End Raising a 'RuntimeError Exception' to halt program NOW"); + PDEBUG("# 240: REPL:End Raising a 'RuntimeError Exception' to halt program NOW"); { void (*npe)() = NULL; npe(); @@ -228,7 +254,7 @@ PDEBUG("# 164:" __FILE__); // unreachable. */ - PDEBUG("# 240: no line-repl requested, exiting and keeping runtime alive"); + PDEBUG("# 248: no line-repl requested, exiting and keeping runtime alive"); } @@ -240,13 +266,13 @@ AsyncPostgresSingleUserMain(int argc, char *argv[], const char *username, int as const char *dbname = NULL; PDEBUG("# 254:"__FILE__); - /* Initialize startup process environment. */ +// if (!async_restart) /* Initialize startup process environment. */ InitStandaloneProcess(argv[0]); PDEBUG("# 254:"__FILE__); - /* Set default values for command-line options. */ +// if (!async_restart) /* Set default values for command-line options. */ InitializeGUCOptions(); PDEBUG("# 257:"__FILE__); - /* Parse command-line options. */ +// if (!async_restart) /* Parse command-line options. */ process_postgres_switches(argc, argv, PGC_POSTMASTER, &dbname); PDEBUG("# 260:"__FILE__); /* Must have gotten a database name, or have a default (the username) */ diff --git a/pglite-REL_17_4_WASM/pgl_stubs.h b/pglite-REL_17_4_WASM/pgl_stubs.h index c1812dc1dcfe0..2ad262c7f0096 100644 --- a/pglite-REL_17_4_WASM/pgl_stubs.h +++ b/pglite-REL_17_4_WASM/pgl_stubs.h @@ -1,12 +1,26 @@ #pragma once + +// wasi only stubs +#if defined(__wasi__) +# undef PQEXPBUFFER_H +# include "../src/interfaces/libpq/pqexpbuffer.h" + +#else +# include "../src/interfaces/libpq/pqexpbuffer.h" +#endif + + +// option_parse_int parse_sync_method +#include "../src/fe_utils/option_utils.c" + + + static void -init_locale(const char *categoryname, int category, const char *locale) -{ +init_locale(const char *categoryname, int category, const char *locale) { if (pg_perm_setlocale(category, locale) == NULL && pg_perm_setlocale(category, "C") == NULL) - elog(FATAL, "could not adopt \"%s\" locale nor C locale for %s", - locale, categoryname); + elog(FATAL, "could not adopt \"%s\" locale nor C locale for %s", locale, categoryname); } @@ -24,12 +38,6 @@ startup_hacks(const char *progname) { } -void pg_repl_raf() { - puts("pg_repl_raf: STUB"); -} - - - // embedded initdb requirements void @@ -56,15 +64,12 @@ char * pg_strdup(const char *in) { char *tmp; - if (!in) - { - fprintf(stderr, - _("cannot duplicate null pointer (internal error)\n")); + if (!in) { + fprintf(stderr, _("cannot duplicate null pointer (internal error)\n")); exit(EXIT_FAILURE); } tmp = strdup(in); - if (!tmp) - { + if (!tmp) { fprintf(stderr, _("out of memory\n")); exit(EXIT_FAILURE); } @@ -81,7 +86,7 @@ simple_prompt(const char *prompt, bool echo) { #ifndef PG16 int ProcessStartupPacket(Port *port, bool ssl_done, bool gss_done) { - puts("# 89:"__FILE__" ProcessStartupPacket: STUB"); + PDEBUG("# 89:" __FILE__ " ProcessStartupPacket: STUB"); return STATUS_OK; } @@ -91,12 +96,10 @@ select_default_timezone(const char *share_path) { return getenv("TZ"); } -#include "../src/interfaces/libpq/pqexpbuffer.h" -#include "../src/fe_utils/option_utils.c" + bool -appendShellStringNoError(PQExpBuffer buf, const char *str) -{ +appendShellStringNoError(PQExpBuffer buf, const char *str) { bool ok = true; const char *p; @@ -126,13 +129,10 @@ appendShellStringNoError(PQExpBuffer buf, const char *str) } void -appendShellString(PQExpBuffer buf, const char *str) -{ +appendShellString(PQExpBuffer buf, const char *str) { if (!appendShellStringNoError(buf, str)) { - fprintf(stderr, - _("shell command argument contains a newline or carriage return: \"%s\"\n"), - str); + fprintf(stderr, _("shell command argument contains a newline or carriage return: \"%s\"\n"), str); exit(EXIT_FAILURE); } } diff --git a/pglite-REL_17_4_WASM/pgl_tools.h b/pglite-REL_17_4_WASM/pgl_tools.h index bb531a47c644d..295586f60ef14 100644 --- a/pglite-REL_17_4_WASM/pgl_tools.h +++ b/pglite-REL_17_4_WASM/pgl_tools.h @@ -12,6 +12,17 @@ void mkdirp(const char *p) { } } +#if FIXME +#warning "some FIXME are used" +bool startswith(const char *str, const char *prefix) { + // Check if the length of prefix is greater than the string + if (strlen(prefix) > strlen(str)) { + return false; + } + // Compare the beginning of the string with the prefix + return strncmp(str, prefix, strlen(prefix)) == 0; +} +#endif void strconcat(char*p, const char *head, const char *tail) { diff --git a/pglite-REL_17_4_WASM/repl.html b/pglite-REL_17_4_WASM/repl.html index 197e353f72291..e73657bdd77e8 100644 --- a/pglite-REL_17_4_WASM/repl.html +++ b/pglite-REL_17_4_WASM/repl.html @@ -46,21 +46,21 @@ } + + + + + + - - - - - - + @@ -165,6 +165,12 @@ document.getElementById('resize').checked)"> +
+pglite.wasi: Download file
+
+pglite.wasi + filesystem: Download file
+
+
@@ -179,10 +185,34 @@ window.buffer_stdout = "" window.flushed_stdout = false + const PGUSER = "postgres" + + // unsure + try { + globalThis.window.is_es6 = true + } catch (x) { + globalThis.window.is_es6 = false + + } + console.warn("ES6 ?", globalThis.window.is_es6) + + function jsimport(url, sync) { + const jsloader=document.createElement('script') + jsloader.setAttribute("type","text/javascript") + jsloader.setAttribute("src", url) + if (!sync) + jsloader.setAttribute('async', true); + document.head.appendChild(jsloader) + } + async function boot() { function fnc_stdout(code) { + if (stderr_on) { + stderr_on = false + buffer_stdout += '\x1B[0m' + } var flush = (code == 4) @@ -194,7 +224,6 @@ flushed_stdout = false return } - buffer_stdout += "\r\n"; flush = true } @@ -232,8 +261,9 @@ vm.vt.xterm.write('\x1B[0m') } vm.vt.xterm.write("\r\n") - } else + } else { vm.vt.xterm.write( String.fromCharCode(c) ) + } } function fnc_stdin() { @@ -279,7 +309,7 @@ case 1: data = "SELECT now(), current_database(), session_user, current_user;" break - +/* case 2: data = ` CREATE EXTENSION IF NOT EXISTS plpgsql; @@ -312,8 +342,8 @@ case 5: data = ` -CREATE USER test_user WITH PASSWORD 'md5abdbecd56d5fbd2cdaee3d0fa9e4f434'; -ALTER TABLE test2 OWNER TO test_user; +CREATE USER ${PGUSER} WITH PASSWORD 'md5abdbecd56d5fbd2cdaee3d0fa9e4f434'; +ALTER TABLE test2 OWNER TO ${PGUSER}; ` break @@ -321,7 +351,7 @@ data = "NOTIFY template1, 'hello';" break - +*/ /* case 3: @@ -330,33 +360,17 @@ data = "SELECT * FROM pg_extension;" data = "SELECT dictname FROM pg_catalog.pg_ts_dict;" data = "CHECKPOINT;SELECT pg_terminate_backend(42);"; - case 3: - data = "CREATE DATABASE test_user WITH OWNER = test_user;" - break - - case 4: - data = "CREATE SCHEMA test_user;" - break - - case 5: - data = `CREATE TABLE IF NOT EXISTS test_user.test ( + data = "CREATE DATABASE ${PGUSER} WITH OWNER = ${PGUSER};" + data = "CREATE SCHEMA ${PGUSER};" + data = `CREATE TABLE IF NOT EXISTS ${PGUSER}.test ( id SERIAL PRIMARY KEY, number INT ); - INSERT INTO test_user.test (number) VALUES (42); - ALTER TABLE test_user.test OWNER TO test_user; + INSERT INTO ${PGUSER}.test (number) VALUES (42); + ALTER TABLE ${PGUSER}.test OWNER TO ${PGUSER}; `; - case 6: data = "SELECT now(), current_database(), session_user, current_user;" - break -*/ -/* - case 4: - break - - case 6: data = "CREATE EXTENSION pg_stat_statements;" - break */ } @@ -392,6 +406,7 @@ async function delayed_init() { await Module.load_package("vector", "vector.tar.gz") + await Module.load_package("uuid-ossp", "uuid-ossp.tar.gz") // await Module.load_package("pg_stat_statements", "pg_stat_statements.tar.gz") // await Module.load_package("postgis", "postgis-3.tar.gz") @@ -425,7 +440,7 @@ if (idb & (0b0100|0b1000)) { console.log(" #3 found db+user : switch user") // switch role - // vm.readline("SET ROLE test_user;"); + // vm.readline("SET ROLE ${PGUSER};"); } console.error("Invalid user ?"); editconf = false; @@ -433,17 +448,40 @@ console.warn(" TODO: create db+user here / callback / throw ") /* vm.readline(` +` +CREATE OR REPLACE FUNCTION adduser(rolename NAME) RETURNS TEXT AS +$$ +BEGIN + IF NOT EXISTS (SELECT * FROM pg_roles WHERE rolname = rolename) THEN + EXECUTE format('CREATE ROLE %I', rolename); + RETURN 'CREATE ROLE'; + ELSE + RETURN format('ROLE ''%I'' ALREADY EXISTS', rolename); + END IF; +END; +$$ +LANGUAGE plpgsql; +`; + CREATE EXTENSION IF NOT EXISTS postgis; -CREATE USER test_user WITH PASSWORD 'md5abdbecd56d5fbd2cdaee3d0fa9e4f434'; +CREATE USER ${PGUSER} WITH PASSWORD 'md5abdbecd56d5fbd2cdaee3d0fa9e4f434'; CREATE TABLE IF NOT EXISTS test ( id SERIAL PRIMARY KEY, number INT ); INSERT INTO test (number) VALUES (42); - ALTER TABLE test OWNER TO test_user; -SET ROLE test_user; + ALTER TABLE test OWNER TO ${PGUSER}; +SET ROLE ${PGUSER}; `); */ + +/* + vm.readline(` +CREATE USER ${PGUSER} WITH PASSWORD 'md5abdbecd56d5fbd2cdaee3d0fa9e4f434'; +SET ROLE ${PGUSER}; +`); +*/ + } vm.PGDATA = PGDATA @@ -549,8 +587,8 @@ load_pg_extension: load_pg_extension, load_package: load_package, -// "PGDATABASE=test_user", - arguments : ["--single", "postgres", "--", "PGDATA=/tmp/pglite/base", "PREFIX=/tmp/pglite", "PGUSER=test_user", "REPL=N"], +// "PGDATABASE=${PGUSER}", + arguments : ["--single", "postgres", "--", "PGDATA=/tmp/pglite/base", "PREFIX=/tmp/pglite", `PGUSER=${PGUSER}`, "REPL=N"], config : { cdn : "https://pygame-web.github.io/archives/0.9/", @@ -656,17 +694,12 @@ Module.bc.onmessage(event) } -/* vm._use_wire(0) - vm._interactive_one() - send_to_pglite(ud, Module.FS) -*/ } } }; - Module.setStatus('Downloading...'); window.onerror = () => { Module.setStatus('Exception thrown, see JavaScript console'); @@ -676,28 +709,12 @@ }; }; - try { - globalThis.window.is_es6 = true - } catch (x) { - globalThis.window.is_es6 = false - - } - console.warn("ES6 ?", globalThis.window.is_es6) - - function jsimport(url, sync) { - const jsloader=document.createElement('script') - jsloader.setAttribute("type","text/javascript") - jsloader.setAttribute("src", url) - if (!sync) - jsloader.setAttribute('async', true); - document.head.appendChild(jsloader) - } //jsimport("tinytar.min.js") window.vm = Module - const { WasmTerminal } = await /**/ import("./vtx.js") + const { WasmTerminal } = await /**/ import("https://pmp-p.ddns.net/pglite-web/vtx.js") Module.vt = new WasmTerminal("repl", 200, 60, 10) @@ -708,22 +725,38 @@ const pg_in = "/tmp/pglite/base/.s.PGSQL.5432.in" const pg_out = "/tmp/pglite/base/.s.PGSQL.5432.out" - function send_to_pglite(ev, FS) { - if (0) { // as socket file - FS.writeFile(pg_lck, ev.data) + function send_to_pglite(encoded, FS, port) { + const count = encoded.byteLength + var buf + if (port<0) { + /* as socket file */ + Module._interactive_write(0) + FS.writeFile(pg_lck, encoded) FS.rename(pg_lck, pg_in) + console.log("send_to_pglite(sf):", count) + Module._interactive_one() + try { + const fstat = FS.stat(pg_out) + console.log("pgreply", fstat.size) + var stream = FS.open(pg_out, 'r'); + buf = new Uint8Array(fstat.size); + FS.read(stream, buf, 0, fstat.size, 0); + FS.close(stream); + FS.unlink(pg_out) + } catch (x) { + console.log("REPL: no reply, see stdout", x) + } + } else { + /* CMA */ + Module._interactive_write(count) + Module.HEAPU8.set(encoded, port) + console.log("send_to_pglite(cma):", count, "sent to", port) Module._interactive_one() - const fstat = FS.stat(pg_out) - console.log("pgreply", fstat.size) - var stream = FS.open(pg_out, 'r'); - var buf = new Uint8Array(fstat.size); - FS.read(stream, buf, 0, fstat.size, 0); - FS.close(stream); - FS.unlink(pg_out) - Module.bc.postMessage(buf) - - } else { /* CMA */ + const msg_start = count + 2 + const msg_end = msg_start + Module._interactive_read() + buf = Module.HEAPU8.subarray(msg_start, msg_end) } + //Module.bc.postMessage(buf) } Module.bc.onmessage = (ev) => { @@ -731,32 +764,21 @@ //console.warn("bc.onmessage:", event) if (ev.userData) { var data = ev.userData["data"] - var resultArray; + var encodedArray; if (data.byteLength) { console.log("binary data, using wire") Module._use_wire(1) - resultArray.set(data) + encodedArray.set(data) } else { console.log("string data, using repl/no wire") Module._use_wire(0) const encoder = new TextEncoder(); - resultArray = encoder.encode(data); - - // const utf8Array = encoder.encode(data+"\n"); - //resultArray = new Uint8Array(utf8Array.byteLength + 1); - //resultArray.set(utf8Array); - //resultArray[utf8Array.byteLength] = 0; // Append a null byte + encodedArray = encoder.encode(data); } - const count = resultArray.byteLength - - var port = Module.cma_port || 1 - // Module._ping() - - Module._interactive_write(count) - console.log("send_to_pglite(cma):", count, "sent to", port) - Module.HEAPU8.set(resultArray, port) - Module._interactive_one() - +// sf + send_to_pglite(encodedArray, FS, -1) +// cma +// send_to_pglite(encodedArray, FS, Module.cma_port || 1) } else { console.warn("BC(srv):", ev); } @@ -772,7 +794,7 @@ #endif { window.Module = Module - jsimport("postgres.js") + jsimport("pglite.js") console.log("main:", Module ) } diff --git a/wasm-build/build-with-docker.sh b/wasm-build/build-with-docker.sh index 2da234e657338..4b48e4cf47c0a 100755 --- a/wasm-build/build-with-docker.sh +++ b/wasm-build/build-with-docker.sh @@ -19,6 +19,12 @@ source .buildconfig cat .buildconfig +[ -f postgres-pglite/configure ] || ln -s $(pwd) postgres-pglite + +mkdir -p \ + postgres-pglite/dist/pglite \ + postgres-pglite/dist/extensions-emsdk + docker run \ --rm \ --env-file .buildconfig \ From 8c21abab1dabdbe90c2b6c1d74a55a5a13db95a2 Mon Sep 17 00:00:00 2001 From: pmp-p Date: Tue, 29 Apr 2025 13:28:32 +0200 Subject: [PATCH 03/21] fix EH in pipelining single mode (cma) --- pglite-REL_17_4_WASM/pgl_sjlj.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pglite-REL_17_4_WASM/pgl_sjlj.c b/pglite-REL_17_4_WASM/pgl_sjlj.c index e85acda6e8f22..8518625de53b6 100644 --- a/pglite-REL_17_4_WASM/pgl_sjlj.c +++ b/pglite-REL_17_4_WASM/pgl_sjlj.c @@ -49,10 +49,12 @@ if (!ignore_till_sync) send_ready_for_query = true; - - if (!is_wire) +#if PGDEBUG + if (is_repl) pg_prompt(); - +#endif + if (pipelining) + goto incoming; goto wire_flush; #endif } From 36b23329ce8df42545760c64110b5582c312e20c Mon Sep 17 00:00:00 2001 From: pmp-p Date: Tue, 29 Apr 2025 13:56:26 +0200 Subject: [PATCH 04/21] link stage fix --- wasm-build.sh | 279 +++++--- wasm-build/build-pgcore.sh | 141 +++-- wasm-build/pack_extension.py | 4 +- wasm-build/sdk.sh | 4 +- .../sdk_port-wasi/sdk_port-wasi-dlfcn.c | 198 ++++++ wasm-build/sdk_port-wasi/sdk_port-wasi.c | 595 ++++++++++++++++++ wasm-build/sdk_port.h | 290 +++++++++ 7 files changed, 1360 insertions(+), 151 deletions(-) create mode 100644 wasm-build/sdk_port-wasi/sdk_port-wasi-dlfcn.c create mode 100644 wasm-build/sdk_port-wasi/sdk_port-wasi.c create mode 100644 wasm-build/sdk_port.h diff --git a/wasm-build.sh b/wasm-build.sh index 24176b10a7a54..773b74d508e61 100755 --- a/wasm-build.sh +++ b/wasm-build.sh @@ -11,56 +11,85 @@ export PORTABLE=${PORTABLE:-$(pwd)/wasm-build} export SDKROOT=${SDKROOT:-/tmp/sdk} export GETZIC=${GETZIC:-true} +# systems default may not be in path export ZIC=${ZIC:-/usr/sbin/zic} - # data transfer zone this is == (wire query size + result size ) + 2 # expressed in EMSDK MB, max is 13MB on emsdk 3.1.74+ export CMA_MB=${CMA_MB:-12} export TOTAL_MEMORY=${TOTAL_MEMORY:-180MB} -export WASI=${WASI:-false} + export WORKSPACE=${GITHUB_WORKSPACE:-$(pwd)} export PGROOT=${PGROOT:-/tmp/pglite} export WEBROOT=${WEBROOT:-/tmp/web} +export PG_BUILD=${BUILD:-/tmp/sdk/build} +export PGL_BUILD_NATIVE="${PG_BUILD}/pglite-native" + export PG_DIST=${DIST:-/tmp/sdk/dist} export PG_DIST_EXT="${PG_DIST}/extensions-emsdk" +export PGL_DIST_JS="${PG_DIST}/pglite-js" -export PGL_DIST_WEB="${PG_DIST}/pglite-sandbox" +export PGL_DIST_NATIVE="${PG_DIST}/pglite-native" +export PGL_DIST_WEB="${PG_DIST}/pglite-web" export DEBUG=${DEBUG:-true} export USE_ICU=${USE_ICU:-false} export PGUSER=${PGUSER:-postgres} -[ -f /portable.opts ] && . /portable.opts +[ -f /tmp/portable.opts ] && . /tmp/portable.opts +[ -f /tmp/portable.dev ] && . /tmp/portable.dev +# can override from cmd line +export WASI=${WASI:-false} +export WASI_SDK=${WASI_SDK:-25.0} +export PYBUILD=${PYBUILD:-3.13} -if $DEBUG + +if $WASI then - export COPTS=${COPTS:-"-O2 -g3"} - export LOPTS=${LOPTS:-"-O2 -g3 --no-wasm-opt -sASSERTIONS=1"} + BUILD=wasi + if $DEBUG + then + export COPTS=${COPTS:-"-O2 -g3"} + export LOPTS=${LOPTS:-"-O2 -g3"} + else + export COPTS=${COPTS:-"-Oz -g0"} + export LOPTS=${LOPTS:-"-Oz -g0"} + fi else - # DO NOT CHANGE COPTS - optimized wasm corruption fix - export COPTS=${COPTS:-"-O2 -g3 --no-wasm-opt"} - export LOPTS=${LOPTS:-'-Oz -g0 --closure=0 --closure-args=--externs=/tmp/externs.js -sASSERTIONS=0'} + BUILD=emscripten + if $DEBUG + then + export COPTS="-O2 -g3 --no-wasm-opt" + export LOPTS=${LOPTS:-"-O2 -g3 --no-wasm-opt -sASSERTIONS=1"} + else + # DO NOT CHANGE COPTS - optimized wasm corruption fix + export COPTS="-O2 -g3 --no-wasm-opt" + export LOPTS=${LOPTS:-"-Oz -g0 --closure=0 --closure-args=--externs=/tmp/externs.js -sASSERTIONS=0"} + fi fi +export BUILD +export BUILD_PATH=${PG_BUILD}/${BUILD} + export PGDATA=${PGROOT}/base export PGPATCH=${WORKSPACE}/patches -chmod +x ${PORTABLE}/*.sh ${PORTABLE}/extra/*.sh - -# exit on error -EOE=false +chmod +x ${PORTABLE}/*.sh +[ -d ${PORTABLE}/extra ] && ${PORTABLE}/extra/*.sh +# this was set to false on 16.4 to skip some harmless exceptions without messing with core code. +# exit on error +EOE=true # default to user writeable paths in /tmp/ . -if mkdir -p ${PGROOT} ${PG_DIST} ${PG_DIST_EXT} ${PGL_DIST_WEB} +if mkdir -p ${PGROOT} ${PG_DIST} ${PG_DIST_EXT} ${PGL_DIST_JS} ${PGL_DIST_WEB} then echo "checking for valid prefix ${PGROOT} ${PG_DIST}" else @@ -104,11 +133,9 @@ System node/pnpm ( may interfer) : if ${WASI} then - echo "Wasi build (experimental)" - export WASI_SDK=25.0 - export WASI_SDK_PREFIX=${SDKROOT}/wasisdk/wasi-sdk-${WASI_SDK}-x86_64-linux - #export WASI_SDK_PREFIX=${SDKROOT}/wasisdk/upstream - export WASI_SYSROOT=${WASI_SDK_PREFIX}/share/wasi-sysroot + pushd ${SDKROOT} + . wasisdk/wasisdk_env.sh + popd if [ -f ${WASI_SYSROOT}/extra ] then @@ -126,20 +153,6 @@ then touch ${WASI_SYSROOT}/extra fi - - if false - then - . ${SDKROOT}/wasisdk/wasisdk_env.sh - env|grep WASI - export CC=${WASI_SDK_DIR}/bin/clang - export CPP=${WASI_SDK_DIR}/bin/clang-cpp - export CXX=${WASI_SDK_DIR}/bin/clang++ - export CFLAGS="-D_WASI_EMULATED_SIGNAL" - export LDFLAGS="-lwasi-emulated-signal" - else - . ${SDKROOT}/wasm32-wasi-shell.sh - fi - # wasi does not use -sGLOBAL_BASE CC_PGLITE="-DCMA_MB=${CMA_MB}" @@ -261,7 +274,7 @@ else #define I_PGDEBUG #define WASM_USERNAME "$PGUSER" #define PGDEBUG 1 -#define PDEBUG(string) fputs(string, stdout) +#define PDEBUG(string) fputs(string, stderr) #define JSDEBUG(string) {EM_ASM({ console.log(string); });} #define ADEBUG(string) { PDEBUG(string); JSDEBUG(string) } #endif @@ -289,6 +302,37 @@ END # store all pg options that have impact on cmd line initdb/boot cat > ${PGROOT}/pgopts.sh <> ${PGROOT}/pgopts.sh echo "export PGLITE=${PGLITE}" >> ${PGROOT}/pgopts.sh + [ -f /tmp/portable.opts ] && cat /tmp/portable.opts >> ${PGROOT}/pgopts.sh + [ -f /tmp/portable.dev ] && cat /tmp/portable.dev >> ${PGROOT}/pgopts.sh . ${PGROOT}/pgopts.sh @@ -342,7 +389,7 @@ fi # put local zic in the path from build dir # put emsdk-shared and also pg_config from the install dir. -export PATH=${WORKSPACE}/build/postgres/bin:${PGROOT}/bin:$PATH +export PATH=${WORKSPACE}/${BUILD_PATH}/bin:${PGROOT}/bin:$PATH # At this stage, PG should be installed to PREFIX and ready for linking @@ -380,7 +427,7 @@ then ]" fi - for extdir in postgresql/contrib/* + for extdir in postgresql-${PG_BRANCH}/contrib/* do if [ -f ${PGROOT}/dumps/dump.vector ] then @@ -403,7 +450,7 @@ then Building contrib extension : $ext : begin " - pushd build/postgres/contrib/$ext + pushd ${BUILD_PATH}/contrib/$ext if PATH=$PREFIX/bin:$PATH emmake make install 2>&1 >/dev/null then echo " @@ -428,47 +475,48 @@ then fi - # only build extra when targeting pglite-wasm . - -# TODO link the good tag -ln -s ${WORKSPACE}/pglite-REL_17_4_WASM ${WORKSPACE}/pglite-wasm - +pwd if [ -f ${WORKSPACE}/pglite-wasm/build.sh ] then - - if echo " $*"|grep -q " extra" + if $WASI then - for extra_ext in ${EXTRA_EXT:-"vector"} - do - if $CI - then - #if [ -d $PREFIX/include/X11 ] - if true + echo " + * WASI build : skipping extra extensions and FS +" + else + + if echo " $*"|grep -q " extra" + then + for extra_ext in ${EXTRA_EXT:-"vector"} + do + if $CI then - echo -n - else - # install EXTRA sdk - . /etc/lsb-release - DISTRIB="${DISTRIB_ID}-${DISTRIB_RELEASE}" - CIVER=${CIVER:-$DISTRIB} - SDK_URL=https://github.com/pygame-web/python-wasm-sdk-extra/releases/download/$SDK_VERSION/python3.13-emsdk-sdk-extra-${CIVER}.tar.lz4 - echo "Installing $SDK_URL" - curl -sL --retry 5 $SDK_URL | tar xvP --use-compress-program=lz4 | pv -p -l -s 15000 >/dev/null - chmod +x ./extra/*.sh + #if [ -d $PREFIX/include/X11 ] + if true + then + echo -n + else + # install EXTRA sdk + . /etc/lsb-release + DISTRIB="${DISTRIB_ID}-${DISTRIB_RELEASE}" + CIVER=${CIVER:-$DISTRIB} + SDK_URL=https://github.com/pygame-web/python-wasm-sdk-extra/releases/download/$SDK_VERSION/python3.13-emsdk-sdk-extra-${CIVER}.tar.lz4 + echo "Installing $SDK_URL" + curl -sL --retry 5 $SDK_URL | tar xvP --use-compress-program=lz4 | pv -p -l -s 15000 >/dev/null + chmod +x ./extra/*.sh + fi fi - fi - echo "======================= ${extra_ext} : $(pwd) ===================" - - ./extra/${extra_ext}.sh || exit 400 + echo "======================= ${extra_ext} : $(pwd) ===================" - python3 ${PORTABLE}/pack_extension.py - done - fi + ./extra/${extra_ext}.sh || exit 480 - # build pglite initdb/loop/transport/repl + python3 ${PORTABLE}/pack_extension.py + done + fi - export PGPRELOAD="\ + # this is for initial emscripten MEMFS + export PGPRELOAD="\ --preload-file ${PGROOT}/share/postgresql@${PGROOT}/share/postgresql \ --preload-file ${PGROOT}/lib/postgresql@${PGROOT}/lib/postgresql \ --preload-file ${PGROOT}/password@${PGROOT}/password \ @@ -476,12 +524,91 @@ then --preload-file placeholder@${PGROOT}/bin/postgres \ --preload-file placeholder@${PGROOT}/bin/initdb\ " - - ${WORKSPACE}/pglite-wasm/build.sh + fi - for file in /tmp/sdk/dist/extensions-emsdk/*.tar; do gzip -9 -k -f "$file"; done -else - echo "Could not find a pglite tag matching $PG_BRANCH" - exit 480 + echo " + * building + linking pglite-wasm (initdb/loop/transport/repl/backend) +" + if ${WORKSPACE}/pglite-wasm/build.sh + then + if $WASI + then + echo "TODO: wasi pack/tests" + else +cat > pglite-link.sh </dev/null # do nothing if it is a submodule - [ -d postgresql ] || ln -s $(realpath postgresql-${PG_VERSION}) postgresql + [ -d postgresql ] || ln -s postgresql-${PG_BRANCH} postgresql fi -export PGSRC=$(realpath postgresql) +export PGSRC=$(realpath postgresql-${PG_BRANCH}) echo " Building $ARCHIVE (patched) from $PGSRC WASI=$WASI -build-pgcore:begin +build-pgcore: begin($BUILD) ___________________________________________________ CC_PGLITE=$CC_PGLITE @@ -67,30 +67,28 @@ CC_PGLITE=$CC_PGLITE " -if [ -f ${PGROOT}/pg.installed ] +if [ -f ${PGROOT}/pg.${BUILD}.installed ] then - echo " * skipping pg build, using previous install from ${PGROOT}" + echo " + * skipping pg build, using previous install from ${PGROOT} +" else - mkdir -p build/postgres - pushd build/postgres + mkdir -p ${BUILD_PATH} + pushd ${BUILD_PATH} # create empty package.json to avoid emsdk node conflicts # with root package.json of project echo "{}" > package.json - if $CI + if [ -f Makefile ] then - echo "CI : using build cache" - else - if [ -f Makefile ] - then - echo "Cleaning up previous build ..." - make distclean 2>&1 > /dev/null - fi + echo "Cleaning up previous build ..." + make distclean 2>&1 > /dev/null fi + # TODO: --with-libxml xml2 >= 2.6.23 # TODO: --with-libxslt add to sdk # --disable-atomics https://github.com/WebAssembly/threads/pull/147 "Allow atomic operations on unshared memories" @@ -98,21 +96,27 @@ else COMMON_CFLAGS="${CC_PGLITE} -fpic -Wno-declaration-after-statement -Wno-macro-redefined -Wno-unused-function -Wno-missing-prototypes -Wno-incompatible-pointer-types" + # common to all wasm flavour + cp ${PGSRC}/src/include/port/wasm_common.h ${PGROOT}/include/wasm_common.h + + # wasm os implementation router + cp ${PORTABLE}/sdk_port.h ${PGROOT}/include/sdk_port.h + + # specific implementation for wasm os flavour + [ -d ${PORTABLE}/sdk_port-${BUILD} ] && cp ${PORTABLE}/sdk_port-${BUILD}/* ${PGROOT}/include/ + if ${WASI} then - BUILD=wasi - echo "WASI BUILD: turning off xml/xslt support" + echo "WASI BUILD: turning off xml/xslt support" XML2="" UUID="" - cp ${PORTABLE}/wasi/wasi_port.c /tmp/pglite/include/sdk_port.c - WASM_LDFLAGS="-lwasi-emulated-getpid -lwasi-emulated-mman -lwasi-emulated-signal -lwasi-emulated-process-clocks" - WASM_CFLAGS="-DSDK_PORT=/tmp/pglite/include/sdk_port.c ${COMMON_CFLAGS} -D_WASI_EMULATED_MMAN -D_WASI_EMULATED_SIGNAL -D_WASI_EMULATED_PROCESS_CLOCKS -D_WASI_EMULATED_GETPID" + # -lwasi-emulated-signal -D_WASI_EMULATED_SIGNAL -lwasi-emulated-getpid -D_WASI_EMULATED_GETPID + WASM_LDFLAGS="-lwasi-emulated-mman -lwasi-emulated-pthread -lwasi-emulated-process-clocks" + WASM_CFLAGS="-I${WASISDK}/hotfix -DSDK_PORT=${PREFIX}/include/sdk_port-wasi.c ${COMMON_CFLAGS} -D_WASI_EMULATED_PTHREAD -D_WASI_EMULATED_MMAN -D_WASI_EMULATED_PROCESS_CLOCKS" export MAIN_MODULE="" else - BUILD=emscripten - # --with-libxml does not fit with --without-zlib if $CI then @@ -137,8 +141,6 @@ else fi - cp ${PGSRC}/src/include/port/wasm_common.h /tmp/pglite/include/wasm_common.h - [ -f ${PREFIX}/devices/emsdk/usr/lib/libossp-uuid.a ] && rm ${PREFIX}/devices/emsdk/usr/lib/libossp-uuid.a [ -f ${PREFIX}/devices/emsdk/usr/lib/libouuid.a ] && rm ${PREFIX}/devices/emsdk/usr/lib/libuuid.a @@ -154,6 +156,7 @@ else GETZIC=${GETZIC:-true} + EMCC_NODE="-sEXIT_RUNTIME=1 -DEXIT_RUNTIME -sNODERAWFS -sENVIRONMENT=node" @@ -171,13 +174,12 @@ END #. ${SDKROOT}/wasm32-wasi-shell.sh TZ=UTC PGTZ=UTC $(command -v wasi-run) $(pwd)/src/timezone/zic.wasi \$@ END - else - echo " + else + echo " * Using system ZIC from ${ZIC:-/usr/sbin/zic} " - cp ${ZIC:-/usr/sbin/zic} bin/ + cp ${ZIC:-/usr/sbin/zic} bin/ fi - else export EXT=wasm cat > ${PGROOT}/config.site < /tmp/disable-shared.log @@ -313,15 +309,23 @@ END # Keep a shell script for fast rebuild with env -i from cmdline + # same script handle emcc and wasi + cat > pg-make.sh <> pg-make.sh cat >> pg-make.sh <&1 > /tmp/install.log then echo install ok if $WASI then - # remove unlinked server - for todel in src/backend/postgres src/backend/postgres.wasi $PGROOT/bin/postgres $PGROOT/bin/postgres.wasi - do - [ -f $todel ] && rm $todel - done - - pushd ../.. - chmod +x ./wasm-build/linkwasi.sh - # WASI_CFLAGS="$WASI_CFLAGS" - ./wasm-build/linkwasi.sh || exit 251 - popd - cp src/backend/postgres.wasi $PGROOT/bin/ || exit 253 + cp src/backend/postgres.wasi $PGROOT/bin/ || exit 365 else if $DEBUG then # built with EMCC_CFLAGS="-sEXIT_RUNTIME=1 -DEXIT_RUNTIME -sNODERAWFS -sENVIRONMENT=node" emmake make -C cp src/bin/initdb/initdb.wasm $PGROOT/bin/ - cp src/backend/postgres.wasm $PGROOT/bin/ fi @@ -381,7 +380,7 @@ END pushd ${PGROOT} - find . -type f | grep -v plpgsql > ${PGROOT}/pg.installed + find . -type f | grep -v plpgsql > ${PGROOT}/pg.${BUILD}.installed popd goback=$(pwd) @@ -390,13 +389,13 @@ END pushd $goback pushd ${PGROOT} - find . -type f > ${PGROOT}/pg.installed + find . -type f > ${PGROOT}/pg.${BUILD}.installed popd else cat /tmp/install.log echo "install failed" - exit 384 + exit 400 fi python3 > ${PGROOT}/PGPASSFILE < 0: diff --git a/wasm-build/sdk.sh b/wasm-build/sdk.sh index 4f9086bf70090..0401c546b3515 100755 --- a/wasm-build/sdk.sh +++ b/wasm-build/sdk.sh @@ -9,11 +9,11 @@ fi # https://github.com/emscripten-core/emscripten/blob/ac676d5e437525d15df5fd46bc2c208ec6d376a3/tools/link.py#L1652-L1658 -if python3 -V +if python3 -V 2>/dev/null then echo using installed python3 else - echo wil use python for build as system python. + echo will use python for build as system python. ln -sf $SDKROOT/devices/$(arch)/usr/bin/python3 /usr/bin/python3 fi diff --git a/wasm-build/sdk_port-wasi/sdk_port-wasi-dlfcn.c b/wasm-build/sdk_port-wasi/sdk_port-wasi-dlfcn.c new file mode 100644 index 0000000000000..44dac9e5e3cd2 --- /dev/null +++ b/wasm-build/sdk_port-wasi/sdk_port-wasi-dlfcn.c @@ -0,0 +1,198 @@ +// dict helper for dlfcn tables +// TODO: use QSTR + +#include "postgres.h" + +#include "fmgr.h" + +typedef struct dict_entry_s { + const char *key; + int value; +} dict_entry_s; + +typedef struct dict_s { + int len; + int cap; + dict_entry_s *entry; +} dict_s, *dict_t; + +static int +dict_find_index(dict_t dict, const char *key) { + for (int i = 0; i < dict->len; i++) { + if (!strcmp(dict->entry[i].key, key)) { + return i; + } + } + return -1; +} + +static int +dict_find(dict_t dict, const char *key, int def) { + int idx = dict_find_index(dict, key); + return idx == -1 ? def : dict->entry[idx].value; +} + +static void +dict_add(dict_t dict, const char *key, int value) { + int idx = dict_find_index(dict, key); + if (idx != -1) { + dict->entry[idx].value = value; + return; + } + if (dict->len == dict->cap) { + dict->cap *= 2; + dict->entry = realloc(dict->entry, dict->cap * sizeof(dict_entry_s)); + } + dict->entry[dict->len].key = strdup(key); + dict->entry[dict->len].value = value; + dict->len++; +} + +static dict_t +dict_new(void) { + dict_s proto = {0, 10, malloc(10 * sizeof(dict_entry_s))}; + dict_t d = malloc(sizeof(dict_s)); + *d = proto; + return d; +} + +/* dlclose stub +static void +dict_free(dict_t dict) { + for (int i = 0; i < dict->len; i++) { + free(dict->entry[i].key); + } + free(dict->entry); + free(dict); +} +*/ + +static inline int +ends_with(const char *str, const char *suffix) +{ + if (!str || !suffix) + return 0; + size_t lenstr = strlen(str); + size_t lensuffix = strlen(suffix); + if (lensuffix > lenstr) + return 0; + return strncmp(str + lenstr - lensuffix, suffix, lensuffix) == 0; +} + +// dlfcn.h + +volatile dict_t* dltab[]; +volatile int dltab_index = 0; + +void * +sym_stub(void) { + puts("DLSYM STUB"); + return NULL; +} + +char * +dlerror(void) { + return (char *)dlerror; +} + +static const Pg_magic_struct Pg_magic_data = PG_MODULE_MAGIC_DATA; + +const Pg_magic_struct * +STUB_Pg_magic_func(void) { + return &Pg_magic_data; +} + +void STUB__PG_init(void) { +} + +void +STUB__PG_fini(void) { +} + +#define PG_FUNCTION_ARGS FunctionCallInfo fcinfo + +extern Datum dsnowball_init(PG_FUNCTION_ARGS); +extern void pg_finfo_dsnowball_init(void); +extern void pg_finfo_dsnowball_lexize(void); + +extern void pg_finfo_plpgsql_call_handler(void); +extern void pg_finfo_plpgsql_inline_handler(void); +extern void pg_finfo_plpgsql_validator(void); + +// listed in .sql +extern void plpgsql_call_handler(void); +extern void plpgsql_inline_handler(void); +extern void plpgsql_validator(void); + +extern void _PG_init(void); + + +void * +dlopen(const char *filename, int flags) { + dict_t tab = NULL; + fprintf(stderr,"void *dlopen(const char *filename = %s, int flags=%d)\n", filename, flags); + for (int i=0; i< dltab_index; i++) { + if ( dict_find_index(dltab[i], filename) > 0 ) + return (void *)i; + } + printf("dlopen: new lib '%s'\n", filename ); + if ( ends_with(filename,"/plpgsql.so") ){ + puts(" ========= CALLING _PG_init ========="); + _PG_init(); + } + + tab = dict_new(); + dict_add(tab, filename, dltab_index++ ); + dltab[dltab_index] = tab; + + return (void *)dltab_index; +} + +void * +dlsym(void *__restrict handle, const char *__restrict symbol) { + void *sym = NULL; + if ( !strcmp(symbol, "Pg_magic_func") ) { + sym = &STUB_Pg_magic_func; + goto report; + } + + if ( !strcmp(symbol, "_PG_init") ) + // sym = &STUB__PG_init; + return &STUB__PG_init; + + if ( !strcmp(symbol, "dsnowball_init") ) + //sym = &dsnowball_init; + return &dsnowball_init; + + if ( !strcmp(symbol, "pg_finfo_dsnowball_init") ) + return &pg_finfo_dsnowball_init; + + if ( !strcmp(symbol, "pg_finfo_dsnowball_lexize") ) + return &pg_finfo_dsnowball_lexize; + + if ( !strcmp(symbol, "pg_finfo_plpgsql_call_handler") ) + return &pg_finfo_plpgsql_call_handler; + + if ( !strcmp(symbol, "pg_finfo_plpgsql_inline_handler") ) + return &pg_finfo_plpgsql_inline_handler; + + if ( !strcmp(symbol, "pg_finfo_plpgsql_validator") ) + return &pg_finfo_plpgsql_validator; + + if ( !strcmp(symbol, "plpgsql_call_handler") ) + return &plpgsql_call_handler; + + if ( !strcmp(symbol, "plpgsql_inline_handler") ) + return &plpgsql_inline_handler; + + if ( !strcmp(symbol, "plpgsql_validator") ) { + sym = &plpgsql_validator; + goto report; + } + +report:; + fprintf(stderr, "void *dlsym(void *handle = %p, const char *symbol = %s) => %p\n", handle, symbol, sym); + return sym; +} + + diff --git a/wasm-build/sdk_port-wasi/sdk_port-wasi.c b/wasm-build/sdk_port-wasi/sdk_port-wasi.c new file mode 100644 index 0000000000000..bf0d75b402e21 --- /dev/null +++ b/wasm-build/sdk_port-wasi/sdk_port-wasi.c @@ -0,0 +1,595 @@ + +// WIP: signal here +// ================================================================== +#include + +/* Convenience type when working with signal handlers. */ +typedef void (*sa_handler_t) (int); + +/* Return the handler of a signal, as a sa_handler_t value regardless + of its true type. The resulting function can be compared to + special values like SIG_IGN but it is not portable to call it. */ +// static inline sa_handler_t; + +/* +struct sigaction { + __sighandler_t sa_handler; + unsigned long sa_flags; +#ifdef SA_RESTORER + __sigrestore_t sa_restorer; +#endif + sigset_t sa_mask; +}; +*/ + +#ifdef SIGABRT_COMPAT +#define SIGABRT_COMPAT_MASK (1U << SIGABRT_COMPAT) +#else +#define SIGABRT_COMPAT_MASK 0 +#endif + + +/* Present to allow compilation, but unsupported by gnulib. */ +#if 0 +union sigval +{ + int sival_int; + void *sival_ptr; +}; + + +struct siginfo_t +{ + int si_signo; + int si_code; + int si_errno; + pid_t si_pid; + uid_t si_uid; + void *si_addr; + int si_status; + long si_band; + union sigval si_value; +}; + +typedef struct siginfo_t siginfo_t; + +struct sigaction_x +{ + union + { + void (*_sa_handler) (int); + /* Present to allow compilation, but unsupported by gnulib. POSIX + says that implementations may, but not must, make sa_sigaction + overlap with sa_handler, but we know of no implementation where + they do not overlap. */ + void (*_sa_sigaction) (int, siginfo_t *, void *); + } _sa_func; + sigset_t sa_mask; + /* Not all POSIX flags are supported. */ + int sa_flags; +}; +#endif // 0 + +#define sa_handler _sa_func._sa_handler +#define sa_sigaction _sa_func._sa_sigaction + +/* Unsupported flags are not present. */ +#define SA_RESETHAND 1 +#define SA_NODEFER 2 +#define SA_RESTART 4 + +#define SIG_BLOCK 0 +#define SIG_UNBLOCK 1 + + + +/* Set of current actions. If sa_handler for an entry is NULL, then + that signal is not currently handled by the sigaction handler. */ +struct sigaction volatile action_array[NSIG] /* = 0 */; + +// typedef void (*__sighandler_t) (int); + +# define _SIGSET_NWORDS (1024 / (8 * sizeof (unsigned long int))) + + +/* Set of currently blocked signals. */ +volatile sigset_t blocked_set /* = 0 */; + +/* Set of currently blocked and pending signals. */ +volatile sig_atomic_t pending_array[NSIG] /* = { 0 } */; + +/* The previous signal handlers. + Only the array elements corresponding to blocked signals are relevant. */ +volatile handler_t old_handlers[NSIG]; + + +int +sigemptyset(sigset_t *set) { + *set = 0; + return 0; +} + +int +sigaction(int signum, const struct sigaction *act, struct sigaction *oldact) { + puts("# 96: sigaction STUB"); + return 0; +} + +int +sigfillset (sigset_t *set) { + *set = ((2U << (NSIG - 1)) - 1) & ~ SIGABRT_COMPAT_MASK; + return 0; +} + +int +sigaddset (sigset_t *set, int sig) { + if (sig >= 0 && sig < NSIG) + { + #ifdef SIGABRT_COMPAT + if (sig == SIGABRT_COMPAT) + sig = SIGABRT; + #endif + + *set |= 1U << sig; + return 0; + } + else + { + errno = EINVAL; + return -1; + } +} + +int +sigdelset (sigset_t *set, int sig) { + if (sig >= 0 && sig < NSIG) + { + #ifdef SIGABRT_COMPAT + if (sig == SIGABRT_COMPAT) + sig = SIGABRT; + #endif + + *set &= ~(1U << sig); + return 0; + } + else + { + errno = EINVAL; + return -1; + } +} + +/* Signal handler that is installed for blocked signals. */ +void +blocked_handler (int sig) +{ + /* Reinstall the handler, in case the signal occurs multiple times + while blocked. There is an inherent race where an asynchronous + signal in between when the kernel uninstalled the handler and + when we reinstall it will trigger the default handler; oh + well. */ + signal (sig, blocked_handler); + if (sig >= 0 && sig < NSIG) + pending_array[sig] = 1; +} + +int +sigprocmask (int operation, const sigset_t *set, sigset_t *old_set) { + if (old_set != NULL) + *old_set = blocked_set; + + if (set != NULL) + { + sigset_t new_blocked_set; + sigset_t to_unblock; + sigset_t to_block; + + switch (operation) + { + case SIG_BLOCK: + new_blocked_set = blocked_set | *set; + break; + case SIG_SETMASK: + new_blocked_set = *set; + break; + case SIG_UNBLOCK: + new_blocked_set = blocked_set & ~*set; + break; + default: + errno = EINVAL; + return -1; + } + to_unblock = blocked_set & ~new_blocked_set; + to_block = new_blocked_set & ~blocked_set; + + if (to_block != 0) + { + int sig; + + for (sig = 0; sig < NSIG; sig++) + if ((to_block >> sig) & 1) + { + pending_array[sig] = 0; + if ((old_handlers[sig] = signal (sig, blocked_handler)) != SIG_ERR) + blocked_set |= 1U << sig; + } + } + + if (to_unblock != 0) + { + sig_atomic_t received[NSIG]; + int sig; + + for (sig = 0; sig < NSIG; sig++) + if ((to_unblock >> sig) & 1) + { + if (signal (sig, old_handlers[sig]) != blocked_handler) + /* The application changed a signal handler while the signal + was blocked, bypassing our rpl_signal replacement. + We don't support this. */ + abort (); + received[sig] = pending_array[sig]; + blocked_set &= ~(1U << sig); + pending_array[sig] = 0; + } + else + received[sig] = 0; + + for (sig = 0; sig < NSIG; sig++) + if (received[sig]) + raise (sig); + } + } + return 0; +} + + +// STUBS +int sigismember(const sigset_t *set, int signum) { + return -1; +} + +int +pthread_sigmask(int how, const sigset_t *set, sigset_t *oldset) { + return 0; +} + +int sigpending(sigset_t *set) { + return -1; +} + +int sigwait(const sigset_t *restrict set, int *restrict sig) { + return 0; +} + +unsigned int alarm(unsigned int seconds) { + return 0; +} + + +// WIP : shm +// ======================================================================================== +volatile int shm_index = 0; + +#include +void get_shm_path(char *tmpnam, const char *name) { + const char *shm = getenv("SHM"); + if (shm) { + printf("# 281 SHM=%s.%d", shm, shm_index); + snprintf(tmpnam, 128, "%s.%d", shm, shm_index++); + } else { + snprintf(tmpnam, 128, "/tmp%s", name); + } +} + +int shm_open(const char *name, int oflag, mode_t mode) { + char tmpnam[128]; + int fd; + get_shm_path(&tmpnam, name); + fd=fileno(fopen(tmpnam, "w+")); + fprintf(stderr, "# 287: shm_open(%s) => %d\n", tmpnam, fd); + return fd; +} + +int shm_unlink(const char *name) { + char tmpnam[128]; + if (getenv("SHM")) { + fprintf(stderr, "# 294: shm_unlink(%s) STUB\n", name); + return 0; + } + get_shm_path(&tmpnam, name); + return remove(tmpnam); +} + + +// popen +// ======================================================================================== + + +#include // FILE+fprintf +extern FILE* IDB_PIPE_FP; +extern FILE* SOCKET_FILE; +extern int SOCKET_DATA; +extern int IDB_STAGE; + + +static inline int +ends_with(const char *str, const char *suffix) +{ + if (!str || !suffix) + return 0; + size_t lenstr = strlen(str); + size_t lensuffix = strlen(suffix); + if (lensuffix > lenstr) + return 0; + return strncmp(str + lenstr - lensuffix, suffix, lensuffix) == 0; +} + +int +system_wasi(const char *command) { + fprintf(stderr, "# 164: system('%s')\n", command); + return -1; +} + +// pthread.h +// ======================================================================================== + + +/* +int pthread_create(pthread_t *restrict thread, + const pthread_attr_t *restrict attr, + void *(*start_routine)(void *), + void *restrict arg) { + puts("# 327: pthread_create STUB"); + return 0; +} + +int pthread_join(pthread_t thread, void **retval) { + return 0; +} + + + +int sdk_pthread_mutex_lock(void *mutex) { + return 0; +} + +int pthread_mutex_lock(pthread_mutex_t *mutex) { + return 0; +} + +int pthread_mutex_unlock(pthread_mutex_t *mutex) { + return 0; +} + +*/ + + +void wait(); + +// present in share/wasi-sysroot/lib/wasm32-wasi/libwasi-emulated-signal.a(signal.o) +// void __SIG_IGN(int param) { } + + +FILE *tmpfile(void) { + return fopen(mktemp("/tmp/tmpfile"),"w"); +} + + + + + +// unix socket via file emulation using sched_yiedl for event pump. +// ================================================================================================= + +#include + +volatile int stage = 0; +volatile int fd_queue = 0; +volatile int fd_out=2; +volatile FILE *fd_FILE = NULL; + +// default fd is stderr +int socket(int domain, int type, int protocol) { +#if 0 + printf("# 404 : domain =%d type=%d proto=%d -> FORCE FD to 3 \n", domain , type, protocol); + return 3; +#else + printf("# 408 : domain =%d type=%d proto=%d\n", domain , type, protocol); +#endif + if (domain|AF_UNIX) { + fd_FILE = fopen(PGS_ILOCK, "w"); + if (fd_FILE) { + fd_out = fileno(fd_FILE); + printf("# 414: AF_UNIX sock=%d (fd_sock write) FILE=%s\n", fd_out, PGS_ILOCK); + } else { + printf("# 416: AF_UNIX ERROR OPEN (w/w+) FILE=%s\n", PGS_ILOCK); + abort(); + } + } + return fd_out; +} + +int connect(int socket, void *address, socklen_t address_len) { +#if 1 + puts("# 425: connect STUB"); + //fd_out = 3; + return 0; +#else + puts("# 429: connect EINPROGRESS"); + errno = EINPROGRESS; + return -1; +#endif +} + +ssize_t sendto(int sockfd, const void *buf, size_t len, int flags, void *dest_addr, socklen_t addrlen) { + int sent = write( fd_out, buf, len); + + printf("# 438: send/sendto(%d ?= %ld )/%zu sockfd=%d fno=%d fd_out=%d)\n", sent, ftell(fd_FILE), len, sockfd, fileno(fd_FILE), fd_out); + fd_queue+=sent; + return sent; +} + +ssize_t sdk_send(int sockfd, const void *buf, size_t len, int flags) { + return sendto(sockfd, buf, len, flags, NULL, 0); +} + +volatile bool web_warned = false; + +void sock_flush() { + if (fd_queue) { + printf(" -- 451 sockflush : AIO YIELD, expecting %s filled on return --\n", PGS_OUT); + if (!fd_FILE) { + if (!web_warned) { + puts("# 454: WARNING: fd_FILE not set but queue not empty, assuming web"); + web_warned = true; + } + + } else { + printf("# 459: SENT=%ld/%d fd_out=%d fno=%d\n", ftell(fd_FILE), fd_queue, fd_out, fileno(fd_FILE)); + fclose(fd_FILE); + rename(PGS_ILOCK, PGS_IN); +//freopen(PGS_ILOCK, "w+", fd_FILE); + fd_FILE = fopen(PGS_ILOCK, "w"); + fd_out = fileno(fd_FILE); + printf("# 465: fd_out=%d fno=%d\n", fd_out, fileno(fd_FILE)); + } + fd_queue = 0; + sched_yield(); + return; + } + + printf(" -- 472 sockflush[%d] : NO YIELD --\n",stage); + + // limit inf loops + if (stage++ > 1024) { + puts("# 476 sock_flush : busy looping ?"); + abort(); + } +} + + +volatile int fd_current_pos = 0; +volatile int fd_filesize = 0; + + +ssize_t recvfrom_bc(int socket, void *buffer, size_t length, int flags, void *address, socklen_t *address_len) { +// int busy = 0; + int rcv = -1; + sock_flush(); +/* + while (access(PGS_OUT, F_OK) != 0) { + if (!(++busy % 555111)) { + printf("# 471: FIXME: busy wait (%d) for input stream %s\n", busy, PGS_OUT); + } + if (busy>1665334) { + errno = EINTR; + return -1; + } + } +*/ + FILE *sock_in = fopen(PGS_OUT,"r"); + if (sock_in) { + if (!fd_filesize) { + fseek(sock_in, 0L, SEEK_END); + fd_filesize = ftell(sock_in); + } + fseek(sock_in, fd_current_pos, SEEK_SET); + + char *buf = buffer; + buf[0] = 0; + rcv = fread(buf, 1, length, sock_in); + + if (rcv1665334) { + errno = EINTR; + return -1; + } + } +*/ + FILE *sock_in = fopen(PGS_OUT,"r"); + if (sock_in) { + if (!fd_filesize) { + fseek(sock_in, 0L, SEEK_END); + fd_filesize = ftell(sock_in); + } + fseek(sock_in, fd_current_pos, SEEK_SET); + + char *buf = buffer; + buf[0] = 0; + rcv = fread(buf, 1, length, sock_in); + + if (rcv + +#elif defined(__wasi__) + + +#ifndef I_WASI +#define I_WASI + +#undef HAVE_PTHREAD + +#if defined(HAVE_SETSID) +#undef HAVE_SETSID +#endif + +#if defined(HAVE_GETRLIMIT) +#undef HAVE_GETRLIMIT +#endif + + +#define PLATFORM_DEFAULT_SYNC_METHOD SYNC_METHOD_FDATASYNC + +#define EMSCRIPTEN_KEEPALIVE __attribute__((used)) +#define __declspec( dllimport ) __attribute__((used)) + +#define em_callback_func void +#define emscripten_set_main_loop(...) +#define emscripten_force_exit(...) +#define EM_JS(...) + +#include "/tmp/pglite/include/wasm_common.h" + + +static pid_t +fork(void) { + puts("# 31: fork -1"); + return -1; +} + +// ======== signal ======================== +#define SA_RESTART 4 +#define SIG_SETMASK 2 + +#define SIG_BLOCK 0 +#define SIG_UNBLOCK 1 + +/* A signal handler. */ +typedef void (*handler_t) (int signal); +typedef unsigned char sigset_t; +typedef void (*__sighandler_t) (int); + +struct sigaction { + __sighandler_t sa_handler; + unsigned long sa_flags; +#ifdef SA_RESTORER + __sigrestore_t sa_restorer; +#endif + sigset_t sa_mask; +}; +extern int sigemptyset(sigset_t *set); +extern int sigaction(int signum, const struct sigaction *act, struct sigaction *oldact); +extern int sigdelset (sigset_t *set, int sig); +extern int sigfillset (sigset_t *set); +extern int sigprocmask (int operation, const sigset_t *set, sigset_t *old_set); +extern int sigaddset (sigset_t *set, int sig); + +// STUBS +extern int pthread_sigmask(int how, const sigset_t *set, sigset_t *oldset); +extern int sigismember(const sigset_t *set, int signum); +extern int sigpending(sigset_t *set); +extern int sigwait(const sigset_t *restrict set, int *restrict sig); + +// ==================================================== +// unistd +extern unsigned int alarm(unsigned int seconds); + +// ==================================================== + + +#include + +static int +sigsetjmp(sigjmp_buf env, int savesigs) { +// puts("# 120: sigsetjmp"); + return 0; +} + +static void +siglongjmp(sigjmp_buf env, int val) { + puts("# 120: siglongjmp"); +} + + + +// WIP : + +#include +static uid_t +getuid(void) { + return 1000; +} + +static int +dup(int fd) { + puts("# 128: dup"); + return fd; +} +static int +dup2(int old, int new) { + puts("# 140: dup2"); + return -1; +} +static int +pipe(int fd[2]) { + puts("# 145: pipe"); + abort(); + return -1; +} + +#include +#define RLIMIT_NOFILE 7 +#define RLIMIT_STACK 3 +#define RLIM_INFINITY ((unsigned long int)(~0UL)) + +struct rlimit { + unsigned long rlim_cur; + unsigned long rlim_max; +}; +static int +getrlimit(int resource, struct rlimit *rlim) { + return -1; +} + + +static const char *gai_strerror_msg = "# 165: gai_strerror_msg"; +static const char * +gai_strerror(int errcode) { + return gai_strerror_msg; +} + + +static int +getrusage(int who, struct rusage *usage) { + return -1; +} + +// WIP: semaphores here +// ================================================================== +#include + +static int +semctl(int semid, int semnum, int cmd, ...) { + return 0; // -1; +} + +static int +semget(key_t key, int nsems, int semflg) { +#if 0 // PGDEBUG + printf("# 213: semget(key_t key = %d, int nsems=%d, int semflg=%d)\n", key, nsems, semflg); +#endif + return 1; +} + +static int +semop(int semid, struct sembuf *sops, size_t nsops) { + return 0; // -1; +} + + + +#include +#if defined(PYDK) +extern int shm_open(const char *name, int oflag, mode_t mode); +extern int shm_unlink(const char *name); +#else +static int +shm_open(const char *name, int oflag, mode_t mode) { + char tmpnam[128]; + int fd; + snprintf(tmpnam, 128, "/tmp%s", name); + fd=fileno(fopen(tmpnam, "w+")); + fprintf(stderr, "# 212: shm_open(%s) => %d\n", tmpnam, fd); + return fd; +} + +static int +shm_unlink(const char *name) { + char tmpnam[128]; + snprintf(tmpnam, 128, "/tmp%s", name); + fprintf(stderr, "# 220: shm_unlink(%s) STUB\n", tmpnam); + return remove(tmpnam); // -1 +} + +#endif + +// initdb chmod +#if defined(__wasi__) + #define chmod(...) 0 +#endif + + + +#define system(command) system_wasi(command) +extern int system_wasi(const char *command); + + + +// time.h + +static void +tzset(void) { + puts("# 241: tzset(void) STUB"); +} + +#if defined(PG_INITDB) || defined(FE_UTILS_PRINT) || defined(PG_DUMP_PARALLEL) +static void +__SIG_IGN(int param) { +} +#endif + + +extern void sock_flush(); + + +// TODO: socket here +// ================================================================== + +#include + +extern ssize_t sdk_recv(int sockfd, void *buf, size_t len, int flags); +extern ssize_t recvfrom(int socket, void *buffer, size_t length, int flags, void *address, socklen_t *address_len); + +extern ssize_t sdk_send(int sockfd, const void *buf, size_t len, int flags); +extern ssize_t sendto(int sockfd, const void *buf, size_t len, int flags, void *dest_addr, socklen_t addrlen); + +#define recv(sockfd, buf, len, flags) sdk_recv(sockfd, buf, len, flags) + + + + +static int +listen(int sockfd, int backlog) { + return 0; +} + +static struct group *_Nullable +getgrnam(const char *_Nullable name) { + return NULL; +} + +static int +getsockname(int sockfd, struct sockaddr *restrict addr, socklen_t *restrict addrlen) { + return -1; +} + +static int +getaddrinfo(const char *restrict node, + const char *restrict service, + void *restrict hints, + void **restrict res) { + puts("# 60: getaddrinfo"); + return -1; +} +static void +freeaddrinfo(void *res) { + puts("# 65: freeaddrinfo"); +} + +extern ssize_t recvfrom_bc(int socket, void *buffer, size_t length, int flags, void *address, socklen_t *address_len); + + +#define getpid sdk_getpid +extern pid_t sdk_getpid(void); + + +//#define pthread_mutex_lock(mut) sdk_pthread_mutex_lock(mut) +//extern int sdk_pthread_mutex_lock(void *mutex); + +/* + int pthread_mutex_lock(pthread_mutex_t *mutex); + int pthread_mutex_trylock(pthread_mutex_t *mutex); + int pthread_mutex_unlock(pthread_mutex_t *mutex); +*/ + + +#endif // I_WASI + +#else + #error "unknown port mode should be __EMSCRIPTEN__ or __wasi__" +#endif // __EMSCRIPTEN__ From 07a2339710578cebd7f159798387e15b10568d3e Mon Sep 17 00:00:00 2001 From: pmp-p Date: Tue, 29 Apr 2025 14:13:26 +0200 Subject: [PATCH 05/21] fix dist path --- wasm-build/build-with-docker.sh | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/wasm-build/build-with-docker.sh b/wasm-build/build-with-docker.sh index 4b48e4cf47c0a..30340896363e7 100755 --- a/wasm-build/build-with-docker.sh +++ b/wasm-build/build-with-docker.sh @@ -19,11 +19,9 @@ source .buildconfig cat .buildconfig -[ -f postgres-pglite/configure ] || ln -s $(pwd) postgres-pglite +[ -f postgres-pglite/configure ] || ln -s . postgres-pglite -mkdir -p \ - postgres-pglite/dist/pglite \ - postgres-pglite/dist/extensions-emsdk +mkdir -p dist/pglite dist/extensions-emsdk docker run \ --rm \ @@ -32,3 +30,4 @@ docker run \ -v ./dist:/tmp/sdk/dist:rw \ $IMG_NAME:$IMG_TAG \ bash -c "source ${SDKROOT}/wasm32-bi-emscripten-shell.sh && ./wasm-build.sh ${WHAT:-\"contrib extra\"}" + From ea234c603adcf553f551abf3a77aa6a5af000071 Mon Sep 17 00:00:00 2001 From: pmp-p Date: Tue, 29 Apr 2025 14:54:33 +0200 Subject: [PATCH 06/21] paths --- .buildconfig | 2 +- wasm-build/build-with-docker.sh | 13 ++++--------- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/.buildconfig b/.buildconfig index f0ef5f8928e12..82c6fe810e116 100644 --- a/.buildconfig +++ b/.buildconfig @@ -2,7 +2,7 @@ PG_VERSION=17.4 PG_BRANCH=REL_17_4_WASM SDK_VERSION=3.1.74.6bi -WASI_SDK_VERSION=24.0.4 +WASI_SDK_VERSION=25.0.0 SDKROOT=/tmp/sdk PG_DIST=/tmp/sdk/dist diff --git a/wasm-build/build-with-docker.sh b/wasm-build/build-with-docker.sh index 30340896363e7..7478106208947 100755 --- a/wasm-build/build-with-docker.sh +++ b/wasm-build/build-with-docker.sh @@ -1,6 +1,6 @@ #!/bin/bash -# export WORKSPACE=${GITHUB_WORKSPACE:-/workspace} +export WORKSPACE=${GITHUB_WORKSPACE:-/workspace} # we are using a custom emsdk to build pglite wasm # this is available as a docker image under electricsql/pglite-builder @@ -11,12 +11,6 @@ IMG_TAG="17.4_3.1.61.7bi" source .buildconfig -# if [[ -z "$SDKROOT" || -z "$PG_VERSION" ]]; then -# echo "Missing SDKROOT and PG_VERSION env vars." -# echo "Source them from .buildconfig" -# exit 1 -# fi - cat .buildconfig [ -f postgres-pglite/configure ] || ln -s . postgres-pglite @@ -26,8 +20,9 @@ mkdir -p dist/pglite dist/extensions-emsdk docker run \ --rm \ --env-file .buildconfig \ - -v .:/workspace:rw \ - -v ./dist:/tmp/sdk/dist:rw \ + --workdir=/workspace \ + -v ${WORKSPACE}:/workspace:rw \ + -v ${WORKSPACE}/dist:/tmp/sdk/dist:rw \ $IMG_NAME:$IMG_TAG \ bash -c "source ${SDKROOT}/wasm32-bi-emscripten-shell.sh && ./wasm-build.sh ${WHAT:-\"contrib extra\"}" From 81a4b01e64a881a395bf722eb1d9af88b4123bb7 Mon Sep 17 00:00:00 2001 From: pmp-p Date: Tue, 29 Apr 2025 14:58:32 +0200 Subject: [PATCH 07/21] paths --- wasm-build/build-with-docker.sh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/wasm-build/build-with-docker.sh b/wasm-build/build-with-docker.sh index 7478106208947..f842355cf5257 100755 --- a/wasm-build/build-with-docker.sh +++ b/wasm-build/build-with-docker.sh @@ -1,7 +1,5 @@ #!/bin/bash -export WORKSPACE=${GITHUB_WORKSPACE:-/workspace} - # we are using a custom emsdk to build pglite wasm # this is available as a docker image under electricsql/pglite-builder IMG_NAME="electricsql/pglite-builder" @@ -15,14 +13,18 @@ cat .buildconfig [ -f postgres-pglite/configure ] || ln -s . postgres-pglite +cd $(realpath ${WORKSPACE}/postgres-pglite) + +export WORKSPACE=${GITHUB_WORKSPACE:-$(pwd)} + mkdir -p dist/pglite dist/extensions-emsdk docker run \ --rm \ --env-file .buildconfig \ --workdir=/workspace \ - -v ${WORKSPACE}:/workspace:rw \ - -v ${WORKSPACE}/dist:/tmp/sdk/dist:rw \ + -v ${WORKSPACE}/postgres-pglite:/workspace:rw \ + -v ${WORKSPACE}/postgres-pglite/dist:/tmp/sdk/dist:rw \ $IMG_NAME:$IMG_TAG \ bash -c "source ${SDKROOT}/wasm32-bi-emscripten-shell.sh && ./wasm-build.sh ${WHAT:-\"contrib extra\"}" From caba2d4f2f665cd402d51510e18ddb0f4334d6bc Mon Sep 17 00:00:00 2001 From: pmp-p Date: Tue, 29 Apr 2025 16:18:33 +0200 Subject: [PATCH 08/21] check pipeline inside EH --- pglite-REL_17_4_WASM/pgl_sjlj.c | 5 +++-- wasm-build.sh | 37 +++------------------------------ 2 files changed, 6 insertions(+), 36 deletions(-) diff --git a/pglite-REL_17_4_WASM/pgl_sjlj.c b/pglite-REL_17_4_WASM/pgl_sjlj.c index 8518625de53b6..f3b14e625cba6 100644 --- a/pglite-REL_17_4_WASM/pgl_sjlj.c +++ b/pglite-REL_17_4_WASM/pgl_sjlj.c @@ -53,9 +53,10 @@ if (is_repl) pg_prompt(); #endif - if (pipelining) + if (pq_buffer_remaining_data()>0) goto incoming; - goto wire_flush; + else + goto wire_flush; #endif } diff --git a/wasm-build.sh b/wasm-build.sh index 773b74d508e61..07250cc4aabee 100755 --- a/wasm-build.sh +++ b/wasm-build.sh @@ -535,7 +535,7 @@ then then echo "TODO: wasi pack/tests" else -cat > pglite-link.sh < pglite-link.sh < Date: Mon, 5 May 2025 08:25:42 +0200 Subject: [PATCH 09/21] sync --- src/backend/Makefile | 16 ++ src/backend/commands/async.c | 2 +- src/backend/libpq/pqcomm.c | 12 +- src/backend/tcop/postgres.c | 3 +- src/backend/utils/error/elog.c | 20 +- src/bin/initdb/initdb.c | 19 +- src/include/port/emscripten.h | 10 +- src/include/port/wasi.h | 500 +-------------------------------- src/port/pqsignal.c | 11 +- 9 files changed, 72 insertions(+), 521 deletions(-) diff --git a/src/backend/Makefile b/src/backend/Makefile index 55e8a81e04940..dbc1d4a148f3a 100644 --- a/src/backend/Makefile +++ b/src/backend/Makefile @@ -87,6 +87,22 @@ postgres: $(OBJS) COPTS="$(LOPTS)" $(CC) $(MAIN_MODULE) $(CFLAGS) $(LDFLAGS) -o $@ $(LIBPGCORE) $(top_builddir)/libpgmain.a $(LIBS) endif +ifeq ($(PORTNAME), wasi) +AR ?= llvm-ar +LIBPGCORE ?= $(top_builddir)/libpgcore.a +LIBPG = $(top_builddir)/libpostgres.a +PGCORE = $(top_builddir)/src/common/libpgcommon_srv.a $(top_builddir)/src/port/libpgport_srv.a $(LIBPG) +PGMAIN = main/main.o tcop/postgres.o +postgres: $(OBJS) + $(AR) rcs $(top_builddir)/libpgmain.a $(PGMAIN) + $(AR) rcs $(LIBPG) $(filter-out $(PGMAIN),$(call expand_subsys,$(ONLYOBJS))) + $(CC) -r -o $(top_builddir)/libpgcore.o -Wl,--whole-archive $(PGCORE) + $(AR) rcs $(LIBPGCORE) $(top_builddir)/libpgcore.o + COPTS="$(LOPTS)" $(CC) $(MAIN_MODULE) $(CFLAGS) $(LDFLAGS) -nostartfiles -o $@ $(LIBPGCORE) $(top_builddir)/libpgmain.a $(LIBS) + mv $@ $@.wasi + touch $@ +endif + ifeq ($(PORTNAME), cygwin) postgres: $(OBJS) diff --git a/src/backend/commands/async.c b/src/backend/commands/async.c index 9843a91032083..172692281bc29 100644 --- a/src/backend/commands/async.c +++ b/src/backend/commands/async.c @@ -1652,7 +1652,7 @@ SignalBackends(void) * just log a low-level debug message if it happens. */ #if defined(__EMSCRIPTEN__) || defined(__wasi__) - HandleNotifyInterrupt(); + HandleNotifyInterrupt(); #else if (SendProcSignal(pid, PROCSIG_NOTIFY_INTERRUPT, procnos[i]) < 0) elog(DEBUG3, "could not signal backend with PID %d: %m", pid); diff --git a/src/backend/libpq/pqcomm.c b/src/backend/libpq/pqcomm.c index f8233d225201f..86d447f2a8a49 100644 --- a/src/backend/libpq/pqcomm.c +++ b/src/backend/libpq/pqcomm.c @@ -320,8 +320,8 @@ PDEBUG("# 285:" __FILE__); AddWaitEventToSet(FeBeWaitSet, WL_POSTMASTER_DEATH, PGINVALID_SOCKET, NULL, NULL); #else /* WASM */ - PDEBUG("# 220: FIXME: socketfile"); - #pragma message "FIXME: socketfile" + PDEBUG("# 323: FIXME: socketfile"); + #pragma message "FIXME: use socketfile when overflowing PqRecvBuffer_static" /* because we fill before starting reading message */ PqRecvBuffer = &PqRecvBuffer_static[0]; #endif /* WASM */ @@ -1150,14 +1150,14 @@ pq_buffer_remaining_data(void) * This must be called before any of the pq_get* functions. * -------------------------------- */ -#if defined(I_EMSCRIPTEN) || defined(I_WASI) +#if defined(__EMSCRIPTEN__) || defined(__wasi__) EMSCRIPTEN_KEEPALIVE void pq_recvbuf_fill(FILE* fp, int packetlen) { fread( PqRecvBuffer, packetlen, 1, fp); PqRecvPointer = 0; PqRecvLength = packetlen; #if PDEBUG - printf("# 1199: pq_recvbuf_fill cma_rsize=%d PqRecvLength=%d buf=%p reply=%p\n", cma_rsize, PqRecvLength, &PqRecvBuffer[0], &PqSendBuffer[0]); + printf("# 1160: pq_recvbuf_fill cma_rsize=%d PqRecvLength=%d buf=%p reply=%p\n", cma_rsize, PqRecvLength, &PqRecvBuffer[0], &PqSendBuffer[0]); #endif } @@ -1175,7 +1175,7 @@ pq_startmsgread(void) ereport(FATAL, (errcode(ERRCODE_PROTOCOL_VIOLATION), errmsg("terminating connection because protocol synchronization was lost"))); -#if defined(I_EMSCRIPTEN) || defined(I_WASI) +#if defined(__EMSCRIPTEN__) || defined(__wasi__) if (!pq_buffer_remaining_data()) { if (cma_rsize) { PqRecvPointer = 0; @@ -1326,7 +1326,7 @@ extern int SOCKET_DATA; static int internal_putbytes(const char *s, size_t len) { if (PqSendPointer >= PqSendBufferSize) { - fprintf(stderr, "# 1329: overflow %d >= %d cma_rsize=%d CMA=%d\n", PqSendPointer, PqSendBufferSize,cma_rsize, CMA_MB); + fprintf(stderr, "# 1329: overflow %zu >= %d cma_rsize=%d CMA=%d\n", PqSendPointer, PqSendBufferSize,cma_rsize, CMA_MB); } if (!cma_rsize) { diff --git a/src/backend/tcop/postgres.c b/src/backend/tcop/postgres.c index d2b543ce29b20..767cda0ceebcd 100644 --- a/src/backend/tcop/postgres.c +++ b/src/backend/tcop/postgres.c @@ -5146,7 +5146,7 @@ ShowUsage(const char *title) (long) sys.tv_sec, (long) sys.tv_usec); #ifndef WIN32 - +#if !defined(__wasi__) /* * The following rusage fields are not defined by POSIX, but they're * present on all current Unix-like systems so we use them without any @@ -5188,6 +5188,7 @@ ShowUsage(const char *title) r.ru_nvcsw - Save_r.ru_nvcsw, r.ru_nivcsw - Save_r.ru_nivcsw, r.ru_nvcsw, r.ru_nivcsw); +#endif /* !__wasi__ */ #endif /* !WIN32 */ /* remove trailing newline */ diff --git a/src/backend/utils/error/elog.c b/src/backend/utils/error/elog.c index 3721d120c1a0f..1875155c2102a 100644 --- a/src/backend/utils/error/elog.c +++ b/src/backend/utils/error/elog.c @@ -543,9 +543,14 @@ errfinish(const char *filename, int lineno, const char *funcname) */ recursion_depth--; -#if 0 //defined(__EMSCRIPTEN__) || defined(__wasi__) - fprintf(stderr, "# 547: PG_RE_THROW(ERROR : %d) ignored\n", recursion_depth); - trap(); +#if defined(__wasi__) + fprintf(stderr, "# 547: PG_RE_THROW(ERROR : %d) custom handling\n", recursion_depth); + EmitErrorReport(); + FreeErrorDataContents(edata); + errordata_stack_depth--; + MemoryContextSwitchTo(oldcontext); + recursion_depth--; + abort(); #else fprintf(stderr, "# 549: PG_RE_THROW(ERROR : %d)\n", recursion_depth); PG_RE_THROW(); @@ -598,7 +603,7 @@ errfinish(const char *filename, int lineno, const char *funcname) * worthy of panic, depending on which subprocess returns it. */ #if defined(__EMSCRIPTEN__) || defined(__wasi__) - puts("# 599: proc_exit(FATAL) ignored"); + PDEBUG("# 601: proc_exit(FATAL) ignored"); #else proc_exit(1); #endif @@ -2015,6 +2020,13 @@ ReThrowError(ErrorData *edata) void pg_re_throw(void) { +#if defined(__wasi__) + if (PG_exception_stack != NULL) + PDEBUG("# 2020: pg_re_throw(void) [ ex stack ! ]"); + else + PDEBUG("# 2022: pg_re_throw(void) [ NO STACK ]"); + return; +#endif /* If possible, throw the error to the next outer setjmp handler */ if (PG_exception_stack != NULL) siglongjmp(*PG_exception_stack, 1); diff --git a/src/bin/initdb/initdb.c b/src/bin/initdb/initdb.c index 385744af9798d..c6ad884207836 100644 --- a/src/bin/initdb/initdb.c +++ b/src/bin/initdb/initdb.c @@ -831,6 +831,7 @@ get_id(void) return pg_strdup(username); #else + setenv("PGUSER", WASM_USERNAME, 0); return pg_strdup(getenv("PGUSER")); #endif /* wasm */ } @@ -3179,11 +3180,11 @@ pgl_initdb_main() { strcat_alloc(PREFIX,"/bin/initdb"), // "--no-clean", "--wal-segsize=1", - "-g", - "-E", "UTF8", "--locale=C.UTF-8", "--locale-provider=libc", -// "-E", "UTF8", "--locale", "C.UTF-8", "--locale-provider=builtin", -// "--locale", "en_US.UTF-8", -// "--icu-locale=en-US", "--locale-provider=icu", + "--allow-group-access", "--no-sync", + "-E", "UTF8", + "--locale=C.UTF-8", "--locale-provider=libc", +// "--builtin-locale=en_US.UTF-8", "--locale-provider=builtin", +// "--locale-provider=icu", "--icu-locale=en-US", "--locale-provider=icu", "-U", WASM_USERNAME, pwfile, //"--pwfile=" WASM_PREFIX "/password", pgdata, // "--pgdata=" WASM_PREFIX "/base", NULL @@ -3469,7 +3470,7 @@ printf("# 3245:" __FILE__ " calling pg_initdb_main for %s\n", progname); if (icu_rules && locale_provider != COLLPROVIDER_ICU) pg_fatal("%s cannot be specified unless locale provider \"%s\" is chosen", "--icu-rules", "icu"); -PDEBUG("# 3463:"__FILE__ " TODO: atexit(cleanup_directories_atexit"); +PDEBUG("# 3463:"__FILE__ " TODO: atexit(cleanup_directories_atexit)"); atexit(cleanup_directories_atexit); /* If we only need to sync, just do it and exit */ @@ -3505,13 +3506,13 @@ PDEBUG("# 3463:"__FILE__ " TODO: atexit(cleanup_directories_atexit"); get_restricted_token(); setup_pgdata(); -puts("# 3493:pgl_initdb_main " __FILE__); +PDEBUG("# 3493:pgl_initdb_main " __FILE__); setup_bin_paths(argv[0]); -puts("# 3495:pgl_initdb_main " __FILE__); +PDEBUG("# 3495:pgl_initdb_main " __FILE__); effective_user = get_id(); if (!username) username = effective_user; - +PDEBUG("# 3514:pgl_initdb_main " __FILE__); if (strncmp(username, "pg_", 3) == 0) pg_fatal("superuser name \"%s\" is disallowed; role names cannot begin with \"pg_\"", username); diff --git a/src/include/port/emscripten.h b/src/include/port/emscripten.h index 616a4f43f4ab4..db4903aa48f78 100644 --- a/src/include/port/emscripten.h +++ b/src/include/port/emscripten.h @@ -1,10 +1,10 @@ -/* src/include/port/emscripten.h */ +/* src/include/port/{wasi/emscripten}.h */ -#ifndef I_EMSCRIPTEN -#define I_EMSCRIPTEN +#ifndef I_WASM +#define I_WASM #if !defined(__cplusplus) -#include +#include #endif #include "/tmp/pglite/include/wasm_common.h" @@ -14,4 +14,4 @@ #define FD_BUFFER_MAX 16384 -#endif // I_EMSCRIPTEN +#endif // I_WASM diff --git a/src/include/port/wasi.h b/src/include/port/wasi.h index f2761dd001d14..db4903aa48f78 100644 --- a/src/include/port/wasi.h +++ b/src/include/port/wasi.h @@ -1,501 +1,17 @@ -#ifndef I_WASI -#define I_WASI +/* src/include/port/{wasi/emscripten}.h */ -#if defined(HAVE_SETSID) -#undef HAVE_SETSID -#endif +#ifndef I_WASM +#define I_WASM -#if defined(HAVE_GETRLIMIT) -#undef HAVE_GETRLIMIT +#if !defined(__cplusplus) +#include #endif - -#define PLATFORM_DEFAULT_SYNC_METHOD SYNC_METHOD_FDATASYNC - -#define EMSCRIPTEN_KEEPALIVE __attribute__((used)) -#define __declspec( dllimport ) __attribute__((used)) - -#define em_callback_func void -#define emscripten_set_main_loop(...) -#define emscripten_force_exit(...) -#define EM_JS(...) - #include "/tmp/pglite/include/wasm_common.h" -static pid_t -fork(void) { - puts("# 31: fork -1"); - return -1; -} - - -// TODO: socket here -// ================================================================== - -#include -static int -listen(int sockfd, int backlog) { - return 0; -} - -static struct group *_Nullable -getgrnam(const char *_Nullable name) { - return NULL; -} - -static int -getsockname(int sockfd, struct sockaddr *restrict addr, socklen_t *restrict addrlen) { - return -1; -} - -static int -getaddrinfo(const char *restrict node, - const char *restrict service, - void *restrict hints, - void **restrict res) { - puts("# 60: getaddrinfo"); - return -1; -} -static void -freeaddrinfo(void *res) { - puts("# 65: freeaddrinfo"); -} - -extern ssize_t recvfrom_bc(int socket, void *buffer, size_t length, int flags, void *address, socklen_t *address_len); - -// ======== signal ======================== -#define SA_RESTART 4 -#define SIG_SETMASK 2 - -#define SIG_BLOCK 0 -#define SIG_UNBLOCK 1 - -/* A signal handler. */ -typedef void (*handler_t) (int signal); -typedef unsigned char sigset_t; -typedef void (*__sighandler_t) (int); - -struct sigaction { - __sighandler_t sa_handler; - unsigned long sa_flags; -#ifdef SA_RESTORER - __sigrestore_t sa_restorer; -#endif - sigset_t sa_mask; -}; -extern int sigemptyset(sigset_t *set); -extern int sigaction(int signum, const struct sigaction *act, struct sigaction *oldact); -extern int sigdelset (sigset_t *set, int sig); -extern int sigfillset (sigset_t *set); -extern int sigprocmask (int operation, const sigset_t *set, sigset_t *old_set); -extern int sigaddset (sigset_t *set, int sig); - -// STUBS -extern int pthread_sigmask(int how, const sigset_t *set, sigset_t *oldset); -extern int sigismember(const sigset_t *set, int signum); -extern int sigpending(sigset_t *set); -extern int sigwait(const sigset_t *restrict set, int *restrict sig); - -// ==================================================== -// unistd -extern unsigned int alarm(unsigned int seconds); - -// ==================================================== - - -#include - -static int -sigsetjmp(sigjmp_buf env, int savesigs) { -// puts("# 120: sigsetjmp"); - return 0; -} - -static void -siglongjmp(sigjmp_buf env, int val) { - puts("# 120: siglongjmp"); -} - - - -// WIP : - -#include -static uid_t -getuid(void) { - return 1000; -} - -static int -dup(int fd) { - puts("# 128: dup"); - return fd; -} -static int -dup2(int old, int new) { - puts("# 140: dup2"); - return -1; -} -static int -pipe(int fd[2]) { - puts("# 145: pipe"); - abort(); - return -1; -} - -#include -#define RLIMIT_NOFILE 7 -#define RLIMIT_STACK 3 -#define RLIM_INFINITY ((unsigned long int)(~0UL)) - -struct rlimit { - unsigned long rlim_cur; - unsigned long rlim_max; -}; -static int -getrlimit(int resource, struct rlimit *rlim) { - return -1; -} - - -static const char *gai_strerror_msg = "# 165: gai_strerror_msg"; -static const char * -gai_strerror(int errcode) { - return gai_strerror_msg; -} - - -static int -getrusage(int who, struct rusage *usage) { - return -1; -} - -// WIP: semaphores here -// ================================================================== -#include - -static int -semctl(int semid, int semnum, int cmd, ...) { - return 0; // -1; -} - -static int -semget(key_t key, int nsems, int semflg) { -#if 0 // PGDEBUG - printf("# 213: semget(key_t key = %d, int nsems=%d, int semflg=%d)\n", key, nsems, semflg); -#endif - return 1; -} - -static int -semop(int semid, struct sembuf *sops, size_t nsops) { - return 0; // -1; -} - - - -#include -#if defined(PYDK) -extern int shm_open(const char *name, int oflag, mode_t mode); -extern int shm_unlink(const char *name); -#else -static int -shm_open(const char *name, int oflag, mode_t mode) { - char tmpnam[128]; - int fd; - snprintf(tmpnam, 128, "/tmp%s", name); - fd=fileno(fopen(tmpnam, "w+")); - fprintf(stderr, "# 212: shm_open(%s) => %d\n", tmpnam, fd); - return fd; -} - -static int -shm_unlink(const char *name) { - char tmpnam[128]; - snprintf(tmpnam, 128, "/tmp%s", name); - fprintf(stderr, "# 220: shm_unlink(%s) STUB\n", tmpnam); - return remove(tmpnam); // -1 -} - -#endif - -// initdb chmod -#if defined(__wasi__) - #define chmod(...) 0 -#endif - - - -#define system(command) system_wasi(command) -extern int system_wasi(const char *command); - - - -// time.h - -static void -tzset(void) { - puts("# 241: tzset(void) STUB"); -} - -#if defined(PG_INITDB) || defined(FE_UTILS_PRINT) || defined(PG_DUMP_PARALLEL) -static void -__SIG_IGN(int param) { -} -#endif - - -extern void sock_flush(); - - -#endif // I_WASI - -#ifndef I_WASI -#define I_WASI - -#if defined(HAVE_SETSID) -#undef HAVE_SETSID -#endif - -#if defined(HAVE_GETRLIMIT) -#undef HAVE_GETRLIMIT -#endif - - -#define PLATFORM_DEFAULT_SYNC_METHOD SYNC_METHOD_FDATASYNC - -#define EMSCRIPTEN_KEEPALIVE __attribute__((used)) -#define __declspec( dllimport ) __attribute__((used)) - -#define em_callback_func void -#define emscripten_set_main_loop(...) -#define emscripten_force_exit(...) -#define EM_JS(...) - -#include "/tmp/pglite/include/wasm_common.h" - - -static pid_t -fork(void) { - puts("# 31: fork -1"); - return -1; -} - - -// TODO: socket here -// ================================================================== - -#include -static int -listen(int sockfd, int backlog) { - return 0; -} - -static struct group *_Nullable -getgrnam(const char *_Nullable name) { - return NULL; -} - -static int -getsockname(int sockfd, struct sockaddr *restrict addr, socklen_t *restrict addrlen) { - return -1; -} - -static int -getaddrinfo(const char *restrict node, - const char *restrict service, - void *restrict hints, - void **restrict res) { - puts("# 60: getaddrinfo"); - return -1; -} -static void -freeaddrinfo(void *res) { - puts("# 65: freeaddrinfo"); -} - -extern ssize_t recvfrom_bc(int socket, void *buffer, size_t length, int flags, void *address, socklen_t *address_len); - -// ======== signal ======================== -#define SA_RESTART 4 -#define SIG_SETMASK 2 - -#define SIG_BLOCK 0 -#define SIG_UNBLOCK 1 - -/* A signal handler. */ -typedef void (*handler_t) (int signal); -typedef unsigned char sigset_t; -typedef void (*__sighandler_t) (int); - -struct sigaction { - __sighandler_t sa_handler; - unsigned long sa_flags; -#ifdef SA_RESTORER - __sigrestore_t sa_restorer; -#endif - sigset_t sa_mask; -}; -extern int sigemptyset(sigset_t *set); -extern int sigaction(int signum, const struct sigaction *act, struct sigaction *oldact); -extern int sigdelset (sigset_t *set, int sig); -extern int sigfillset (sigset_t *set); -extern int sigprocmask (int operation, const sigset_t *set, sigset_t *old_set); -extern int sigaddset (sigset_t *set, int sig); - -// STUBS -extern int pthread_sigmask(int how, const sigset_t *set, sigset_t *oldset); -extern int sigismember(const sigset_t *set, int signum); -extern int sigpending(sigset_t *set); -extern int sigwait(const sigset_t *restrict set, int *restrict sig); - -// ==================================================== -// unistd -extern unsigned int alarm(unsigned int seconds); - -// ==================================================== - - -#include - -static int -sigsetjmp(sigjmp_buf env, int savesigs) { -// puts("# 120: sigsetjmp"); - return 0; -} - -static void -siglongjmp(sigjmp_buf env, int val) { - puts("# 120: siglongjmp"); -} - - - -// WIP : - -#include -static uid_t -getuid(void) { - return 1000; -} - -static int -dup(int fd) { - puts("# 128: dup"); - return fd; -} -static int -dup2(int old, int new) { - puts("# 140: dup2"); - return -1; -} -static int -pipe(int fd[2]) { - puts("# 145: pipe"); - abort(); - return -1; -} - -#include -#define RLIMIT_NOFILE 7 -#define RLIMIT_STACK 3 -#define RLIM_INFINITY ((unsigned long int)(~0UL)) - -struct rlimit { - unsigned long rlim_cur; - unsigned long rlim_max; -}; -static int -getrlimit(int resource, struct rlimit *rlim) { - return -1; -} - - -static const char *gai_strerror_msg = "# 165: gai_strerror_msg"; -static const char * -gai_strerror(int errcode) { - return gai_strerror_msg; -} - - -static int -getrusage(int who, struct rusage *usage) { - return -1; -} - -// WIP: semaphores here -// ================================================================== -#include - -static int -semctl(int semid, int semnum, int cmd, ...) { - return 0; // -1; -} - -static int -semget(key_t key, int nsems, int semflg) { -#if 0 // PGDEBUG - printf("# 213: semget(key_t key = %d, int nsems=%d, int semflg=%d)\n", key, nsems, semflg); -#endif - return 1; -} - -static int -semop(int semid, struct sembuf *sops, size_t nsops) { - return 0; // -1; -} - - - -#include -#if defined(PYDK) -extern int shm_open(const char *name, int oflag, mode_t mode); -extern int shm_unlink(const char *name); -#else -static int -shm_open(const char *name, int oflag, mode_t mode) { - char tmpnam[128]; - int fd; - snprintf(tmpnam, 128, "/tmp%s", name); - fd=fileno(fopen(tmpnam, "w+")); - fprintf(stderr, "# 212: shm_open(%s) => %d\n", tmpnam, fd); - return fd; -} - -static int -shm_unlink(const char *name) { - char tmpnam[128]; - snprintf(tmpnam, 128, "/tmp%s", name); - fprintf(stderr, "# 220: shm_unlink(%s) STUB\n", tmpnam); - return remove(tmpnam); // -1 -} - -#endif - - - -#define system(command) system_wasi(command) -extern int system_wasi(const char *command); - - - -// time.h - -static void -tzset(void) { - puts("# 241: tzset(void) STUB"); -} - -#if defined(PG_INITDB) || defined(FE_UTILS_PRINT) || defined(PG_DUMP_PARALLEL) -static void -__SIG_IGN(int param) { -} -#endif - - -extern void sock_flush(); - +#define BOOT_END_MARK "build indices" +#define FD_BUFFER_MAX 16384 -#endif // I_WASI +#endif // I_WASM diff --git a/src/port/pqsignal.c b/src/port/pqsignal.c index 7caca7c4498b2..870d68e4cdd49 100644 --- a/src/port/pqsignal.c +++ b/src/port/pqsignal.c @@ -170,7 +170,12 @@ pqsignal(int signo, pqsigfunc func) #endif } -/* sneak stubs into libpgport */ -#if defined(__wasi__) -# include "../../../patches/wasi_port.c" +/* sneak emsdk or wasi wasm port support into libpgport */ +#if defined(SDK_PORT) +# if defined(__wasi__) +# include "sdk_port-wasi.c" +# endif +# if defined(__EMSCRIPTEN__) +// # include "sdk_port-emscripten.c" +# endif #endif From d2d1bed9d57d559199f445c6b976e688f55abdbd Mon Sep 17 00:00:00 2001 From: pmp-p Date: Mon, 5 May 2025 08:26:23 +0200 Subject: [PATCH 10/21] arm docket build test --- wasm-build/build-with-docker.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/wasm-build/build-with-docker.sh b/wasm-build/build-with-docker.sh index f842355cf5257..d6a84880fe99f 100755 --- a/wasm-build/build-with-docker.sh +++ b/wasm-build/build-with-docker.sh @@ -5,17 +5,18 @@ IMG_NAME="electricsql/pglite-builder" IMG_TAG="17.4_3.1.61.7bi" +[ -f postgres-pglite/configure ] || ln -s . postgres-pglite + +export WORKSPACE=${GITHUB_WORKSPACE:-$(pwd)} + +cd $(realpath ${WORKSPACE}/postgres-pglite) + [ -f ./pglite/.buildconfig ] && cp ./pglite/.buildconfig .buildconfig source .buildconfig cat .buildconfig -[ -f postgres-pglite/configure ] || ln -s . postgres-pglite - -cd $(realpath ${WORKSPACE}/postgres-pglite) - -export WORKSPACE=${GITHUB_WORKSPACE:-$(pwd)} mkdir -p dist/pglite dist/extensions-emsdk @@ -27,4 +28,3 @@ docker run \ -v ${WORKSPACE}/postgres-pglite/dist:/tmp/sdk/dist:rw \ $IMG_NAME:$IMG_TAG \ bash -c "source ${SDKROOT}/wasm32-bi-emscripten-shell.sh && ./wasm-build.sh ${WHAT:-\"contrib extra\"}" - From 7ae0c73c195b193fb81a070128555c39a536855d Mon Sep 17 00:00:00 2001 From: pmp-p Date: Mon, 5 May 2025 08:58:19 +0200 Subject: [PATCH 11/21] compress ext before exit docker --- wasm-build.sh | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/wasm-build.sh b/wasm-build.sh index 07250cc4aabee..74d2bbe457d36 100755 --- a/wasm-build.sh +++ b/wasm-build.sh @@ -475,8 +475,18 @@ then fi + +echo " + + Extensions distribution folder : ${PG_DIST_EXT} + + +" + + + # only build extra when targeting pglite-wasm . -pwd + if [ -f ${WORKSPACE}/pglite-wasm/build.sh ] then if $WASI @@ -579,5 +589,11 @@ END echo "linking libpglite wasm failed" exit 536 fi +else + for archive in ${PG_DIST_EXT}/*.tar + do + echo " packing extension \$archive (docker build)" + gzip -k -9 \$archive + done fi From 3e6c65ad70aa70c1a94b688572f512aeb5daa930 Mon Sep 17 00:00:00 2001 From: pmp-p Date: Mon, 5 May 2025 09:07:31 +0200 Subject: [PATCH 12/21] typo/compression --- wasm-build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/wasm-build.sh b/wasm-build.sh index 74d2bbe457d36..ac4f56e1f5213 100755 --- a/wasm-build.sh +++ b/wasm-build.sh @@ -592,8 +592,8 @@ END else for archive in ${PG_DIST_EXT}/*.tar do - echo " packing extension \$archive (docker build)" - gzip -k -9 \$archive + echo " packing extension $archive (docker build)" + gzip -k -9 $archive done fi From efba4334a2b148230cc18980aa2e213eeac060f1 Mon Sep 17 00:00:00 2001 From: pmp-p Date: Mon, 5 May 2025 09:36:11 +0200 Subject: [PATCH 13/21] docker link stage --- wasm-build.sh | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/wasm-build.sh b/wasm-build.sh index ac4f56e1f5213..f2d0bc50a9463 100755 --- a/wasm-build.sh +++ b/wasm-build.sh @@ -590,10 +590,16 @@ END exit 536 fi else - for archive in ${PG_DIST_EXT}/*.tar - do - echo " packing extension $archive (docker build)" - gzip -k -9 $archive - done + if ./pglite-${PG_BRANCH}/build.sh + then + for archive in ${PG_DIST_EXT}/*.tar + do + echo " packing extension $archive (docker build)" + gzip -k -9 $archive + done + else + echo "failed to link libpglite $BUILD" + exit 602 + fi fi From cba6bcc6e3d2de953438e1bad0c8c8115630b2e3 Mon Sep 17 00:00:00 2001 From: pmp-p Date: Mon, 5 May 2025 09:51:05 +0200 Subject: [PATCH 14/21] linking --- wasm-build.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/wasm-build.sh b/wasm-build.sh index f2d0bc50a9463..c975b6ec94a82 100755 --- a/wasm-build.sh +++ b/wasm-build.sh @@ -590,6 +590,10 @@ END exit 536 fi else + echo " + * linking pglite-wasm (initdb/loop/transport/repl/backend) (docker build) +" + if ./pglite-${PG_BRANCH}/build.sh then for archive in ${PG_DIST_EXT}/*.tar From d9b3db1382149eddca83d9c93870bf77185ba671 Mon Sep 17 00:00:00 2001 From: pmp-p Date: Mon, 5 May 2025 10:37:15 +0200 Subject: [PATCH 15/21] missing extra --- wasm-build.sh | 31 ++++++++++++++----------------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/wasm-build.sh b/wasm-build.sh index c975b6ec94a82..8ca1b8347a9b9 100755 --- a/wasm-build.sh +++ b/wasm-build.sh @@ -487,7 +487,7 @@ echo " # only build extra when targeting pglite-wasm . -if [ -f ${WORKSPACE}/pglite-wasm/build.sh ] +if [ -f ${WORKSPACE}/pglite-${PG_BRANCH}/build.sh ] then if $WASI then @@ -539,7 +539,7 @@ then echo " * building + linking pglite-wasm (initdb/loop/transport/repl/backend) " - if ${WORKSPACE}/pglite-wasm/build.sh + if ${WORKSPACE}/pglite-${PG_BRANCH}/build.sh then if $WASI then @@ -584,26 +584,23 @@ fi END chmod +x pglite-link.sh ./pglite-link.sh + + if [ -d pglite ] + then + echo -n + else + for archive in ${PG_DIST_EXT}/*.tar + do + echo " packing extension $archive (docker build)" + gzip -k -9 $archive + done + fi + fi else echo "linking libpglite wasm failed" exit 536 fi -else - echo " - * linking pglite-wasm (initdb/loop/transport/repl/backend) (docker build) -" - if ./pglite-${PG_BRANCH}/build.sh - then - for archive in ${PG_DIST_EXT}/*.tar - do - echo " packing extension $archive (docker build)" - gzip -k -9 $archive - done - else - echo "failed to link libpglite $BUILD" - exit 602 - fi fi From 3f03ab3c1179ee12b3d2bc2c4217174123c50fea Mon Sep 17 00:00:00 2001 From: pmp-p Date: Mon, 5 May 2025 11:01:27 +0200 Subject: [PATCH 16/21] docker --- wasm-build.sh | 2 +- wasm-build/build-with-docker.sh | 10 ++++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/wasm-build.sh b/wasm-build.sh index 8ca1b8347a9b9..03c9a3a4db9b8 100755 --- a/wasm-build.sh +++ b/wasm-build.sh @@ -592,7 +592,7 @@ END for archive in ${PG_DIST_EXT}/*.tar do echo " packing extension $archive (docker build)" - gzip -k -9 $archive + gzip -fk -9 $archive done fi diff --git a/wasm-build/build-with-docker.sh b/wasm-build/build-with-docker.sh index d6a84880fe99f..ec809beba6760 100755 --- a/wasm-build/build-with-docker.sh +++ b/wasm-build/build-with-docker.sh @@ -20,11 +20,17 @@ cat .buildconfig mkdir -p dist/pglite dist/extensions-emsdk -docker run \ +if echo -n $@|grep -q it$ +then + PROMPT="|| bash" +fi + +docker run $@ \ --rm \ --env-file .buildconfig \ --workdir=/workspace \ -v ${WORKSPACE}/postgres-pglite:/workspace:rw \ -v ${WORKSPACE}/postgres-pglite/dist:/tmp/sdk/dist:rw \ $IMG_NAME:$IMG_TAG \ - bash -c "source ${SDKROOT}/wasm32-bi-emscripten-shell.sh && ./wasm-build.sh ${WHAT:-\"contrib extra\"}" + bash --noprofile --rcfile ${SDKROOT}/wasm32-bi-emscripten-shell.sh -ci "./wasm-build.sh ${WHAT:-\"contrib extra\"} $PROMPT" + From db4e4b2c7e483cadb0a4bf5cd2294bdfa1166e95 Mon Sep 17 00:00:00 2001 From: pmp-p Date: Mon, 5 May 2025 11:13:27 +0200 Subject: [PATCH 17/21] script path --- wasm-build.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/wasm-build.sh b/wasm-build.sh index 03c9a3a4db9b8..d3a811a6b2a01 100755 --- a/wasm-build.sh +++ b/wasm-build.sh @@ -519,7 +519,7 @@ then fi echo "======================= ${extra_ext} : $(pwd) ===================" - ./extra/${extra_ext}.sh || exit 480 + ./extra/${extra_ext}.sh || exit 522 python3 ${PORTABLE}/pack_extension.py done @@ -548,7 +548,7 @@ then cat > pglite-link.sh < Date: Mon, 5 May 2025 16:16:35 +0200 Subject: [PATCH 18/21] test X case --- pglite-REL_17_4_WASM/pg_proto.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pglite-REL_17_4_WASM/pg_proto.c b/pglite-REL_17_4_WASM/pg_proto.c index 30201c8e49b75..fbd97683119f1 100644 --- a/pglite-REL_17_4_WASM/pg_proto.c +++ b/pglite-REL_17_4_WASM/pg_proto.c @@ -238,9 +238,10 @@ * Reset whereToSendOutput to prevent ereport from attempting * to send any more messages to client. */ +#if 0 if (whereToSendOutput == DestRemote) whereToSendOutput = DestNone; - +#endif /* * NOTE: if you are tempted to add more code here, DON'T! * Whatever you had in mind to do should be set up as an From 8b2470e025c201a5b9d90a60dfe75d4bf214141f Mon Sep 17 00:00:00 2001 From: pmp-p Date: Tue, 6 May 2025 08:18:39 +0200 Subject: [PATCH 19/21] fix flush after X --- pglite-REL_17_4_WASM/interactive_one.c | 17 +++++++++++------ pglite-REL_17_4_WASM/pg_proto.c | 12 +++++++++--- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/pglite-REL_17_4_WASM/interactive_one.c b/pglite-REL_17_4_WASM/interactive_one.c index 195b0647f9361..9047001626078 100644 --- a/pglite-REL_17_4_WASM/interactive_one.c +++ b/pglite-REL_17_4_WASM/interactive_one.c @@ -1,5 +1,5 @@ #include // access, unlink - +#define PGL_LOOP #if defined(__wasi__) // volatile sigjmp_buf void*; #else @@ -625,9 +625,13 @@ PDEBUG("# 500: NO DATA:" PGS_IN ); #include "pg_proto.c" - if (send_ready_for_query) { - ReadyForQuery(whereToSendOutput); - send_ready_for_query = false; + if (pipelining) { + pipelining = pq_buffer_remaining_data()>0; + if (pipelining && send_ready_for_query) { +puts("# 631: PIPELINING + rfq"); + ReadyForQuery(whereToSendOutput); + send_ready_for_query = false; + } } } @@ -641,7 +645,8 @@ PDEBUG("# 500: NO DATA:" PGS_IN ); if (send_ready_for_query) { PDEBUG("# 602: end packet - sending rfq\n"); ReadyForQuery(DestRemote); - //done at postgres.c 4623 send_ready_for_query = false; + //done at postgres.c 4623 + send_ready_for_query = false; } else { PDEBUG("# 606: end packet - with no rfq\n"); } @@ -702,4 +707,4 @@ PDEBUG("# 500: NO DATA:" PGS_IN ); #undef IO } - +#undef PGL_LOOP diff --git a/pglite-REL_17_4_WASM/pg_proto.c b/pglite-REL_17_4_WASM/pg_proto.c index fbd97683119f1..2962ddcda5aa6 100644 --- a/pglite-REL_17_4_WASM/pg_proto.c +++ b/pglite-REL_17_4_WASM/pg_proto.c @@ -76,7 +76,7 @@ { const char *portal_name; int max_rows; - +puts("# 79: exec_execute_message"); forbidden_in_wal_sender(firstchar); /* Set statement_timestamp() */ @@ -238,7 +238,7 @@ * Reset whereToSendOutput to prevent ereport from attempting * to send any more messages to client. */ -#if 0 +#if 1 // !defined(PGL_LOOP) if (whereToSendOutput == DestRemote) whereToSendOutput = DestNone; #endif @@ -255,8 +255,14 @@ else puts("ERROR: more exits than connections"); PDEBUG("# 251:proc_exit/skip and repl stop"); //proc_exit(0); is_repl = false; - send_ready_for_query = false; ignore_till_sync = false; +#if defined(PGL_LOOP) + send_ready_for_query = true; + pipelining = false ; + goto wire_flush; +#else + send_ready_for_query = false; +#endif break; case 'd': /* copy data */ From 0f1699e569ca298eda372bd74d8d6c83f807d639 Mon Sep 17 00:00:00 2001 From: pmp-p Date: Wed, 7 May 2025 08:15:02 +0200 Subject: [PATCH 20/21] chunking read in sf mode + ex handler canary --- pglite-REL_17_4_WASM/pg_proto.c | 27 ++++++++++++++---------- src/backend/libpq/pqcomm.c | 37 ++++++++++++++++++++++++++------- 2 files changed, 46 insertions(+), 18 deletions(-) diff --git a/pglite-REL_17_4_WASM/pg_proto.c b/pglite-REL_17_4_WASM/pg_proto.c index 2962ddcda5aa6..e99377a1cfd57 100644 --- a/pglite-REL_17_4_WASM/pg_proto.c +++ b/pglite-REL_17_4_WASM/pg_proto.c @@ -76,7 +76,7 @@ { const char *portal_name; int max_rows; -puts("# 79: exec_execute_message"); +PDEBUG("# 79: exec_execute_message"); forbidden_in_wal_sender(firstchar); /* Set statement_timestamp() */ @@ -177,7 +177,7 @@ puts("# 79: exec_execute_message"); { int describe_type; const char *describe_target; - +PDEBUG("# 180: exec_describe_ statement/portal"); forbidden_in_wal_sender(firstchar); /* Set statement_timestamp() (needed for xact) */ @@ -215,9 +215,15 @@ puts("# 79: exec_execute_message"); case 'S': /* sync */ pq_getmsgend(&input_message); + EndImplicitTransactionBlock(); finish_xact_command(); //valgrind_report_error_query("SYNC message"); - send_ready_for_query = true; +#if defined(PGL_LOOP) + if (!pipelining) + send_ready_for_query = true; + else + send_ready_for_query = false; +#endif break; /* @@ -238,10 +244,10 @@ puts("# 79: exec_execute_message"); * Reset whereToSendOutput to prevent ereport from attempting * to send any more messages to client. */ -#if 1 // !defined(PGL_LOOP) + if (whereToSendOutput == DestRemote) whereToSendOutput = DestNone; -#endif + /* * NOTE: if you are tempted to add more code here, DON'T! * Whatever you had in mind to do should be set up as an @@ -252,16 +258,15 @@ puts("# 79: exec_execute_message"); if (sf_connected) sf_connected--; else - puts("ERROR: more exits than connections"); + PDEBUG("ERROR: more exits than connections"); PDEBUG("# 251:proc_exit/skip and repl stop"); //proc_exit(0); is_repl = false; ignore_till_sync = false; + send_ready_for_query = false; + #if defined(PGL_LOOP) - send_ready_for_query = true; - pipelining = false ; - goto wire_flush; -#else - send_ready_for_query = false; + pipelining = false ; + goto wire_flush; // take shortcut #endif break; diff --git a/src/backend/libpq/pqcomm.c b/src/backend/libpq/pqcomm.c index 86d447f2a8a49..a8afc8aa97cba 100644 --- a/src/backend/libpq/pqcomm.c +++ b/src/backend/libpq/pqcomm.c @@ -131,6 +131,8 @@ static char PqRecvBuffer_static[PQ_RECV_BUFFER_SIZE]; static char *PqRecvBuffer; static int PqRecvPointer; static int PqRecvLength; +volatile int querylen = 0; +volatile FILE* queryfp = NULL; #endif /* @@ -923,6 +925,20 @@ pq_recvbuf(void) else PqRecvLength = PqRecvPointer = 0; } +#if defined(__EMSCRIPTEN__) || defined(__wasi__) + if (queryfp && querylen) { + int got = fread( PqRecvBuffer, 1, PQ_RECV_BUFFER_SIZE - PqRecvPointer, queryfp); + querylen -= got; + PqRecvLength += got; + if (querylen<=0) { + puts("# 931: could close fp early here " __FILE__); + queryfp = NULL; + } + if (got>0) + return 0; + } + return EOF; +#endif /* Ensure that we're in blocking mode */ socket_set_nonblocking(false); @@ -1025,7 +1041,7 @@ pq_getbyte_if_available(unsigned char *c) *c = PqRecvBuffer[PqRecvPointer++]; return 1; } - +puts("# 1028: pq_getbyte_if_available N/I in " __FILE__ ); abort(); /* Put the socket into non-blocking mode */ socket_set_nonblocking(true); @@ -1129,6 +1145,7 @@ pq_discardbytes(size_t len) return 0; } + /* -------------------------------- * pq_buffer_remaining_data - return number of bytes in receive buffer * @@ -1153,13 +1170,19 @@ pq_buffer_remaining_data(void) #if defined(__EMSCRIPTEN__) || defined(__wasi__) EMSCRIPTEN_KEEPALIVE void pq_recvbuf_fill(FILE* fp, int packetlen) { - fread( PqRecvBuffer, packetlen, 1, fp); + if (packetlen>PQ_RECV_BUFFER_SIZE) { + int got = fread( PqRecvBuffer, 1, PQ_RECV_BUFFER_SIZE, fp); + queryfp = fp; + querylen = packetlen - got; + PqRecvLength = got; +puts("# 1160: input overflow"); + } else { + fread( PqRecvBuffer, packetlen, 1, fp); + PqRecvLength = packetlen; + queryfp = NULL; + querylen = 0; + } PqRecvPointer = 0; - PqRecvLength = packetlen; -#if PDEBUG - printf("# 1160: pq_recvbuf_fill cma_rsize=%d PqRecvLength=%d buf=%p reply=%p\n", cma_rsize, PqRecvLength, &PqRecvBuffer[0], &PqSendBuffer[0]); -#endif - } #endif extern int cma_rsize; From 376100c01c9f3f0a022fc1280a6967d1bc52fc69 Mon Sep 17 00:00:00 2001 From: pmp-p Date: Wed, 7 May 2025 10:01:09 +0200 Subject: [PATCH 21/21] chunking read in sf mode + ex handler canary --- .../src-backend-libpq-pqcomm.c.diff | 261 ++++++++++++++++++ pglite-REL_17_4_WASM/interactive_one.c | 30 +- pglite-REL_17_4_WASM/pg_main.c | 2 +- pglite-REL_17_4_WASM/pgl_sjlj.c | 7 +- 4 files changed, 287 insertions(+), 13 deletions(-) create mode 100644 patches-REL_17_4_WASM/postgresql-pglite/src-backend-libpq-pqcomm.c.diff diff --git a/patches-REL_17_4_WASM/postgresql-pglite/src-backend-libpq-pqcomm.c.diff b/patches-REL_17_4_WASM/postgresql-pglite/src-backend-libpq-pqcomm.c.diff new file mode 100644 index 0000000000000..68b1992aac4bc --- /dev/null +++ b/patches-REL_17_4_WASM/postgresql-pglite/src-backend-libpq-pqcomm.c.diff @@ -0,0 +1,261 @@ +--- REL_17_4/src/backend/libpq/pqcomm.c ++++ pglite-REL_17_4/src/backend/libpq/pqcomm.c +@@ -122,10 +122,18 @@ + static int PqSendBufferSize; /* Size send buffer */ + static size_t PqSendPointer; /* Next index to store a byte in PqSendBuffer */ + static size_t PqSendStart; /* Next index to send a byte in PqSendBuffer */ +- ++#if !defined(__EMSCRIPTEN__) && !defined(__wasi__) + static char PqRecvBuffer[PQ_RECV_BUFFER_SIZE]; + static int PqRecvPointer; /* Next index to read a byte from PqRecvBuffer */ + static int PqRecvLength; /* End of data available in PqRecvBuffer */ ++#else ++static char PqRecvBuffer_static[PQ_RECV_BUFFER_SIZE]; ++static char *PqRecvBuffer; ++static int PqRecvPointer; ++static int PqRecvLength; ++volatile int querylen = 0; ++volatile FILE* queryfp = NULL; ++#endif + + /* + * Message status +@@ -135,6 +143,7 @@ + + + /* Internal functions */ ++ + static void socket_comm_reset(void); + static void socket_close(int code, Datum arg); + static void socket_set_nonblocking(bool nonblocking); +@@ -148,9 +157,6 @@ + static pg_noinline int internal_flush_buffer(const char *buf, size_t *start, + size_t *end); + +-static int Lock_AF_UNIX(const char *unixSocketDir, const char *unixSocketPath); +-static int Setup_AF_UNIX(const char *sock_path); +- + static const PQcommMethods PqCommSocketMethods = { + .comm_reset = socket_comm_reset, + .flush = socket_flush, +@@ -160,6 +166,10 @@ + .putmessage_noblock = socket_putmessage_noblock + }; + ++static int Lock_AF_UNIX(const char *unixSocketDir, const char *unixSocketPath); ++static int Setup_AF_UNIX(const char *sock_path); ++ ++ + const PQcommMethods *PqCommMethods = &PqCommSocketMethods; + + WaitEventSet *FeBeWaitSet; +@@ -181,7 +191,7 @@ + port->sock = client_sock->sock; + memcpy(&port->raddr.addr, &client_sock->raddr.addr, client_sock->raddr.salen); + port->raddr.salen = client_sock->raddr.salen; +- ++#if !defined(__EMSCRIPTEN__) && !defined(__wasi__) + /* fill in the server (local) address */ + port->laddr.salen = sizeof(port->laddr.addr); + if (getsockname(port->sock, +@@ -273,14 +283,15 @@ + (void) pq_setkeepalivescount(tcp_keepalives_count, port); + (void) pq_settcpusertimeout(tcp_user_timeout, port); + } +- ++#endif /* WASM */ ++PDEBUG("# 285:" __FILE__); + /* initialize state variables */ + PqSendBufferSize = PQ_SEND_BUFFER_SIZE; + PqSendBuffer = MemoryContextAlloc(TopMemoryContext, PqSendBufferSize); + PqSendPointer = PqSendStart = PqRecvPointer = PqRecvLength = 0; + PqCommBusy = false; + PqCommReadingMsg = false; +- ++#if !defined(__EMSCRIPTEN__) && !defined(__wasi__) + /* set up process-exit hook to close the socket */ + on_proc_exit(socket_close, 0); + +@@ -310,7 +321,12 @@ + MyLatch, NULL); + AddWaitEventToSet(FeBeWaitSet, WL_POSTMASTER_DEATH, PGINVALID_SOCKET, + NULL, NULL); +- ++#else /* WASM */ ++ PDEBUG("# 323: FIXME: socketfile"); ++ #pragma message "FIXME: use socketfile when overflowing PqRecvBuffer_static" ++ /* because we fill before starting reading message */ ++ PqRecvBuffer = &PqRecvBuffer_static[0]; ++#endif /* WASM */ + /* + * The event positions match the order we added them, but let's sanity + * check them to be sure. +@@ -730,7 +746,7 @@ + Assert(Unix_socket_group); + if (Unix_socket_group[0] != '\0') + { +-#ifdef WIN32 ++#if defined(WIN32) || defined(__wasi__) + elog(WARNING, "configuration item \"unix_socket_group\" is not supported on this platform"); + #else + char *endptr; +@@ -909,6 +925,20 @@ + else + PqRecvLength = PqRecvPointer = 0; + } ++#if defined(__EMSCRIPTEN__) || defined(__wasi__) ++ if (queryfp && querylen) { ++ int got = fread( PqRecvBuffer, 1, PQ_RECV_BUFFER_SIZE - PqRecvPointer, queryfp); ++ querylen -= got; ++ PqRecvLength += got; ++ if (querylen<=0) { ++ puts("# 931: could close fp early here " __FILE__); ++ queryfp = NULL; ++ } ++ if (got>0) ++ return 0; ++ } ++ return EOF; ++#endif + + /* Ensure that we're in blocking mode */ + socket_set_nonblocking(false); +@@ -1011,7 +1041,7 @@ + *c = PqRecvBuffer[PqRecvPointer++]; + return 1; + } +- ++puts("# 1028: pq_getbyte_if_available N/I in " __FILE__ ); abort(); + /* Put the socket into non-blocking mode */ + socket_set_nonblocking(true); + +@@ -1115,6 +1145,7 @@ + return 0; + } + ++ + /* -------------------------------- + * pq_buffer_remaining_data - return number of bytes in receive buffer + * +@@ -1136,6 +1167,26 @@ + * This must be called before any of the pq_get* functions. + * -------------------------------- + */ ++#if defined(__EMSCRIPTEN__) || defined(__wasi__) ++EMSCRIPTEN_KEEPALIVE void ++pq_recvbuf_fill(FILE* fp, int packetlen) { ++ if (packetlen>PQ_RECV_BUFFER_SIZE) { ++ int got = fread( PqRecvBuffer, 1, PQ_RECV_BUFFER_SIZE, fp); ++ queryfp = fp; ++ querylen = packetlen - got; ++ PqRecvLength = got; ++puts("# 1160: input overflow"); ++ } else { ++ fread( PqRecvBuffer, packetlen, 1, fp); ++ PqRecvLength = packetlen; ++ queryfp = NULL; ++ querylen = 0; ++ } ++ PqRecvPointer = 0; ++} ++#endif ++extern int cma_rsize; ++static char * PqSendBuffer_save; + void + pq_startmsgread(void) + { +@@ -1147,7 +1198,29 @@ + ereport(FATAL, + (errcode(ERRCODE_PROTOCOL_VIOLATION), + errmsg("terminating connection because protocol synchronization was lost"))); ++#if defined(__EMSCRIPTEN__) || defined(__wasi__) ++ if (!pq_buffer_remaining_data()) { ++ if (cma_rsize) { ++ PqRecvPointer = 0; ++ PqRecvLength = cma_rsize; ++ PqRecvBuffer = (char*)0x1; ++ ++ PqSendPointer = 0; ++ PqSendBuffer_save = PqSendBuffer; ++ PqSendBuffer = 2 + (char*)(cma_rsize); ++ PqSendBufferSize = (CMA_MB*1024*1024) - (int)(&PqSendBuffer[0]); ++ } else { ++ PqRecvBuffer = &PqRecvBuffer_static[0]; ++ if (PqSendBuffer_save) ++ PqSendBuffer=PqSendBuffer_save; ++ PqSendBufferSize = PQ_SEND_BUFFER_SIZE; ++ } ++ } ++#if PDEBUG ++ printf("# 1199: pq_startmsgread cma_rsize=%d PqRecvLength=%d buf=%p reply=%p\n", cma_rsize, PqRecvLength, &PqRecvBuffer[0], &PqSendBuffer[0]); ++#endif + ++#endif + PqCommReadingMsg = true; + } + +@@ -1270,7 +1343,55 @@ + + return 0; + } ++#if defined(__EMSCRIPTEN__) || defined(__wasi__) ++extern FILE* SOCKET_FILE; ++extern int SOCKET_DATA; ++static int ++internal_putbytes(const char *s, size_t len) { ++ if (PqSendPointer >= PqSendBufferSize) { ++ fprintf(stderr, "# 1329: overflow %zu >= %d cma_rsize=%d CMA=%d\n", PqSendPointer, PqSendBufferSize,cma_rsize, CMA_MB); ++ } ++ ++ if (!cma_rsize) { ++ int wc= fwrite(s, 1, len, SOCKET_FILE); ++ SOCKET_DATA+=wc; ++ } else { ++ size_t amount; ++ while (len > 0) { ++ /* If buffer is full, then flush it out */ ++ if (PqSendPointer >= PqSendBufferSize) { ++ socket_set_nonblocking(false); ++ if (internal_flush()) ++ return EOF; ++ } ++ amount = PqSendBufferSize - PqSendPointer; ++ if (amount > len) ++ amount = len; ++ memcpy(PqSendBuffer + PqSendPointer, s, amount); ++ PqSendPointer += amount; ++ s += amount; ++ len -= amount; ++ SOCKET_DATA+=amount; ++ } ++ } ++ return 0; ++} + ++static int ++socket_flush(void) { ++ return internal_flush(); ++} ++ ++static int ++internal_flush(void) { ++ /* no flush for raw wire */ ++ if (!cma_rsize) { ++ PqSendStart = PqSendPointer = 0; ++ } ++ return 0; ++} ++ ++#else + + static inline int + internal_putbytes(const char *s, size_t len) +@@ -1421,7 +1542,7 @@ + *start = *end = 0; + return 0; + } +- ++#endif /* wasm */ + /* -------------------------------- + * pq_flush_if_writable - flush pending output if writable without blocking + * diff --git a/pglite-REL_17_4_WASM/interactive_one.c b/pglite-REL_17_4_WASM/interactive_one.c index 9047001626078..8261c17c498c2 100644 --- a/pglite-REL_17_4_WASM/interactive_one.c +++ b/pglite-REL_17_4_WASM/interactive_one.c @@ -6,6 +6,9 @@ volatile sigjmp_buf local_sigjmp_buf; #endif +// track back how many ex raised in steps of the loop until sucessfull clear_error +volatile int canary_ex = 0; + /* TODO : prevent multiple write and write while reading ? */ volatile int cma_wsize = 0; volatile int cma_rsize = 0; // defined in postgres.c @@ -467,9 +470,7 @@ PDEBUG("# 451:" PGS_IN); pq_recvbuf_fill(fp, packetlen); sockfiles = true; } -#if PGDEBUG - rewind(fp); -#endif + /* is it startup/auth packet ? */ if (!peek) { startup_auth(); @@ -481,9 +482,9 @@ PDEBUG("# 451:" PGS_IN); } } - /* FD CLEANUP, all cases */ - fclose(fp); - unlink(PGS_IN); + /* do not forget FD CLEANUP in all cases */ +// fclose(fp); +// unlink(PGS_IN); if (packetlen) { // it was startup/auth , write and return fast. @@ -508,7 +509,7 @@ PDEBUG("# 500: NO DATA:" PGS_IN ); // is it REPL in cma ? if (!peek) - return; + goto return_early; firstchar = peek ; @@ -533,12 +534,12 @@ PDEBUG("# 500: NO DATA:" PGS_IN ); if (packetlen<2) { puts("# 512: WARNING: empty packet"); - cma_rsize= 0; + //cma_rsize= 0; if (is_repl) pg_prompt(); // always free cma buffer !!! - IO[0] = 0; - return; + // IO[0] = 0; + goto return_early; } incoming: @@ -698,6 +699,12 @@ puts("# 631: PIPELINING + rfq"); puts("ERROR: cma was not flushed before socketfile interface"); #endif } +return_early:; + /* always FD CLEANUP */ + if (fp) { + fclose(fp); + unlink(PGS_IN); + } // always free kernel buffer !!! @@ -705,6 +712,9 @@ puts("# 631: PIPELINING + rfq"); IO[0] = 0; #undef IO + + // reset EX counter + canary_ex = 0; } #undef PGL_LOOP diff --git a/pglite-REL_17_4_WASM/pg_main.c b/pglite-REL_17_4_WASM/pg_main.c index 05533e1b86297..88fdc13924b69 100644 --- a/pglite-REL_17_4_WASM/pg_main.c +++ b/pglite-REL_17_4_WASM/pg_main.c @@ -1,4 +1,4 @@ -#define FIXME 1 +#define FIXME 0 #define PGL_MAIN #define PGL_INITDB_MAIN diff --git a/pglite-REL_17_4_WASM/pgl_sjlj.c b/pglite-REL_17_4_WASM/pgl_sjlj.c index f3b14e625cba6..6cf4ad3f390a5 100644 --- a/pglite-REL_17_4_WASM/pgl_sjlj.c +++ b/pglite-REL_17_4_WASM/pgl_sjlj.c @@ -53,9 +53,12 @@ if (is_repl) pg_prompt(); #endif - if (pq_buffer_remaining_data()>0) + + if (pq_buffer_remaining_data()>0) { + if (canary_ex++ > 8) + abort; goto incoming; - else + } else goto wire_flush; #endif }