diff --git a/.buildconfig b/.buildconfig index f0ef5f8928e12..71005f852466a 100644 --- a/.buildconfig +++ b/.buildconfig @@ -1,14 +1,8 @@ PG_VERSION=17.4 PG_BRANCH=REL_17_4_WASM - SDK_VERSION=3.1.74.6bi WASI_SDK_VERSION=24.0.4 SDKROOT=/tmp/sdk - -PG_DIST=/tmp/sdk/dist -PG_DIST_EXT=/tmp/sdk/dist/extensions-emsdk -DEBUG=false -CI=true - GETZIC=false ZIC=/usr/sbin/zic + diff --git a/patches-REL_17_4_WASM/postgresql-emscripten/src-backend-commands-async.c.diff b/patches-REL_17_4_WASM/postgresql-emscripten/src-backend-commands-async.c.diff index 17646ac8cf3f2..0299b57028d65 100644 --- a/patches-REL_17_4_WASM/postgresql-emscripten/src-backend-commands-async.c.diff +++ b/patches-REL_17_4_WASM/postgresql-emscripten/src-backend-commands-async.c.diff @@ -5,7 +5,7 @@ * just log a low-level debug message if it happens. */ +#if defined(__EMSCRIPTEN__) || defined(__wasi__) -+ HandleNotifyInterrupt(); ++ HandleNotifyInterrupt(); +#else if (SendProcSignal(pid, PROCSIG_NOTIFY_INTERRUPT, procnos[i]) < 0) elog(DEBUG3, "could not signal backend with PID %d: %m", pid); diff --git a/patches-REL_17_4_WASM/postgresql-pglite/src-bin-initdb-initdb.c.diff b/patches-REL_17_4_WASM/postgresql-pglite/src-bin-initdb-initdb.c.diff index cbb044acc6b00..3e110b9c43ff3 100644 --- a/patches-REL_17_4_WASM/postgresql-pglite/src-bin-initdb-initdb.c.diff +++ b/patches-REL_17_4_WASM/postgresql-pglite/src-bin-initdb-initdb.c.diff @@ -20,17 +20,18 @@ const char *username; #ifndef WIN32 -@@ -825,6 +830,9 @@ +@@ -825,6 +830,10 @@ username = get_user_name_or_exit(progname); return pg_strdup(username); +#else ++ setenv("PGUSER", WASM_USERNAME, 0); + return pg_strdup(getenv("PGUSER")); +#endif /* wasm */ } static char * -@@ -1070,6 +1078,9 @@ +@@ -1070,6 +1079,9 @@ static const char * choose_dsm_implementation(void) { @@ -40,7 +41,7 @@ #if defined(HAVE_SHM_OPEN) && !defined(__sun__) int ntries = 10; pg_prng_state prng_state; -@@ -1608,10 +1619,9 @@ +@@ -1608,10 +1620,9 @@ } PG_CMD_CLOSE(); @@ -53,7 +54,7 @@ check_ok(); } -@@ -1711,16 +1721,16 @@ +@@ -1711,16 +1722,16 @@ setup_run_file(FILE *cmdfd, const char *filename) { char **lines; @@ -73,7 +74,7 @@ free(lines); } -@@ -2636,8 +2646,13 @@ +@@ -2636,8 +2647,13 @@ strlcpy(full_path, progname, sizeof(full_path)); if (ret == -1) @@ -87,7 +88,7 @@ else pg_fatal("program \"%s\" was found by \"%s\" but was not the same version as %s", "postgres", full_path, progname); -@@ -3096,7 +3111,7 @@ +@@ -3096,7 +3112,7 @@ initPQExpBuffer(&cmd); printfPQExpBuffer(&cmd, "\"%s\" %s %s template1 >%s", backend_exec, backend_options, extra_options, DEVNULL); @@ -96,7 +97,7 @@ PG_CMD_OPEN(cmd.data); setup_auth(cmdfd); -@@ -3134,14 +3149,53 @@ +@@ -3134,14 +3150,53 @@ PG_CMD_CLOSE(); termPQExpBuffer(&cmd); @@ -133,11 +134,11 @@ + strcat_alloc(PREFIX,"/bin/initdb"), + // "--no-clean", + "--wal-segsize=1", -+ "-g", -+ "-E", "UTF8", "--locale=C.UTF-8", "--locale-provider=libc", -+// "-E", "UTF8", "--locale", "C.UTF-8", "--locale-provider=builtin", -+// "--locale", "en_US.UTF-8", -+// "--icu-locale=en-US", "--locale-provider=icu", ++ "--allow-group-access", "--no-sync", ++ "-E", "UTF8", ++ "--locale=C.UTF-8", "--locale-provider=libc", ++// "--builtin-locale=en_US.UTF-8", "--locale-provider=builtin", ++// "--locale-provider=icu", "--icu-locale=en-US", "--locale-provider=icu", + "-U", WASM_USERNAME, pwfile, //"--pwfile=" WASM_PREFIX "/password", + pgdata, // "--pgdata=" WASM_PREFIX "/base", + NULL @@ -153,7 +154,7 @@ static struct option long_options[] = { {"pgdata", required_argument, NULL, 'D'}, {"encoding", required_argument, NULL, 'E'}, -@@ -3201,9 +3255,15 @@ +@@ -3201,9 +3256,15 @@ * POSIX says we must do this before any other usage of these files. */ setvbuf(stdout, NULL, PG_IOLBF, 0); @@ -170,28 +171,33 @@ set_pglocale_pgservice(argv[0], PG_TEXTDOMAIN("initdb")); if (argc > 1) -@@ -3409,7 +3469,7 @@ +@@ -3409,7 +3470,7 @@ if (icu_rules && locale_provider != COLLPROVIDER_ICU) pg_fatal("%s cannot be specified unless locale provider \"%s\" is chosen", "--icu-rules", "icu"); - -+PDEBUG("# 3463:"__FILE__ " TODO: atexit(cleanup_directories_atexit"); ++PDEBUG("# 3463:"__FILE__ " TODO: atexit(cleanup_directories_atexit)"); atexit(cleanup_directories_atexit); /* If we only need to sync, just do it and exit */ -@@ -3445,9 +3505,9 @@ +@@ -3445,13 +3506,13 @@ get_restricted_token(); setup_pgdata(); - -+puts("# 3493:pgl_initdb_main " __FILE__); ++PDEBUG("# 3493:pgl_initdb_main " __FILE__); setup_bin_paths(argv[0]); - -+puts("# 3495:pgl_initdb_main " __FILE__); ++PDEBUG("# 3495:pgl_initdb_main " __FILE__); effective_user = get_id(); if (!username) username = effective_user; -@@ -3479,7 +3539,7 @@ +- ++PDEBUG("# 3514:pgl_initdb_main " __FILE__); + if (strncmp(username, "pg_", 3) == 0) + pg_fatal("superuser name \"%s\" is disallowed; role names cannot begin with \"pg_\"", username); + +@@ -3479,7 +3540,7 @@ get_su_pwd(); printf("\n"); @@ -200,7 +206,7 @@ initialize_data_directory(); if (do_sync) -@@ -3535,7 +3595,7 @@ +@@ -3535,7 +3596,7 @@ destroyPQExpBuffer(start_db_cmd); } diff --git a/pglite-REL_17_4_WASM/build.sh b/pglite-REL_17_4_WASM/build.sh index 66fb6366e6f6f..ee11b258f29cc 100755 --- a/pglite-REL_17_4_WASM/build.sh +++ b/pglite-REL_17_4_WASM/build.sh @@ -1,26 +1,145 @@ -#!/bin/bash -echo "pglite/build: begin" +export WASI=${WASI:-false} + +if ${WASI} +then + BUILD=wasi +else + BUILD=emscripten + WEBROOT=${PG_DIST}/web +fi + +echo "pglite/build-$BUILD: begin target BUILD_PATH=$BUILD_PATH" WORKSPACE=$(pwd) PGROOT=/tmp/pglite -PGSRC=${WORKSPACE} -PGBUILD=${WORKSPACE}/build/postgres +if [ -d ${WORKSPACE}/src/fe_utils ] +then + PGSRC=${WORKSPACE} +else + PGSRC=${WORKSPACE}/postgresql-${PG_BRANCH} +fi -LIBPGCORE=${PGBUILD}/libpgcore.a +LIBPGCORE=${BUILD_PATH}/libpgcore.a -WEBROOT=${PGBUILD}/web -PGINC="-I/tmp/pglite/include \ - -I${PGSRC}/src/include -I${PGSRC}/src/interfaces/libpq \ - -I${PGBUILD}/src/include" + +PGINC=" -I${BUILD_PATH}/src/include \ +-I${PGROOT}/include -I${PGROOT}/include/postgresql/server \ +-I${PGSRC}/src/include -I${PGSRC}/src/interfaces/libpq -I${PGSRC}/src" + + +GLOBAL_BASE_B=$(python3 -c "print(${CMA_MB}*1024*1024)") if $WASI then - echo TODO +echo " +_______________________ PG_BRANCH=${PG_BRANCH} _____________________ + +wasi : $(which wasi-c) $(wasi-c -v) +python : $(which python3) $(python3 -V) +wasmtime : $(which wasmtime) + +CC=${CC:-undefined} + +Linking to libpgcore static from $LIBPGCORE + +Folders : + source : $PGSRC + build : $BUILD_PATH + target : $PGROOT + retarget : ${PGL_DIST_C} + native : ${PGL_DIST_NATIVE} + + + CPOPTS : $COPTS + DEBUG : $DEBUG + LOPTS : $LOPTS + + CMA_MB : $CMA_MB +GLOBAL_BASE : $GLOBAL_BASE_B + + CC_PGLITE : $CC_PGLITE + + ICU i18n : $USE_ICU + +INCLUDES: $PGINC +________________________________________________________ + + +" + + + if ${CC} -ferror-limit=1 ${CC_PGLITE} \ + ${PGINC} \ + -DPOSTGRES_C=\"../postgresql/src/backend/tcop/postgres.c\" \ + -DPQEXPBUFFER_H=\"../postgresql/src/interfaces/libpq/pqexpbuffer.h\" \ + -DOPTION_UTILS_C=\"../postgresql/src/fe_utils/option_utils.c\" \ + -o ${BUILD_PATH}/pglite.o -c ${WORKSPACE}/pglite-wasm/pg_main.c \ + -Wno-incompatible-pointer-types-discards-qualifiers + then + if ${CC} -fpic -ferror-limit=1 ${CC_PGLITE} ${PGINC} \ + -o ${BUILD_PATH}/sdk_port-wasi.o \ + -c wasm-build/sdk_port-wasi/sdk_port-wasi-dlfcn.c \ + -Wno-incompatible-pointer-types + then + COPTS="$LOPTS" ${CC} ${CC_PGLITE} -ferror-limit=1 -Wl,--global-base=${GLOBAL_BASE_B} -o ${PG_DIST}/pglite.wasi \ + -nostartfiles ${PGINC} ${BUILD_PATH}/pglite.o \ + ${BUILD_PATH}/sdk_port-wasi.o \ + $LINKER $LIBPGCORE \ + $LINK_ICU \ + ${PG_BUILD}/${BUILD}/src/backend/snowball/libdict_snowball.a \ + ${PG_BUILD}/${BUILD}/src/pl/plpgsql/src/libplpgsql.a \ + -lxml2 -lz + else + echo "compilation of libpglite ${BUILD} support failed" + fi + + if [ -f ${PG_DIST}/pglite.wasi ] + then + echo "building minimal wasi FS" + cp ${PG_DIST}/pglite.wasi ${PGROOT}/bin/ + touch ${PGROOT}/bin/initdb ${PGROOT}/bin/postgres + tar -cvJ --files-from=${WORKSPACE}/wasmfs.txt > ${PG_DIST}/pglite-wasi.tar.xz + mkdir -p ${PGL_BUILD_NATIVE} + cat > ${PGL_BUILD_NATIVE}/pglite-native.sh < ${PREFIX}/${cleanup} - done - - mkdir -p ${PG_DIST:-/tmp/sdk/dist}/pglite - COPTS="$LOPTS" ${CC} ${CC_PGLITE} -sGLOBAL_BASE=${CMA_MB}MB -o ${PG_DIST:-/tmp/sdk/dist}/pglite/pglite.html -ferror-limit=1 --shell-file ${WORKSPACE}/pglite-wasm/repl.html \ - $PGPRELOAD \ - -sFORCE_FILESYSTEM=1 -sNO_EXIT_RUNTIME=1 -sENVIRONMENT=node,web \ - -sMODULARIZE=1 -sEXPORT_ES6=1 -sEXPORT_NAME=Module \ - -sALLOW_TABLE_GROWTH -sALLOW_MEMORY_GROWTH -sERROR_ON_UNDEFINED_SYMBOLS \ - -sEXPORTED_RUNTIME_METHODS=${EXPORTED_RUNTIME_METHODS} \ - ${PGINC} ${PGBUILD}/pglite.o \ - $LINKER $LIBPGCORE \ - $LINK_ICU \ - -lnodefs.js -lidbfs.js -lxml2 -lz - + else + echo "compilation of libpglite ${PG_BRANCH} failed" + exit 220 + fi fi -du -hs ${PG_DIST:-/tmp/sdk/dist}/pglite/pglite.* -echo "pglite/build: end" +du -hs ${PG_DIST}/* + +echo "pglite/build($BUILD): end" + diff --git a/pglite-REL_17_4_WASM/interactive_one.c b/pglite-REL_17_4_WASM/interactive_one.c index 07e86475ad7ab..69caf7dce68ab 100644 --- a/pglite-REL_17_4_WASM/interactive_one.c +++ b/pglite-REL_17_4_WASM/interactive_one.c @@ -1,9 +1,9 @@ #include // access, unlink -#if !defined(__wasi__) -volatile sigjmp_buf local_sigjmp_buf; +#if defined(__wasi__) +// volatile sigjmp_buf void*; #else -volatile sigjmp_buf void*; +volatile sigjmp_buf local_sigjmp_buf; #endif /* TODO : prevent multiple write and write while reading ? */ @@ -152,7 +152,7 @@ static void io_init(bool in_auth, bool out_auth) { whereToSendOutput = DestRemote; /* now safe to ereport to client */ if (!MyProcPort) { - PDEBUG("# 137: io_init --------- NO CLIENT (oom) ---------"); + PDEBUG("# 155: io_init --------- NO CLIENT (oom) ---------"); abort(); } #ifdef PG16 @@ -162,18 +162,18 @@ static void io_init(bool in_auth, bool out_auth) { SOCKET_FILE = NULL; SOCKET_DATA = 0; - PDEBUG("\n\n\n\n# 147: io_init --------- Ready for CLIENT ---------"); + PDEBUG("\n\n\n# 165: io_init --------- Ready for CLIENT ---------"); } -volatile int sf_connected = 0; + volatile bool sockfiles = false; volatile bool is_wire = true; extern char * cma_port; +extern void pq_startmsgread(void); - -__attribute__((export_name("interactive_write"))) // EMSCRIPTEN_KEEPALIVE +__attribute__((export_name("interactive_write"))) void interactive_write(int size) { cma_rsize = size; @@ -193,18 +193,16 @@ void use_wire(int state) { #if PGDEBUG force_echo=true; -#else - force_echo=false; #endif if (state>0) { #if PGDEBUG - printf("176: wire mode, repl off, echo %d\n", force_echo); + printf("# 199: wire mode, repl off, echo %d\n", force_echo); #endif is_wire = true; is_repl = false; } else { #if PGDEBUG - printf("181: repl mode, no wire, echo %d\n", force_echo); + printf("# 205: repl mode, no wire, echo %d\n", force_echo); #endif is_wire = false; is_repl = true; @@ -278,14 +276,17 @@ void discard_input(){ void startup_auth() { /* code is in handshake/auth domain so read whole msg now */ + send_ready_for_query = false; if (ProcessStartupPacket(MyProcPort, true, true) != STATUS_OK) { - PDEBUG("# 283: ProcessStartupPacket !OK"); + PDEBUG("# 271: ProcessStartupPacket !OK"); } else { - PDEBUG("# 285: sending auth request"); + + sf_connected++; + PDEBUG("# 273: sending auth request"); //ClientAuthentication(MyProcPort); - discard_input(); - + discard_input(); + ClientAuthInProgress = true; md5Salt[0]=0x01; md5Salt[1]=0x23; @@ -309,7 +310,7 @@ startup_pass(bool check) { // auth 'p' if (check) { char *passwd = recv_password_packet(MyProcPort); - printf("# 312: auth recv password: %s\n", "md5***" ); + PDEBUG("# 223: auth recv password: md5***"); /* // TODO: CheckMD5Auth if (passwd == NULL) @@ -321,8 +322,8 @@ startup_pass(bool check) { */ pfree(passwd); } else { - PDEBUG("# 324: auth skip"); - discard_input(); + PDEBUG("# 310: auth skip"); + discard_input(); } ClientAuthInProgress = false; @@ -342,15 +343,15 @@ startup_pass(bool check) { pq_sendint32(&buf, (int32) MyCancelKey); pq_endmessage(&buf); } -PDEBUG("# 305: TODO: set a pg_main started flag"); - sf_connected++; +PDEBUG("# 330: TODO: set a pgl started flag"); send_ready_for_query = true; - + ignore_till_sync = false; + volatile int sf_connected = 0; } extern void pg_startcma(); -EMSCRIPTEN_KEEPALIVE void +__attribute__((export_name("interactive_one"))) void interactive_one() { int peek = -1; /* preview of firstchar with no pos change */ int firstchar = 0; /* character read from getc() */ @@ -358,22 +359,27 @@ interactive_one() { StringInfoData input_message; StringInfoData *inBuf; FILE *stream ; - FILE *fp; + FILE *fp = NULL; int packetlen; bool had_notification = notifyInterruptPending; bool notified = false; -if (notifyInterruptPending) - PDEBUG("# 327: has notification !"); + send_ready_for_query = false; if (!MyProcPort) { + PDEBUG("# 353: client created"); io_init(is_wire, false); } +#if PGDEBUG + if (notifyInterruptPending) + PDEBUG("# 359: has notification !"); +#endif + // this could be pg_flush in sync mode. // but in fact we are writing socket data that was piled up previous frame async. if (SOCKET_DATA>0) { - puts("331: ERROR flush after frame"); + puts("# 361: ERROR flush after frame"); goto wire_flush; } @@ -385,7 +391,7 @@ if (notifyInterruptPending) } } - doing_extended_query_message = false; + MemoryContextSwitchTo(MessageContext); MemoryContextResetAndDeleteChildren(MessageContext); @@ -398,32 +404,28 @@ if (notifyInterruptPending) if (send_ready_for_query) { if (IsAbortedTransactionBlockState()) { - puts("@@@@ TODO 356: idle in transaction (aborted)"); + puts("@@@@ TODO 390: idle in transaction (aborted)"); } else if (IsTransactionOrTransactionBlock()) { - puts("@@@@ TODO 359: idle in transaction"); + puts("@@@@ TODO 393: idle in transaction"); } else { if (notifyInterruptPending) { ProcessNotifyInterrupt(false); -PDEBUG("# 367: notified ?"); notified = true; } } send_ready_for_query = false; } - - // postgres.c 4627 DoingCommandRead = true; #if defined(EMUL_CMA) - #define IO ((char *)(1+(int)cma_port)) // temp fix for -O0 but less efficient than literal - #error "inefficient" + // temp fix for -O0 but less efficient than literal + #define IO ((char *)(1+(int)cma_port)) #else #define IO ((char *)(1)) #endif - /* * in cma mode (cma_rsize>0), client call the wire loop itself waiting synchronously for the results * in socketfiles mode, the wire loop polls a pseudo socket made from incoming and outgoing files. @@ -438,16 +440,16 @@ PDEBUG("# 367: notified ?"); if (!is_repl) { whereToSendOutput = DestRemote; if (!is_wire) - PDEBUG("repl message in cma buffer !"); + PDEBUG("# 426: repl message in cma buffer !"); } else { if (is_wire) - PDEBUG("wire message in cma buffer for REPL !"); + PDEBUG("# 429: wire message in cma buffer for REPL !"); whereToSendOutput = DestDebug; } } else { fp = fopen(PGS_IN, "r"); - // read as a socket. + // read file in socket buffer for SocketBackend to consumme. if (fp) { fseek(fp, 0L, SEEK_END); packetlen = ftell(fp); @@ -483,13 +485,13 @@ PDEBUG("# 367: notified ?"); if (packetlen) { // it was startup/auth , write and return fast. if (peek<0) { - PDEBUG("# 438: handshake/auth/pass skip"); + PDEBUG("# 471: handshake/auth/pass skip"); goto wire_flush; } /* else it was wire msg */ #if PGDEBUG - printf("# 444: node+repl is_wire -> true : %c\n", peek); + printf("# 477: node+repl is_wire -> true : %c\n", peek); force_echo = true; #endif firstchar = peek; @@ -511,8 +513,8 @@ PDEBUG("# 367: notified ?"); #if PGDEBUG if (packetlen) - IO[packetlen]=0; - printf("# 479: fd=%d is_embed=%d is_repl=%d is_wire=%d fd %s,len=%d peek=%d [%s]\n", MyProcPort->sock, is_embed, is_repl, is_wire, PGS_OLOCK, packetlen, peek, IO); + IO[packetlen]=0; // wire blocks are not zero terminated + printf("\n# 500: fd=%d is_embed=%d is_repl=%d is_wire=%d fd %s,len=%d cma=%d peek=%d [%s]\n", MyProcPort->sock, is_embed, is_repl, is_wire, PGS_OLOCK, packetlen,cma_rsize, peek, IO); #endif // buffer query TODO: direct access ? @@ -524,7 +526,7 @@ PDEBUG("# 367: notified ?"); } if (packetlen<2) { - puts("# 491: WARNING: empty packet"); + puts("# 512: WARNING: empty packet"); cma_rsize= 0; if (is_repl) pg_prompt(); @@ -540,26 +542,32 @@ PDEBUG("# 367: notified ?"); #error "sigsetjmp unsupported" #endif // wasi + while (pipelining) { + if (is_repl) { - // TODO: are we sure repl could not pipeline ? + // are we sure repl could not pipeline ? pipelining = false; /* stdio node repl */ - printf("# 512: enforcing REPL mode, wire off, echo %d\n", force_echo); +#if PGDEBUG + printf("# 533: enforcing REPL mode, wire off, echo %d\n", force_echo); +#endif whereToSendOutput = DestDebug; } + DoingCommandRead = true; if (is_wire) { - /* wire on a socket or cma may auth, not handled by pg_proto block */ + /* wire on a socket or cma may auth */ + /* would be handled as error by pg_proto block */ if (peek==0) { - PDEBUG("# 519: handshake/auth"); + PDEBUG("# 540: handshake/auth"); startup_auth(); - PDEBUG("# 521: auth request"); + PDEBUG("# 542: auth request"); break; } if (peek==112) { - PDEBUG("# 525: password"); + PDEBUG("# 547: password"); startup_pass(true); break; } @@ -567,11 +575,13 @@ PDEBUG("# 367: notified ?"); firstchar = SocketBackend(inBuf); pipelining = pq_buffer_remaining_data()>0; +#if PGDEBUG if (!pipelining) { - printf("# 535: end of wire, rfq=%d\n", send_ready_for_query); + printf("# 556: end of wire, rfq=%d\n", send_ready_for_query); } else { - printf("# 537: no end of wire -> pipelining, rfq=%d\n", send_ready_for_query); + printf("# 558: no end of wire -> pipelining, rfq=%d\n", send_ready_for_query); } +#endif } else { /* nowire */ pipelining = false; @@ -582,28 +592,37 @@ PDEBUG("# 367: notified ?"); firstchar = 'Q'; } } + DoingCommandRead = false; - #if PGDEBUG +#if PGDEBUG if (!pipelining) { - printf("# 552: wire=%d 1stchar=%c Q: %s\n", is_wire, firstchar, inBuf->data); + printf("# 573: wire=%d 1stchar=%c Q: %s\n", is_wire, firstchar, inBuf->data); force_echo = false; } else { - printf("# 555: PIPELINING [%c]!\n", firstchar); + printf("# 576: PIPELINING [%c]!\n", firstchar); } - #endif +#endif - if (!ignore_till_sync) + if (!ignore_till_sync) { + /* initially, or after error */ send_ready_for_query = true; - - if (ignore_till_sync && firstchar != EOF) { - puts("@@@@@@@@@@@@@ 562 TODO: postgres.c 4684 : continue"); - } else { /* process notifications (ASYNC) */ - if (notifyInterruptPending) { PDEBUG("# 565: @@@ has notification @@@@\n"); + if (notifyInterruptPending) ProcessClientReadInterrupt(true); + } else { + /* ignoring till sync will skip all pipeline */ + if (firstchar != EOF) { + if (firstchar != 'S') { + continue; + } } } #include "pg_proto.c" + + if (send_ready_for_query) { + ReadyForQuery(whereToSendOutput); + send_ready_for_query = false; + } } if (!is_repl) { @@ -614,14 +633,14 @@ PDEBUG("# 367: notified ?"); ProcessNotifyInterrupt(false); if (send_ready_for_query) { - PDEBUG("# 581: end packet - sending rfq"); + PDEBUG("# 602: end packet - sending rfq\n"); ReadyForQuery(DestRemote); //done at postgres.c 4623 send_ready_for_query = false; } else { - PDEBUG("# 585: end packet - with no rfq"); + PDEBUG("# 606: end packet - with no rfq\n"); } } else { - PDEBUG("# 588: end packet (ClientAuthInProgress - no rfq) "); + PDEBUG("# 609: end packet (ClientAuthInProgress - no rfq)\n"); } if (SOCKET_DATA>0) { @@ -638,16 +657,16 @@ PDEBUG("# 367: notified ?"); SOCKET_FILE = NULL; SOCKET_DATA = 0; if (cma_wsize) - PDEBUG("# 605: cma and sockfile ???"); + PDEBUG("# 626: cma and sockfile ???\n"); if (sockfiles) { #if PGDEBUG - printf("# 608: client:ready -> read(%d) " PGS_OLOCK "->" PGS_OUT"\n", outb); + printf("# 629: client:ready -> read(%d) " PGS_OLOCK "->" PGS_OUT"\n", outb); #endif rename(PGS_OLOCK, PGS_OUT); } } else { #if PGDEBUG - printf("# 614: out queue : %d flushed\n", cma_wsize); + printf("\n# 635: in[%d] out[%d] flushed\n", cma_rsize, cma_wsize); #endif SOCKET_DATA = 0; } diff --git a/pglite-REL_17_4_WASM/native.sh b/pglite-REL_17_4_WASM/native.sh new file mode 100755 index 0000000000000..d24d5ea4d20c7 --- /dev/null +++ b/pglite-REL_17_4_WASM/native.sh @@ -0,0 +1,185 @@ +#!/bin/bash + +apk add clang gcc +mkdir -p ${SDKROOT}/src +if [ -d ${SDKROOT}/src/w2c2 ] +then + echo " using local w2c2" +else + pushd ${SDKROOT}/src + git clone https://github.com/pygame-web/w2c2 + popd +fi + +mv ${PGROOT}/bin/pglite.wasi pglite.wasi +${SDKROOT}/native/w2c2/w2c2 -f 0 -t 1 -p pglite.wasi pglite.c +mv pglite.wasi ${PGL_DIST_NATIVE}/ + +cat > tmp.py <0: + try: + bodies.write( makefunc(l) ) + except Exception as e: + print("?", l, e) + continue + +print("="*70) +with open('${WORKSPACE}/pglite-wasm/pglite-modpython.c','r') as source: + with open('tmp.c','w') as out: + out.write( source.read().replace('\${WASM2C}','${WASM2C}') ) +END + +python3 tmp.py + +echo " building loader ..." + + +PYLD="-lm" +PYEXT="" + +if echo $CC|grep gcc +then + COPTS="-O0 -g0" + CCOPTS="-Wno-attributes" +else + COPTS="-O0 -g0" + CCOPTS="-fbracket-depth=4096 -Wno-unknown-attributes" +fi + +COMPILE="$CC -fPIC $PYINC $COPTS $CCOPTS -I${SDKROOT}/src/w2c2 -I${SDKROOT}/src/w2c2/w2c2 -o ${WASM2C}$PYEXT tmp.c ${SDKROOT}/native/wasi/libw2c2wasi.a $PYLD -lc" +echo $COMPILE + +PYVER=$($PYTHON -V|cut -d' ' -f2|cut -d. -f1-2) +PYVER=${PYVER}$(${PYTHON}-config --abiflags) + +PYINC="-D__PYDK__=1 -shared $(${PYTHON}-config --includes)" +PYEXT=$(${PYTHON}-config --extension-suffix) +PYLD="-lpython$PYVER $(${PYTHON}-config --ldflags)" + +echo " +======================================================================== +WASM2C=$WASM2C +COPTS=$COPTS + +PYVER=$PYVER +PYEXT=$PYEXT + +PYINC=$PYINC +PYLD=$PYLD + +PGL_BUILD_NATIVE=${PGL_BUILD_NATIVE} +PGL_DIST_NATIVE=${PGL_DIST_NATIVE} +TARGET: ${WASM2C}$PYEXT +======================================================================== +" + +$COMPILE + + +[ -f ${WASM2C}$PYEXT ] && rm ${WASM2C}$PYEXT + + +COMPILE="$CC -fPIC -Os -g0 $PYINC $CCOPTS -I${SDKROOT}/src/w2c2 -I${SDKROOT}/src/w2c2/w2c2 -o ${WASM2C}$PYEXT tmp.c ${SDKROOT}/native/wasi/libw2c2wasi.a $PYLD -lc" +echo $COMPILE + +time $COMPILE + +if [ -f ${WASM2C} ] +then + du -hs ${WASM2C} + # ./${WASM2C} $@ +else + echo build native ${WASM2C} failed +fi + +PY=$(command -v python${PYMAJOR}.${PYMINOR}) + +echo "__________________________________________" +echo $PY +echo "__________________________________________" + + + +if [ -f ${WASM2C}$PYEXT ] +then + env -i $PY <nextOid < ((Oid) FirstNormalObjectId)) { /* IsPostmasterEnvironment is now true - these will be executed when required in varsup.c/GetNewObjectId - TransamVariables->nextOid = FirstNormalObjectId; - TransamVariables->oidCount = 0; - */ + these will be executed when required in varsup.c/GetNewObjectId + TransamVariables->nextOid = FirstNormalObjectId; + TransamVariables->oidCount = 0; + */ #if PGDEBUG - puts("# 382: initdb done, oid base too low but OID range will be set because IsPostmasterEnvironment"); + puts("# 403: initdb done, oid base too low but OID range will be set because IsPostmasterEnvironment"); #endif } -} + } #if defined(__EMSCRIPTEN__) -EMSCRIPTEN_KEEPALIVE + EMSCRIPTEN_KEEPALIVE #else -__attribute__((export_name("pgl_initdb"))) + __attribute__ ((export_name("pgl_initdb"))) #endif -int -pgl_initdb() { - PDEBUG("# 433: pg_initdb()"); + int pgl_initdb() { + PDEBUG("# 412: pg_initdb()"); optind = 1; pgl_idb_status |= IDB_FAILED; - if (!chdir(PGDATA)){ + if (!chdir(PGDATA)) { if (access("PG_VERSION", F_OK) == 0) { - chdir("/"); + chdir("/"); pgl_idb_status |= IDB_HASDB; /* assume auth success for now */ pgl_idb_status |= IDB_HASUSER; #if PGDEBUG - fprintf(stdout, "# 446: pg_initdb: db exists at : %s TODO: test for db name : %s \n", PGDATA, getenv("PGDATABASE")); + fprintf(stdout, "# 427: pg_initdb: db exists at : %s TODO: test for db name : %s \n", PGDATA, getenv("PGDATABASE")); #endif // PGDEBUG async_restart = 0; goto initdb_done; } - chdir("/"); + chdir("/"); #if PGDEBUG - fprintf(stderr, "# 454: pg_initdb no db found at : %s\n", PGDATA ); + fprintf(stderr, "# 435: pg_initdb no db found at : %s\n", PGDATA); #endif // PGDEBUG } else { #if PGDEBUG - fprintf(stderr, "# 458: pg_initdb db folder not found at : %s\n", PGDATA ); + fprintf(stderr, "# 439: pg_initdb db folder not found at : %s\n", PGDATA); #endif // PGDEBUG } int initdb_rc = pgl_initdb_main(); #if PGDEBUG - fprintf(stderr, "\n\n# 465: " __FILE__ "pgl_initdb_main = %d\n", initdb_rc ); + fprintf(stderr, "\n\n# 444: " __FILE__ "pgl_initdb_main = %d\n", initdb_rc); #endif // PGDEBUG - PDEBUG("# 467:" __FILE__); + PDEBUG("# 448:" __FILE__); /* save stdin and use previous initdb output to feed boot mode */ int saved_stdin = dup(STDIN_FILENO); { - PDEBUG("# 471: restarting in boot mode for initdb"); + PDEBUG("# 450: restarting in boot mode for initdb"); freopen(IDB_PIPE_BOOT, "r", stdin); char *boot_argv[] = { g_argv[0], "--boot", "-D", PGDATA, - "-d","3", + "-d", "3", WASM_PGOPTS, "-X", "1048576", NULL }; - int boot_argc = sizeof(boot_argv) / sizeof(char*) - 1; + int boot_argc = sizeof(boot_argv) / sizeof(char *) - 1; - set_pglocale_pgservice(boot_argv[0], PG_TEXTDOMAIN("initdb")); + set_pglocale_pgservice(boot_argv[0], PG_TEXTDOMAIN("initdb")); optind = 1; BootstrapModeMain(boot_argc, boot_argv, false); fclose(stdin); +#if PGDEBUG + puts("BOOT FILE:"); + puts(IDB_PIPE_BOOT); +#else remove(IDB_PIPE_BOOT); +#endif stdin = fdopen(saved_stdin, "r"); - PDEBUG("# 493: initdb faking shutdown to complete WAL/OID states"); + PDEBUG("# 479: initdb faking shutdown to complete WAL/OID states"); pg_proc_exit(66); } @@ -481,7 +488,7 @@ pgl_initdb() { //IsPostmasterEnvironment = true; if (TransamVariables->nextOid < ((Oid) FirstNormalObjectId)) { #if PGDEBUG - puts("# 503: warning oid base too low, will need to set OID range after initdb(bootstrap/single)"); + puts("# 482: warning oid base too low, will need to set OID range after initdb(bootstrap/single)"); #endif } /* @@ -506,110 +513,21 @@ PDEBUG("# 498: initdb faking shutdown to complete WAL/OID states in single mode" async_restart = 1; } */ - async_restart = 1; -initdb_done:; + async_restart = 1; + initdb_done:; pgl_idb_status |= IDB_CALLED; - if (optind>0) { + if (optind > 0) { /* RESET getopt */ optind = 1; /* we did not fail, clear the default failed state */ pgl_idb_status &= IDB_OK; } else { - PDEBUG("# 511: exiting on initdb-single error"); + PDEBUG("# 524: exiting on initdb-single error"); // TODO raise js exception } return pgl_idb_status; -} // pg_initdb - - - - - -int -main_repl() { - bool hadloop_error = false; - - whereToSendOutput = DestNone; - - if (!mkdir(PGDATA, 0700)) { - /* no db : run initdb now. */ -#if defined(PGDEBUG_STARTUP) - fprintf(stderr, "PGDATA=%s not found, running initdb with default=%s\n",PGDATA, WASM_PGDATA ); -#endif - #if defined(PG_INITDB_MAIN) - #warning "web build" - hadloop_error = pg_initdb() & IDB_FAILED; - #else - #warning "node build" - #if defined(__wasi__) - hadloop_error = pg_initdb() & IDB_FAILED; - #endif - #endif - - } else { - // download a db case ? - mkdirp(WASM_PGDATA); - - // db fixup because empty dirs may not be packaged in git case - mksub_dir(WASM_PGDATA, "/pg_wal"); - mksub_dir(WASM_PGDATA, "/pg_wal/archive_status"); - mksub_dir(WASM_PGDATA, "/pg_wal/summaries"); - - mksub_dir(WASM_PGDATA, "/pg_tblspc"); - mksub_dir(WASM_PGDATA, "/pg_snapshots"); - mksub_dir(WASM_PGDATA, "/pg_commit_ts"); - mksub_dir(WASM_PGDATA, "/pg_notify"); - mksub_dir(WASM_PGDATA, "/pg_replslot"); - mksub_dir(WASM_PGDATA, "/pg_twophase"); - - mksub_dir(WASM_PGDATA, "/pg_logical"); - mksub_dir(WASM_PGDATA, "/pg_logical/snapshots"); - mksub_dir(WASM_PGDATA, "/pg_logical/mappings"); - - } - - if (!hadloop_error) { - main_post(); - - /* - * Catch standard options before doing much else, in particular before we - * insist on not being root. - */ - if (g_argc > 1) { - if (strcmp(g_argv[1], "--help") == 0 || strcmp(g_argv[1], "-?") == 0) - { - help(progname); - exit(0); - } - if (strcmp(g_argv[1], "--version") == 0 || strcmp(g_argv[1], "-V") == 0) - { - fputs(PG_BACKEND_VERSIONSTR, stdout); - exit(0); - } - - } - - if (g_argc > 1 && strcmp(g_argv[1], "--check") == 0) { - BootstrapModeMain(g_argc, g_argv, true); - return 0; - } - - if (g_argc > 1 && strcmp(g_argv[1], "--boot") == 0) { - PDEBUG("# 1410: boot: " __FILE__ ); - BootstrapModeMain(g_argc, g_argv, false); - return 0; - } - - PDEBUG("# 570: single: " __FILE__ ); - AsyncPostgresSingleUserMain(g_argc, g_argv, PGUSER, 0); - } - return 0; -} - - - - +} // pgl_initdb @@ -626,59 +544,43 @@ main_repl() { PGOPTIONS */ - - - -// ???? __attribute__((export_name("_main"))) -int -main(int argc, char **argv) -{ - int exit_code = 0; - main_pre(argc,argv); +// __attribute__((export_name("main"))) + int main(int argc, char **argv) { + int exit_code = 0; + main_pre(argc, argv); #if PGDEBUG - printf("# 616: argv0 (%s) PGUSER=%s PGDATA=%s\n PGDATABASE=%s REPL=%s\n", - argv[0], PGUSER, PGDATA, getenv("PGDATABASE"), getenv("REPL") ); + printf("# 550: argv0 (%s) PGUSER=%s PGDATA=%s\n PGDATABASE=%s REPL=%s\n", + argv[0], PGUSER, PGDATA, getenv("PGDATABASE"), getenv("REPL")); #endif - progname = get_progname(argv[0]); - startup_hacks(progname); - g_argv = argv; - g_argc = argc; + progname = get_progname(argv[0]); + startup_hacks(progname); + g_argv = argv; + g_argc = argc; - is_repl = strlen(getenv("REPL")) && getenv("REPL")[0]!='N'; - is_embed = true; + is_repl = strlen(getenv("REPL")) && getenv("REPL")[0] != 'N'; + is_embed = true; - if (!is_repl) { - PDEBUG("# 628: exit with live runtime (nodb)"); - return 0; - } -/* + if (!is_repl) { + PDEBUG("# 562: exit with live runtime (nodb)"); + return 0; + } +#if defined(__wasi__) + + +#else + /* main_post(); - PDEBUG("# 634: repl"); + PDEBUG("# 565: repl"); // so it is repl main_repl(); if (is_node) { - PDEBUG("# 639: node repl"); + PDEBUG("# 570: node repl"); pg_repl_raf(); } -*/ - emscripten_force_exit(exit_code); - return exit_code; + */ + emscripten_force_exit(exit_code); +#endif + return exit_code; } - - - - - - - - - - - - - - - - diff --git a/pglite-REL_17_4_WASM/pg_proto.c b/pglite-REL_17_4_WASM/pg_proto.c index 1abf7024a1a0b..30201c8e49b75 100644 --- a/pglite-REL_17_4_WASM/pg_proto.c +++ b/pglite-REL_17_4_WASM/pg_proto.c @@ -248,8 +248,14 @@ * it will fail to be called during other backend-shutdown * scenarios. */ +if (sf_connected) + sf_connected--; +else + puts("ERROR: more exits than connections"); PDEBUG("# 251:proc_exit/skip and repl stop"); //proc_exit(0); is_repl = false; + send_ready_for_query = false; + ignore_till_sync = false; break; case 'd': /* copy data */ diff --git a/pglite-REL_17_4_WASM/pgl_initdb.c b/pglite-REL_17_4_WASM/pgl_initdb.c index 3fc19685ca549..19bf7fd405457 100644 --- a/pglite-REL_17_4_WASM/pgl_initdb.c +++ b/pglite-REL_17_4_WASM/pgl_initdb.c @@ -36,18 +36,18 @@ pg_chmod(const char * path, int mode_t) { #endif #define FRONTEND -# include "../postgresql/src/common/logging.c" +# include "common/logging.c" #undef FRONTEND -#include "../postgresql/src/interfaces/libpq/pqexpbuffer.c" +#include "interfaces/libpq/pqexpbuffer.c" #define sync_pgdata(...) #define icu_language_tag(loc_str) icu_language_tag_idb(loc_str) #define icu_validate_locale(loc_str) icu_validate_locale_idb(loc_str) -#include "../postgresql/src/bin/initdb/initdb.c" +#include "bin/initdb/initdb.c" void use_socketfile(void) { is_repl = true; diff --git a/pglite-REL_17_4_WASM/pgl_mains.c b/pglite-REL_17_4_WASM/pgl_mains.c index 0f8305b48a214..e09f3e79738d0 100644 --- a/pglite-REL_17_4_WASM/pgl_mains.c +++ b/pglite-REL_17_4_WASM/pgl_mains.c @@ -1,5 +1,6 @@ #include +volatile int sf_connected = 0; FILE * single_mode_feed = NULL; volatile bool inloop = false; volatile sigjmp_buf local_sigjmp_buf; @@ -12,6 +13,17 @@ pg_shutdown() { proc_exit(66); } +__attribute__((export_name("pgl_closed"))) +int +pgl_closed() { + if (sf_connected>0) + return 1; + return 0; +} + +#if FIXME +extern bool startswith(const char *str, const char *prefix); +#endif void interactive_file() { @@ -20,7 +32,8 @@ interactive_file() { StringInfoData input_message; StringInfoData *inBuf; FILE *stream ; - + int sql_line=1; + bool sql_skip = false; /* * At top of loop, reset extended-query-message flag, so that any * errors encountered in "idle" state don't provoke skip. @@ -45,6 +58,7 @@ interactive_file() { while ((c = getc(stream)) != EOF) { if (c == '\n') { + sql_line++; if (UseSemiNewlineNewline) { /* @@ -93,14 +107,26 @@ interactive_file() { /* Add '\0' to make it look the same as message case. */ appendStringInfoChar(inBuf, (char) '\0'); firstchar = 'Q'; -PDEBUG(inBuf->data); - +#if FIXME +#warning "FIXME: REVOKE ALL ON pg_largeobject FROM PUBLIC;" +#warning "FIXME: REVOKE CREATE,TEMPORARY ON DATABASE template1 FROM public;" + sql_skip |= startswith(inBuf->data , "REVOKE ALL ON pg_largeobject FROM PUBLIC;"); + sql_skip |= startswith(inBuf->data , "REVOKE CREATE,TEMPORARY ON DATABASE template1 FROM public;"); + if (sql_skip) { + fprintf(stdout, "# 106: SKIPPED: %d: %s\n", sql_line, inBuf->data); + sql_skip = false; + continue; + } else { + // fprintf(stderr, "%d: %s\n", sql_line, inBuf->data); + } +#endif // ??? if (ignore_till_sync && firstchar != EOF) continue; #include "pg_proto.c" } + PDEBUG("# 115: interactive_file: end"); } void @@ -220,7 +246,7 @@ PDEBUG("# 164:" __FILE__); /* while (repl) { interactive_file(); } - PDEBUG("# 232: REPL:End Raising a 'RuntimeError Exception' to halt program NOW"); + PDEBUG("# 240: REPL:End Raising a 'RuntimeError Exception' to halt program NOW"); { void (*npe)() = NULL; npe(); @@ -228,7 +254,7 @@ PDEBUG("# 164:" __FILE__); // unreachable. */ - PDEBUG("# 240: no line-repl requested, exiting and keeping runtime alive"); + PDEBUG("# 248: no line-repl requested, exiting and keeping runtime alive"); } @@ -240,13 +266,13 @@ AsyncPostgresSingleUserMain(int argc, char *argv[], const char *username, int as const char *dbname = NULL; PDEBUG("# 254:"__FILE__); - /* Initialize startup process environment. */ +// if (!async_restart) /* Initialize startup process environment. */ InitStandaloneProcess(argv[0]); PDEBUG("# 254:"__FILE__); - /* Set default values for command-line options. */ +// if (!async_restart) /* Set default values for command-line options. */ InitializeGUCOptions(); PDEBUG("# 257:"__FILE__); - /* Parse command-line options. */ +// if (!async_restart) /* Parse command-line options. */ process_postgres_switches(argc, argv, PGC_POSTMASTER, &dbname); PDEBUG("# 260:"__FILE__); /* Must have gotten a database name, or have a default (the username) */ diff --git a/pglite-REL_17_4_WASM/pgl_stubs.h b/pglite-REL_17_4_WASM/pgl_stubs.h index c1812dc1dcfe0..2ad262c7f0096 100644 --- a/pglite-REL_17_4_WASM/pgl_stubs.h +++ b/pglite-REL_17_4_WASM/pgl_stubs.h @@ -1,12 +1,26 @@ #pragma once + +// wasi only stubs +#if defined(__wasi__) +# undef PQEXPBUFFER_H +# include "../src/interfaces/libpq/pqexpbuffer.h" + +#else +# include "../src/interfaces/libpq/pqexpbuffer.h" +#endif + + +// option_parse_int parse_sync_method +#include "../src/fe_utils/option_utils.c" + + + static void -init_locale(const char *categoryname, int category, const char *locale) -{ +init_locale(const char *categoryname, int category, const char *locale) { if (pg_perm_setlocale(category, locale) == NULL && pg_perm_setlocale(category, "C") == NULL) - elog(FATAL, "could not adopt \"%s\" locale nor C locale for %s", - locale, categoryname); + elog(FATAL, "could not adopt \"%s\" locale nor C locale for %s", locale, categoryname); } @@ -24,12 +38,6 @@ startup_hacks(const char *progname) { } -void pg_repl_raf() { - puts("pg_repl_raf: STUB"); -} - - - // embedded initdb requirements void @@ -56,15 +64,12 @@ char * pg_strdup(const char *in) { char *tmp; - if (!in) - { - fprintf(stderr, - _("cannot duplicate null pointer (internal error)\n")); + if (!in) { + fprintf(stderr, _("cannot duplicate null pointer (internal error)\n")); exit(EXIT_FAILURE); } tmp = strdup(in); - if (!tmp) - { + if (!tmp) { fprintf(stderr, _("out of memory\n")); exit(EXIT_FAILURE); } @@ -81,7 +86,7 @@ simple_prompt(const char *prompt, bool echo) { #ifndef PG16 int ProcessStartupPacket(Port *port, bool ssl_done, bool gss_done) { - puts("# 89:"__FILE__" ProcessStartupPacket: STUB"); + PDEBUG("# 89:" __FILE__ " ProcessStartupPacket: STUB"); return STATUS_OK; } @@ -91,12 +96,10 @@ select_default_timezone(const char *share_path) { return getenv("TZ"); } -#include "../src/interfaces/libpq/pqexpbuffer.h" -#include "../src/fe_utils/option_utils.c" + bool -appendShellStringNoError(PQExpBuffer buf, const char *str) -{ +appendShellStringNoError(PQExpBuffer buf, const char *str) { bool ok = true; const char *p; @@ -126,13 +129,10 @@ appendShellStringNoError(PQExpBuffer buf, const char *str) } void -appendShellString(PQExpBuffer buf, const char *str) -{ +appendShellString(PQExpBuffer buf, const char *str) { if (!appendShellStringNoError(buf, str)) { - fprintf(stderr, - _("shell command argument contains a newline or carriage return: \"%s\"\n"), - str); + fprintf(stderr, _("shell command argument contains a newline or carriage return: \"%s\"\n"), str); exit(EXIT_FAILURE); } } diff --git a/pglite-REL_17_4_WASM/pgl_tools.h b/pglite-REL_17_4_WASM/pgl_tools.h index bb531a47c644d..295586f60ef14 100644 --- a/pglite-REL_17_4_WASM/pgl_tools.h +++ b/pglite-REL_17_4_WASM/pgl_tools.h @@ -12,6 +12,17 @@ void mkdirp(const char *p) { } } +#if FIXME +#warning "some FIXME are used" +bool startswith(const char *str, const char *prefix) { + // Check if the length of prefix is greater than the string + if (strlen(prefix) > strlen(str)) { + return false; + } + // Compare the beginning of the string with the prefix + return strncmp(str, prefix, strlen(prefix)) == 0; +} +#endif void strconcat(char*p, const char *head, const char *tail) { diff --git a/pglite-REL_17_4_WASM/pglite-modpython.c b/pglite-REL_17_4_WASM/pglite-modpython.c new file mode 100644 index 0000000000000..8dc492eab99ad --- /dev/null +++ b/pglite-REL_17_4_WASM/pglite-modpython.c @@ -0,0 +1,218 @@ +#include +#include +#include + +#ifdef __wii__ +# include +# include +# include +#endif + +#include "w2c2/w2c2_base.h" +#include "wasi/wasi.h" +#include "${WASM2C}.h" + +void +trap(Trap trap) { + fprintf(stderr, "TRAP: %s\n", trapDescription(trap)); +#ifdef __wii__ + VIDEO_WaitVSync(); +#endif + abort(); +} + +U32 +wasi_snapshot_preview1__fd_renumber(void* ctx, U32 from, U32 to) { + return -1; +} + +wasmMemory* +wasiMemory(void* instance) { + return ${WASM2C}_memory((${WASM2C}Instance*)instance); +} + +extern char **environ; +volatile int skip_main = 0; +//char** __environ = NULL; +${WASM2C}Instance instance; + +#include "${WASM2C}.c" + +#define WASM_PREFIX "/tmp/pglite" +#define WASM_USERNAME "postgres" +#define PGDB WASM_PREFIX "/base" + + +char **g_argv; +int g_argc; + +char **copy_argv(int argc, char *argv[]) { + // calculate the contiguous argv buffer size + int length=0; + size_t ptr_args = argc + 1; + for (int i = 0; i < argc; i++) { + length += (strlen(argv[i]) + 1); + } + char** new_argv = (char**)malloc((ptr_args) * sizeof(char*) + length); + + // copy argv into the contiguous buffer + length = 0; + for (int i = 0; i < argc; i++) { + new_argv[i] = &(((char*)new_argv)[(ptr_args * sizeof(char*)) + length]); + strcpy(new_argv[i], argv[i]); + length += (strlen(argv[i]) + 1); + } + + // insert NULL terminating ptr at the end of the ptr array + new_argv[ptr_args-1] = NULL; + return (new_argv); +} + +int pre_main(int tmp_argc, char* tmp_argv[]) { + puts("# 84: pre_main"); + // __environ = environ; + + setenv("EMBED", "wasi", 1); + setenv("REPL", "N", 1); + + setenv("PGSYSCONFDIR", WASM_PREFIX, 1); + setenv("PGCLIENTENCODING", "UTF8", 1); + + setenv("TZ", "UTC", 1); + setenv("PGTZ", "UTC", 1); + setenv("PGUSER", WASM_USERNAME , 1); + setenv("PGDATA", PGDB , 1); + setenv("PGDATABASE", "template1" , 1); + setenv("PG_COLOR", "always", 0); + puts("# 87: global argc/argv"); + g_argv = copy_argv(tmp_argc, tmp_argv); + g_argc = sizeof(g_argv) / sizeof(char*) - 1; + + +# if defined(__MWERKS__) && defined(macintosh) + MaxApplZone(); + MoreMasters(); + MoreMasters(); + argc = ccommand(&argv); +# elif defined(__wii__) + /* TODO: get interactive console working */ + wiiInitVideo(); + fatInitDefault(); +# endif + + /* Initialize WASI */ + if (!wasiInit(g_argc, g_argv, environ)) { + fprintf(stderr, "failed to init WASI\n"); + return 1; + } + + if (!wasiFileDescriptorAdd(-1, "/", NULL)) { + fprintf(stderr, "failed to add preopen\n"); + return 1; + } + +# ifdef __MSL__ + SIOUXSetTitle("\p${WASM2C}"); +# endif + + ${WASM2C}Instantiate(&instance, NULL); + + return 0; +} + +// #define REACTOR + +#if defined(REACTOR) + #define STARTPROC(i) ${WASM2C}_setup(i) + // #define STARTPROC(i) ${WASM2C}_pg_initdb(i) +#else + #define STARTPROC(i) ${WASM2C}__start(&instance); +#endif + + +int main(int argc, char* argv[]); + +void do_main() { + puts("# 128: do_main Begin"); + STARTPROC(&instance); + puts("# 134: do_main End"); +} + +#if defined(__PYDK__) +# include "Python.h" + + static PyObject * ${WASM2C}_info(PyObject *self, PyObject *args, PyObject *kwds) + { + puts("${WASM2C} test function : return 42"); + return Py_BuildValue("i", 42); + } + + static PyObject * Begin(PyObject *self, PyObject *args, PyObject *kwds) { + puts("PyInit_${WASM2C}"); + + char *tmp_argv[] = { "/tmp/pglite/bin/postgres", "--single", "template1", NULL}; + int tmp_argc = sizeof(tmp_argv) / sizeof(char*) - 1; +puts("167"); + pre_main(tmp_argc, tmp_argv); +puts("169"); + +//${WASM2C}__start(&instance); + Py_RETURN_NONE; + } + + static PyObject * End(PyObject *self, PyObject *args, PyObject *kwds) { +puts("164: do main"); + do_main(); +puts("FREE"); + ${WASM2C}FreeInstance(&instance); + Py_RETURN_NONE; + } + +# include "${WASM2C}.pymod" + + static PyMethodDef mod_${WASM2C}_methods[] = { +#include "${WASM2C}.def" + {NULL, NULL, 0, NULL} + }; + + static struct PyModuleDef mod_${WASM2C} = { + PyModuleDef_HEAD_INIT, + "${WASM2C}", + NULL, + -1, + mod_${WASM2C}_methods, + NULL, // m_slots + NULL, // m_traverse + NULL, // m_clear + NULL, // m_free + }; + + + PyMODINIT_FUNC + PyInit_${WASM2C}(void) { + PyObject *${WASM2C}_mod = PyModule_Create(&mod_${WASM2C}); +# ifdef Py_GIL_DISABLED + PyUnstable_Module_SetGIL(${WASM2C}_mod, Py_MOD_GIL_NOT_USED); +# endif + return ${WASM2C}_mod; + } +#endif + + +int main(int argc, char* argv[]) { + if (!skip_main) { +puts("216"); + char *tmp_argv[] = { "/tmp/pglite/bin/postgres", "--single", "template1", NULL}; + int tmp_argc = sizeof(tmp_argv) / sizeof(char*) - 1; + skip_main = pre_main(tmp_argc, tmp_argv); +puts("220"); + if (!skip_main) { + do_main(); +puts("FREE"); + ${WASM2C}FreeInstance(&instance); + } + } else { + puts(" -- main skipped --"); + } + return skip_main; +} diff --git a/pglite-REL_17_4_WASM/repl.html b/pglite-REL_17_4_WASM/repl.html index 197e353f72291..44a8c93226b7e 100644 --- a/pglite-REL_17_4_WASM/repl.html +++ b/pglite-REL_17_4_WASM/repl.html @@ -47,20 +47,22 @@ - + - + + + + + - @@ -165,6 +167,12 @@ document.getElementById('resize').checked)"> +
+pglite.wasi: Download file
+
+pglite.wasi + filesystem: Download file
+

+

@@ -179,6 +187,8 @@ window.buffer_stdout = "" window.flushed_stdout = false + const PGUSER = "postgres" + async function boot() { @@ -279,7 +289,7 @@ case 1: data = "SELECT now(), current_database(), session_user, current_user;" break - +/* case 2: data = ` CREATE EXTENSION IF NOT EXISTS plpgsql; @@ -312,8 +322,8 @@ case 5: data = ` -CREATE USER test_user WITH PASSWORD 'md5abdbecd56d5fbd2cdaee3d0fa9e4f434'; -ALTER TABLE test2 OWNER TO test_user; +CREATE USER ${PGUSER} WITH PASSWORD 'md5abdbecd56d5fbd2cdaee3d0fa9e4f434'; +ALTER TABLE test2 OWNER TO ${PGUSER}; ` break @@ -321,7 +331,7 @@ data = "NOTIFY template1, 'hello';" break - +*/ /* case 3: @@ -330,33 +340,17 @@ data = "SELECT * FROM pg_extension;" data = "SELECT dictname FROM pg_catalog.pg_ts_dict;" data = "CHECKPOINT;SELECT pg_terminate_backend(42);"; - case 3: - data = "CREATE DATABASE test_user WITH OWNER = test_user;" - break - - case 4: - data = "CREATE SCHEMA test_user;" - break - - case 5: - data = `CREATE TABLE IF NOT EXISTS test_user.test ( + data = "CREATE DATABASE ${PGUSER} WITH OWNER = ${PGUSER};" + data = "CREATE SCHEMA ${PGUSER};" + data = `CREATE TABLE IF NOT EXISTS ${PGUSER}.test ( id SERIAL PRIMARY KEY, number INT ); - INSERT INTO test_user.test (number) VALUES (42); - ALTER TABLE test_user.test OWNER TO test_user; + INSERT INTO ${PGUSER}.test (number) VALUES (42); + ALTER TABLE ${PGUSER}.test OWNER TO ${PGUSER}; `; - case 6: data = "SELECT now(), current_database(), session_user, current_user;" - break -*/ -/* - case 4: - break - - case 6: data = "CREATE EXTENSION pg_stat_statements;" - break */ } @@ -425,7 +419,7 @@ if (idb & (0b0100|0b1000)) { console.log(" #3 found db+user : switch user") // switch role - // vm.readline("SET ROLE test_user;"); + // vm.readline("SET ROLE ${PGUSER};"); } console.error("Invalid user ?"); editconf = false; @@ -433,17 +427,40 @@ console.warn(" TODO: create db+user here / callback / throw ") /* vm.readline(` +` +CREATE OR REPLACE FUNCTION adduser(rolename NAME) RETURNS TEXT AS +$$ +BEGIN + IF NOT EXISTS (SELECT * FROM pg_roles WHERE rolname = rolename) THEN + EXECUTE format('CREATE ROLE %I', rolename); + RETURN 'CREATE ROLE'; + ELSE + RETURN format('ROLE ''%I'' ALREADY EXISTS', rolename); + END IF; +END; +$$ +LANGUAGE plpgsql; +`; + CREATE EXTENSION IF NOT EXISTS postgis; -CREATE USER test_user WITH PASSWORD 'md5abdbecd56d5fbd2cdaee3d0fa9e4f434'; +CREATE USER ${PGUSER} WITH PASSWORD 'md5abdbecd56d5fbd2cdaee3d0fa9e4f434'; CREATE TABLE IF NOT EXISTS test ( id SERIAL PRIMARY KEY, number INT ); INSERT INTO test (number) VALUES (42); - ALTER TABLE test OWNER TO test_user; -SET ROLE test_user; + ALTER TABLE test OWNER TO ${PGUSER}; +SET ROLE ${PGUSER}; +`); +*/ + +/* + vm.readline(` +CREATE USER ${PGUSER} WITH PASSWORD 'md5abdbecd56d5fbd2cdaee3d0fa9e4f434'; +SET ROLE ${PGUSER}; `); */ + } vm.PGDATA = PGDATA @@ -549,8 +566,8 @@ load_pg_extension: load_pg_extension, load_package: load_package, -// "PGDATABASE=test_user", - arguments : ["--single", "postgres", "--", "PGDATA=/tmp/pglite/base", "PREFIX=/tmp/pglite", "PGUSER=test_user", "REPL=N"], +// "PGDATABASE=${PGUSER}", + arguments : ["--single", "postgres", "--", "PGDATA=/tmp/pglite/base", "PREFIX=/tmp/pglite", `PGUSER=${PGUSER}`, "REPL=N"], config : { cdn : "https://pygame-web.github.io/archives/0.9/", @@ -697,7 +714,7 @@ window.vm = Module - const { WasmTerminal } = await /**/ import("./vtx.js") + const { WasmTerminal } = await /**/ import("https://pygame-web.github.io/archives/0.9/vtx.js") Module.vt = new WasmTerminal("repl", 200, 60, 10) @@ -750,7 +767,7 @@ const count = resultArray.byteLength var port = Module.cma_port || 1 - // Module._ping() + Module._ping() Module._interactive_write(count) console.log("send_to_pglite(cma):", count, "sent to", port) diff --git a/portable/Dockerfile b/portable/Dockerfile index 2f70f37e0d75d..c4b6f251867b2 100755 --- a/portable/Dockerfile +++ b/portable/Dockerfile @@ -6,10 +6,10 @@ ENV PIP_NO_CACHE_DIR 1 # nb: the python for build from sdk built with clang directly is used as SYS_PYTHON for emsdk RUN apk add --no-cache --virtual .build-deps \ - tar file lz4 \ + tar file lz4 xz \ git patch bison flex \ findutils binutils coreutils \ - libffi curl perl nodejs \ + libffi tzdata-utils curl perl nodejs \ make autoconf automake libtool pkgconfig \ ; diff --git a/portable/portable.sh b/portable/portable.sh index 557eda934ca25..c084a8c35c973 100755 --- a/portable/portable.sh +++ b/portable/portable.sh @@ -1,11 +1,13 @@ #!/usr/bin/env bash -export PG_VERSION=${PG_BRANCH:-17.4} +export PG_VERSION=${PG_VERSION:-17.4} export PG_BRANCH=${PG_BRANCH:-REL_17_4_WASM} export PORTABLE=$(realpath $(dirname $0)) export ROOT=$(realpath $(pwd)) export SDKROOT=${SDKROOT:-/tmp/sdk} +export WASI=${WASI:-false} echo " + ================================================================================================== ================================================================================================== @@ -16,6 +18,7 @@ PG_BRANCH=$PG_BRANCH SDKROOT=$SDKROOT DEBUG=$DEBUG USE_ICU=$USE_ICU +WASI=$WASI ================================================================================================== ================================================================================================== @@ -24,15 +27,14 @@ USE_ICU=$USE_ICU - export PATH=$PORTABLE:$PATH export WORKDIR=${ROOT} export CONTAINER_PATH=${CONTAINER_PATH:-/tmp/fs} export HOME=/tmp export PROOT=${PORTABLE}/proot -# git remove empty dirs -mkdir -p ${WORKDIR}/sdk/dist +# git would remove empty dirs +mkdir -p ${WORKDIR}/dist-${PG_BRANCH} ${WORKDIR}/build-${PG_BRANCH} # -------------------------------------------------------- # "docker emulation" @@ -371,11 +373,13 @@ __start() { COMMANDS+=" -b /proc/self/fd/1:/dev/stdout" COMMANDS+=" -b /proc/self/fd/2:/dev/stderr" COMMANDS+=" -b ${WORKDIR}:/workspace" - COMMANDS+=" -b ${WORKDIR}/dist:/tmp/sdk/dist" + COMMANDS+=" -b ${WORKDIR}/build-${PG_BRANCH}:/tmp/sdk/build" + COMMANDS+=" -b ${WORKDIR}/dist-${PG_BRANCH}:/tmp/sdk/dist" for f in stat version loadavg vmstat uptime do [ -f "$CONTAINER_PATH/proc/.$f" ] && COMMANDS+=" -b $CONTAINER_PATH/proc/.$f:/proc/$f" done +# --change-id=uid:gid COMMANDS+=" -r $CONTAINER_PATH -0 -w /root" COMMANDS+=" -b $CONTAINER_PATH/root:/dev/shm" @@ -407,7 +411,10 @@ __start() { } -if git checkout ${PG_BRANCH}-pglite + + +# git checkout ${PG_BRANCH} +if cd ${WORKDIR}/postgresql-${PG_BRANCH} then if [ -f postgresql-${PG_BRANCH}.patched ] then @@ -417,12 +424,13 @@ then Patching branch ${PG_BRANCH} with : -$(find patches-${PG_BRANCH}/postgresql-*) +$(find ${WORKDIR}/patches-${PG_BRANCH}/postgresql-*) " - # these don't exist in a released postgres. + + # these initially don't exist in a released postgres. touch ./src/template/emscripten \ ./src/include/port/emscripten.h \ ./src/include/port/wasm_common.h \ @@ -434,9 +442,9 @@ $(find patches-${PG_BRANCH}/postgresql-*) postgresql-wasi \ postgresql-pglite do - if [ -d patches-${PG_BRANCH}/$patchdir ] + if [ -d ${WORKDIR}/patches-${PG_BRANCH}/$patchdir ] then - for one in patches-${PG_BRANCH}/$patchdir/*.diff + for one in ${WORKDIR}/patches-${PG_BRANCH}/$patchdir/*.diff do if cat $one | patch -p1 then @@ -446,7 +454,7 @@ $(find patches-${PG_BRANCH}/postgresql-*) Fatal: failed to apply patch : $one " - exit 366 + exit 453 fi done fi @@ -454,21 +462,39 @@ Fatal: failed to apply patch : $one touch postgresql-${PG_BRANCH}.patched fi + if [ -d $CONTAINER_PATH/${SDKROOT} ] then - echo using cached version + echo using cached sdk version from $CONTAINER_PATH/${SDKROOT} else SDK_URL=https://github.com/pygame-web/portable-sdk/releases/download/3.1.74.7bi/python3.13-wasm-sdk-alpine-3.21.tar.lz4 echo "setting up sdk $SDK_URL" pushd $CONTAINER_PATH - mkdir -p /tmp/sdk - tmpfile=/tmp/sdk/python3.13-wasm-sdk-alpine-3.21.tar.lz4 - [ -f /opt/python3.13-wasm-sdk-alpine-3.21.tar.lz4 ] && cp -f /opt/python3.13-wasm-sdk-alpine-3.21.tar.lz4 $tmpfile - [ -f /tmp/sdk/python3.13-wasm-sdk-alpine-3.21.tar.lz4 ] || wget -q $SDK_URL -O$tmpfile + mkdir -p tmp + tmpfile=tmp/python3.13-wasm-sdk-alpine-3.21.tar.lz4 + # local cache + [ -f $PORTABLE/python3.13-wasm-sdk-alpine-3.21.tar.lz4 ] && cp -f $PORTABLE/python3.13-wasm-sdk-alpine-3.21.tar.lz4 $tmpfile + [ -f $tmpfile ] || wget -q $SDK_URL -O$tmpfile cat $tmpfile | tar x --use-compress-program=lz4 + tar xf $PORTABLE/wasi-sdk-25.tar.xz $PORTABLE/wasi-sdk-25.0-$(arch)-linux.tar.xz + mv tmp/sdk/wasisdk/wasi-sdk-25.0-$(arch)-linux/* tmp/sdk/wasisdk/upstream/ + if [ -f $CONTAINER_PATH/usr/bin/python3 ] + then + echo "system python found" + else + if [ -L $CONTAINER_PATH/usr/bin/python3 ] + then + echo "linked python for build found" + else + echo "Setting python for build as system python3" + mkdir -p $CONTAINER_PATH/usr/bin + ln -s $SDKROOT/devices/$(arch)/usr/bin/python3 $CONTAINER_PATH/usr/bin/python3 + fi + fi popd fi + # prevent erasing touch ${CONTAINER_PATH}${SDKROOT}/dev @@ -479,5 +505,5 @@ Fatal: failed to apply patch : $one alpineproot "apk add bash;/bin/bash --init-file /initrc" fi else - echo Error need PG_BRANCH=$PG_BRANCH set to a valid branch + echo "Error need PG_BRANCH=$PG_BRANCH set to a valid postgres-pglite WASM branch" fi diff --git a/portable/wasi-sdk-25.0-aarch64-linux.tar.xz b/portable/wasi-sdk-25.0-aarch64-linux.tar.xz new file mode 100644 index 0000000000000..cd8f21105d418 Binary files /dev/null and b/portable/wasi-sdk-25.0-aarch64-linux.tar.xz differ diff --git a/portable/wasi-sdk-25.0-x86_64-linux.tar.xz b/portable/wasi-sdk-25.0-x86_64-linux.tar.xz new file mode 100644 index 0000000000000..7d2c005e3ed0b Binary files /dev/null and b/portable/wasi-sdk-25.0-x86_64-linux.tar.xz differ diff --git a/portable/wasi-sdk-25.tar.xz b/portable/wasi-sdk-25.tar.xz new file mode 100644 index 0000000000000..3ac8a50f27d09 Binary files /dev/null and b/portable/wasi-sdk-25.tar.xz differ diff --git a/wasm-build.sh b/wasm-build.sh index 24176b10a7a54..2e060ec7d26ef 100755 --- a/wasm-build.sh +++ b/wasm-build.sh @@ -11,56 +11,85 @@ export PORTABLE=${PORTABLE:-$(pwd)/wasm-build} export SDKROOT=${SDKROOT:-/tmp/sdk} export GETZIC=${GETZIC:-true} +# systems default may not be in path export ZIC=${ZIC:-/usr/sbin/zic} - # data transfer zone this is == (wire query size + result size ) + 2 # expressed in EMSDK MB, max is 13MB on emsdk 3.1.74+ export CMA_MB=${CMA_MB:-12} export TOTAL_MEMORY=${TOTAL_MEMORY:-180MB} -export WASI=${WASI:-false} + export WORKSPACE=${GITHUB_WORKSPACE:-$(pwd)} export PGROOT=${PGROOT:-/tmp/pglite} export WEBROOT=${WEBROOT:-/tmp/web} +export PG_BUILD=${BUILD:-/tmp/sdk/build} +export PGL_BUILD_NATIVE="${PG_BUILD}/pglite-native" + export PG_DIST=${DIST:-/tmp/sdk/dist} export PG_DIST_EXT="${PG_DIST}/extensions-emsdk" +export PGL_DIST_JS="${PG_DIST}/pglite-js" -export PGL_DIST_WEB="${PG_DIST}/pglite-sandbox" +export PGL_DIST_NATIVE="${PG_DIST}/pglite-native" +export PGL_DIST_WEB="${PG_DIST}/pglite-web" export DEBUG=${DEBUG:-true} export USE_ICU=${USE_ICU:-false} export PGUSER=${PGUSER:-postgres} -[ -f /portable.opts ] && . /portable.opts +[ -f /tmp/portable.opts ] && . /tmp/portable.opts +[ -f /tmp/portable.dev ] && . /tmp/portable.dev +# can override from cmd line +export WASI=${WASI:-false} +export WASI_SDK=${WASI_SDK:-25.0} +export PYBUILD=${PYBUILD:-3.13} -if $DEBUG + +if $WASI then - export COPTS=${COPTS:-"-O2 -g3"} - export LOPTS=${LOPTS:-"-O2 -g3 --no-wasm-opt -sASSERTIONS=1"} + BUILD=wasi + if $DEBUG + then + export COPTS=${COPTS:-"-O2 -g3"} + export LOPTS=${LOPTS:-"-O2 -g3"} + else + export COPTS=${COPTS:-"-Oz -g0"} + export LOPTS=${LOPTS:-"-Oz -g0"} + fi else - # DO NOT CHANGE COPTS - optimized wasm corruption fix - export COPTS=${COPTS:-"-O2 -g3 --no-wasm-opt"} - export LOPTS=${LOPTS:-'-Oz -g0 --closure=0 --closure-args=--externs=/tmp/externs.js -sASSERTIONS=0'} + BUILD=emscripten + if $DEBUG + then + export COPTS="-O2 -g3 --no-wasm-opt" + export LOPTS=${LOPTS:-"-O2 -g3 --no-wasm-opt -sASSERTIONS=1"} + else + # DO NOT CHANGE COPTS - optimized wasm corruption fix + export COPTS="-O2 -g3 --no-wasm-opt" + export LOPTS=${LOPTS:-"-Oz -g0 --closure=0 --closure-args=--externs=/tmp/externs.js -sASSERTIONS=0"} + fi fi +export BUILD +export BUILD_PATH=${PG_BUILD}/${BUILD} + export PGDATA=${PGROOT}/base export PGPATCH=${WORKSPACE}/patches -chmod +x ${PORTABLE}/*.sh ${PORTABLE}/extra/*.sh - -# exit on error -EOE=false +chmod +x ${PORTABLE}/*.sh +[ -d ${PORTABLE}/extra ] && ${PORTABLE}/extra/*.sh +# this was set to false on 16.4 to skip some harmless exceptions without messing with core code. +# exit on error +EOE=true # default to user writeable paths in /tmp/ . -if mkdir -p ${PGROOT} ${PG_DIST} ${PG_DIST_EXT} ${PGL_DIST_WEB} +if mkdir -p ${PGROOT} ${PG_DIST} ${PG_DIST_EXT} ${PGL_DIST_JS} ${PGL_DIST_WEB} then echo "checking for valid prefix ${PGROOT} ${PG_DIST}" else @@ -104,11 +133,9 @@ System node/pnpm ( may interfer) : if ${WASI} then - echo "Wasi build (experimental)" - export WASI_SDK=25.0 - export WASI_SDK_PREFIX=${SDKROOT}/wasisdk/wasi-sdk-${WASI_SDK}-x86_64-linux - #export WASI_SDK_PREFIX=${SDKROOT}/wasisdk/upstream - export WASI_SYSROOT=${WASI_SDK_PREFIX}/share/wasi-sysroot + pushd ${SDKROOT} + . wasisdk/wasisdk_env.sh + popd if [ -f ${WASI_SYSROOT}/extra ] then @@ -126,20 +153,6 @@ then touch ${WASI_SYSROOT}/extra fi - - if false - then - . ${SDKROOT}/wasisdk/wasisdk_env.sh - env|grep WASI - export CC=${WASI_SDK_DIR}/bin/clang - export CPP=${WASI_SDK_DIR}/bin/clang-cpp - export CXX=${WASI_SDK_DIR}/bin/clang++ - export CFLAGS="-D_WASI_EMULATED_SIGNAL" - export LDFLAGS="-lwasi-emulated-signal" - else - . ${SDKROOT}/wasm32-wasi-shell.sh - fi - # wasi does not use -sGLOBAL_BASE CC_PGLITE="-DCMA_MB=${CMA_MB}" @@ -261,7 +274,7 @@ else #define I_PGDEBUG #define WASM_USERNAME "$PGUSER" #define PGDEBUG 1 -#define PDEBUG(string) fputs(string, stdout) +#define PDEBUG(string) fputs(string, stderr) #define JSDEBUG(string) {EM_ASM({ console.log(string); });} #define ADEBUG(string) { PDEBUG(string); JSDEBUG(string) } #endif @@ -289,6 +302,37 @@ END # store all pg options that have impact on cmd line initdb/boot cat > ${PGROOT}/pgopts.sh <> ${PGROOT}/pgopts.sh echo "export PGLITE=${PGLITE}" >> ${PGROOT}/pgopts.sh + [ -f /tmp/portable.opts ] && cat /tmp/portable.opts >> ${PGROOT}/pgopts.sh + [ -f /tmp/portable.dev ] && cat /tmp/portable.dev >> ${PGROOT}/pgopts.sh . ${PGROOT}/pgopts.sh @@ -342,7 +389,7 @@ fi # put local zic in the path from build dir # put emsdk-shared and also pg_config from the install dir. -export PATH=${WORKSPACE}/build/postgres/bin:${PGROOT}/bin:$PATH +export PATH=${WORKSPACE}/${BUILD_PATH}/bin:${PGROOT}/bin:$PATH # At this stage, PG should be installed to PREFIX and ready for linking @@ -380,7 +427,7 @@ then ]" fi - for extdir in postgresql/contrib/* + for extdir in postgresql-${PG_BRANCH}/contrib/* do if [ -f ${PGROOT}/dumps/dump.vector ] then @@ -403,7 +450,7 @@ then Building contrib extension : $ext : begin " - pushd build/postgres/contrib/$ext + pushd ${BUILD_PATH}/contrib/$ext if PATH=$PREFIX/bin:$PATH emmake make install 2>&1 >/dev/null then echo " @@ -428,47 +475,48 @@ then fi - # only build extra when targeting pglite-wasm . - -# TODO link the good tag -ln -s ${WORKSPACE}/pglite-REL_17_4_WASM ${WORKSPACE}/pglite-wasm - +pwd if [ -f ${WORKSPACE}/pglite-wasm/build.sh ] then - - if echo " $*"|grep -q " extra" + if $WASI then - for extra_ext in ${EXTRA_EXT:-"vector"} - do - if $CI - then - #if [ -d $PREFIX/include/X11 ] - if true + echo " + * WASI build : skipping extra extensions and FS +" + else + + if echo " $*"|grep -q " extra" + then + for extra_ext in ${EXTRA_EXT:-"vector"} + do + if $CI then - echo -n - else - # install EXTRA sdk - . /etc/lsb-release - DISTRIB="${DISTRIB_ID}-${DISTRIB_RELEASE}" - CIVER=${CIVER:-$DISTRIB} - SDK_URL=https://github.com/pygame-web/python-wasm-sdk-extra/releases/download/$SDK_VERSION/python3.13-emsdk-sdk-extra-${CIVER}.tar.lz4 - echo "Installing $SDK_URL" - curl -sL --retry 5 $SDK_URL | tar xvP --use-compress-program=lz4 | pv -p -l -s 15000 >/dev/null - chmod +x ./extra/*.sh + #if [ -d $PREFIX/include/X11 ] + if true + then + echo -n + else + # install EXTRA sdk + . /etc/lsb-release + DISTRIB="${DISTRIB_ID}-${DISTRIB_RELEASE}" + CIVER=${CIVER:-$DISTRIB} + SDK_URL=https://github.com/pygame-web/python-wasm-sdk-extra/releases/download/$SDK_VERSION/python3.13-emsdk-sdk-extra-${CIVER}.tar.lz4 + echo "Installing $SDK_URL" + curl -sL --retry 5 $SDK_URL | tar xvP --use-compress-program=lz4 | pv -p -l -s 15000 >/dev/null + chmod +x ./extra/*.sh + fi fi - fi - echo "======================= ${extra_ext} : $(pwd) ===================" + echo "======================= ${extra_ext} : $(pwd) ===================" - ./extra/${extra_ext}.sh || exit 400 - - python3 ${PORTABLE}/pack_extension.py - done - fi + ./extra/${extra_ext}.sh || exit 480 - # build pglite initdb/loop/transport/repl + python3 ${PORTABLE}/pack_extension.py + done + fi - export PGPRELOAD="\ + # this is for initial emscripten MEMFS + export PGPRELOAD="\ --preload-file ${PGROOT}/share/postgresql@${PGROOT}/share/postgresql \ --preload-file ${PGROOT}/lib/postgresql@${PGROOT}/lib/postgresql \ --preload-file ${PGROOT}/password@${PGROOT}/password \ @@ -476,12 +524,81 @@ then --preload-file placeholder@${PGROOT}/bin/postgres \ --preload-file placeholder@${PGROOT}/bin/initdb\ " - - ${WORKSPACE}/pglite-wasm/build.sh + fi - for file in /tmp/sdk/dist/extensions-emsdk/*.tar; do gzip -9 -k -f "$file"; done -else - echo "Could not find a pglite tag matching $PG_BRANCH" - exit 480 + echo " + * building + linking pglite-wasm (initdb/loop/transport/repl/backend) +" + if ${WORKSPACE}/pglite-wasm/build.sh + then + if $WASI + then + echo "TODO: wasi pack/tests" + else +cat > pglite-link.sh </dev/null # do nothing if it is a submodule - [ -d postgresql ] || ln -s $(realpath postgresql-${PG_VERSION}) postgresql + [ -d postgresql ] || ln -s postgresql-${PG_BRANCH} postgresql fi -export PGSRC=$(realpath postgresql) +export PGSRC=$(realpath postgresql-${PG_BRANCH}) echo " Building $ARCHIVE (patched) from $PGSRC WASI=$WASI -build-pgcore:begin +build-pgcore: begin($BUILD) ___________________________________________________ CC_PGLITE=$CC_PGLITE @@ -67,30 +67,28 @@ CC_PGLITE=$CC_PGLITE " -if [ -f ${PGROOT}/pg.installed ] +if [ -f ${PGROOT}/pg.${BUILD}.installed ] then - echo " * skipping pg build, using previous install from ${PGROOT}" + echo " + * skipping pg build, using previous install from ${PGROOT} +" else - mkdir -p build/postgres - pushd build/postgres + mkdir -p ${BUILD_PATH} + pushd ${BUILD_PATH} # create empty package.json to avoid emsdk node conflicts # with root package.json of project echo "{}" > package.json - if $CI + if [ -f Makefile ] then - echo "CI : using build cache" - else - if [ -f Makefile ] - then - echo "Cleaning up previous build ..." - make distclean 2>&1 > /dev/null - fi + echo "Cleaning up previous build ..." + make distclean 2>&1 > /dev/null fi + # TODO: --with-libxml xml2 >= 2.6.23 # TODO: --with-libxslt add to sdk # --disable-atomics https://github.com/WebAssembly/threads/pull/147 "Allow atomic operations on unshared memories" @@ -98,21 +96,27 @@ else COMMON_CFLAGS="${CC_PGLITE} -fpic -Wno-declaration-after-statement -Wno-macro-redefined -Wno-unused-function -Wno-missing-prototypes -Wno-incompatible-pointer-types" + # common to all wasm flavour + cp ${PGSRC}/src/include/port/wasm_common.h ${PGROOT}/include/wasm_common.h + + # wasm os implementation router + cp ${PORTABLE}/sdk_port.h ${PGROOT}/include/sdk_port.h + + # specific implementation for wasm os flavour + [ -d ${PORTABLE}/sdk_port-${BUILD} ] && cp ${PORTABLE}/sdk_port-${BUILD}/* ${PGROOT}/include/ + if ${WASI} then - BUILD=wasi - echo "WASI BUILD: turning off xml/xslt support" + echo "WASI BUILD: turning off xml/xslt support" XML2="" UUID="" - cp ${PORTABLE}/wasi/wasi_port.c /tmp/pglite/include/sdk_port.c - WASM_LDFLAGS="-lwasi-emulated-getpid -lwasi-emulated-mman -lwasi-emulated-signal -lwasi-emulated-process-clocks" - WASM_CFLAGS="-DSDK_PORT=/tmp/pglite/include/sdk_port.c ${COMMON_CFLAGS} -D_WASI_EMULATED_MMAN -D_WASI_EMULATED_SIGNAL -D_WASI_EMULATED_PROCESS_CLOCKS -D_WASI_EMULATED_GETPID" + # -lwasi-emulated-signal -D_WASI_EMULATED_SIGNAL -lwasi-emulated-getpid -D_WASI_EMULATED_GETPID + WASM_LDFLAGS="-lwasi-emulated-mman -lwasi-emulated-pthread -lwasi-emulated-process-clocks" + WASM_CFLAGS="-I${WASISDK}/hotfix -DSDK_PORT=${PREFIX}/include/sdk_port-wasi.c ${COMMON_CFLAGS} -D_WASI_EMULATED_PTHREAD -D_WASI_EMULATED_MMAN -D_WASI_EMULATED_PROCESS_CLOCKS" export MAIN_MODULE="" else - BUILD=emscripten - # --with-libxml does not fit with --without-zlib if $CI then @@ -137,8 +141,6 @@ else fi - cp ${PGSRC}/src/include/port/wasm_common.h /tmp/pglite/include/wasm_common.h - [ -f ${PREFIX}/devices/emsdk/usr/lib/libossp-uuid.a ] && rm ${PREFIX}/devices/emsdk/usr/lib/libossp-uuid.a [ -f ${PREFIX}/devices/emsdk/usr/lib/libouuid.a ] && rm ${PREFIX}/devices/emsdk/usr/lib/libuuid.a @@ -154,6 +156,7 @@ else GETZIC=${GETZIC:-true} + EMCC_NODE="-sEXIT_RUNTIME=1 -DEXIT_RUNTIME -sNODERAWFS -sENVIRONMENT=node" @@ -171,13 +174,12 @@ END #. ${SDKROOT}/wasm32-wasi-shell.sh TZ=UTC PGTZ=UTC $(command -v wasi-run) $(pwd)/src/timezone/zic.wasi \$@ END - else - echo " + else + echo " * Using system ZIC from ${ZIC:-/usr/sbin/zic} " - cp ${ZIC:-/usr/sbin/zic} bin/ + cp ${ZIC:-/usr/sbin/zic} bin/ fi - else export EXT=wasm cat > ${PGROOT}/config.site < /tmp/disable-shared.log @@ -313,15 +309,23 @@ END # Keep a shell script for fast rebuild with env -i from cmdline + # same script handle emcc and wasi + cat > pg-make.sh <> pg-make.sh cat >> pg-make.sh <&1 > /tmp/install.log then echo install ok if $WASI then - # remove unlinked server - for todel in src/backend/postgres src/backend/postgres.wasi $PGROOT/bin/postgres $PGROOT/bin/postgres.wasi - do - [ -f $todel ] && rm $todel - done - - pushd ../.. - chmod +x ./wasm-build/linkwasi.sh - # WASI_CFLAGS="$WASI_CFLAGS" - ./wasm-build/linkwasi.sh || exit 251 - popd - cp src/backend/postgres.wasi $PGROOT/bin/ || exit 253 + cp src/backend/postgres.wasi $PGROOT/bin/ || exit 365 else if $DEBUG then # built with EMCC_CFLAGS="-sEXIT_RUNTIME=1 -DEXIT_RUNTIME -sNODERAWFS -sENVIRONMENT=node" emmake make -C cp src/bin/initdb/initdb.wasm $PGROOT/bin/ - cp src/backend/postgres.wasm $PGROOT/bin/ fi @@ -381,7 +380,7 @@ END pushd ${PGROOT} - find . -type f | grep -v plpgsql > ${PGROOT}/pg.installed + find . -type f | grep -v plpgsql > ${PGROOT}/pg.${BUILD}.installed popd goback=$(pwd) @@ -390,13 +389,13 @@ END pushd $goback pushd ${PGROOT} - find . -type f > ${PGROOT}/pg.installed + find . -type f > ${PGROOT}/pg.${BUILD}.installed popd else cat /tmp/install.log echo "install failed" - exit 384 + exit 400 fi python3 > ${PGROOT}/PGPASSFILE < 0: diff --git a/wasm-build/sdk.sh b/wasm-build/sdk.sh index 4f9086bf70090..0401c546b3515 100755 --- a/wasm-build/sdk.sh +++ b/wasm-build/sdk.sh @@ -9,11 +9,11 @@ fi # https://github.com/emscripten-core/emscripten/blob/ac676d5e437525d15df5fd46bc2c208ec6d376a3/tools/link.py#L1652-L1658 -if python3 -V +if python3 -V 2>/dev/null then echo using installed python3 else - echo wil use python for build as system python. + echo will use python for build as system python. ln -sf $SDKROOT/devices/$(arch)/usr/bin/python3 /usr/bin/python3 fi diff --git a/wasm-build/sdk_port.h b/wasm-build/sdk_port.h new file mode 100644 index 0000000000000..82bff990f1f82 --- /dev/null +++ b/wasm-build/sdk_port.h @@ -0,0 +1,290 @@ +#if defined(__EMSCRIPTEN__) +#include + +#elif defined(__wasi__) + + +#ifndef I_WASI +#define I_WASI + +#undef HAVE_PTHREAD + +#if defined(HAVE_SETSID) +#undef HAVE_SETSID +#endif + +#if defined(HAVE_GETRLIMIT) +#undef HAVE_GETRLIMIT +#endif + + +#define PLATFORM_DEFAULT_SYNC_METHOD SYNC_METHOD_FDATASYNC + +#define EMSCRIPTEN_KEEPALIVE __attribute__((used)) +#define __declspec( dllimport ) __attribute__((used)) + +#define em_callback_func void +#define emscripten_set_main_loop(...) +#define emscripten_force_exit(...) +#define EM_JS(...) + +#include "/tmp/pglite/include/wasm_common.h" + + +static pid_t +fork(void) { + puts("# 31: fork -1"); + return -1; +} + +// ======== signal ======================== +#define SA_RESTART 4 +#define SIG_SETMASK 2 + +#define SIG_BLOCK 0 +#define SIG_UNBLOCK 1 + +/* A signal handler. */ +typedef void (*handler_t) (int signal); +typedef unsigned char sigset_t; +typedef void (*__sighandler_t) (int); + +struct sigaction { + __sighandler_t sa_handler; + unsigned long sa_flags; +#ifdef SA_RESTORER + __sigrestore_t sa_restorer; +#endif + sigset_t sa_mask; +}; +extern int sigemptyset(sigset_t *set); +extern int sigaction(int signum, const struct sigaction *act, struct sigaction *oldact); +extern int sigdelset (sigset_t *set, int sig); +extern int sigfillset (sigset_t *set); +extern int sigprocmask (int operation, const sigset_t *set, sigset_t *old_set); +extern int sigaddset (sigset_t *set, int sig); + +// STUBS +extern int pthread_sigmask(int how, const sigset_t *set, sigset_t *oldset); +extern int sigismember(const sigset_t *set, int signum); +extern int sigpending(sigset_t *set); +extern int sigwait(const sigset_t *restrict set, int *restrict sig); + +// ==================================================== +// unistd +extern unsigned int alarm(unsigned int seconds); + +// ==================================================== + + +#include + +static int +sigsetjmp(sigjmp_buf env, int savesigs) { +// puts("# 120: sigsetjmp"); + return 0; +} + +static void +siglongjmp(sigjmp_buf env, int val) { + puts("# 120: siglongjmp"); +} + + + +// WIP : + +#include +static uid_t +getuid(void) { + return 1000; +} + +static int +dup(int fd) { + puts("# 128: dup"); + return fd; +} +static int +dup2(int old, int new) { + puts("# 140: dup2"); + return -1; +} +static int +pipe(int fd[2]) { + puts("# 145: pipe"); + abort(); + return -1; +} + +#include +#define RLIMIT_NOFILE 7 +#define RLIMIT_STACK 3 +#define RLIM_INFINITY ((unsigned long int)(~0UL)) + +struct rlimit { + unsigned long rlim_cur; + unsigned long rlim_max; +}; +static int +getrlimit(int resource, struct rlimit *rlim) { + return -1; +} + + +static const char *gai_strerror_msg = "# 165: gai_strerror_msg"; +static const char * +gai_strerror(int errcode) { + return gai_strerror_msg; +} + + +static int +getrusage(int who, struct rusage *usage) { + return -1; +} + +// WIP: semaphores here +// ================================================================== +#include + +static int +semctl(int semid, int semnum, int cmd, ...) { + return 0; // -1; +} + +static int +semget(key_t key, int nsems, int semflg) { +#if 0 // PGDEBUG + printf("# 213: semget(key_t key = %d, int nsems=%d, int semflg=%d)\n", key, nsems, semflg); +#endif + return 1; +} + +static int +semop(int semid, struct sembuf *sops, size_t nsops) { + return 0; // -1; +} + + + +#include +#if defined(PYDK) +extern int shm_open(const char *name, int oflag, mode_t mode); +extern int shm_unlink(const char *name); +#else +static int +shm_open(const char *name, int oflag, mode_t mode) { + char tmpnam[128]; + int fd; + snprintf(tmpnam, 128, "/tmp%s", name); + fd=fileno(fopen(tmpnam, "w+")); + fprintf(stderr, "# 212: shm_open(%s) => %d\n", tmpnam, fd); + return fd; +} + +static int +shm_unlink(const char *name) { + char tmpnam[128]; + snprintf(tmpnam, 128, "/tmp%s", name); + fprintf(stderr, "# 220: shm_unlink(%s) STUB\n", tmpnam); + return remove(tmpnam); // -1 +} + +#endif + +// initdb chmod +#if defined(__wasi__) + #define chmod(...) 0 +#endif + + + +#define system(command) system_wasi(command) +extern int system_wasi(const char *command); + + + +// time.h + +static void +tzset(void) { + puts("# 241: tzset(void) STUB"); +} + +#if defined(PG_INITDB) || defined(FE_UTILS_PRINT) || defined(PG_DUMP_PARALLEL) +static void +__SIG_IGN(int param) { +} +#endif + + +extern void sock_flush(); + + +// TODO: socket here +// ================================================================== + +#include + +extern ssize_t sdk_recv(int sockfd, void *buf, size_t len, int flags); +extern ssize_t recvfrom(int socket, void *buffer, size_t length, int flags, void *address, socklen_t *address_len); + +extern ssize_t sdk_send(int sockfd, const void *buf, size_t len, int flags); +extern ssize_t sendto(int sockfd, const void *buf, size_t len, int flags, void *dest_addr, socklen_t addrlen); + +#define recv(sockfd, buf, len, flags) sdk_recv(sockfd, buf, len, flags) + + + + +static int +listen(int sockfd, int backlog) { + return 0; +} + +static struct group *_Nullable +getgrnam(const char *_Nullable name) { + return NULL; +} + +static int +getsockname(int sockfd, struct sockaddr *restrict addr, socklen_t *restrict addrlen) { + return -1; +} + +static int +getaddrinfo(const char *restrict node, + const char *restrict service, + void *restrict hints, + void **restrict res) { + puts("# 60: getaddrinfo"); + return -1; +} +static void +freeaddrinfo(void *res) { + puts("# 65: freeaddrinfo"); +} + +extern ssize_t recvfrom_bc(int socket, void *buffer, size_t length, int flags, void *address, socklen_t *address_len); + + +#define getpid sdk_getpid +extern pid_t sdk_getpid(void); + + +//#define pthread_mutex_lock(mut) sdk_pthread_mutex_lock(mut) +//extern int sdk_pthread_mutex_lock(void *mutex); + +/* + int pthread_mutex_lock(pthread_mutex_t *mutex); + int pthread_mutex_trylock(pthread_mutex_t *mutex); + int pthread_mutex_unlock(pthread_mutex_t *mutex); +*/ + + +#endif // I_WASI + +#else + #error "unknown port mode should be __EMSCRIPTEN__ or __wasi__" +#endif // __EMSCRIPTEN__