From 3cf7254d7260f58c2d75971f3df1197ecb2efac9 Mon Sep 17 00:00:00 2001 From: Dax Raad Date: Thu, 10 Oct 2024 23:49:22 -0400 Subject: [PATCH] sst.aws.Function: build system update. do not serialize links into code, encrypt them into a seperate file. experimental esbuild plugins support. --- bun.lockb | Bin 476928 -> 476752 bytes go.mod | 1 + go.sum | 2 + pkg/runtime/node/build.go | 16 +- pkg/runtime/node/node.go | 22 ++ pkg/runtime/node/plugin.go | 350 +++++++++++++++++-- pkg/runtime/runtime.go | 45 ++- pkg/runtime/worker/worker.go | 3 + platform/functions/nodejs-runtime/plugin.mjs | 77 ++-- platform/package.json | 2 +- platform/src/components/aws/function.ts | 99 +++--- sdk/js/src/resource.ts | 16 + 12 files changed, 513 insertions(+), 120 deletions(-) diff --git a/bun.lockb b/bun.lockb index 7e74fc40022c4733b255ef0d010f874cc44be016..6b6d7b86c6f16b224be3540903c65ad694f4f4b6 100755 GIT binary patch delta 4041 zcmXYz3s}|l8OQ$z4;(IO-g47|)DltSwxytf2y#(U!#iP`p@|wKI5SM^;04GF3Znic zOC_}=8cUZQt+naa8mpGJ1?jd&O>0lf_B=kzHaDMLKJPi-_wnWZ{yx9=cmMtV$C~Z2 z=eET*CJk+i&n*3K+1bnQmK+=1vFty8?HVw8P}7Egw_Tf2@WG^IT zr+2fbZd7w+eq?=hiYHK=64rHmg17k7ojHB3PbkXi8`eEJ{!B@D>(ydeL*20%cX_k9>QkZ4+ma1D~<6vQw<8WULf3OVY-HncPQ>W6!7oM^)vjN~Nm5SOv^B_Xqd@u@%buA#9*~hA#Y{v9#MLR#9a; zGaBE|7W5>p78`@!3bX65RxAx&54)XM1G9rk$LnCdi5tbnqHAG&tWInkdJe3w zD7aUNIk0PJdmk-CdlM7!0<_s1%F9Jp+S0LOVtMEkvEwj1gnT>$X8pXWyaIFq;|J}8 z;3P`3@$chr!K}e#e6HB_PNG1K3%yDd^H5VJF3=QeG?O{{(0=oQ6Lq z_y;9EM0ulKee6@QB6O`-m)LYzomjWn3|JStI+*ww%uaSD{*BnhP@dnJ1^k=fB{jAk z=wMbuiI-uv3v=)fZKl{im1nzh!QLEpQ>+-B!({Fy{tISJ9u^x$yd`G4mlMgE#JitN z=OGNI{6Ez_qWlD+hq+lF^TkFGdx$L%yN?(y_9(hq@689Zd%O@|BNi$47|iB*KQStt zhhT%03fe1-QHP7rwv%bZoV26_XOGoHa1+W9DTnvz!JnN(4)mhh&>5QH)aoE zq~HqlJwW?Tj1sFvC)wv3yHBhNoi3Irwi1>hmL&EgShWUD7Fz|g#n~q&h3L2Xr*K=H z!89eVM%(I)rHeg{-h{Sq(O9u+bRF7koY)%lLt+nzJp&sgmLbM#^gDC8d}f(~>(KUF zVm4mvS#+UT7L4l~e6>?x6VW*^yGOR0tE>{sRofbLgIJ!}dRU@0uon4(8z>elu|RAi zEDM%ToFujh{Y|iO=&Nd%uomwUD^y+`>;kNS_@G!l`cKN6V$5dzJpO0FsX&vP@ol)7 zZKbg-_(J7PSBG0+1@?1_%@DJ8lf-6T~o3>`fVmr~}#2yxFhV>LH5!(fODs)->&OD$kYB#<%1fBUXJBdB`Cb5NT?7|)q zdrWLE>`}2&u@=}_n7xceVlSZ2DQ|I*vH$ykhXo&3;)^ib8G9~E#9l&gLEDWlBif|* z<8~Bw6 zx(;TSal2RmeGz6CYKPcK^eLEGqu43*F|j7Gw_&#Zc29QN`P&TN!Dq3Tt;A-rpP^@q z?Gk$zwgq;GxLfS!=s1`i*B-H7pik-OT(Q%zT$mlc7`vA z{R(Y&#Exd4*n8-5^6VI1gdv?@x0?j5+R z+uC<<{HHgzy$mq-Q~DQff+4N}y$AcvXAMQTxHg61v0PYq`EbZgO{cTXb=Eh+dvp zf2BJ-aHTuqLb7L!JLK!|A@0ElU!*&)GooMMW`yr(X4K%^(B(~W+j4zr(bX!R&vn1f s_3e*}%nMZycAN5iSXtjTl#nCDv%b5iBgc|Nybo_CnLG~c)P|GDGG`2YX_ delta 4182 zcmXY!d306P6~=GE3lG99Rv8LHA_!PQ2v87$3??B21cJ;WAOfa9NPK{_ug{$rF3*TPs?C<{0KIfikzxSwI*&&K4AF3wpv?AWk`?>nu_ zI@7hRG(90+nx1&^NZ%uea`v?9mv|=q%k=I)%)fm9S5$r6>YszZIh2=@dScOs z1xtE-)P2x}{zYA`<=q%Mys+8HA-5Y8TphUZ-lsJ4!gAV@&{1n+B>W_BN}u~{dx`(crAjfO}@$~wELDY||&34#om zdN#wqc4BwuHHUo@Ue^$Rb<-nmYC+R&vz}%xVRy`W!4i3X0FQ>9A7_YKbme*Fs$^^mO*)^BcwLxQJJsoF z=z7kZ?z~R0`eBKNXyB%qZt6@^BeO!Y4A{M9vtZiXF8F=nNrs3ubyJDcyU}!srPU?Q zhG}Phj&EZN>JjIebw|Gn({*^!tOvRb)_}MWrh|C|Uj}PPTw>M}y%g3c2x38*VK3k) z;620@Fl3O4PnN*ez_d-h@qAcg;#&886kW+o?2P%(7sAN2_MGI%cB>pk+sinsg3ctnI*#nGHav#|+;C>SS{9 zOjr}*aVHK$D_M5JEDt>rS4F-L(*y?Lqs`7aZ!pi}aM^jY$I+7nu-F%dLwJ~6je_8U z*-)P6n_Yxyg2V8a%`Q1FpXbFY2kf%haP(5MD`ro?mYMzeE{g>tfEU@;*2Jqo9k*8Y zt=SEC9tHb{*-bO8_HAaBO1uTrE{wrH)l9KF&KrxqLLJI}G}Es1ufo z(Wb8BtJ{L$Az}jW6no079kGVl1he+UB(sTTD*ve2Br}zGZL_D*r7pfYNfd(a$z=R_ z!@4AjJ%jgW*HoGH++xTz$o|oJWzxsyhp28@`BMg6>1i zF?$KU3@saA_A+{?S+1FOQ^zRN=a?3^03Xc-m*tr)L_bx{=rPD}krJFZ7)Ff*i}As* zY@%MKidcfrQ6rY`zDv=o%!ZpSgLPB~>{Zwm*g)bKvz6$p&KqmC3ihShIGE&WJf>vXc*9?zCpmF~JD0--==&L)XtoBO zYc|PjEld|vpKVW@tuvcSoNTt^~|0% z+X$QMn_d9Znl|C{&8EX@sQxwsmlzhh^A^~6Gkr^RbKV}bE}yKF zsJu6ET{PJXM2)u>*PW8Z<|A|=_u+?p`(K8s^M3p#vjs581Nbn+BBIJv16JVqW=qZ9 zf(Ca{xt(HCYIf0vEXeU_Gt##awoont}uJW>=3LUOx2>@&}0td-)pAWYMA5^ z{1!shQf~Gx`X)@5eXYkkivB8*^H*uCGdzaY_Q=+|^Lyw&;VPHcofk*XU_ez%Xm%WZ zM>D`Sn4Lhk;y7g+&E7|sa2IqzH^mG;;GrC-8@k!-*XU(1UC=FNzd?Tm(s*+s%Fl)AsKnzF`(S$%B5*Yt=gpKSGZ*+iCWDn0}w?PV6%K1G*(l$FF4@J+MR=-Y|ZpU!Zv;U{R#P-nQ$?2pp8j%L4^j$yX%t+mM-@?fH z#BgR&qyE z(!4C1&^4`dx3u(lZo8z73AMCCviRTHCzVvx8B@iiu*+3RVXv%cvoLlsX@7F9OLu8` zR#`?>Qaq!o=9Nwf-NGFmYNmwcwWHCn;&jc%@!XoxeZ6Z}X5GEk*TVli7VVzgF8l7| w^6ao@R&-nK<9@CTuV+R3Bsa;q+nk;gcF2xSs8itQ;+$xBc3h-QNdAWZ19FnrY5)KL diff --git a/go.mod b/go.mod index 484086302..0c3b90f56 100644 --- a/go.mod +++ b/go.mod @@ -30,6 +30,7 @@ require ( github.com/fsnotify/fsnotify v1.7.0 github.com/gdamore/tcell/v2 v2.7.4 github.com/gorilla/websocket v1.5.0 + github.com/iancoleman/strcase v0.3.0 github.com/joho/godotenv v1.5.1 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 github.com/manifoldco/promptui v0.9.0 diff --git a/go.sum b/go.sum index 29b57a9d9..16d3125c5 100644 --- a/go.sum +++ b/go.sum @@ -200,6 +200,8 @@ github.com/hashicorp/go-retryablehttp v0.7.5 h1:bJj+Pj19UZMIweq/iie+1u5YCdGrnxCT github.com/hashicorp/go-retryablehttp v0.7.5/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8= github.com/hashicorp/hcl/v2 v2.20.1 h1:M6hgdyz7HYt1UN9e61j+qKJBqR3orTWbI1HKBJEdxtc= github.com/hashicorp/hcl/v2 v2.20.1/go.mod h1:TZDqQ4kNKCbh1iJp99FdPiUaVDDUPivbqxZulxDYqL4= +github.com/iancoleman/strcase v0.3.0 h1:nTXanmYxhfFAMjZL34Ov6gkzEsSJZ5DbhxWjvSASxEI= +github.com/iancoleman/strcase v0.3.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= diff --git a/pkg/runtime/node/build.go b/pkg/runtime/node/build.go index c76b149ab..5fdaced2c 100644 --- a/pkg/runtime/node/build.go +++ b/pkg/runtime/node/build.go @@ -57,13 +57,14 @@ func (r *Runtime) Build(ctx context.Context, input *runtime.BuildInput) (*runtim } plugins := []esbuild.Plugin{} + if properties.Plugins != "" { + plugins = append(plugins, plugin(properties.Plugins)) + } external := append(forceExternal, properties.Install...) external = append(external, properties.ESBuild.External...) - serializedLinks, err := json.Marshal(input.Links) if err != nil { return nil, err } - slog.Info("serialized links", "links", string(serializedLinks)) options := esbuild.BuildOptions{ EntryPoints: []string{file}, Platform: esbuild.PlatformNode, @@ -87,7 +88,6 @@ func (r *Runtime) Build(ctx context.Context, input *runtime.BuildInput) (*runtim `import { fileURLToPath as topLevelFileUrlToPath, URL as topLevelURL } from "url"`, `const __filename = topLevelFileUrlToPath(import.meta.url)`, `const __dirname = topLevelFileUrlToPath(new topLevelURL(".", import.meta.url))`, - `globalThis.$SST_LINKS = ` + string(serializedLinks) + `;`, properties.Banner, }, "\n"), }, @@ -97,12 +97,10 @@ func (r *Runtime) Build(ctx context.Context, input *runtime.BuildInput) (*runtim options.Format = esbuild.FormatCommonJS options.Target = esbuild.ESNext options.MainFields = []string{"main"} - options.Banner = map[string]string{ - "js": strings.Join([]string{ - `globalThis.$SST_LINKS = ` + string(serializedLinks) + `;`, - properties.Banner, - }, "\n"), - } + } + + if properties.ESBuild.Target != 0 { + options.Target = properties.ESBuild.Target } if properties.Splitting { diff --git a/pkg/runtime/node/node.go b/pkg/runtime/node/node.go index 401131868..c3720d70c 100644 --- a/pkg/runtime/node/node.go +++ b/pkg/runtime/node/node.go @@ -32,6 +32,28 @@ var loaderMap = map[string]api.Loader{ "binary": api.LoaderBinary, } +var LoaderToString = []string{ + "none", + "base64", + "binary", + "copy", + "css", + "dataurl", + "default", + "empty", + "file", + "global-css", + "js", + "json", + "json", + "jsx", + "local-css", + "text", + "ts", + "ts", + "tsx", +} + type Runtime struct { cfgPath string contexts map[string]esbuild.BuildContext diff --git a/pkg/runtime/node/plugin.go b/pkg/runtime/node/plugin.go index 323f48750..d28af6c8c 100644 --- a/pkg/runtime/node/plugin.go +++ b/pkg/runtime/node/plugin.go @@ -2,18 +2,28 @@ package node import ( "encoding/json" - "fmt" + "errors" + "io" "log/slog" "os" "os/exec" "path/filepath" "github.com/evanw/esbuild/pkg/api" + "github.com/sst/ion/internal/util" + "golang.org/x/sync/errgroup" ) -type NodeLoadResult struct { - api.OnLoadResult - Loader string `json:"loader"` +type message struct { + ID int `json:"id"` + Command string `json:"command"` + Value map[string]any `json:"value"` +} + +type request struct { + Command string + Chan chan map[string]any + Value map[string]any } func plugin(path string) api.Plugin { @@ -24,6 +34,9 @@ func plugin(path string) api.Plugin { Setup: func(build api.PluginBuild) { slog.Info("nodejs plugin", "path", path) cmd := exec.Command("node", ".sst/platform/functions/nodejs-runtime/plugin.mjs", path) + util.SetProcessGroupID(cmd) + var wg errgroup.Group + // cmd.Stderr = os.Stderr stdin, err := cmd.StdinPipe() if err != nil { return @@ -35,45 +48,328 @@ func plugin(path string) api.Plugin { if err := cmd.Start(); err != nil { return } + requests := make(chan request, 0) + responses := make(chan message, 0) + + request := func(command string, input map[string]any) map[string]any { + c := make(chan map[string]any) + requests <- request{Command: command, Chan: c, Value: input} + return <-c + } + + wg.Go(func() error { + count := 0 + encoder := json.NewEncoder(stdin) + pending := map[int]chan map[string]any{} + for { + select { + case req, ok := <-requests: + if !ok { + return nil + } + encoder.Encode(message{ + Command: req.Command, + ID: count, + Value: req.Value, + }) + pending[count] = req.Chan + count++ + case reply, ok := <-responses: + if !ok { + return nil + } + match, ok := pending[reply.ID] + if !ok { + continue + } + delete(pending, reply.ID) + match <- reply.Value + } + } + }) + wg.Go(func() error { + decoder := json.NewDecoder(stdout) + for { + var reply message + err := decoder.Decode(&reply) + if err != nil { + if err == io.EOF { + return nil + } + return err + } + responses <- reply + } + }) build.OnResolve(api.OnResolveOptions{Filter: ".*"}, func(args api.OnResolveArgs) (api.OnResolveResult, error) { - request := map[string]interface{}{ - "type": "resolve", - "path": args.Path, - "importer": args.Importer, - "namespace": args.Namespace, + with := make(map[string]interface{}, len(args.With)) + for k, v := range args.With { + with[k] = v + } + response := request("resolve", map[string]interface{}{ + "path": args.Path, + "importer": args.Importer, + "namespace": args.Namespace, + "resolveDir": args.ResolveDir, + "kind": resolveKindToString(args.Kind), + "pluginData": args.PluginData, + "with": with, + }) + result := api.OnResolveResult{} + if value, ok := response["error"]; ok { + return result, errors.New(value.(string)) + } + if value, ok := response["pluginName"]; ok { + result.PluginName = value.(string) + } + if value, ok := response["path"]; ok { + result.Path = value.(string) + } + if value, ok := response["namespace"]; ok { + result.Namespace = value.(string) + } + if value, ok := response["suffix"]; ok { + result.Suffix = value.(string) + } + if value, ok := response["external"]; ok { + result.External = value.(bool) + } + if value, ok := response["sideEffects"]; ok { + if value.(bool) { + result.SideEffects = api.SideEffectsTrue + } else { + result.SideEffects = api.SideEffectsFalse + } + } + if value, ok := response["pluginData"]; ok { + result.PluginData = value.(int) } - if err := json.NewEncoder(stdin).Encode(request); err != nil { - return api.OnResolveResult{}, fmt.Errorf("error sending resolve request: %w", err) + if value, ok := response["errors"]; ok { + result.Errors = decodeMessages(value.([]interface{})) } - var result api.OnResolveResult - if err := json.NewDecoder(stdout).Decode(&result); err != nil { - return api.OnResolveResult{}, fmt.Errorf("error reading resolve response: %w", err) + if value, ok := response["warnings"]; ok { + result.Warnings = decodeMessages(value.([]interface{})) + } + if value, ok := response["watchFiles"]; ok { + result.WatchFiles = decodeStringArray(value.([]interface{})) + } + if value, ok := response["watchDirs"]; ok { + result.WatchDirs = decodeStringArray(value.([]interface{})) } - slog.Info("result", "result", result) return result, nil }) build.OnLoad(api.OnLoadOptions{Filter: ".*"}, func(args api.OnLoadArgs) (api.OnLoadResult, error) { - request := map[string]interface{}{ - "type": "load", - "path": args.Path, - "namespace": args.Namespace, + result := api.OnLoadResult{} + + with := make(map[string]interface{}, len(args.With)) + for k, v := range args.With { + with[k] = v + } + + response := request("load", map[string]interface{}{ + "path": args.Path, + "namespace": args.Namespace, + "suffix": args.Suffix, + "pluginData": args.PluginData, + "with": with, + }) + + if value, ok := response["error"]; ok { + return result, errors.New(value.(string)) + } + if value, ok := response["pluginName"]; ok { + result.PluginName = value.(string) + } + if value, ok := response["loader"]; ok { + loader, _ := loaderMap[value.(string)] + result.Loader = loader } - if err := json.NewEncoder(stdin).Encode(request); err != nil { - return api.OnLoadResult{}, fmt.Errorf("error sending load request: %w", err) + if value, ok := response["contents"]; ok { + contents := value.(string) + result.Contents = &contents } - var nodeResult NodeLoadResult - if err := json.NewDecoder(stdout).Decode(&nodeResult); err != nil { - return api.OnLoadResult{}, fmt.Errorf("error reading load response: %w", err) + if value, ok := response["resolveDir"]; ok { + result.ResolveDir = value.(string) + } + if value, ok := response["pluginData"]; ok { + result.PluginData = value.(int) + } + if value, ok := response["errors"]; ok { + result.Errors = decodeMessages(value.([]interface{})) + } + if value, ok := response["warnings"]; ok { + result.Warnings = decodeMessages(value.([]interface{})) + } + if value, ok := response["watchFiles"]; ok { + result.WatchFiles = decodeStringArray(value.([]interface{})) + } + if value, ok := response["watchDirs"]; ok { + result.WatchDirs = decodeStringArray(value.([]interface{})) + } + return result, nil + }) + build.OnEnd(func(result *api.BuildResult) (api.OnEndResult, error) { + req := map[string]interface{}{ + "errors": encodeMessages(result.Errors), + "warnings": encodeMessages(result.Warnings), } - nodeResult.OnLoadResult.Loader = loaderMap[nodeResult.Loader] - return nodeResult.OnLoadResult, nil + req["outputFiles"] = encodeOutputFiles(result.OutputFiles) + req["metafile"] = result.Metafile + req["mangleCache"] = result.MangleCache + return api.OnEndResult{}, nil }) build.OnDispose(func() { stdin.Close() stdout.Close() - cmd.Process.Kill() + close(requests) + close(responses) + util.TerminateProcess(cmd.Process.Pid) + wg.Wait() }) }, } +} + +func encodeStringArray(strings []string) []interface{} { + values := make([]interface{}, len(strings)) + for i, value := range strings { + values[i] = value + } + return values +} + +func decodeStringArray(values []interface{}) []string { + strings := make([]string, len(values)) + for i, value := range values { + strings[i] = value.(string) + } + return strings +} + +func encodeOutputFiles(outputFiles []api.OutputFile) []interface{} { + values := make([]interface{}, len(outputFiles)) + for i, outputFile := range outputFiles { + values[i] = map[string]interface{}{ + "path": outputFile.Path, + "contents": outputFile.Contents, + "hash": outputFile.Hash, + } + } + return values +} + +func encodeLocation(loc *api.Location) interface{} { + if loc == nil { + return nil + } + return map[string]interface{}{ + "file": loc.File, + "namespace": loc.Namespace, + "line": loc.Line, + "column": loc.Column, + "length": loc.Length, + "lineText": loc.LineText, + "suggestion": loc.Suggestion, + } +} + +func encodeMessages(msgs []api.Message) []interface{} { + values := make([]interface{}, len(msgs)) + for i, msg := range msgs { + value := map[string]interface{}{ + "id": msg.ID, + "pluginName": msg.PluginName, + "text": msg.Text, + "location": encodeLocation(msg.Location), + } + values[i] = value + + notes := make([]interface{}, len(msg.Notes)) + for j, note := range msg.Notes { + notes[j] = map[string]interface{}{ + "text": note.Text, + "location": encodeLocation(note.Location), + } + } + value["notes"] = notes + // Send "-1" to mean "undefined" + detail, ok := msg.Detail.(int) + if !ok { + detail = -1 + } + value["detail"] = detail + } + return values +} + +func decodeLocation(value interface{}) *api.Location { + if value == nil { + return nil + } + loc := value.(map[string]interface{}) + namespace := loc["namespace"].(string) + if namespace == "" { + namespace = "file" + } + return &api.Location{ + File: loc["file"].(string), + Namespace: namespace, + Line: loc["line"].(int), + Column: loc["column"].(int), + Length: loc["length"].(int), + LineText: loc["lineText"].(string), + Suggestion: loc["suggestion"].(string), + } +} + +func decodeMessages(values []interface{}) []api.Message { + msgs := make([]api.Message, len(values)) + for i, value := range values { + obj := value.(map[string]interface{}) + msg := api.Message{ + ID: obj["id"].(string), + PluginName: obj["pluginName"].(string), + Text: obj["text"].(string), + Location: decodeLocation(obj["location"]), + Detail: obj["detail"].(int), + } + for _, note := range obj["notes"].([]interface{}) { + noteObj := note.(map[string]interface{}) + msg.Notes = append(msg.Notes, api.Note{ + Text: noteObj["text"].(string), + Location: decodeLocation(noteObj["location"]), + }) + } + msgs[i] = msg + } + return msgs +} + +func resolveKindToString(kind api.ResolveKind) string { + switch kind { + case api.ResolveEntryPoint: + return "entry-point" + + // JS + case api.ResolveJSImportStatement: + return "import-statement" + case api.ResolveJSRequireCall: + return "require-call" + case api.ResolveJSDynamicImport: + return "dynamic-import" + case api.ResolveJSRequireResolve: + return "require-resolve" + + // CSS + case api.ResolveCSSImportRule: + return "import-rule" + case api.ResolveCSSComposesFrom: + return "composes-from" + case api.ResolveCSSURLToken: + return "url-token" + + default: + panic("Internal error") + } } diff --git a/pkg/runtime/runtime.go b/pkg/runtime/runtime.go index 31784e07b..31e4cac49 100644 --- a/pkg/runtime/runtime.go +++ b/pkg/runtime/runtime.go @@ -2,6 +2,9 @@ package runtime import ( "context" + "crypto/aes" + "crypto/cipher" + "encoding/base64" "encoding/json" "fmt" "io" @@ -25,14 +28,15 @@ type Worker interface { } type BuildInput struct { - CfgPath string - Dev bool `json:"dev"` - FunctionID string `json:"functionID"` - Handler string `json:"handler"` - Runtime string `json:"runtime"` - Properties json.RawMessage `json:"properties"` - Links map[string]json.RawMessage `json:"links"` - CopyFiles []struct { + CfgPath string + Dev bool `json:"dev"` + FunctionID string `json:"functionID"` + Handler string `json:"handler"` + Runtime string `json:"runtime"` + Properties json.RawMessage `json:"properties"` + Links map[string]json.RawMessage `json:"links"` + EncryptionKey string `json:"encryptionKey"` + CopyFiles []struct { From string `json:"from"` To string `json:"to"` } `json:"copyFiles"` @@ -144,6 +148,31 @@ func (c *Collection) Build(ctx context.Context, input *BuildInput) (*BuildOutput } } + if input.EncryptionKey != "" { + key, err := base64.StdEncoding.DecodeString(input.EncryptionKey) + if err != nil { + return nil, err + } + json, err := json.Marshal(input.Links) + if err != nil { + return nil, err + } + block, err := aes.NewCipher(key) + if err != nil { + return nil, err + } + gcm, err := cipher.NewGCM(block) + if err != nil { + return nil, err + } + ciphertext := gcm.Seal(nil, make([]byte, 12), json, nil) + err = os.WriteFile(filepath.Join(result.Out, "resource.enc"), ciphertext, 0644) + os.WriteFile(filepath.Join(result.Out, input.EncryptionKey), ciphertext, 0644) + if err != nil { + return nil, err + } + } + return result, nil } diff --git a/pkg/runtime/worker/worker.go b/pkg/runtime/worker/worker.go index d02128f21..bf4ecdc1e 100644 --- a/pkg/runtime/worker/worker.go +++ b/pkg/runtime/worker/worker.go @@ -97,6 +97,9 @@ func (w *Runtime) Build(ctx context.Context, input *runtime.BuildInput) (*runtim ResolveDir: filepath.Dir(abs), Loader: esbuild.LoaderTS, }, + NodePaths: []string{ + filepath.Join(path.ResolvePlatformDir(input.CfgPath), "node_modules"), + }, Alias: w.unenv.Alias, External: []string{"node:*", "cloudflare:workers"}, Conditions: []string{"workerd", "worker", "browser"}, diff --git a/platform/functions/nodejs-runtime/plugin.mjs b/platform/functions/nodejs-runtime/plugin.mjs index 57745af16..00fb35dca 100644 --- a/platform/functions/nodejs-runtime/plugin.mjs +++ b/platform/functions/nodejs-runtime/plugin.mjs @@ -1,15 +1,33 @@ import { createInterface } from "readline"; import { stdin as input, stdout as output } from "process"; -import fs from "fs/promises"; -// open file and append to it -// create file and open it for writing -const file = await fs.open("out", "w"); +const loaderToString = [ + "none", + "base64", + "binary", + "copy", + "css", + "dataurl", + "default", + "empty", + "file", + "global-css", + "js", + "json", + "json", + "jsx", + "local-css", + "text", + "ts", + "ts", + "tsx", +]; const plugins = await import(process.argv[2]); const onResolve = []; const onLoad = []; +const onEnd = []; const stubAPI = { onResolve(options, callback) { @@ -18,6 +36,9 @@ const stubAPI = { onLoad(options, callback) { onLoad.push({ options, callback }); }, + onEnd(callback) { + onEnd.push(callback); + }, }; for (const plugin of plugins.default) { @@ -27,28 +48,40 @@ for (const plugin of plugins.default) { const rl = createInterface({ input, output, terminal: false }); rl.on("line", async (line) => { - const request = JSON.parse(line); - await file.write("request: " + JSON.stringify(request) + "\n"); - let result; - - if (request.type === "resolve") { - for (const { options, callback } of onResolve) { - if (new RegExp(options.filter).test(request.path)) { - result = callback(request); - if (result) break; + const msg = JSON.parse(line); + + new Promise(async () => { + let reply; + + if (msg.command === "resolve") { + for (const { options, callback } of onResolve) { + if (new RegExp(options.filter).test(msg.path)) { + reply = await callback(msg.value); + if (reply) break; + } } } - } - if (request.type === "load") { - for (const { options, callback } of onLoad) { - if (new RegExp(options.filter).test(request.path)) { - result = callback(request); - if (result) break; + if (msg.command === "load") { + for (const { options, callback } of onLoad) { + if (new RegExp(options.filter).test(msg.path)) { + reply = await callback(msg.value); + if (reply) break; + } + } + } + if (msg.command === "end") { + for (const callback of onEnd) { + reply = await callback(msg.value); } } - } - await file.write("result: " + JSON.stringify(result) + "\n"); - output.write(JSON.stringify(result || {}) + "\n"); + reply = reply || {}; + output.write( + JSON.stringify({ + id: msg.id, + value: reply, + }) + "\n", + ); + }); }); diff --git a/platform/package.json b/platform/package.json index 1f6d1af8d..77ff97a64 100644 --- a/platform/package.json +++ b/platform/package.json @@ -28,7 +28,7 @@ "@pulumi/cloudflare": "5.37.1", "@pulumi/docker-build": "0.0.6", "@pulumi/pulumi": "3.134.1", - "@pulumi/random": "4.15.0", + "@pulumi/random": "^4.16.6", "@pulumi/tls": "5.0.1", "@smithy/smithy-client": "2.1.18", "@tsconfig/node18": "18.2.2", diff --git a/platform/src/components/aws/function.ts b/platform/src/components/aws/function.ts index f7829b5b0..5d6732fe6 100644 --- a/platform/src/components/aws/function.ts +++ b/platform/src/components/aws/function.ts @@ -13,7 +13,6 @@ import { unsecret, secret, } from "@pulumi/pulumi"; -import { buildNode } from "../../runtime/node.js"; import { bootstrap } from "./helpers/bootstrap.js"; import { Duration, DurationMinutes, toSeconds } from "../duration.js"; import { Size, toMBs } from "../size.js"; @@ -39,6 +38,8 @@ import { buildPython, buildPythonContainer } from "../../runtime/python.js"; import { Image } from "@pulumi/docker-build"; import { rpc } from "../rpc/rpc.js"; import { parseRoleArn } from "./helpers/arn.js"; +import { RandomBytes } from "@pulumi/random"; +import { lazy } from "../../util/lazy.js"; /** * Helper type to define function ARN type @@ -651,6 +652,7 @@ export interface FunctionArgs { * cold starts. */ nodejs?: Input<{ + plugins: Input; /** * Configure additional esbuild loaders for other file extensions. This is useful * when your code is importing non-JS files like `.png`, `.css`, etc. @@ -1138,6 +1140,13 @@ export class Function extends Component implements Link.Linkable { private fnUrl: Output; private missingSourcemap?: boolean; + private static readonly encryptionKey = lazy( + () => + new RandomBytes("LambdaEncryptionKey", { + length: 32, + }), + ); + constructor( name: string, args: FunctionArgs, @@ -1187,6 +1196,7 @@ export class Function extends Component implements Link.Linkable { functionID: name, handler: args.handler, bundle: args.bundle, + encryptionKey: Function.encryptionKey().base64, runtime, links: output(linkData).apply((input) => Object.fromEntries(input.map((item) => [item.name, item.properties])), @@ -1269,25 +1279,30 @@ export class Function extends Component implements Link.Linkable { } function normalizeEnvironment() { - return all([args.environment, dev, bootstrapData]).apply( - ([environment, dev, bootstrap]) => { - const result = environment ?? {}; - result.SST_RESOURCE_App = JSON.stringify({ - name: $app.name, - stage: $app.stage, - }); - if (dev) { - result.SST_REGION = process.env.SST_AWS_REGION!; - result.SST_FUNCTION_ID = name; - result.SST_APP = $app.name; - result.SST_STAGE = $app.stage; - result.SST_ASSET_BUCKET = bootstrap.asset; - if (process.env.SST_FUNCTION_TIMEOUT) - result.SST_FUNCTION_TIMEOUT = process.env.SST_FUNCTION_TIMEOUT; - } - return result; - }, - ); + return all([ + args.environment, + dev, + bootstrapData, + Function.encryptionKey().base64, + ]).apply(([environment, dev, bootstrap, key]) => { + const result = environment ?? {}; + result.SST_RESOURCE_App = JSON.stringify({ + name: $app.name, + stage: $app.stage, + }); + result.SST_KEY = key; + result.SST_KEY_FILE = "resource.enc"; + if (dev) { + result.SST_REGION = process.env.SST_AWS_REGION!; + result.SST_FUNCTION_ID = name; + result.SST_APP = $app.name; + result.SST_STAGE = $app.stage; + result.SST_ASSET_BUCKET = bootstrap.asset; + if (process.env.SST_FUNCTION_TIMEOUT) + result.SST_FUNCTION_TIMEOUT = process.env.SST_FUNCTION_TIMEOUT; + } + return result; + }); } function normalizeStreaming() { @@ -1449,24 +1464,6 @@ export class Function extends Component implements Link.Linkable { }; } - if (false) { - const buildResult = buildInput.apply(async (input) => { - const result = await rpc.call<{ - handler: string; - out: string; - errors: string[]; - }>("Runtime.Build", input); - if (result.errors.length > 0) { - throw new Error(result.errors.join("\n")); - } - return result; - }); - return { - handler: buildResult.handler, - out: buildResult.out, - }; - } - if (args.bundle) { return { bundle: output(args.bundle), @@ -1474,21 +1471,17 @@ export class Function extends Component implements Link.Linkable { }; } - const buildResult = all([args, linkData]).apply( - async ([args, linkData]) => { - const result = await buildNode(name, { - ...args, - links: linkData, - }); - if (result.type === "error") { - throw new VisibleError( - `Failed to build function "${args.handler}": ` + - result.errors.join("\n").trim(), - ); - } - return result; - }, - ); + const buildResult = buildInput.apply(async (input) => { + const result = await rpc.call<{ + handler: string; + out: string; + errors: string[]; + }>("Runtime.Build", input); + if (result.errors.length > 0) { + throw new Error(result.errors.join("\n")); + } + return result; + }); return { handler: buildResult.handler, bundle: buildResult.out, diff --git a/sdk/js/src/resource.ts b/sdk/js/src/resource.ts index cb27def62..9a1b50e36 100644 --- a/sdk/js/src/resource.ts +++ b/sdk/js/src/resource.ts @@ -1,4 +1,6 @@ import { env } from "process"; +import { readFileSync } from "fs"; +import crypto from "crypto"; export interface Resource { App: { @@ -23,6 +25,20 @@ for (const [key, value] of Object.entries(environment)) { } } +if (env.SST_KEY_FILE && env.SST_KEY) { + const key = Buffer.from(env.SST_KEY, "base64"); + const encryptedData = readFileSync(env.SST_KEY_FILE); + const nonce = Buffer.alloc(12, 0); + const decipher = crypto.createDecipheriv("aes-256-gcm", key, nonce); + const authTag = encryptedData.subarray(-16); + const actualCiphertext = encryptedData.subarray(0, -16); + decipher.setAuthTag(authTag); + let decrypted = decipher.update(actualCiphertext); + decrypted = Buffer.concat([decrypted, decipher.final()]); + const decryptedData = JSON.parse(decrypted.toString()); + Object.assign(raw, decryptedData); +} + export function fromCloudflareEnv(input: any) { for (let [key, value] of Object.entries(input)) { if (typeof value === "string") {