diff --git a/Cargo.lock b/Cargo.lock index 751a0d4cd0eca..3fbe5fd7ed630 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10069,8 +10069,8 @@ version = "1.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675" dependencies = [ - "cfg-if 1.0.0", - "rand 0.8.5", + "cfg-if 0.1.10", + "rand 0.4.6", "static_assertions", ] diff --git a/cli/internal/ffi/ffi.go b/cli/internal/ffi/ffi.go index 6b478bee94a10..641d6ed79880c 100644 --- a/cli/internal/ffi/ffi.go +++ b/cli/internal/ffi/ffi.go @@ -220,6 +220,8 @@ func toPackageManager(packageManager string) ffi_proto.PackageManager { return ffi_proto.PackageManager_PNPM case "yarn": return ffi_proto.PackageManager_YARN + case "bun": + return ffi_proto.PackageManager_BUN default: panic(fmt.Sprintf("Invalid package manager string: %s", packageManager)) } diff --git a/cli/internal/ffi/proto/messages.pb.go b/cli/internal/ffi/proto/messages.pb.go index 9dac0a04fba62..989c237f4f08c 100644 --- a/cli/internal/ffi/proto/messages.pb.go +++ b/cli/internal/ffi/proto/messages.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.28.1 -// protoc v3.21.12 +// protoc-gen-go v1.31.0 +// protoc v4.23.4 // source: turborepo-ffi/messages.proto package proto @@ -27,6 +27,7 @@ const ( PackageManager_BERRY PackageManager = 1 PackageManager_PNPM PackageManager = 2 PackageManager_YARN PackageManager = 3 + PackageManager_BUN PackageManager = 4 ) // Enum value maps for PackageManager. @@ -36,12 +37,14 @@ var ( 1: "BERRY", 2: "PNPM", 3: "YARN", + 4: "BUN", } PackageManager_value = map[string]int32{ "NPM": 0, "BERRY": 1, "PNPM": 2, "YARN": 3, + "BUN": 4, } ) @@ -196,6 +199,7 @@ type GlobResp struct { unknownFields protoimpl.UnknownFields // Types that are assignable to Response: + // // *GlobResp_Files // *GlobResp_Error Response isGlobResp_Response `protobuf_oneof:"response"` @@ -394,6 +398,7 @@ type ChangedFilesResp struct { unknownFields protoimpl.UnknownFields // Types that are assignable to Response: + // // *ChangedFilesResp_Files // *ChangedFilesResp_Error Response isChangedFilesResp_Response `protobuf_oneof:"response"` @@ -584,6 +589,7 @@ type PreviousContentResp struct { unknownFields protoimpl.UnknownFields // Types that are assignable to Response: + // // *PreviousContentResp_Content // *PreviousContentResp_Error Response isPreviousContentResp_Response `protobuf_oneof:"response"` @@ -884,6 +890,7 @@ type TransitiveDepsResponse struct { unknownFields protoimpl.UnknownFields // Types that are assignable to Response: + // // *TransitiveDepsResponse_Dependencies // *TransitiveDepsResponse_Error Response isTransitiveDepsResponse_Response `protobuf_oneof:"response"` @@ -1200,6 +1207,7 @@ type SubgraphResponse struct { unknownFields protoimpl.UnknownFields // Types that are assignable to Response: + // // *SubgraphResponse_Contents // *SubgraphResponse_Error Response isSubgraphResponse_Response `protobuf_oneof:"response"` @@ -1335,6 +1343,7 @@ type PatchesResponse struct { unknownFields protoimpl.UnknownFields // Types that are assignable to Response: + // // *PatchesResponse_Patches // *PatchesResponse_Error Response isPatchesResponse_Response `protobuf_oneof:"response"` @@ -1753,6 +1762,7 @@ type VerifySignatureResponse struct { unknownFields protoimpl.UnknownFields // Types that are assignable to Response: + // // *VerifySignatureResponse_Verified // *VerifySignatureResponse_Error Response isVerifySignatureResponse_Response `protobuf_oneof:"response"` @@ -1896,6 +1906,7 @@ type GetPackageFileHashesResponse struct { unknownFields protoimpl.UnknownFields // Types that are assignable to Response: + // // *GetPackageFileHashesResponse_Hashes // *GetPackageFileHashesResponse_Error Response isGetPackageFileHashesResponse_Response `protobuf_oneof:"response"` @@ -2039,6 +2050,7 @@ type GetHashesForFilesResponse struct { unknownFields protoimpl.UnknownFields // Types that are assignable to Response: + // // *GetHashesForFilesResponse_Hashes // *GetHashesForFilesResponse_Error Response isGetHashesForFilesResponse_Response `protobuf_oneof:"response"` @@ -2221,6 +2233,7 @@ type FromWildcardsResponse struct { unknownFields protoimpl.UnknownFields // Types that are assignable to Response: + // // *FromWildcardsResponse_EnvVars // *FromWildcardsResponse_Error Response isFromWildcardsResponse_Response `protobuf_oneof:"response"` @@ -2513,6 +2526,7 @@ type GetGlobalHashableEnvVarsResponse struct { unknownFields protoimpl.UnknownFields // Types that are assignable to Response: + // // *GetGlobalHashableEnvVarsResponse_DetailedMap // *GetGlobalHashableEnvVarsResponse_Error Response isGetGlobalHashableEnvVarsResponse_Response `protobuf_oneof:"response"` @@ -2884,11 +2898,12 @@ var file_turborepo_ffi_messages_proto_rawDesc = []byte{ 0x0b, 0x64, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x4d, 0x61, 0x70, 0x12, 0x16, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x42, 0x0a, 0x0a, 0x08, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x2a, 0x38, 0x0a, 0x0e, 0x50, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x4d, 0x61, 0x6e, 0x61, 0x67, + 0x2a, 0x41, 0x0a, 0x0e, 0x50, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x4d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x12, 0x07, 0x0a, 0x03, 0x4e, 0x50, 0x4d, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x42, 0x45, 0x52, 0x52, 0x59, 0x10, 0x01, 0x12, 0x08, 0x0a, 0x04, 0x50, 0x4e, 0x50, 0x4d, 0x10, 0x02, - 0x12, 0x08, 0x0a, 0x04, 0x59, 0x41, 0x52, 0x4e, 0x10, 0x03, 0x42, 0x0b, 0x5a, 0x09, 0x66, 0x66, - 0x69, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x12, 0x08, 0x0a, 0x04, 0x59, 0x41, 0x52, 0x4e, 0x10, 0x03, 0x12, 0x07, 0x0a, 0x03, 0x42, 0x55, + 0x4e, 0x10, 0x04, 0x42, 0x0b, 0x5a, 0x09, 0x66, 0x66, 0x69, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/cli/internal/lockfile/bun_lockfile.go b/cli/internal/lockfile/bun_lockfile.go new file mode 100644 index 0000000000000..058c4b3dd493e --- /dev/null +++ b/cli/internal/lockfile/bun_lockfile.go @@ -0,0 +1,38 @@ +package lockfile + +import ( + "github.com/vercel/turbo/cli/internal/turbopath" +) + +// BunLockfile representation of bun lockfile +type BunLockfile struct { + contents []byte +} + +var _ Lockfile = (*BunLockfile)(nil) + +// ResolvePackage Given a package and version returns the key, resolved version, and if it was found +func (l *BunLockfile) ResolvePackage(_ turbopath.AnchoredUnixPath, _ string, _ string) (Package, error) { + // This is only used when doing calculating the transitive deps, but Rust + // implementations do this calculation on the Rust side. + panic("Unreachable") +} + +// AllDependencies Given a lockfile key return all (dev/optional/peer) dependencies of that package +func (l *BunLockfile) AllDependencies(_ string) (map[string]string, bool) { + // This is only used when doing calculating the transitive deps, but Rust + // implementations do this calculation on the Rust side. + panic("Unreachable") +} + +// DecodeBunLockfile Takes the contents of a bun lockfile and returns a struct representation +func DecodeBunLockfile(contents []byte) (*BunLockfile, error) { + return &BunLockfile{contents: contents}, nil +} + +// GlobalChange checks if there are any differences between lockfiles that would completely invalidate +// the cache. +func (l *BunLockfile) GlobalChange(other Lockfile) bool { + _, ok := other.(*BunLockfile) + return !ok +} diff --git a/cli/internal/lockfile/lockfile.go b/cli/internal/lockfile/lockfile.go index d701801434aa1..9d8a2143ca82e 100644 --- a/cli/internal/lockfile/lockfile.go +++ b/cli/internal/lockfile/lockfile.go @@ -78,6 +78,9 @@ func AllTransitiveClosures( if lf, ok := lockFile.(*YarnLockfile); ok { return rustTransitiveDeps(lf.contents, "yarn", workspaces, nil) } + if lf, ok := lockFile.(*BunLockfile); ok { + return rustTransitiveDeps(lf.contents, "bun", workspaces, nil) + } g := new(errgroup.Group) c := make(chan closureMsg, len(workspaces)) diff --git a/cli/internal/packagemanager/berry.go b/cli/internal/packagemanager/berry.go index 8f69498c02765..668a95ebd4ae3 100644 --- a/cli/internal/packagemanager/berry.go +++ b/cli/internal/packagemanager/berry.go @@ -9,12 +9,14 @@ import ( "github.com/vercel/turbo/cli/internal/turbopath" ) +const berryLockfile = "yarn.lock" + var nodejsBerry = PackageManager{ Name: "nodejs-berry", Slug: "yarn", Command: "yarn", Specfile: "package.json", - Lockfile: "yarn.lock", + Lockfile: berryLockfile, PackageDir: "node_modules", ArgSeparator: func(_userArgs []string) []string { return nil }, @@ -43,6 +45,18 @@ var nodejsBerry = PackageManager{ return true, nil }, + GetLockfileName: func(_ turbopath.AbsoluteSystemPath) string { + return berryLockfile + }, + + GetLockfilePath: func(projectDirectory turbopath.AbsoluteSystemPath) turbopath.AbsoluteSystemPath { + return projectDirectory.UntypedJoin(berryLockfile) + }, + + GetLockfileContents: func(projectDirectory turbopath.AbsoluteSystemPath) ([]byte, error) { + return projectDirectory.UntypedJoin(berryLockfile).ReadFile() + }, + UnmarshalLockfile: func(rootPackageJSON *fs.PackageJSON, contents []byte) (lockfile.Lockfile, error) { var resolutions map[string]string if untypedResolutions, ok := rootPackageJSON.RawJSON["resolutions"]; ok { diff --git a/cli/internal/packagemanager/bun.go b/cli/internal/packagemanager/bun.go new file mode 100644 index 0000000000000..4f908ef2f3d2c --- /dev/null +++ b/cli/internal/packagemanager/bun.go @@ -0,0 +1,81 @@ +package packagemanager + +import ( + "fmt" + "os/exec" + + "github.com/vercel/turbo/cli/internal/fs" + "github.com/vercel/turbo/cli/internal/lockfile" + "github.com/vercel/turbo/cli/internal/turbopath" +) + +const command = "bun" +const bunLockfile = "bun.lockb" + +func getLockfilePath(rootPath turbopath.AbsoluteSystemPath) turbopath.AbsoluteSystemPath { + return rootPath.UntypedJoin(bunLockfile) +} + +var nodejsBun = PackageManager{ + Name: "nodejs-bun", + Slug: "bun", + Command: command, + Specfile: "package.json", + Lockfile: bunLockfile, + PackageDir: "node_modules", + ArgSeparator: func(userArgs []string) []string { + // Bun swallows a single "--" token. If the user is passing "--", we need + // to prepend our own so that the user's doesn't get swallowed. If they are not + // passing their own, we don't need the "--" token and can avoid the warning. + for _, arg := range userArgs { + if arg == "--" { + return []string{"--"} + } + } + return nil + }, + + getWorkspaceGlobs: func(rootpath turbopath.AbsoluteSystemPath) ([]string, error) { + pkg, err := fs.ReadPackageJSON(rootpath.UntypedJoin("package.json")) + if err != nil { + return nil, fmt.Errorf("package.json: %w", err) + } + if len(pkg.Workspaces) == 0 { + return nil, fmt.Errorf("package.json: no workspaces found. Turborepo requires Bun workspaces to be defined in the root package.json") + } + return pkg.Workspaces, nil + }, + + getWorkspaceIgnores: func(pm PackageManager, rootpath turbopath.AbsoluteSystemPath) ([]string, error) { + // Matches upstream values: + // Key code: https://github.com/oven-sh/bun/blob/f267c1d097923a2d2992f9f60a6dd365fe706512/src/install/lockfile.zig#L3057 + return []string{ + "**/node_modules", + "**/.git", + }, nil + }, + + canPrune: func(cwd turbopath.AbsoluteSystemPath) (bool, error) { + return false, nil + }, + + GetLockfileName: func(rootPath turbopath.AbsoluteSystemPath) string { + return bunLockfile + }, + + GetLockfilePath: func(rootPath turbopath.AbsoluteSystemPath) turbopath.AbsoluteSystemPath { + return getLockfilePath(rootPath) + }, + + GetLockfileContents: func(projectDirectory turbopath.AbsoluteSystemPath) ([]byte, error) { + lockfilePath := getLockfilePath(projectDirectory) + cmd := exec.Command(command, lockfilePath.ToString()) + cmd.Dir = projectDirectory.ToString() + + return cmd.Output() + }, + + UnmarshalLockfile: func(_rootPackageJSON *fs.PackageJSON, contents []byte) (lockfile.Lockfile, error) { + return lockfile.DecodeBunLockfile(contents) + }, +} diff --git a/cli/internal/packagemanager/npm.go b/cli/internal/packagemanager/npm.go index 5c8e4d08d469c..8c082b962213b 100644 --- a/cli/internal/packagemanager/npm.go +++ b/cli/internal/packagemanager/npm.go @@ -8,12 +8,14 @@ import ( "github.com/vercel/turbo/cli/internal/turbopath" ) +const npmLockfile = "package-lock.json" + var nodejsNpm = PackageManager{ Name: "nodejs-npm", Slug: "npm", Command: "npm", Specfile: "package.json", - Lockfile: "package-lock.json", + Lockfile: npmLockfile, PackageDir: "node_modules", ArgSeparator: func(_userArgs []string) []string { return []string{"--"} }, @@ -42,6 +44,18 @@ var nodejsNpm = PackageManager{ return true, nil }, + GetLockfileName: func(_ turbopath.AbsoluteSystemPath) string { + return npmLockfile + }, + + GetLockfilePath: func(projectDirectory turbopath.AbsoluteSystemPath) turbopath.AbsoluteSystemPath { + return projectDirectory.UntypedJoin(npmLockfile) + }, + + GetLockfileContents: func(projectDirectory turbopath.AbsoluteSystemPath) ([]byte, error) { + return projectDirectory.UntypedJoin(npmLockfile).ReadFile() + }, + UnmarshalLockfile: func(_rootPackageJSON *fs.PackageJSON, contents []byte) (lockfile.Lockfile, error) { return lockfile.DecodeNpmLockfile(contents) }, diff --git a/cli/internal/packagemanager/packagemanager.go b/cli/internal/packagemanager/packagemanager.go index 70c9533887758..de5c52587889b 100644 --- a/cli/internal/packagemanager/packagemanager.go +++ b/cli/internal/packagemanager/packagemanager.go @@ -51,6 +51,15 @@ type PackageManager struct { // Detect if Turbo knows how to produce a pruned workspace for the project canPrune func(cwd turbopath.AbsoluteSystemPath) (bool, error) + // Gets lockfile name. + GetLockfileName func(projectDirectory turbopath.AbsoluteSystemPath) string + + // Gets lockfile path. + GetLockfilePath func(projectDirectory turbopath.AbsoluteSystemPath) turbopath.AbsoluteSystemPath + + // Read from disk a lockfile for a package manager. + GetLockfileContents func(projectDirectory turbopath.AbsoluteSystemPath) ([]byte, error) + // Read a lockfile for a given package manager UnmarshalLockfile func(rootPackageJSON *fs.PackageJSON, contents []byte) (lockfile.Lockfile, error) @@ -64,6 +73,7 @@ var packageManagers = []PackageManager{ nodejsNpm, nodejsPnpm, nodejsPnpm6, + nodejsBun, } // GetPackageManager reads the package manager name sent by the Rust side @@ -71,6 +81,8 @@ func GetPackageManager(name string) (packageManager *PackageManager, err error) switch name { case "yarn": return &nodejsYarn, nil + case "bun": + return &nodejsBun, nil case "berry": return &nodejsBerry, nil case "npm": @@ -119,13 +131,13 @@ func (pm PackageManager) ReadLockfile(projectDirectory turbopath.AbsoluteSystemP if pm.UnmarshalLockfile == nil { return nil, nil } - contents, err := projectDirectory.UntypedJoin(pm.Lockfile).ReadFile() + contents, err := pm.GetLockfileContents(projectDirectory) if err != nil { - return nil, fmt.Errorf("reading %s: %w", pm.Lockfile, err) + return nil, fmt.Errorf("reading %s: %w", pm.GetLockfilePath(projectDirectory).ToString(), err) } lf, err := pm.UnmarshalLockfile(rootPackageJSON, contents) if err != nil { - return nil, errors.Wrapf(err, "error in %v", pm.Lockfile) + return nil, errors.Wrapf(err, "error in %v", pm.GetLockfilePath(projectDirectory).ToString()) } return lf, nil } diff --git a/cli/internal/packagemanager/packagemanager_test.go b/cli/internal/packagemanager/packagemanager_test.go index 3be3b57cfedff..c22733a6c1413 100644 --- a/cli/internal/packagemanager/packagemanager_test.go +++ b/cli/internal/packagemanager/packagemanager_test.go @@ -26,6 +26,7 @@ func Test_GetWorkspaces(t *testing.T) { repoRoot, err := fs.GetCwd(cwd) assert.NilError(t, err, "GetCwd") rootPath := map[string]turbopath.AbsoluteSystemPath{ + "nodejs-bun": repoRoot.UntypedJoin("../../../examples/with-yarn"), "nodejs-npm": repoRoot.UntypedJoin("../../../examples/with-yarn"), "nodejs-berry": repoRoot.UntypedJoin("../../../examples/with-yarn"), "nodejs-yarn": repoRoot.UntypedJoin("../../../examples/with-yarn"), @@ -34,6 +35,13 @@ func Test_GetWorkspaces(t *testing.T) { } want := map[string][]string{ + "nodejs-bun": { + filepath.ToSlash(filepath.Join(cwd, "../../../examples/with-yarn/apps/docs/package.json")), + filepath.ToSlash(filepath.Join(cwd, "../../../examples/with-yarn/apps/web/package.json")), + filepath.ToSlash(filepath.Join(cwd, "../../../examples/with-yarn/packages/eslint-config-custom/package.json")), + filepath.ToSlash(filepath.Join(cwd, "../../../examples/with-yarn/packages/tsconfig/package.json")), + filepath.ToSlash(filepath.Join(cwd, "../../../examples/with-yarn/packages/ui/package.json")), + }, "nodejs-npm": { filepath.ToSlash(filepath.Join(cwd, "../../../examples/with-yarn/apps/docs/package.json")), filepath.ToSlash(filepath.Join(cwd, "../../../examples/with-yarn/apps/web/package.json")), @@ -117,6 +125,7 @@ func Test_GetWorkspaceIgnores(t *testing.T) { cwd, err := fs.GetCwd(cwdRaw) assert.NilError(t, err, "GetCwd") want := map[string][]string{ + "nodejs-bun": {"**/node_modules", "**/.git"}, "nodejs-npm": {"**/node_modules/**"}, "nodejs-berry": {"**/node_modules", "**/.git", "**/.yarn"}, "nodejs-yarn": {"apps/*/node_modules/**", "packages/*/node_modules/**"}, diff --git a/cli/internal/packagemanager/pnpm.go b/cli/internal/packagemanager/pnpm.go index f1f6069801c89..96d7382592c6c 100644 --- a/cli/internal/packagemanager/pnpm.go +++ b/cli/internal/packagemanager/pnpm.go @@ -12,6 +12,8 @@ import ( "github.com/vercel/turbo/cli/internal/yaml" ) +const pnpmLockfile = "pnpm-lock.yaml" + // PnpmWorkspaces is a representation of workspace package globs found // in pnpm-workspace.yaml type PnpmWorkspaces struct { @@ -79,7 +81,7 @@ var nodejsPnpm = PackageManager{ Slug: "pnpm", Command: "pnpm", Specfile: "package.json", - Lockfile: "pnpm-lock.yaml", + Lockfile: pnpmLockfile, PackageDir: "node_modules", // pnpm v7+ changed their handling of '--'. We no longer need to pass it to pass args to // the script being run, and in fact doing so will cause the '--' to be passed through verbatim, @@ -98,6 +100,18 @@ var nodejsPnpm = PackageManager{ return true, nil }, + GetLockfileName: func(_ turbopath.AbsoluteSystemPath) string { + return pnpmLockfile + }, + + GetLockfilePath: func(projectDirectory turbopath.AbsoluteSystemPath) turbopath.AbsoluteSystemPath { + return projectDirectory.UntypedJoin(pnpmLockfile) + }, + + GetLockfileContents: func(projectDirectory turbopath.AbsoluteSystemPath) ([]byte, error) { + return projectDirectory.UntypedJoin(pnpmLockfile).ReadFile() + }, + UnmarshalLockfile: func(_rootPackageJSON *fs.PackageJSON, contents []byte) (lockfile.Lockfile, error) { return lockfile.DecodePnpmLockfile(contents) }, diff --git a/cli/internal/packagemanager/pnpm6.go b/cli/internal/packagemanager/pnpm6.go index 489962dc11433..d19076910283e 100644 --- a/cli/internal/packagemanager/pnpm6.go +++ b/cli/internal/packagemanager/pnpm6.go @@ -6,6 +6,8 @@ import ( "github.com/vercel/turbo/cli/internal/turbopath" ) +const pnpm6Lockfile = "pnpm-lock.yaml" + // Pnpm6Workspaces is a representation of workspace package globs found // in pnpm-workspace.yaml type Pnpm6Workspaces struct { @@ -17,7 +19,7 @@ var nodejsPnpm6 = PackageManager{ Slug: "pnpm", Command: "pnpm", Specfile: "package.json", - Lockfile: "pnpm-lock.yaml", + Lockfile: pnpm6Lockfile, PackageDir: "node_modules", ArgSeparator: func(_userArgs []string) []string { return []string{"--"} }, WorkspaceConfigurationPath: "pnpm-workspace.yaml", @@ -30,6 +32,18 @@ var nodejsPnpm6 = PackageManager{ return true, nil }, + GetLockfileName: func(_ turbopath.AbsoluteSystemPath) string { + return pnpm6Lockfile + }, + + GetLockfilePath: func(projectDirectory turbopath.AbsoluteSystemPath) turbopath.AbsoluteSystemPath { + return projectDirectory.UntypedJoin(pnpm6Lockfile) + }, + + GetLockfileContents: func(projectDirectory turbopath.AbsoluteSystemPath) ([]byte, error) { + return projectDirectory.UntypedJoin(pnpm6Lockfile).ReadFile() + }, + UnmarshalLockfile: func(_rootPackageJSON *fs.PackageJSON, contents []byte) (lockfile.Lockfile, error) { return lockfile.DecodePnpmLockfile(contents) }, diff --git a/cli/internal/packagemanager/yarn.go b/cli/internal/packagemanager/yarn.go index 4044b21eb903a..53241b1194a10 100644 --- a/cli/internal/packagemanager/yarn.go +++ b/cli/internal/packagemanager/yarn.go @@ -17,12 +17,14 @@ func (e *NoWorkspacesFoundError) Error() string { return "package.json: no workspaces found. Turborepo requires Yarn workspaces to be defined in the root package.json" } +const yarnLockfile = "yarn.lock" + var nodejsYarn = PackageManager{ Name: "nodejs-yarn", Slug: "yarn", Command: "yarn", Specfile: "package.json", - Lockfile: "yarn.lock", + Lockfile: yarnLockfile, PackageDir: "node_modules", ArgSeparator: func(userArgs []string) []string { // Yarn warns and swallows a "--" token. If the user is passing "--", we need @@ -79,6 +81,18 @@ var nodejsYarn = PackageManager{ return true, nil }, + GetLockfileName: func(_ turbopath.AbsoluteSystemPath) string { + return yarnLockfile + }, + + GetLockfilePath: func(projectDirectory turbopath.AbsoluteSystemPath) turbopath.AbsoluteSystemPath { + return projectDirectory.UntypedJoin(yarnLockfile) + }, + + GetLockfileContents: func(projectDirectory turbopath.AbsoluteSystemPath) ([]byte, error) { + return projectDirectory.UntypedJoin(yarnLockfile).ReadFile() + }, + UnmarshalLockfile: func(_rootPackageJSON *fs.PackageJSON, contents []byte) (lockfile.Lockfile, error) { return lockfile.DecodeYarnLockfile(contents) }, diff --git a/cli/internal/run/global_hash.go b/cli/internal/run/global_hash.go index 32a9eb4baa5f8..55173796f0523 100644 --- a/cli/internal/run/global_hash.go +++ b/cli/internal/run/global_hash.go @@ -121,8 +121,9 @@ func getGlobalHashInputs( if lockFile == nil { // If we don't have lockfile information available, add the specfile and lockfile to global deps globalDeps.Add(filepath.Join(rootpath.ToStringDuringMigration(), packageManager.Specfile)) - if rootpath.UntypedJoin(packageManager.Lockfile).Exists() { - globalDeps.Add(filepath.Join(rootpath.ToStringDuringMigration(), packageManager.Lockfile)) + lockfilePath := packageManager.GetLockfilePath(rootpath) + if lockfilePath.Exists() { + globalDeps.Add(lockfilePath.ToString()) } } diff --git a/cli/internal/scope/scope.go b/cli/internal/scope/scope.go index 3615f9e1e142d..1804c7df78b8b 100644 --- a/cli/internal/scope/scope.go +++ b/cli/internal/scope/scope.go @@ -205,7 +205,7 @@ func calculateInference(repoRoot turbopath.AbsoluteSystemPath, pkgInferencePath }, nil } -func (o *Opts) getPackageChangeFunc(scm scm.SCM, cwd turbopath.AbsoluteSystemPath, ctx *context.Context) scope_filter.PackagesChangedInRange { +func (o *Opts) getPackageChangeFunc(scm scm.SCM, repoRoot turbopath.AbsoluteSystemPath, ctx *context.Context) scope_filter.PackagesChangedInRange { return func(fromRef string, toRef string) (util.Set, error) { // We could filter changed files at the git level, since it's possible // that the changes we're interested in are scoped, but we need to handle @@ -213,7 +213,7 @@ func (o *Opts) getPackageChangeFunc(scm scm.SCM, cwd turbopath.AbsoluteSystemPat // scope changed files more deeply if we know there are no global dependencies. var changedFiles []string if fromRef != "" { - scmChangedFiles, err := scm.ChangedFiles(fromRef, toRef, cwd.ToStringDuringMigration()) + scmChangedFiles, err := scm.ChangedFiles(fromRef, toRef, repoRoot.ToStringDuringMigration()) if err != nil { return nil, err } @@ -239,7 +239,7 @@ func (o *Opts) getPackageChangeFunc(scm scm.SCM, cwd turbopath.AbsoluteSystemPat } changedPkgs := getChangedPackages(filteredChangedFiles, ctx.WorkspaceInfos) - if lockfileChanges, fullChanges := getChangesFromLockfile(scm, ctx, changedFiles, fromRef); !fullChanges { + if lockfileChanges, fullChanges := getChangesFromLockfile(repoRoot, scm, ctx, changedFiles, fromRef); !fullChanges { for _, pkg := range lockfileChanges { changedPkgs.Add(pkg) } @@ -251,8 +251,8 @@ func (o *Opts) getPackageChangeFunc(scm scm.SCM, cwd turbopath.AbsoluteSystemPat } } -func getChangesFromLockfile(scm scm.SCM, ctx *context.Context, changedFiles []string, fromRef string) ([]string, bool) { - lockfileFilter, err := filter.Compile([]string{ctx.PackageManager.Lockfile}) +func getChangesFromLockfile(repoRoot turbopath.AbsoluteSystemPath, scm scm.SCM, ctx *context.Context, changedFiles []string, fromRef string) ([]string, bool) { + lockfileFilter, err := filter.Compile([]string{ctx.PackageManager.GetLockfileName(repoRoot)}) if err != nil { panic(fmt.Sprintf("Lockfile is invalid glob: %v", err)) } @@ -271,7 +271,8 @@ func getChangesFromLockfile(scm scm.SCM, ctx *context.Context, changedFiles []st return nil, true } - prevContents, err := scm.PreviousContent(fromRef, ctx.PackageManager.Lockfile) + // FIXME: If you move your bun lockfile then we don't track that move into the history. + prevContents, err := scm.PreviousContent(fromRef, ctx.PackageManager.GetLockfileName(repoRoot)) if err != nil { // unable to reconstruct old lockfile, assume everything changed return nil, true diff --git a/cli/internal/scope/scope_test.go b/cli/internal/scope/scope_test.go index ba03a68bf457c..5f0525b244b92 100644 --- a/cli/internal/scope/scope_test.go +++ b/cli/internal/scope/scope_test.go @@ -530,7 +530,7 @@ func TestResolvePackages(t *testing.T) { }, root, scm, &context.Context{ WorkspaceInfos: workspaceInfos, WorkspaceNames: packageNames, - PackageManager: &packagemanager.PackageManager{Lockfile: tc.lockfile, UnmarshalLockfile: readLockfile}, + PackageManager: &packagemanager.PackageManager{Lockfile: tc.lockfile, UnmarshalLockfile: readLockfile, GetLockfileName: func(_ turbopath.AbsoluteSystemPath) string { return tc.lockfile }}, WorkspaceGraph: graph, RootNode: "root", Lockfile: tc.currLockfile, diff --git a/crates/turborepo-ffi/messages.proto b/crates/turborepo-ffi/messages.proto index 890ca5b0ac30c..de27736c5f3d5 100644 --- a/crates/turborepo-ffi/messages.proto +++ b/crates/turborepo-ffi/messages.proto @@ -59,6 +59,7 @@ enum PackageManager { BERRY = 1; PNPM = 2; YARN = 3; + BUN = 4; } message PackageDependency { diff --git a/crates/turborepo-ffi/src/lockfile.rs b/crates/turborepo-ffi/src/lockfile.rs index 9f1600f1dd498..42f340f208883 100644 --- a/crates/turborepo-ffi/src/lockfile.rs +++ b/crates/turborepo-ffi/src/lockfile.rs @@ -5,7 +5,8 @@ use std::{ use thiserror::Error; use turborepo_lockfiles::{ - self, BerryLockfile, Lockfile, LockfileData, NpmLockfile, Package, PnpmLockfile, Yarn1Lockfile, + self, BerryLockfile, BunLockfile, Lockfile, LockfileData, NpmLockfile, Package, PnpmLockfile, + Yarn1Lockfile, }; use super::{proto, Buffer}; @@ -54,6 +55,7 @@ fn transitive_closure_inner(buf: Buffer) -> Result berry_transitive_closure_inner(request), proto::PackageManager::Pnpm => pnpm_transitive_closure_inner(request), proto::PackageManager::Yarn => yarn_transitive_closure_inner(request), + proto::PackageManager::Bun => bun_transitive_closure_inner(request), } } @@ -126,6 +128,23 @@ fn yarn_transitive_closure_inner( Ok(dependencies.into()) } +fn bun_transitive_closure_inner( + request: proto::TransitiveDepsRequest, +) -> Result { + let proto::TransitiveDepsRequest { + contents, + workspaces, + .. + } = request; + let lockfile = + BunLockfile::from_bytes(contents.as_slice()).map_err(turborepo_lockfiles::Error::from)?; + let dependencies = turborepo_lockfiles::all_transitive_closures( + &lockfile, + workspaces.into_iter().map(|(k, v)| (k, v.into())).collect(), + )?; + Ok(dependencies.into()) +} + #[no_mangle] pub extern "C" fn subgraph(buf: Buffer) -> Buffer { use proto::subgraph_response::Response; @@ -162,6 +181,9 @@ fn subgraph_inner(buf: Buffer) -> Result, Error> { turborepo_lockfiles::pnpm_subgraph(&contents, &workspaces, &packages)? } proto::PackageManager::Yarn => turborepo_lockfiles::yarn_subgraph(&contents, &packages)?, + proto::PackageManager::Bun => { + return Err(Error::UnsupportedPackageManager(proto::PackageManager::Bun)) + } }; Ok(contents) } @@ -227,6 +249,7 @@ fn global_change_inner(buf: Buffer) -> Result { &request.curr_contents, )?), proto::PackageManager::Yarn => Ok(false), + proto::PackageManager::Bun => Ok(false), } } @@ -271,6 +294,7 @@ impl fmt::Display for proto::PackageManager { proto::PackageManager::Berry => "berry", proto::PackageManager::Pnpm => "pnpm", proto::PackageManager::Yarn => "yarn", + proto::PackageManager::Bun => "bun", }) } } diff --git a/crates/turborepo-lib/src/commands/prune.rs b/crates/turborepo-lib/src/commands/prune.rs index f15dc23d0bda6..f4e8c7970645b 100644 --- a/crates/turborepo-lib/src/commands/prune.rs +++ b/crates/turborepo-lib/src/commands/prune.rs @@ -40,6 +40,8 @@ pub enum Error { MissingWorkspace(WorkspaceName), #[error("Cannot prune without parsed lockfile")] MissingLockfile, + #[error("Prune is not supported for Bun")] + BunUnsupported, } // Files that should be copied from root and if they're required for install @@ -71,6 +73,13 @@ pub fn prune( ) -> Result<(), Error> { let prune = Prune::new(base, scope, docker, output_dir)?; + if matches!( + prune.package_graph.package_manager(), + crate::package_manager::PackageManager::Bun + ) { + return Err(Error::BunUnsupported); + } + println!( "Generating pruned monorepo for {} in {}", base.ui.apply(BOLD.apply_to(scope.join(", "))), diff --git a/crates/turborepo-lib/src/package_manager/bun.rs b/crates/turborepo-lib/src/package_manager/bun.rs new file mode 100644 index 0000000000000..4ed64cb207d46 --- /dev/null +++ b/crates/turborepo-lib/src/package_manager/bun.rs @@ -0,0 +1,63 @@ +use turbopath::AbsoluteSystemPath; + +use crate::package_manager::{Error, PackageManager}; + +pub const LOCKFILE: &str = "bun.lockb"; + +pub struct BunDetector<'a> { + repo_root: &'a AbsoluteSystemPath, + found: bool, +} + +impl<'a> BunDetector<'a> { + pub fn new(repo_root: &'a AbsoluteSystemPath) -> Self { + Self { + repo_root, + found: false, + } + } +} + +impl<'a> Iterator for BunDetector<'a> { + type Item = Result; + + fn next(&mut self) -> Option { + if self.found { + return None; + } + + self.found = true; + let package_json = self.repo_root.join_component(LOCKFILE); + + if package_json.exists() { + Some(Ok(PackageManager::Bun)) + } else { + None + } + } +} + +#[cfg(test)] +mod tests { + use std::fs::File; + + use anyhow::Result; + use tempfile::tempdir; + use turbopath::AbsoluteSystemPathBuf; + + use super::LOCKFILE; + use crate::package_manager::PackageManager; + + #[test] + fn test_detect_bun() -> Result<()> { + let repo_root = tempdir()?; + let repo_root_path = AbsoluteSystemPathBuf::try_from(repo_root.path())?; + + let lockfile_path = repo_root.path().join(LOCKFILE); + File::create(lockfile_path)?; + let package_manager = PackageManager::detect_package_manager(&repo_root_path)?; + assert_eq!(package_manager, PackageManager::Bun); + + Ok(()) + } +} diff --git a/crates/turborepo-lib/src/package_manager/mod.rs b/crates/turborepo-lib/src/package_manager/mod.rs index d9f65e7a1db96..0cc9d7a43dbc5 100644 --- a/crates/turborepo-lib/src/package_manager/mod.rs +++ b/crates/turborepo-lib/src/package_manager/mod.rs @@ -1,3 +1,4 @@ +mod bun; mod npm; mod pnpm; mod yarn; @@ -6,6 +7,7 @@ use std::{ backtrace, fmt::{self, Display}, fs, + process::Command, }; use globwalk::fix_glob_pattern; @@ -18,10 +20,11 @@ use turbopath::{AbsoluteSystemPath, AbsoluteSystemPathBuf, RelativeUnixPath}; use turborepo_lockfiles::Lockfile; use turborepo_ui::{UI, UNDERLINE}; use wax::{Any, Glob, Pattern}; +use which::which; use crate::{ package_json::PackageJson, - package_manager::{npm::NpmDetector, pnpm::PnpmDetector, yarn::YarnDetector}, + package_manager::{bun::BunDetector, npm::NpmDetector, pnpm::PnpmDetector, yarn::YarnDetector}, }; #[derive(Debug, Deserialize)] @@ -67,6 +70,7 @@ pub enum PackageManager { Pnpm, Pnpm6, Yarn, + Bun, } impl Display for PackageManager { @@ -79,6 +83,7 @@ impl Display for PackageManager { PackageManager::Pnpm => write!(f, "pnpm"), PackageManager::Pnpm6 => write!(f, "pnpm6"), PackageManager::Yarn => write!(f, "yarn"), + PackageManager::Bun => write!(f, "bun"), } } } @@ -218,6 +223,10 @@ impl Display for MissingWorkspaceError { "package.json: no workspaces found. Turborepo requires npm workspaces to be \ defined in the root package.json" } + PackageManager::Bun => { + "package.json: no workspaces found. Turborepo requires bun workspaces to be \ + defined in the root package.json" + } }; write!(f, "{}", err) } @@ -277,7 +286,7 @@ pub enum Error { } static PACKAGE_MANAGER_PATTERN: Lazy = - lazy_regex!(r"(?Pnpm|pnpm|yarn)@(?P\d+\.\d+\.\d+(-.+)?)"); + lazy_regex!(r"(?Pbun|npm|pnpm|yarn)@(?P\d+\.\d+\.\d+(-.+)?)"); impl PackageManager { /// Returns the set of globs for the workspace. @@ -304,6 +313,7 @@ impl PackageManager { ["**/node_modules/**", "**/bower_components/**"].as_slice() } PackageManager::Npm => ["**/node_modules/**"].as_slice(), + PackageManager::Bun => ["**/node_modules", "**/.git"].as_slice(), PackageManager::Berry => ["**/node_modules", "**/.git", "**/.yarn"].as_slice(), PackageManager::Yarn => [].as_slice(), // yarn does its own handling above }; @@ -325,7 +335,10 @@ impl PackageManager { pnpm_workspace.packages } } - PackageManager::Berry | PackageManager::Npm | PackageManager::Yarn => { + PackageManager::Berry + | PackageManager::Npm + | PackageManager::Yarn + | PackageManager::Bun => { let package_json_text = fs::read_to_string(root_path.join_component("package.json"))?; let package_json: PackageJsonWorkspaces = serde_json::from_str(&package_json_text)?; @@ -374,6 +387,7 @@ impl PackageManager { let version = version.parse()?; let manager = match manager { "npm" => Some(PackageManager::Npm), + "bun" => Some(PackageManager::Bun), "yarn" => Some(YarnDetector::detect_berry_or_yarn(&version)?), "pnpm" => Some(PnpmDetector::detect_pnpm6_or_pnpm(&version)?), _ => None, @@ -386,6 +400,7 @@ impl PackageManager { let mut detected_package_managers = PnpmDetector::new(repo_root) .chain(NpmDetector::new(repo_root)) .chain(YarnDetector::new(repo_root)) + .chain(BunDetector::new(repo_root)) .collect::, Error>>()?; match detected_package_managers.len() { @@ -433,6 +448,7 @@ impl PackageManager { pub fn lockfile_name(&self) -> &'static str { match self { PackageManager::Npm => npm::LOCKFILE, + PackageManager::Bun => bun::LOCKFILE, PackageManager::Pnpm | PackageManager::Pnpm6 => pnpm::LOCKFILE, PackageManager::Yarn | PackageManager::Berry => yarn::LOCKFILE, } @@ -441,7 +457,10 @@ impl PackageManager { pub fn workspace_configuration_path(&self) -> Option<&'static str> { match self { PackageManager::Pnpm | PackageManager::Pnpm6 => Some("pnpm-workspace.yaml"), - PackageManager::Npm | PackageManager::Berry | PackageManager::Yarn => None, + PackageManager::Npm + | PackageManager::Berry + | PackageManager::Yarn + | PackageManager::Bun => None, } } @@ -450,7 +469,17 @@ impl PackageManager { root_path: &AbsoluteSystemPath, root_package_json: &PackageJson, ) -> Result, Error> { - let contents = root_path.join_component(self.lockfile_name()).read()?; + let lockfile_path = self.lockfile_path(root_path); + let contents = match self { + PackageManager::Bun => { + Command::new(which("bun")?) + .arg(lockfile_path.to_string()) + .current_dir(root_path.to_string()) + .output()? + .stdout + } + _ => lockfile_path.read()?, + }; self.parse_lockfile(root_package_json, &contents) } @@ -467,6 +496,9 @@ impl PackageManager { PackageManager::Yarn => { Box::new(turborepo_lockfiles::Yarn1Lockfile::from_bytes(contents)?) } + PackageManager::Bun => { + Box::new(turborepo_lockfiles::BunLockfile::from_bytes(contents)?) + } PackageManager::Berry => Box::new(turborepo_lockfiles::BerryLockfile::load( contents, Some(turborepo_lockfiles::BerryManifest::with_resolutions( @@ -490,8 +522,8 @@ impl PackageManager { PackageManager::Pnpm6 | PackageManager::Pnpm => { pnpm::prune_patches(package_json, patches) } - PackageManager::Yarn | PackageManager::Npm => { - unreachable!("npm and yarn 1 don't have a concept of patches") + PackageManager::Yarn | PackageManager::Npm | PackageManager::Bun => { + unreachable!("bun, npm, and yarn 1 don't have a concept of patches") } } } @@ -545,6 +577,7 @@ mod tests { PackageManager::Berry, PackageManager::Yarn, PackageManager::Npm, + PackageManager::Bun, ] { let found = mgr.get_package_jsons(&with_yarn).unwrap(); let found: HashSet = HashSet::from_iter(found); @@ -588,6 +621,7 @@ mod tests { let expected: &[&str] = match mgr { PackageManager::Npm => &["**/node_modules/**"], PackageManager::Berry => &["**/node_modules", "**/.git", "**/.yarn"], + PackageManager::Bun => &["**/node_modules", "**/.git"], PackageManager::Yarn => &["apps/*/node_modules/**", "packages/*/node_modules/**"], PackageManager::Pnpm | PackageManager::Pnpm6 => &[ "**/node_modules/**", @@ -667,6 +701,13 @@ mod tests { expected_version: "111.0.1".to_owned(), expected_error: false, }, + TestCase { + name: "supports bun".to_owned(), + package_manager: "bun@1.0.1".to_owned(), + expected_manager: "bun".to_owned(), + expected_version: "1.0.1".to_owned(), + expected_error: false, + }, ]; for case in tests { @@ -706,6 +747,10 @@ mod tests { let package_manager = PackageManager::read_package_manager(&package_json)?; assert_eq!(package_manager, Some(PackageManager::Pnpm)); + package_json.package_manager = Some("bun@1.0.1".to_string()); + let package_manager = PackageManager::read_package_manager(&package_json)?; + assert_eq!(package_manager, Some(PackageManager::Bun)); + Ok(()) } diff --git a/crates/turborepo-lib/src/shim.rs b/crates/turborepo-lib/src/shim.rs index c541ba8b97022..6e045fa120f75 100644 --- a/crates/turborepo-lib/src/shim.rs +++ b/crates/turborepo-lib/src/shim.rs @@ -299,6 +299,7 @@ pub struct LocalTurboState { impl LocalTurboState { // Hoisted strategy: + // - `bun install` // - `npm install` // - `yarn` // - `yarn install --flat` diff --git a/crates/turborepo-lockfiles/src/bun/de.rs b/crates/turborepo-lockfiles/src/bun/de.rs new file mode 100644 index 0000000000000..b0da7d08e3089 --- /dev/null +++ b/crates/turborepo-lockfiles/src/bun/de.rs @@ -0,0 +1,317 @@ +use std::sync::OnceLock; + +use nom::{ + branch::alt, + bytes::complete::{escaped_transform, is_not, tag, take_till}, + character::complete::{anychar, char as nom_char, crlf, newline, none_of, satisfy, space1}, + combinator::{all_consuming, map, not, opt, peek, recognize, value}, + multi::{count, many0, many1}, + sequence::{delimited, pair, preceded, separated_pair, terminated, tuple}, + IResult, +}; +use regex::Regex; +use serde_json::Value; + +// regex for trimming spaces from start and end +fn pseudostring_replace() -> &'static Regex { + static RE: OnceLock = OnceLock::new(); + RE.get_or_init(|| Regex::new(r"^ *| *$").unwrap()) +} + +pub fn parse_syml(input: &str) -> Result { + match all_consuming(property_statements(0))(input) { + Ok((_, value)) => Ok(value), + Err(e) => Err(super::Error::SymlParse(e.to_string())), + } +} + +// Array and map types +fn item_statements(level: usize) -> impl Fn(&str) -> IResult<&str, Value> { + move |i: &str| map(many0(item_statement(level)), Value::Array)(i) +} + +fn item_statement(level: usize) -> impl Fn(&str) -> IResult<&str, Value> { + move |i: &str| { + let (i, _) = indent(level)(i)?; + let (i, _) = nom_char('-')(i)?; + let (i, _) = blankspace(i)?; + expression(level)(i) + } +} + +fn property_statements(level: usize) -> impl Fn(&str) -> IResult<&str, Value> { + move |i: &str| { + let (i, properties) = many0(property_statement(level))(i)?; + let mut map = serde_json::Map::new(); + for (key, value) in properties.into_iter().flatten() { + map.insert(key, value); + } + Ok((i, Value::Object(map))) + } +} + +fn property_statement(level: usize) -> impl Fn(&str) -> IResult<&str, Vec<(String, Value)>> { + move |i: &str| { + alt(( + value( + vec![], + tuple(( + opt(blankspace), + opt(pair(nom_char('#'), many1(pair(not(eol), anychar)))), + many1(eol_any), + )), + ), + map( + preceded( + indent(level), + separated_pair(name, wrapped_colon, expression(level)), + ), + |entry| vec![entry], + ), + // legacy names + map( + preceded( + indent(level), + separated_pair(legacy_name, wrapped_colon, expression(level)), + ), + |entry| vec![entry], + ), + // legacy prop without colon + map( + preceded( + indent(level), + separated_pair( + legacy_name, + blankspace, + terminated(legacy_literal, many1(eol_any)), + ), + ), + |entry| vec![entry], + ), + multikey_property_statement(level), + ))(i) + } +} + +fn multikey_property_statement( + level: usize, +) -> impl Fn(&str) -> IResult<&str, Vec<(String, Value)>> { + move |i: &str| { + let (i, ()) = indent(level)(i)?; + let (i, property) = legacy_name(i)?; + let (i, others) = many1(preceded( + delimited(opt(blankspace), nom_char(','), opt(blankspace)), + legacy_name, + ))(i)?; + let (i, _) = wrapped_colon(i)?; + let (i, value) = expression(level)(i)?; + + Ok(( + i, + std::iter::once(property) + .chain(others) + .map(|key| (key, value.clone())) + .collect(), + )) + } +} + +fn wrapped_colon(i: &str) -> IResult<&str, char> { + delimited(opt(blankspace), nom_char(':'), opt(blankspace))(i) +} + +fn expression(level: usize) -> impl Fn(&str) -> IResult<&str, Value> { + move |i: &str| { + alt(( + preceded( + tuple(( + peek(tuple((eol, indent(level + 1), nom_char('-'), blankspace))), + eol_any, + )), + item_statements(level + 1), + ), + preceded(eol, property_statements(level + 1)), + terminated(literal, many1(eol_any)), + ))(i) + } +} + +fn indent(level: usize) -> impl Fn(&str) -> IResult<&str, ()> { + move |i: &str| { + let (i, _) = count(nom_char(' '), level * 2)(i)?; + Ok((i, ())) + } +} + +// Simple types + +fn name(i: &str) -> IResult<&str, String> { + alt((string, pseudostring))(i) +} + +fn legacy_name(i: &str) -> IResult<&str, String> { + alt(( + string, + map(recognize(many1(pseudostring_legacy)), |s| s.to_string()), + ))(i) +} + +fn literal(i: &str) -> IResult<&str, Value> { + alt(( + value(Value::Null, null), + map(boolean, Value::Bool), + map(string, Value::String), + map(pseudostring, Value::String), + ))(i) +} + +fn legacy_literal(i: &str) -> IResult<&str, Value> { + alt(( + value(Value::Null, null), + map(string, Value::String), + map(pseudostring_legacy, Value::String), + ))(i) +} + +fn pseudostring(i: &str) -> IResult<&str, String> { + let (i, pseudo) = recognize(pseudostring_inner)(i)?; + Ok(( + i, + pseudostring_replace().replace_all(pseudo, "").into_owned(), + )) +} + +fn pseudostring_inner(i: &str) -> IResult<&str, ()> { + let (i, _) = none_of("\r\n\t ?:,][{}#&*!|>'\"%@`-")(i)?; + let (i, _) = many0(tuple((opt(blankspace), none_of("\r\n\t ,][{}:#\"'"))))(i)?; + Ok((i, ())) +} + +fn pseudostring_legacy(i: &str) -> IResult<&str, String> { + let (i, pseudo) = recognize(pseudostring_legacy_inner)(i)?; + let replaced = pseudostring_replace().replace_all(pseudo, ""); + Ok((i, replaced.to_string())) +} + +fn pseudostring_legacy_inner(i: &str) -> IResult<&str, ()> { + let (i, _) = opt(tag("--"))(i)?; + let (i, _) = satisfy(|c| c.is_ascii_alphanumeric() || c == '/')(i)?; + let (i, _) = take_till(|c| "\r\n\t :,".contains(c))(i)?; + Ok((i, ())) +} + +// String parsing + +fn null(i: &str) -> IResult<&str, &str> { + tag("null")(i) +} + +fn boolean(i: &str) -> IResult<&str, bool> { + alt((value(true, tag("true")), value(false, tag("false"))))(i) +} + +fn string(i: &str) -> IResult<&str, String> { + alt((empty_string, delimited(tag("\""), syml_chars, tag("\""))))(i) +} + +fn empty_string(i: &str) -> IResult<&str, String> { + let (i, _) = tag(r#""""#)(i)?; + Ok((i, "".to_string())) +} + +fn syml_chars(i: &str) -> IResult<&str, String> { + // The SYML grammar provided by Yarn2+ includes escape sequences that weren't + // supported by the yarn1 parser. We diverge from the Yarn2+ provided + // grammar to match the actual parser used by yarn1. + escaped_transform( + is_not("\"\\"), + '\\', + alt(( + value("\"", tag("\"")), + value("\\", tag("\\")), + value("/", tag("/")), + value("\n", tag("n")), + value("\r", tag("r")), + value("\t", tag("t")), + )), + )(i) +} + +// Spaces +fn blankspace(i: &str) -> IResult<&str, &str> { + space1(i) +} + +fn eol_any(i: &str) -> IResult<&str, &str> { + recognize(tuple((eol, many0(tuple((opt(blankspace), eol))))))(i) +} + +fn eol(i: &str) -> IResult<&str, &str> { + alt((crlf, value("\n", newline), value("\r", nom_char('\r'))))(i) +} + +#[cfg(test)] +mod test { + use serde_json::json; + use test_case::test_case; + + use super::*; + + #[test_case("null", Value::Null ; "null")] + #[test_case("false", Value::Bool(false) ; "literal false")] + #[test_case("true", Value::Bool(true) ; "literal true")] + #[test_case("\"\"", Value::String("".into()) ; "empty string literal")] + #[test_case("\"foo\"", Value::String("foo".into()) ; "quoted string literal")] + #[test_case("foo", Value::String("foo".into()) ; "unquoted string literal")] + fn test_literal(input: &str, expected: Value) { + let (_, actual) = literal(input).unwrap(); + assert_eq!(actual, expected); + } + + #[test_case("name: foo", "name" ; "basic")] + #[test_case("technically a name: foo", "technically a name" ; "multiword name")] + fn test_name(input: &str, expected: &str) { + let (_, actual) = name(input).unwrap(); + assert_eq!(actual, expected); + } + + #[test_case("foo@1:", "foo@1" ; "name with colon terminator")] + #[test_case("\"foo@1\":", "foo@1" ; "qutoed name with colon terminator")] + #[test_case("name foo", "name" ; "name without colon terminator")] + fn test_legacy_name(input: &str, expected: &str) { + let (_, actual) = legacy_name(input).unwrap(); + assert_eq!(actual, expected); + } + + #[test_case("null\n", Value::Null ; "null")] + #[test_case("\"foo\"\n", json!("foo") ; "basic string")] + #[test_case("\n name: foo\n", json!({ "name": "foo" }) ; "basic object")] + fn test_expression(input: &str, expected: Value) { + let (_, actual) = expression(0)(input).unwrap(); + assert_eq!(actual, expected); + } + + #[test_case("# a comment\n", vec![] ; "comment")] + #[test_case("foo: null\n", vec![("foo".into(), Value::Null)] ; "single property")] + #[test_case("name foo\n", vec![("name".into(), json!("foo"))] ; "legacy property")] + fn test_property_statement(input: &str, expected: Vec<(String, Value)>) { + let (_, actual) = property_statement(0)(input).unwrap(); + assert_eq!(actual, expected); + } + + #[test_case("name: foo\n", json!({"name": "foo"}) ; "single property object")] + #[test_case("\"name\": foo\n", json!({"name": "foo"}) ; "single quoted property object")] + #[test_case("name foo\n", json!({"name": "foo"}) ; "single property without colon object")] + #[test_case("# comment\nname: foo\n", json!({"name": "foo"}) ; "comment doesn't affect object")] + #[test_case("name foo\nversion \"1.2.3\"\n", json!({"name": "foo", "version": "1.2.3"}) ; "multi-property object")] + #[test_case("foo:\n version \"1.2.3\"\n", json!({"foo": {"version": "1.2.3"}}) ; "nested object")] + #[test_case("foo, bar, baz:\n version \"1.2.3\"\n", json!({ + "foo": {"version": "1.2.3"}, + "bar": {"version": "1.2.3"}, + "baz": {"version": "1.2.3"}, + }) ; "multi-key object")] + fn test_property_statements(input: &str, expected: Value) { + let (_, actual) = property_statements(0)(input).unwrap(); + assert_eq!(actual, expected); + } +} diff --git a/crates/turborepo-lockfiles/src/bun/mod.rs b/crates/turborepo-lockfiles/src/bun/mod.rs new file mode 100644 index 0000000000000..e7624b34f2f0c --- /dev/null +++ b/crates/turborepo-lockfiles/src/bun/mod.rs @@ -0,0 +1,156 @@ +use std::str::FromStr; + +use serde::Deserialize; + +use crate::Lockfile; + +mod de; + +type Map = std::collections::BTreeMap; + +#[derive(Debug, thiserror::Error)] +pub enum Error { + #[error("unable to parse: {0}")] + SymlParse(String), + #[error("unable to convert to structured syml: {0}")] + SymlStructure(#[from] serde_json::Error), + #[error("unexpected non-utf8 yarn.lock")] + NonUTF8(#[from] std::str::Utf8Error), + #[error("Turborepo cannot serialize Bun lockfiles.")] + NotImplemented(), +} + +pub struct BunLockfile { + inner: Map, +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Default, Deserialize)] +#[serde(rename_all = "camelCase")] +struct Entry { + name: Option, + version: String, + uid: Option, + resolved: Option, + integrity: Option, + registry: Option, + dependencies: Option>, + optional_dependencies: Option>, +} + +impl BunLockfile { + pub fn from_bytes(input: &[u8]) -> Result { + let input = std::str::from_utf8(input).map_err(Error::from)?; + Self::from_str(input) + } +} + +impl FromStr for BunLockfile { + type Err = super::Error; + + fn from_str(s: &str) -> Result { + let value = de::parse_syml(s)?; + let inner = serde_json::from_value(value)?; + Ok(Self { inner }) + } +} + +impl Lockfile for BunLockfile { + fn resolve_package( + &self, + _workspace_path: &str, + name: &str, + version: &str, + ) -> Result, crate::Error> { + for key in possible_keys(name, version) { + if let Some(entry) = self.inner.get(&key) { + return Ok(Some(crate::Package { + key, + version: entry.version.clone(), + })); + } + } + + Ok(None) + } + + fn all_dependencies( + &self, + key: &str, + ) -> Result>, crate::Error> { + let Some(entry) = self.inner.get(key) else { + return Ok(None); + }; + + let all_deps: std::collections::HashMap<_, _> = entry.dependency_entries().collect(); + Ok(match all_deps.is_empty() { + false => Some(all_deps), + true => None, + }) + } + + fn subgraph( + &self, + _workspace_packages: &[String], + packages: &[String], + ) -> Result, super::Error> { + let mut inner = Map::new(); + + for (key, entry) in packages.iter().filter_map(|key| { + let entry = self.inner.get(key)?; + Some((key, entry)) + }) { + inner.insert(key.clone(), entry.clone()); + } + + Ok(Box::new(Self { inner })) + } + + fn encode(&self) -> Result, crate::Error> { + Err(crate::Error::Bun(Error::NotImplemented())) + } + + fn global_change_key(&self) -> Vec { + vec![b'b', b'u', b'n', 0] + } +} + +impl Entry { + fn dependency_entries(&self) -> impl Iterator + '_ { + self.dependencies + .iter() + .flatten() + .chain(self.optional_dependencies.iter().flatten()) + .map(|(k, v)| (k.clone(), v.clone())) + } +} + +const PROTOCOLS: &[&str] = ["", "npm:", "file:", "workspace:", "yarn:"].as_slice(); + +fn possible_keys<'a>(name: &'a str, version: &'a str) -> impl Iterator + 'a { + PROTOCOLS + .iter() + .copied() + .map(move |protocol| format!("{name}@{protocol}{version}")) +} + +#[cfg(test)] +mod test { + use super::*; + const FULL: &str = include_str!("../../fixtures/yarn1full.lock"); + + #[test] + fn test_key_splitting() { + let lockfile = BunLockfile::from_str(FULL).unwrap(); + for key in [ + "@babel/types@^7.18.10", + "@babel/types@^7.18.6", + "@babel/types@^7.19.0", + ] { + assert!( + lockfile.inner.contains_key(key), + "missing {} in lockfile", + key + ); + } + } +} diff --git a/crates/turborepo-lockfiles/src/error.rs b/crates/turborepo-lockfiles/src/error.rs index 84724790da9a8..9a1f95928736f 100644 --- a/crates/turborepo-lockfiles/src/error.rs +++ b/crates/turborepo-lockfiles/src/error.rs @@ -19,6 +19,8 @@ pub enum Error { #[error(transparent)] Yarn1(#[from] crate::yarn1::Error), #[error(transparent)] + Bun(#[from] crate::bun::Error), + #[error(transparent)] Berry(#[from] crate::berry::Error), #[error("lockfile contains invalid path: {0}")] Path(#[from] turbopath::PathError), diff --git a/crates/turborepo-lockfiles/src/lib.rs b/crates/turborepo-lockfiles/src/lib.rs index 606e77c1717e2..571ab475b201f 100644 --- a/crates/turborepo-lockfiles/src/lib.rs +++ b/crates/turborepo-lockfiles/src/lib.rs @@ -1,6 +1,7 @@ #![deny(clippy::all)] mod berry; +mod bun; mod error; mod npm; mod pnpm; @@ -9,6 +10,7 @@ mod yarn1; use std::collections::{HashMap, HashSet}; pub use berry::{Error as BerryError, *}; +pub use bun::BunLockfile; pub use error::Error; pub use npm::*; pub use pnpm::{pnpm_global_change, pnpm_subgraph, PnpmLockfile};