diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 85f51f6..785b79b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,4 +1,4 @@ -# Contributing to astchunk +# Contributing to code-chunk Thank you for your interest in contributing to our project! This guide will help you get started with the development process. @@ -11,8 +11,8 @@ Thank you for your interest in contributing to our project! This guide will help ### Getting Started 1. Fork the repository -2. Clone your fork: `git clone https://github.com/supermemoryai/astchunk.git` -3. Navigate to the project directory: `cd astchunk` +2. Clone your fork: `git clone https://github.com/supermemoryai/code-chunk.git` +3. Navigate to the project directory: `cd code-chunk` 4. Install dependencies: `bun install` 5. Start development: `bun run dev` @@ -53,6 +53,6 @@ Please be respectful and constructive in all interactions within our community. ## Questions? -If you have any questions, please [open an issue](https://github.com/supermemoryai/astchunk/issues/new) for discussion. +If you have any questions, please [open an issue](https://github.com/supermemoryai/code-chunk/issues/new) for discussion. -Thank you for contributing to astchunk! +Thank you for contributing to code-chunk! diff --git a/biome.json b/biome.json index 3b28a1e..b3c7d1f 100644 --- a/biome.json +++ b/biome.json @@ -31,7 +31,7 @@ }, "overrides": [ { - "includes": ["**/test/**/*.ts", "packages/astchunk/test/**/*.ts"], + "includes": ["**/test/**/*.ts", "packages/code-chunk/test/**/*.ts"], "linter": { "rules": { "style": { diff --git a/bun.lock b/bun.lock index 44602ce..a768e33 100644 --- a/bun.lock +++ b/bun.lock @@ -3,13 +3,13 @@ "configVersion": 1, "workspaces": { "": { - "name": "astchunk-monorepo", + "name": "code-chunk-monorepo", "devDependencies": { "@biomejs/biome": "^2.3.8", "typescript": "^5.9.3", }, }, - "packages/astchunk": { + "packages/code-chunk": { "name": "code-chunk", "version": "0.1.0", "dependencies": { @@ -75,7 +75,7 @@ "@bunup/dts": ["@bunup/dts@0.14.43", "", { "dependencies": { "@babel/parser": "^7.28.4", "coffi": "^0.1.37", "oxc-minify": "^0.93.0", "oxc-resolver": "^11.9.0", "oxc-transform": "^0.93.0", "picocolors": "^1.1.1", "std-env": "^3.9.0", "ts-import-resolver": "^0.1.23" }, "peerDependencies": { "typescript": ">=4.5.0" }, "optionalPeers": ["typescript"] }, "sha512-ky/82sy2FLMvMTGxOhegB+EUvru5dH5xW+TkGpx4jmUlEa3IR8GxKOKlCgBmb5jmmcJitOagJrCh1LM6pguReQ=="], - "@bunup/shared": ["@bunup/shared@0.16.8", "", { "peerDependencies": { "typescript": "latest" }, "optionalPeers": ["typescript"] }, "sha512-HBh5RLjhBSKgWTooX1f9WUwIQVLxeGfcNKjpV+04U6z2BolxKLSMn7MxUgYOxU4Q8hfCG6HGVD+Ks5fcnh9wBw=="], + "@bunup/shared": ["@bunup/shared@0.16.10", "", { "peerDependencies": { "typescript": "latest" }, "optionalPeers": ["typescript"] }, "sha512-XDrhTdLS9OnXguHDGDTqrdcce17zlndBF3vOuuyQ5Vy6+T3b8waqMq9cAeYYlvhSTSkNTtDsDGiyWVji0KNPKQ=="], "@emnapi/core": ["@emnapi/core@1.7.1", "", { "dependencies": { "@emnapi/wasi-threads": "1.1.0", "tslib": "^2.4.0" } }, "sha512-o1uhUASyo921r2XtHYOHy7gdkGLge8ghBEQHMWmyJFoXlpU58kIrhhN3w26lpQb6dspetweapMn2CSNwQ8I4wg=="], @@ -211,7 +211,7 @@ "bun-types": ["bun-types@1.3.5", "", { "dependencies": { "@types/node": "*" } }, "sha512-inmAYe2PFLs0SUbFOWSVD24sg1jFlMPxOjOSSCYqUgn4Hsc3rDc7dFvfVYjFPNHtov6kgUeulV4SxbuIV/stPw=="], - "bunup": ["bunup@0.16.10", "", { "dependencies": { "@bunup/dts": "^0.14.37", "@bunup/shared": "0.16.8", "chokidar": "^4.0.3", "coffi": "^0.1.37", "lightningcss": "^1.30.2", "picocolors": "^1.1.1", "tinyexec": "^1.0.1", "tree-kill": "^1.2.2", "zlye": "^0.4.4" }, "peerDependencies": { "typescript": "latest" }, "optionalPeers": ["typescript"], "bin": { "bunup": "dist/cli/index.js" } }, "sha512-rG++Esy3K5BYmZc/cFE58KRlDnzX4kvUPCpTE9gFf8n8ly5SNnCXPgEHMILmZ2VH1+VwJp2uY3tbaOI65TAXHQ=="], + "bunup": ["bunup@0.16.11", "", { "dependencies": { "@bunup/dts": "^0.14.37", "@bunup/shared": "0.16.10", "chokidar": "^5.0.0", "coffi": "^0.1.37", "lightningcss": "^1.30.2", "picocolors": "^1.1.1", "tinyexec": "^1.0.1", "tree-kill": "^1.2.2", "zlye": "^0.4.4" }, "peerDependencies": { "typescript": "latest" }, "optionalPeers": ["typescript"], "bin": { "bunup": "dist/cli/index.js" } }, "sha512-hy8DeA4HAkG+exZJ92UYQoUd6g1ZCwFtZQKguedomMvnrv/k+5OhkSMetQTrUOb/qSlJFwqG0QyRXLeILU9hlg=="], "c12": ["c12@3.3.3", "", { "dependencies": { "chokidar": "^5.0.0", "confbox": "^0.2.2", "defu": "^6.1.4", "dotenv": "^17.2.3", "exsolve": "^1.0.8", "giget": "^2.0.0", "jiti": "^2.6.1", "ohash": "^2.0.11", "pathe": "^2.0.3", "perfect-debounce": "^2.0.0", "pkg-types": "^2.3.0", "rc9": "^2.1.2" }, "peerDependencies": { "magicast": "*" }, "optionalPeers": ["magicast"] }, "sha512-750hTRvgBy5kcMNPdh95Qo+XUBeGo8C7nsKSmedDmaQI+E0r82DwHeM6vBewDe4rGFbnxoa4V9pw+sPh5+Iz8Q=="], @@ -219,11 +219,11 @@ "call-bind-apply-helpers": ["call-bind-apply-helpers@1.0.2", "", { "dependencies": { "es-errors": "^1.3.0", "function-bind": "^1.1.2" } }, "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ=="], - "chokidar": ["chokidar@4.0.3", "", { "dependencies": { "readdirp": "^4.0.1" } }, "sha512-Qgzu8kfBvo+cA4962jnP1KkS6Dop5NS6g7R5LFYJr4b8Ub94PPQXUksCw9PvXoeXPRRddRNC5C1JQUR2SMGtnA=="], + "chokidar": ["chokidar@5.0.0", "", { "dependencies": { "readdirp": "^5.0.0" } }, "sha512-TQMmc3w+5AxjpL8iIiwebF73dRDF4fBIieAqGn9RGCWaEVwQ6Fb2cGe31Yns0RRIzii5goJ1Y7xbMwo1TxMplw=="], "citty": ["citty@0.1.6", "", { "dependencies": { "consola": "^3.2.3" } }, "sha512-tskPPKEs8D2KPafUypv2gxwJP8h/OaJmC82QQGGDQcHvXX43xF2VDACcJVmZ0EuSxkpO9Kc4MlrA3q0+FG58AQ=="], - "code-chunk": ["code-chunk@workspace:packages/astchunk"], + "code-chunk": ["code-chunk@workspace:packages/code-chunk"], "coffi": ["coffi@0.1.37", "", { "dependencies": { "strip-json-comments": "^5.0.3" } }, "sha512-ewO5Xis7sw7g54yI/3lJ/nNV90Er4ZnENeDORZjrs58T70MmwKFLZgevraNCz+RmB4KDKsYT1ui1wDB36iPWqQ=="], @@ -245,7 +245,7 @@ "dunder-proto": ["dunder-proto@1.0.1", "", { "dependencies": { "call-bind-apply-helpers": "^1.0.1", "es-errors": "^1.3.0", "gopd": "^1.2.0" } }, "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A=="], - "effect": ["effect@3.19.12", "", { "dependencies": { "@standard-schema/spec": "^1.0.0", "fast-check": "^3.23.1" } }, "sha512-7F9RGTrCTC3D7nh9Zw+3VlJWwZgo5k33KA+476BAaD0rKIXKZsY/jQ+ipyhR/Avo239Fi6GqAVFs1mqM1IJ7yg=="], + "effect": ["effect@3.19.13", "", { "dependencies": { "@standard-schema/spec": "^1.0.0", "fast-check": "^3.23.1" } }, "sha512-8MZ783YuHRwHZX2Mmm+bpGxq+7XPd88sWwYAz2Ysry80sEKpftDZXs2Hg9ZyjESi1IBTNHF0oDKe0zJRkUlyew=="], "es-define-property": ["es-define-property@1.0.1", "", {}, "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g=="], @@ -363,7 +363,7 @@ "rc9": ["rc9@2.1.2", "", { "dependencies": { "defu": "^6.1.4", "destr": "^2.0.3" } }, "sha512-btXCnMmRIBINM2LDZoEmOogIZU7Qe7zn4BpomSKZ/ykbLObuBdvG+mFq11DL6fjH1DRwHhrlgtYWG96bJiC7Cg=="], - "readdirp": ["readdirp@4.1.2", "", {}, "sha512-GDhwkLfywWL2s6vEjyhri+eXmfH6j1L7JE27WhqLeYzoh/A3DBaYGEj2H/HFZCn/kMfim73FXxEJTw06WtxQwg=="], + "readdirp": ["readdirp@5.0.0", "", {}, "sha512-9u/XQ1pvrQtYyMpZe7DXKv2p5CNvyVwzUB6uhLAnQwHMSgKMBR62lc7AHljaeteeHXn11XTAaLLUVZYVZyuRBQ=="], "semver": ["semver@7.7.3", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q=="], @@ -411,10 +411,6 @@ "zlye": ["zlye@0.4.4", "", { "dependencies": { "picocolors": "^1.1.1" }, "peerDependencies": { "typescript": ">=4.5.0" }, "optionalPeers": ["typescript"] }, "sha512-fwpeC841X3ElOLYRMKXbwX29pitNrsm6nRNvEhDMrRXDl3BhR2i03Bkr0GNrpyYgZJuEzUsBylXAYzgGPXXOCQ=="], - "c12/chokidar": ["chokidar@5.0.0", "", { "dependencies": { "readdirp": "^5.0.0" } }, "sha512-TQMmc3w+5AxjpL8iIiwebF73dRDF4fBIieAqGn9RGCWaEVwQ6Fb2cGe31Yns0RRIzii5goJ1Y7xbMwo1TxMplw=="], - "tree-sitter-typescript/tree-sitter-javascript": ["tree-sitter-javascript@0.23.1", "", { "dependencies": { "node-addon-api": "^8.2.2", "node-gyp-build": "^4.8.2" }, "peerDependencies": { "tree-sitter": "^0.21.1" }, "optionalPeers": ["tree-sitter"] }, "sha512-/bnhbrTD9frUYHQTiYnPcxyHORIw157ERBa6dqzaKxvR/x3PC4Yzd+D1pZIMS6zNg2v3a8BZ0oK7jHqsQo9fWA=="], - - "c12/chokidar/readdirp": ["readdirp@5.0.0", "", {}, "sha512-9u/XQ1pvrQtYyMpZe7DXKv2p5CNvyVwzUB6uhLAnQwHMSgKMBR62lc7AHljaeteeHXn11XTAaLLUVZYVZyuRBQ=="], } } diff --git a/package.json b/package.json index bc88d35..7f14b7d 100644 --- a/package.json +++ b/package.json @@ -1,5 +1,5 @@ { - "name": "astchunk-monorepo", + "name": "code-chunk-monorepo", "private": true, "workspaces": [ "packages/*" diff --git a/packages/astchunk/package.json b/packages/code-chunk/package.json similarity index 86% rename from packages/astchunk/package.json rename to packages/code-chunk/package.json index 27df753..66c8b2e 100644 --- a/packages/astchunk/package.json +++ b/packages/code-chunk/package.json @@ -2,9 +2,9 @@ "name": "code-chunk", "version": "0.1.0", "description": "AST-aware code chunking for semantic search and RAG", - "homepage": "https://github.com/supermemoryai/astchunk#readme", + "homepage": "https://github.com/supermemoryai/code-chunk#readme", "bugs": { - "url": "https://github.com/supermemoryai/astchunk/issues" + "url": "https://github.com/supermemoryai/code-chunk/issues" }, "license": "MIT", "files": [ @@ -12,7 +12,7 @@ ], "repository": { "type": "git", - "url": "git+https://github.com/supermemoryai/astchunk.git" + "url": "git+https://github.com/supermemoryai/code-chunk.git" }, "scripts": { "build": "bunup", diff --git a/packages/astchunk/src/chunk.ts b/packages/code-chunk/src/chunk.ts similarity index 97% rename from packages/astchunk/src/chunk.ts rename to packages/code-chunk/src/chunk.ts index 061c1a8..719b909 100644 --- a/packages/astchunk/src/chunk.ts +++ b/packages/code-chunk/src/chunk.ts @@ -115,7 +115,7 @@ const chunkEffect = ( /** * Chunk source code into pieces with semantic context * - * This is the main entry point for the astchunk library. It takes source code + * This is the main entry point for the code-chunk library. It takes source code * and returns an array of chunks, each with contextual information about the * code's structure. * @@ -128,7 +128,7 @@ const chunkEffect = ( * * @example * ```ts - * import { chunk } from 'astchunk' + * import { chunk } from 'code-chunk' * * const chunks = await chunk('src/utils.ts', sourceCode) * for (const chunk of chunks) { @@ -203,7 +203,7 @@ const prepareChunking = ( * * @example * ```ts - * import { chunkStreamEffect } from 'astchunk' + * import { chunkStreamEffect } from 'code-chunk' * import { Effect, Stream } from 'effect' * * const program = Stream.runForEach( @@ -268,7 +268,7 @@ export const chunkStreamEffect = ( * * @example * ```ts - * import { chunkStream } from 'astchunk' + * import { chunkStream } from 'code-chunk' * * for await (const chunk of chunkStream('src/utils.ts', sourceCode)) { * console.log(chunk.text, chunk.context) diff --git a/packages/astchunk/src/chunker.ts b/packages/code-chunk/src/chunker.ts similarity index 98% rename from packages/astchunk/src/chunker.ts rename to packages/code-chunk/src/chunker.ts index 28c8a4a..475ee1c 100644 --- a/packages/astchunk/src/chunker.ts +++ b/packages/code-chunk/src/chunker.ts @@ -62,7 +62,7 @@ class ChunkerImpl implements Chunker { * * @example * ```ts - * import { createChunker } from 'astchunk' + * import { createChunker } from 'code-chunk' * * const chunker = createChunker({ maxChunkSize: 2048 }) * diff --git a/packages/astchunk/src/chunking/index.ts b/packages/code-chunk/src/chunking/index.ts similarity index 100% rename from packages/astchunk/src/chunking/index.ts rename to packages/code-chunk/src/chunking/index.ts diff --git a/packages/astchunk/src/chunking/merge.ts b/packages/code-chunk/src/chunking/merge.ts similarity index 100% rename from packages/astchunk/src/chunking/merge.ts rename to packages/code-chunk/src/chunking/merge.ts diff --git a/packages/astchunk/src/chunking/nws.ts b/packages/code-chunk/src/chunking/nws.ts similarity index 100% rename from packages/astchunk/src/chunking/nws.ts rename to packages/code-chunk/src/chunking/nws.ts diff --git a/packages/astchunk/src/chunking/oversized.ts b/packages/code-chunk/src/chunking/oversized.ts similarity index 100% rename from packages/astchunk/src/chunking/oversized.ts rename to packages/code-chunk/src/chunking/oversized.ts diff --git a/packages/astchunk/src/chunking/rebuild.ts b/packages/code-chunk/src/chunking/rebuild.ts similarity index 100% rename from packages/astchunk/src/chunking/rebuild.ts rename to packages/code-chunk/src/chunking/rebuild.ts diff --git a/packages/astchunk/src/chunking/windows.ts b/packages/code-chunk/src/chunking/windows.ts similarity index 100% rename from packages/astchunk/src/chunking/windows.ts rename to packages/code-chunk/src/chunking/windows.ts diff --git a/packages/astchunk/src/context/format.ts b/packages/code-chunk/src/context/format.ts similarity index 100% rename from packages/astchunk/src/context/format.ts rename to packages/code-chunk/src/context/format.ts diff --git a/packages/astchunk/src/context/index.ts b/packages/code-chunk/src/context/index.ts similarity index 100% rename from packages/astchunk/src/context/index.ts rename to packages/code-chunk/src/context/index.ts diff --git a/packages/astchunk/src/context/siblings.ts b/packages/code-chunk/src/context/siblings.ts similarity index 100% rename from packages/astchunk/src/context/siblings.ts rename to packages/code-chunk/src/context/siblings.ts diff --git a/packages/astchunk/src/extract/docstring.ts b/packages/code-chunk/src/extract/docstring.ts similarity index 100% rename from packages/astchunk/src/extract/docstring.ts rename to packages/code-chunk/src/extract/docstring.ts diff --git a/packages/astchunk/src/extract/fallback.ts b/packages/code-chunk/src/extract/fallback.ts similarity index 100% rename from packages/astchunk/src/extract/fallback.ts rename to packages/code-chunk/src/extract/fallback.ts diff --git a/packages/astchunk/src/extract/imports.ts b/packages/code-chunk/src/extract/imports.ts similarity index 100% rename from packages/astchunk/src/extract/imports.ts rename to packages/code-chunk/src/extract/imports.ts diff --git a/packages/astchunk/src/extract/index.ts b/packages/code-chunk/src/extract/index.ts similarity index 100% rename from packages/astchunk/src/extract/index.ts rename to packages/code-chunk/src/extract/index.ts diff --git a/packages/astchunk/src/extract/queries.ts b/packages/code-chunk/src/extract/queries.ts similarity index 100% rename from packages/astchunk/src/extract/queries.ts rename to packages/code-chunk/src/extract/queries.ts diff --git a/packages/astchunk/src/extract/signature.ts b/packages/code-chunk/src/extract/signature.ts similarity index 100% rename from packages/astchunk/src/extract/signature.ts rename to packages/code-chunk/src/extract/signature.ts diff --git a/packages/astchunk/src/index.ts b/packages/code-chunk/src/index.ts similarity index 95% rename from packages/astchunk/src/index.ts rename to packages/code-chunk/src/index.ts index 0b50057..f4dd19e 100644 --- a/packages/astchunk/src/index.ts +++ b/packages/code-chunk/src/index.ts @@ -1,5 +1,5 @@ /** - * astchunk - AST-aware code chunking for LLMs + * code-chunk - AST-aware code chunking for LLMs * * This library provides intelligent code chunking that preserves semantic * context by leveraging tree-sitter for AST parsing. Each chunk includes diff --git a/packages/astchunk/src/parser/index.ts b/packages/code-chunk/src/parser/index.ts similarity index 100% rename from packages/astchunk/src/parser/index.ts rename to packages/code-chunk/src/parser/index.ts diff --git a/packages/astchunk/src/parser/languages.ts b/packages/code-chunk/src/parser/languages.ts similarity index 100% rename from packages/astchunk/src/parser/languages.ts rename to packages/code-chunk/src/parser/languages.ts diff --git a/packages/astchunk/src/scope/index.ts b/packages/code-chunk/src/scope/index.ts similarity index 100% rename from packages/astchunk/src/scope/index.ts rename to packages/code-chunk/src/scope/index.ts diff --git a/packages/astchunk/src/scope/tree.ts b/packages/code-chunk/src/scope/tree.ts similarity index 100% rename from packages/astchunk/src/scope/tree.ts rename to packages/code-chunk/src/scope/tree.ts diff --git a/packages/astchunk/src/types.ts b/packages/code-chunk/src/types.ts similarity index 100% rename from packages/astchunk/src/types.ts rename to packages/code-chunk/src/types.ts diff --git a/packages/astchunk/test/chunking.test.ts b/packages/code-chunk/test/chunking.test.ts similarity index 100% rename from packages/astchunk/test/chunking.test.ts rename to packages/code-chunk/test/chunking.test.ts diff --git a/packages/astchunk/test/extract.test.ts b/packages/code-chunk/test/extract.test.ts similarity index 100% rename from packages/astchunk/test/extract.test.ts rename to packages/code-chunk/test/extract.test.ts diff --git a/packages/astchunk/test/integration.test.ts b/packages/code-chunk/test/integration.test.ts similarity index 100% rename from packages/astchunk/test/integration.test.ts rename to packages/code-chunk/test/integration.test.ts diff --git a/packages/astchunk/test/parser.test.ts b/packages/code-chunk/test/parser.test.ts similarity index 100% rename from packages/astchunk/test/parser.test.ts rename to packages/code-chunk/test/parser.test.ts diff --git a/packages/astchunk/test/pipeline-debug.test.ts b/packages/code-chunk/test/pipeline-debug.test.ts similarity index 100% rename from packages/astchunk/test/pipeline-debug.test.ts rename to packages/code-chunk/test/pipeline-debug.test.ts diff --git a/packages/astchunk/test/scope.test.ts b/packages/code-chunk/test/scope.test.ts similarity index 100% rename from packages/astchunk/test/scope.test.ts rename to packages/code-chunk/test/scope.test.ts diff --git a/packages/astchunk/tsconfig.json b/packages/code-chunk/tsconfig.json similarity index 100% rename from packages/astchunk/tsconfig.json rename to packages/code-chunk/tsconfig.json diff --git a/packages/eval/package.json b/packages/eval/package.json index 1962dd8..c54865c 100644 --- a/packages/eval/package.json +++ b/packages/eval/package.json @@ -2,7 +2,7 @@ "name": "@supermemory/eval", "version": "0.1.0", "private": true, - "description": "Evaluation harness for astchunk", + "description": "Evaluation harness for code-chunk", "type": "module", "scripts": { "start": "bun run src/run.ts", diff --git a/packages/eval/src/chunkers/ast.ts b/packages/eval/src/chunkers/ast.ts index 9e8f799..67727ce 100644 --- a/packages/eval/src/chunkers/ast.ts +++ b/packages/eval/src/chunkers/ast.ts @@ -1,7 +1,7 @@ /** * AST-aware chunker wrapper for evaluation * - * Wraps the astchunk library for use in the evaluation harness. + * Wraps the code-chunk library for use in the evaluation harness. * Uses the built-in contextualizedText for better embedding quality. */ diff --git a/tsconfig.json b/tsconfig.json index 7617122..5fb70d8 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -1,7 +1,7 @@ { "files": [], "references": [ - { "path": "./packages/astchunk" }, + { "path": "./packages/code-chunk" }, { "path": "./packages/eval" } ] }