Merge pull request #1 from bamlab/claude/slack-remove-the-limit-pt7mU

Remove npm package limit by fetching @bam.tech packages dynamically

Author: vicprz

scripts/fetch-data.ts

@@ -13,8 +13,8 @@ import { writeFileSync, mkdirSync } from 'fs';
 import { join, dirname } from 'path';
 import { fileURLToPath } from 'url';
 
-import { PACKAGES } from '../src/constants/packages';
-import { fetchDownloads, fetchRegistry, NotFoundError } from '../src/lib/npmApi';
+import { UNSCOPED_PACKAGES } from '../src/constants/packages';
+import { fetchDownloads, fetchRegistry, fetchScopedPackages, NotFoundError } from '../src/lib/npmApi';
 import { fetchStars } from '../src/lib/githubApi';
 import {
   bucketByMonth,
@@ -171,8 +171,21 @@ async function main() {
   const __dirname = dirname(fileURLToPath(import.meta.url));
   const outPath = join(__dirname, '../public/data.json');
 
+  // Fetch @bam.tech scoped packages dynamically
+  console.log('Fetching @bam.tech scoped npm packages...');
+  let scopedPackages: string[] = [];
+  try {
+    scopedPackages = await fetchScopedPackages('bam.tech');
+    console.log(`  Found ${scopedPackages.length} scoped npm packages`);
+  } catch (err) {
+    console.error(`  npm scope fetch failed: ${(err as Error).message}`);
+  }
+
+  // Combine scoped packages with unscoped packages
+  const npmPackages = [...scopedPackages, ...UNSCOPED_PACKAGES];
+
   // Fetch pub.dev package list for bam.tech publisher
-  console.log('Fetching pub.dev packages for bam.tech...');
+  console.log('\nFetching pub.dev packages for bam.tech...');
   let pubPackages: string[] = [];
   try {
     pubPackages = await fetchPubPublisherPackages('bam.tech');
@@ -181,9 +194,9 @@ async function main() {
     console.error(`  pub.dev publisher fetch failed: ${(err as Error).message}`);
   }
 
-  console.log(`\nFetching data for ${PACKAGES.length} npm packages...`);
+  console.log(`\nFetching data for ${npmPackages.length} npm packages...`);
   const npmResults = await Promise.all(
-    PACKAGES.map(async (name) => {
+    npmPackages.map(async (name) => {
       try {
         const result = await processNpmPackage(name);
         const status = result.notFound

src/constants/packages.ts

@@ -1,18 +1,9 @@
-export const PACKAGES = [
-  // Scoped @bam.tech packages
-  '@bam.tech/react-native-image-resizer',
-  '@bam.tech/react-navigation-visualizer-dev-plugin',
-  '@bam.tech/react-native-app-security',
-  '@bam.tech/react-native-make',
-  '@bam.tech/eslint-plugin',
-  '@bam.tech/react-native-config',
-  '@bam.tech/react-native-keyevent-expo-config-plugin',
-  '@bam.tech/lrud',
-  '@bam.tech/flow-navigator',
-  '@bam.tech/react-native-graphql-transformer',
-  '@bam.tech/tanstack-query-detect-waterfall',
-  '@bam.tech/react-native-batch',
-  // Unscoped packages
+/**
+ * Unscoped npm packages maintained by BAM.
+ * These cannot be discovered via the npm scope search API, so they are hardcoded.
+ * Scoped @bam.tech/* packages are fetched dynamically.
+ */
+export const UNSCOPED_PACKAGES = [
   'react-native-performance',
   'react-native-image-resizer',
   'react-native-hide-with-keyboard',
@@ -36,4 +27,4 @@ export const PACKAGES = [
   'react-native-text-input',
 ] as const;
 
-export type PackageName = (typeof PACKAGES)[number];
+export type UnscopedPackageName = (typeof UNSCOPED_PACKAGES)[number];

src/hooks/useAllPackageData.ts

@@ -1,18 +1,6 @@
 import { useQuery } from '@tanstack/react-query';
-import { PACKAGES } from '../constants/packages';
 import type { DataFile, PackageData } from '../types';
 
-const EMPTY_PACKAGES: PackageData[] = PACKAGES.map((name) => ({
-  name,
-  totalDownloads: 0,
-  monthlyDownloads: [],
-  momGrowthPct: null,
-  isGrowing: false,
-  stars: null,
-  description: null,
-  notFound: false,
-}));
-
 export function useAllPackageData(): { packages: PackageData[]; isLoading: boolean; generatedAt: string | null } {
   const { data, isLoading } = useQuery<DataFile>({
     queryKey: ['data.json'],
@@ -22,7 +10,7 @@ export function useAllPackageData(): { packages: PackageData[]; isLoading: boole
   });
 
   return {
-    packages: data?.packages ?? EMPTY_PACKAGES,
+    packages: data?.packages ?? [],
     isLoading,
     generatedAt: data?.generatedAt ?? null,
   };

src/lib/npmApi.ts

@@ -2,6 +2,34 @@ import type { NpmDownloadResponse, NpmRegistryResponse } from '../types';
 
 const DOWNLOADS_BASE = 'https://api.npmjs.org/downloads/range';
 const REGISTRY_BASE = 'https://registry.npmjs.org';
+const SEARCH_BASE = 'https://registry.npmjs.org/-/v1/search';
+
+interface NpmSearchResponse {
+  objects: Array<{ package: { name: string } }>;
+  total: number;
+}
+
+/**
+ * Fetches all npm packages from a given scope (e.g., 'bam.tech' for @bam.tech/*)
+ */
+export async function fetchScopedPackages(scope: string): Promise<string[]> {
+  const allNames: string[] = [];
+  let from = 0;
+  const size = 250; // npm search API max per page
+
+  while (true) {
+    const url = `${SEARCH_BASE}?text=scope:${encodeURIComponent(scope)}&size=${size}&from=${from}`;
+    const res = await fetch(url);
+    if (!res.ok) throw new Error(`npm search API error ${res.status} for scope ${scope}`);
+    const data: NpmSearchResponse = await res.json();
+    if (data.objects.length === 0) break;
+    allNames.push(...data.objects.map((o) => o.package.name));
+    from += data.objects.length;
+    if (from >= data.total) break;
+  }
+
+  return allNames;
+}
 
 // Fetch last 18 months of daily download data
 export async function fetchDownloads(packageName: string): Promise<NpmDownloadResponse> {