Skip to content

Commit

Permalink
using csv for website
Browse files Browse the repository at this point in the history
  • Loading branch information
@remorses
remorses committed Feb 14, 2023
1 parent ff57782 commit 97bce93
Show file tree
Hide file tree
Showing 10 changed files with 161 additions and 277 deletions.
2 changes: 1 addition & 1 deletion .pnpmfile.cjs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ const enforceSingleVersion = [
'react-dom',
'next-auth',
'next',
'tailwindcss',
// 'tailwindcss',
'@prisma/client',
'next-themes',
'@chakra-ui/react',
Expand Down
91 changes: 88 additions & 3 deletions pnpm-lock.yaml
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

14 changes: 7 additions & 7 deletions scraper/data.csv
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ slug ,status ,website ,url
,failed:schema ,https://gqless.com ,https://examples-api.gqless.com/graphql ,Menu ,A GraphQL client without queries
,failed:schema ,https://zaproxy.org ,https://www.zaproxy.org/docs/desktop/addons/graphql-support/ ,OWASP ZAP ,Welcome to ZAP!
,failed:schema ,https://your.domain ,https://your.domain/graphql , ,
,failed:schema ,https://spacex.land ,https://api.spacex.land/graphql/ p ,SpaceX Land ,
,failed:schema ,https://spacex.land ,https://api.spacex.land/graphql/ p ,SpaceX Land ,
,failed:schema ,https://leetcode.com ,https://leetcode.com/graphql ,LeetCode - The World's Leading Online Programming Learning Platform ,Level up your coding skills and quickly land a job. This is the best place to expand your knowledge and get prepared for your next interview.
,failed:schema ,https://github.io ,https://graphql.github.io/graphql-spec/June2018/ ,"GitHub Pages | Websites for you and your projects, hosted directly from your GitHub repository. Just edit, push, and your changes are live." ,"Websites for you and your projects, hosted directly from your GitHub repository. Just edit, push, and your changes are live."
,failed:schema ,https://your.domain ,https://your.domain/graphql/endpoint , ,
Expand Down Expand Up @@ -101,12 +101,12 @@ slug ,status ,website ,url
,failed:schema ,https://prismic.io ,https://qwerty.cdn.prismic.io/graphql ,Prismic: Headless Website Builder - Launch and Iterate Faster ,"Prismic is the headless website builder that lets developers and marketers ship and iterate faster, and build sites that just keep getting better."
,failed:schema ,https://prismic.io ,https://foo.cdn.prismic.io/graphql ,Prismic: Headless Website Builder - Launch and Iterate Faster ,"Prismic is the headless website builder that lets developers and marketers ship and iterate faster, and build sites that just keep getting better."
,failed:schema ,https://enterprise.local ,https://api.github.enterprise.local/graphql , ,
zora , ,https://zora.co ,https://api.zora.co/graphql ,ZORA - Enjoy Ethereum ,Discover. Create. Collect. Enjoy.
wapm-registry , ,https://wapm.io ,https://registry.wapm.io/graphql ,WAPM - WebAssembly Package Manager ,
universe-com , ,https://universe.com ,https://www.universe.com/graphql/beta ,"Sell Tickets, Create Events and Discover Experiences - Universe" ,Universe has the tools and support you need to make your next event a success. Create your event today.
thegraph , ,https://thegraph.com ,https://api.thegraph.com/index-node/graphql ,The Graph ,The Graph is an indexing protocol for organizing blockchain data and making it easily accessible with GraphQL.
tezos-domains , ,https://tezos.domains ,https://api.tezos.domains/graphql ,Tezos Domains ,"Distributed, open and extensible naming system using the Tezos blockchain."
sourcegraph , ,https://sourcegraph.com ,https://sourcegraph.com/.api/graphql ,Sourcegraph ,"Sourcegraph is a web-based code search and navigation tool for dev teams. Search, navigate, and review code. Find answers."
zora ,enabled ,https://zora.co ,https://api.zora.co/graphql ,ZORA - Enjoy Ethereum ,Discover. Create. Collect. Enjoy.
wapm-registry ,enabled ,https://wapm.io ,https://registry.wapm.io/graphql ,WAPM - WebAssembly Package Manager ,
universe-com ,enabled ,https://universe.com ,https://www.universe.com/graphql/beta ,"Sell Tickets, Create Events and Discover Experiences - Universe" ,Universe has the tools and support you need to make your next event a success. Create your event today.
thegraph ,enabled ,https://thegraph.com ,https://api.thegraph.com/index-node/graphql ,The Graph ,The Graph is an indexing protocol for organizing blockchain data and making it easily accessible with GraphQL.
tezos-domains ,enabled ,https://tezos.domains ,https://api.tezos.domains/graphql ,Tezos Domains ,"Distributed, open and extensible naming system using the Tezos blockchain."
sourcegraph ,enabled ,https://sourcegraph.com ,https://sourcegraph.com/.api/graphql ,Sourcegraph ,"Sourcegraph is a web-based code search and navigation tool for dev teams. Search, navigate, and review code. Find answers."
sorare , ,https://sorare.com ,https://api.sorare.com/graphql ,Sorare: Own Your Game ,"Collect, play and win officially licensed digital cards featuring the world's best global football, NBA and MLB players."
snapshot-org , ,https://snapshot.org ,https://hub.snapshot.org/graphql ,Snapshot ,"Snapshot is an off-chain voting platform that allows DAOs, DeFi protocols, or NFT communities to vote easily and without gas fees."
slothpixel , ,https://slothpixel.me ,https://api.slothpixel.me/api/graphql , ,The Slothpixel API provides Hypixel related data.
Expand Down
11 changes: 8 additions & 3 deletions scraper/src/utils/utils.ts
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import Papa from 'papaparse'
import path from 'path'
import { sort } from 'fast-sort'
import posthtml from 'posthtml'
import fs from 'fs'
Expand Down Expand Up @@ -42,9 +43,11 @@ export class CsvStore<T> {
async read(): Promise<T[]> {
let csvData: T[] = []
if (!fs.existsSync(this.path)) {
console.log('file not found', this.path)
return []
}
let csv = fs.readFileSync(this.path, 'utf-8')
// console.log('read', this.path)
return await new Promise((resolve, reject) => {
let r = Papa.parse(csv, {
header: true,
Expand All @@ -65,7 +68,8 @@ export class CsvStore<T> {
if (res.errors.length) {
return reject(new Error(JSON.stringify(res.errors)))
}
csvData = res.data.filter(Boolean) as any
csvData = this.transform(res.data.filter(Boolean) as any)

this.data = csvData
resolve(csvData)
},
Expand Down Expand Up @@ -118,7 +122,8 @@ export function unique<T>(arr: T[], key: (x: T) => string) {
}

export let dataStore = new CsvStore<CsvDataType>(
'data.csv',
path.resolve('../scraper/data.csv'),

(x) => getCleanUrl(x.url),
(data) => {
data = sort(data).desc([(x) => x.status, (x) => x.slug])
Expand All @@ -127,7 +132,7 @@ export let dataStore = new CsvStore<CsvDataType>(
['slug', 'status', 'website', 'url', 'title', 'description'],
)
export let generatedStore = new CsvStore<GeneratedEntry>(
'generated.csv',
path.resolve('../scraper/generated.csv'),
(x) => x.slug.trim(),
(data) => {
data = sort(data).desc([(x) => x.version, (x) => x.slug])
Expand Down
3 changes: 3 additions & 0 deletions website/next.config.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,9 @@ const withTM = require('next-transpile-modules')(['@genql/cli', 'beskar'])

/** @type {import('next').NextConfig} */
const config = {
experimental: {
externalDir: true,
},
rewrites() {
const rewriteDocsTo =
'https://aeea2a8f-f718-4d46.docs-base-path.notaku.site'
Expand Down
1 change: 1 addition & 0 deletions website/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
"dependencies": {
"@emotion/react": "^11.10.5",
"@genql/cli": "workspace:*",
"scraper": "workspace:*",
"@types/js-cookie": "^3.0.2",
"autoprefixer": "^10.4.13",
"baby-i-am-faded": "^4.0.14",
Expand Down
Loading

0 comments on commit 97bce93

Please sign in to comment.