From 8143dce449d08921d4a33229b09333f0718df55a Mon Sep 17 00:00:00 2001 From: Nathan Abondance Date: Sat, 28 Sep 2024 19:42:28 +0200 Subject: [PATCH 01/10] refactor(core): graphql pagination to get all the data --- .github/workflows/ci.yml | 1 + __tests__/getstats.test.js | 231 +++++++++++++++++++++++--- badges/coverage.svg | 2 +- dist/index.js | 325 ++++++++++++++++++++++++++++--------- src/getStats.js | 325 ++++++++++++++++++++++++++++--------- 5 files changed, 710 insertions(+), 174 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8367670..e8b1260 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -84,6 +84,7 @@ jobs: no-commit: true show-skill-number: 8 show-event-badge: 'number' + show-badge: 'icon' - name: Print Output id: output diff --git a/__tests__/getstats.test.js b/__tests__/getstats.test.js index cf593a4..f0bb5e6 100644 --- a/__tests__/getstats.test.js +++ b/__tests__/getstats.test.js @@ -10,7 +10,7 @@ const { fetchTrailblazerEarnedStampsInfo } = require('./../src/getStats') -describe('fetchTrailblazerRankInfo', () => { +describe('fetches successfully data from the API', () => { afterEach(() => { jest.clearAllMocks() }) @@ -18,7 +18,9 @@ describe('fetchTrailblazerRankInfo', () => { it('fetches successfully data from the API: TrailblazerRank', async () => { const mockData = { data: { - /* mock response data */ + profile: { + /* mock profile data */ + } } } axios.post.mockResolvedValue(mockData) @@ -44,7 +46,12 @@ describe('fetchTrailblazerRankInfo', () => { it('fetches successfully data from the API: TrailblazerBadges', async () => { const mockData = { data: { - /* mock response data */ + profile: { + earnedAwards: { + edges: [], + pageInfo: null + } + } } } axios.post.mockResolvedValue(mockData) @@ -56,9 +63,8 @@ describe('fetchTrailblazerRankInfo', () => { { query: expect.any(String), variables: { - count: 10, + count: 100, after: null, - filter: null, hasSlug: true, slug: 'testUsername' } @@ -67,13 +73,18 @@ describe('fetchTrailblazerRankInfo', () => { headers: { 'Content-Type': 'application/json' } } ) - expect(response).toEqual(mockData.data) + expect(response).toEqual(mockData) }) it('fetches successfully data from the API: TrailblazerSuperBadges', async () => { const mockData = { data: { - /* mock response data */ + profile: { + earnedAwards: { + edges: [], + pageInfo: null + } + } } } axios.post.mockResolvedValue(mockData) @@ -96,13 +107,18 @@ describe('fetchTrailblazerRankInfo', () => { headers: { 'Content-Type': 'application/json' } } ) - expect(response).toEqual(mockData.data) + expect(response).toEqual(mockData) }) it('fetches successfully data from the API: TrailblazerEventBadges', async () => { const mockData = { data: { - /* mock response data */ + profile: { + earnedAwards: { + edges: [], + pageInfo: null + } + } } } axios.post.mockResolvedValue(mockData) @@ -125,13 +141,24 @@ describe('fetchTrailblazerRankInfo', () => { headers: { 'Content-Type': 'application/json' } } ) - expect(response).toEqual(mockData.data) + expect(response).toEqual(mockData) }) it('fetches successfully data from the API: TrailblazerCertifs', async () => { const mockData = { data: { - /* mock response data */ + data: { + profile: { + __typename: 'Profile', + id: '123', + credential: { + messages: ['message1', 'message2'], + messagesOnly: ['messageOnly1', 'messageOnly2'], + brands: ['brand1', 'brand2'], + certifications: ['certification1', 'certification2'] + } + } + } } } axios.post.mockResolvedValue(mockData) @@ -144,7 +171,8 @@ describe('fetchTrailblazerRankInfo', () => { query: expect.any(String), variables: { hasSlug: true, - slug: 'testUsername' + slug: 'testUsername', + count: 100 } }, { @@ -157,7 +185,11 @@ describe('fetchTrailblazerRankInfo', () => { it('fetches successfully data from the API: TrailblazerSkills', async () => { const mockData = { data: { - /* mock response data */ + data: { + profile: { + earnedSkills: [] + } + } } } axios.post.mockResolvedValue(mockData) @@ -183,7 +215,12 @@ describe('fetchTrailblazerRankInfo', () => { it('fetches successfully data from the API: EarnedStamps', async () => { const mockData = { data: { - /* mock response data */ + data: { + earnedStamps: { + edges: [], + pageInfo: null + } + } } } axios.post.mockResolvedValue(mockData) @@ -195,7 +232,10 @@ describe('fetchTrailblazerRankInfo', () => { { query: expect.any(String), variables: { - first: 10, + first: 100, + count: 100, + after: null, + hasSlug: true, slug: 'testUsername' } }, @@ -205,6 +245,120 @@ describe('fetchTrailblazerRankInfo', () => { ) expect(response).toEqual(mockData.data) }) +}) + +describe('fetches successfully data from the API in multiple pages', () => { + afterEach(() => { + jest.clearAllMocks() + }) + + // it('fetches successfully data in multiple pages from the API: TrailblazerBadges', async () => { + // const mockDataPage1 = { + // data: { + // profile: { + // profileData : { + // __typename: null, + // trailheadStats: null + // }, + // earnedAwards: { + // edges: ['edge1'], + // pageInfo: { + // hasNextPage: true, + // endCursor: 'cursor1' + // } + // } + // } + // } + // } + + // const mockDataPage2 = { + // data: { + // profile: { + // profileData : { + // __typename: null, + // trailheadStats: null + // }, + // earnedAwards: { + // edges: ['edge2'], + // pageInfo: { + // hasNextPage: false, + // endCursor: 'cursor2' + // } + // } + // } + // } + // } + + // axios.post + // .mockResolvedValueOnce(mockDataPage1) + // .mockResolvedValueOnce(mockDataPage2) + + // const response = await fetchTrailblazerBadgesInfo('testUsername') + + // expect(axios.post).toHaveBeenCalledTimes(2) + // expect(response).toEqual({ + // data: { + // profile: { + // earnedAwards: { + // edges: ['edge1', 'edge2'], + // pageInfo: mockDataPage2.data.profile.earnedAwards.pageInfo + // } + // } + // } + // }) + // }) + + it('fetches successfully data in multiple pages from the API: EarnedStamps', async () => { + const mockDataPage1 = { + data: { + data: { + earnedStamps: { + edges: ['edge1'], + pageInfo: { + hasNextPage: true, + endCursor: 'cursor1' + } + } + } + } + } + + const mockDataPage2 = { + data: { + data: { + earnedStamps: { + edges: ['edge2'], + pageInfo: { + hasNextPage: false, + endCursor: 'cursor2' + } + } + } + } + } + + axios.post + .mockResolvedValueOnce(mockDataPage1) + .mockResolvedValueOnce(mockDataPage2) + + const response = await fetchTrailblazerEarnedStampsInfo('testUsername') + + expect(axios.post).toHaveBeenCalledTimes(2) + expect(response).toEqual({ + data: { + earnedStamps: { + edges: ['edge1', 'edge2'], + pageInfo: mockDataPage2.data.data.earnedStamps.pageInfo + } + } + }) + }) +}) + +describe('handles API error', () => { + afterEach(() => { + jest.clearAllMocks() + }) it('handles API error : TrailblazerRank', async () => { const errorMessage = 'Network Error' @@ -237,6 +391,9 @@ describe('fetchTrailblazerRankInfo', () => { const errorMessage = 'Network Error' axios.post.mockRejectedValue(new Error(errorMessage)) console.error = jest.fn() + const mockErrorResponse = { + data: { profile: { earnedAwards: { edges: [], pageInfo: null } } } + } const response = await fetchTrailblazerBadgesInfo('testUsername') @@ -245,9 +402,8 @@ describe('fetchTrailblazerRankInfo', () => { { query: expect.any(String), variables: { - count: 10, + count: 100, after: null, - filter: null, hasSlug: true, slug: 'testUsername' } @@ -256,7 +412,7 @@ describe('fetchTrailblazerRankInfo', () => { headers: { 'Content-Type': 'application/json' } } ) - expect(response).toBeNull() + expect(response).toEqual(mockErrorResponse) expect(console.error).toHaveBeenCalledWith( 'Error fetching data: ', expect.any(Error) @@ -267,6 +423,9 @@ describe('fetchTrailblazerRankInfo', () => { const errorMessage = 'Network Error' axios.post.mockRejectedValue(new Error(errorMessage)) console.error = jest.fn() + const mockErrorResponse = { + data: { profile: { earnedAwards: { edges: [], pageInfo: null } } } + } const response = await fetchTrailblazerSuperBadgesInfo('testUsername') @@ -286,7 +445,7 @@ describe('fetchTrailblazerRankInfo', () => { headers: { 'Content-Type': 'application/json' } } ) - expect(response).toBeNull() + expect(response).toEqual(mockErrorResponse) expect(console.error).toHaveBeenCalledWith( 'Error fetching data: ', expect.any(Error) @@ -297,6 +456,9 @@ describe('fetchTrailblazerRankInfo', () => { const errorMessage = 'Network Error' axios.post.mockRejectedValue(new Error(errorMessage)) console.error = jest.fn() + const mockErrorResponse = { + data: { profile: { earnedAwards: { edges: [], pageInfo: null } } } + } const response = await fetchTrailblazerEventBadgesInfo('testUsername') @@ -316,7 +478,7 @@ describe('fetchTrailblazerRankInfo', () => { headers: { 'Content-Type': 'application/json' } } ) - expect(response).toBeNull() + expect(response).toEqual(mockErrorResponse) expect(console.error).toHaveBeenCalledWith( 'Error fetching data: ', expect.any(Error) @@ -327,6 +489,15 @@ describe('fetchTrailblazerRankInfo', () => { const errorMessage = 'Network Error' axios.post.mockRejectedValue(new Error(errorMessage)) console.error = jest.fn() + const mockErrorResponse = { + data: { + profile: { + credential: { + certifications: [] + } + } + } + } const response = await fetchTrailblazerCertifsInfo('testUsername') @@ -336,14 +507,15 @@ describe('fetchTrailblazerRankInfo', () => { query: expect.any(String), variables: { hasSlug: true, - slug: 'testUsername' + slug: 'testUsername', + count: 100 } }, { headers: { 'Content-Type': 'application/json' } } ) - expect(response).toBeNull() + expect(response).toEqual(mockErrorResponse) expect(console.error).toHaveBeenCalledWith( 'Error fetching data: ', expect.any(Error) @@ -354,6 +526,9 @@ describe('fetchTrailblazerRankInfo', () => { const errorMessage = 'Network Error' axios.post.mockRejectedValue(new Error(errorMessage)) console.error = jest.fn() + const mockErrorResponse = { + data: { profile: { earnedSkills: [] } } + } const response = await fetchTrailblazerSkillsInfo('testUsername') @@ -370,7 +545,7 @@ describe('fetchTrailblazerRankInfo', () => { headers: { 'Content-Type': 'application/json' } } ) - expect(response).toBeNull() + expect(response).toEqual(mockErrorResponse) expect(console.error).toHaveBeenCalledWith( 'Error fetching data: ', expect.any(Error) @@ -381,6 +556,9 @@ describe('fetchTrailblazerRankInfo', () => { const errorMessage = 'Network Error' axios.post.mockRejectedValue(new Error(errorMessage)) console.error = jest.fn() + const mockErrorResponse = { + data: { earnedStamps: { edges: [], pageInfo: null } } + } const response = await fetchTrailblazerEarnedStampsInfo('testUsername') @@ -389,7 +567,10 @@ describe('fetchTrailblazerRankInfo', () => { { query: expect.any(String), variables: { - first: 10, + after: null, + count: 100, + first: 100, + hasSlug: true, slug: 'testUsername' } }, @@ -397,7 +578,7 @@ describe('fetchTrailblazerRankInfo', () => { headers: { 'Content-Type': 'application/json' } } ) - expect(response).toBeNull() + expect(response).toEqual(mockErrorResponse) expect(console.error).toHaveBeenCalledWith( 'Error fetching data: ', expect.any(Error) diff --git a/badges/coverage.svg b/badges/coverage.svg index 752a3d6..f8e0486 100644 --- a/badges/coverage.svg +++ b/badges/coverage.svg @@ -1 +1 @@ -Coverage: 97.73%Coverage97.73% \ No newline at end of file +Coverage: 94.95%Coverage94.95% \ No newline at end of file diff --git a/dist/index.js b/dist/index.js index 77768fa..c351826 100644 --- a/dist/index.js +++ b/dist/index.js @@ -292259,96 +292259,188 @@ async function fetchTrailblazerRankInfo(trailheadUsername) { async function fetchTrailblazerBadgesInfo(trailheadUsername) { const endpoint = 'https://profile.api.trailhead.com/graphql' + let allEdges = [] + let pageInfo = null + let profileData = null - const graphqlQuery = { - query: GET_TRAILBLAZER_BADGES, - variables: { - count: 10, - after: null, - filter: null, - hasSlug: true, - slug: trailheadUsername + while (true) { + const graphqlQuery = { + query: GET_TRAILBLAZER_BADGES, + variables: { + count: 100, + after: pageInfo ? pageInfo.endCursor : null, + hasSlug: true, + slug: trailheadUsername + } } - } - try { - const response = await axios.post(endpoint, graphqlQuery, { - headers: { - 'Content-Type': 'application/json' + try { + const response = await axios.post(endpoint, graphqlQuery, { + headers: { + 'Content-Type': 'application/json' + } + }) + + if (!profileData) { + profileData = { + __typename: response.data.data.profile.__typename, + trailheadStats: response.data.data.profile.trailheadStats + } } - }) - return response.data - } catch (error) { - console.error('Error fetching data: ', error) - return null + allEdges = allEdges.concat(response.data.data.profile.earnedAwards.edges) + pageInfo = response.data.data.profile.earnedAwards.pageInfo + + if (!pageInfo.hasNextPage) { + break + } + } catch (error) { + console.error('Error fetching data: ', error) + return { + data: { + profile: { + ...profileData, + earnedAwards: { edges: allEdges, pageInfo } + } + } + } + } + } + + return { + data: { + profile: { ...profileData, earnedAwards: { edges: allEdges, pageInfo } } + } } } async function fetchTrailblazerSuperBadgesInfo(trailheadUsername) { const endpoint = 'https://profile.api.trailhead.com/graphql' + let allEdges = [] + let pageInfo = null + let profileData = null - const graphqlQuery = { - query: GET_TRAILBLAZER_BADGES, - variables: { - count: 100, - after: null, - filter: 'SUPERBADGE', - hasSlug: true, - slug: trailheadUsername + while (true) { + const graphqlQuery = { + query: GET_TRAILBLAZER_BADGES, + variables: { + count: 100, + after: pageInfo ? pageInfo.endCursor : null, + filter: 'SUPERBADGE', + hasSlug: true, + slug: trailheadUsername + } } - } - try { - const response = await axios.post(endpoint, graphqlQuery, { - headers: { - 'Content-Type': 'application/json' + try { + const response = await axios.post(endpoint, graphqlQuery, { + headers: { + 'Content-Type': 'application/json' + } + }) + + if (!profileData) { + profileData = { + __typename: response.data.data.profile.__typename, + trailheadStats: response.data.data.profile.trailheadStats + } } - }) - return response.data - } catch (error) { - console.error('Error fetching data: ', error) - return null + allEdges = allEdges.concat(response.data.data.profile.earnedAwards.edges) + pageInfo = response.data.data.profile.earnedAwards.pageInfo + + if (!pageInfo.hasNextPage) { + break + } + } catch (error) { + console.error('Error fetching data: ', error) + return { + data: { + profile: { + ...profileData, + earnedAwards: { edges: allEdges, pageInfo } + } + } + } + } + } + + return { + data: { + profile: { ...profileData, earnedAwards: { edges: allEdges, pageInfo } } + } } } async function fetchTrailblazerEventBadgesInfo(trailheadUsername) { const endpoint = 'https://profile.api.trailhead.com/graphql' + let allEdges = [] + let pageInfo = null + let profileData = null - const graphqlQuery = { - query: GET_TRAILBLAZER_BADGES, - variables: { - count: 100, - after: null, - filter: 'EVENT', - hasSlug: true, - slug: trailheadUsername + while (true) { + const graphqlQuery = { + query: GET_TRAILBLAZER_BADGES, + variables: { + count: 100, + after: pageInfo ? pageInfo.endCursor : null, + filter: 'EVENT', + hasSlug: true, + slug: trailheadUsername + } } - } - try { - const response = await axios.post(endpoint, graphqlQuery, { - headers: { - 'Content-Type': 'application/json' + try { + const response = await axios.post(endpoint, graphqlQuery, { + headers: { + 'Content-Type': 'application/json' + } + }) + + if (!profileData) { + profileData = { + __typename: response.data.data.profile.__typename, + trailheadStats: response.data.data.profile.trailheadStats + } } - }) - return response.data - } catch (error) { - console.error('Error fetching data: ', error) - return null + allEdges = allEdges.concat(response.data.data.profile.earnedAwards.edges) + pageInfo = response.data.data.profile.earnedAwards.pageInfo + + if (!pageInfo.hasNextPage) { + break + } + } catch (error) { + console.error('Error fetching data: ', error) + return { + data: { + profile: { + ...profileData, + earnedAwards: { edges: allEdges, pageInfo } + } + } + } + } + } + + return { + data: { + profile: { ...profileData, earnedAwards: { edges: allEdges, pageInfo } } + } } } async function fetchTrailblazerCertifsInfo(trailheadUsername) { const endpoint = 'https://profile.api.trailhead.com/graphql' + let allCertifications = [] + let profileData = null const graphqlQuery = { query: GET_TRAILBLAZER_CERTIFS, variables: { hasSlug: true, - slug: trailheadUsername + slug: trailheadUsername, + count: 100 } } @@ -292359,15 +292451,55 @@ async function fetchTrailblazerCertifsInfo(trailheadUsername) { } }) - return response.data + if (!profileData) { + profileData = { + __typename: response.data.data.profile.__typename, + id: response.data.data.profile.id, + credential: { + messages: response.data.data.profile.credential.messages, + messagesOnly: response.data.data.profile.credential.messagesOnly, + brands: response.data.data.profile.credential.brands + } + } + } + + allCertifications = allCertifications.concat( + response.data.data.profile.credential.certifications + ) } catch (error) { console.error('Error fetching data: ', error) - return null + profileData = profileData || {} + profileData.credential = profileData.credential || {} + return { + data: { + profile: { + ...profileData, + credential: { + ...profileData.credential, + certifications: allCertifications + } + } + } + } + } + + return { + data: { + profile: { + ...profileData, + credential: { + ...profileData.credential, + certifications: allCertifications + } + } + } } } async function fetchTrailblazerSkillsInfo(trailheadUsername) { const endpoint = 'https://profile.api.trailhead.com/graphql' + let allSkills = [] + let profileData = null const graphqlQuery = { query: GET_TRAILBLAZER_SKILLS, @@ -292384,35 +292516,80 @@ async function fetchTrailblazerSkillsInfo(trailheadUsername) { } }) - return response.data + if (!profileData) { + profileData = { + __typename: response.data.data.profile.__typename, + id: response.data.data.profile.id + } + } + + allSkills = allSkills.concat(response.data.data.profile.earnedSkills) } catch (error) { console.error('Error fetching data: ', error) - return null + return { + data: { + profile: { + ...profileData, + earnedSkills: allSkills + } + } + } + } + + return { + data: { + profile: { + ...profileData, + earnedSkills: allSkills + } + } } } async function fetchTrailblazerEarnedStampsInfo(trailheadUsername) { const endpoint = 'https://mobile.api.trailhead.com/graphql' + let allEdges = [] + let pageInfo = null - const graphqlQuery = { - query: GET_TRAILBLAZER_EARNED_STAMPS, - variables: { - slug: trailheadUsername, - first: 10 + while (true) { + const graphqlQuery = { + query: GET_TRAILBLAZER_EARNED_STAMPS, + variables: { + first: 100, + count: 100, + after: pageInfo ? pageInfo.endCursor : null, + hasSlug: true, + slug: trailheadUsername + } } - } - try { - const response = await axios.post(endpoint, graphqlQuery, { - headers: { - 'Content-Type': 'application/json' + try { + const response = await axios.post(endpoint, graphqlQuery, { + headers: { + 'Content-Type': 'application/json' + } + }) + + allEdges = allEdges.concat(response.data.data.earnedStamps.edges) + pageInfo = response.data.data.earnedStamps.pageInfo + + if (!pageInfo.hasNextPage) { + break } - }) + } catch (error) { + console.error('Error fetching data: ', error) + return { + data: { + earnedStamps: { edges: allEdges, pageInfo } + } + } + } + } - return response.data - } catch (error) { - console.error('Error fetching data: ', error) - return null + return { + data: { + earnedStamps: { edges: allEdges, pageInfo } + } } } diff --git a/src/getStats.js b/src/getStats.js index cd0d07c..8649c7c 100644 --- a/src/getStats.js +++ b/src/getStats.js @@ -34,96 +34,188 @@ async function fetchTrailblazerRankInfo(trailheadUsername) { async function fetchTrailblazerBadgesInfo(trailheadUsername) { const endpoint = 'https://profile.api.trailhead.com/graphql' + let allEdges = [] + let pageInfo = null + let profileData = null - const graphqlQuery = { - query: GET_TRAILBLAZER_BADGES, - variables: { - count: 10, - after: null, - filter: null, - hasSlug: true, - slug: trailheadUsername + while (true) { + const graphqlQuery = { + query: GET_TRAILBLAZER_BADGES, + variables: { + count: 100, + after: pageInfo ? pageInfo.endCursor : null, + hasSlug: true, + slug: trailheadUsername + } } - } - try { - const response = await axios.post(endpoint, graphqlQuery, { - headers: { - 'Content-Type': 'application/json' + try { + const response = await axios.post(endpoint, graphqlQuery, { + headers: { + 'Content-Type': 'application/json' + } + }) + + if (!profileData) { + profileData = { + __typename: response.data.data.profile.__typename, + trailheadStats: response.data.data.profile.trailheadStats + } } - }) - return response.data - } catch (error) { - console.error('Error fetching data: ', error) - return null + allEdges = allEdges.concat(response.data.data.profile.earnedAwards.edges) + pageInfo = response.data.data.profile.earnedAwards.pageInfo + + if (!pageInfo.hasNextPage) { + break + } + } catch (error) { + console.error('Error fetching data: ', error) + return { + data: { + profile: { + ...profileData, + earnedAwards: { edges: allEdges, pageInfo } + } + } + } + } + } + + return { + data: { + profile: { ...profileData, earnedAwards: { edges: allEdges, pageInfo } } + } } } async function fetchTrailblazerSuperBadgesInfo(trailheadUsername) { const endpoint = 'https://profile.api.trailhead.com/graphql' + let allEdges = [] + let pageInfo = null + let profileData = null - const graphqlQuery = { - query: GET_TRAILBLAZER_BADGES, - variables: { - count: 100, - after: null, - filter: 'SUPERBADGE', - hasSlug: true, - slug: trailheadUsername + while (true) { + const graphqlQuery = { + query: GET_TRAILBLAZER_BADGES, + variables: { + count: 100, + after: pageInfo ? pageInfo.endCursor : null, + filter: 'SUPERBADGE', + hasSlug: true, + slug: trailheadUsername + } } - } - try { - const response = await axios.post(endpoint, graphqlQuery, { - headers: { - 'Content-Type': 'application/json' + try { + const response = await axios.post(endpoint, graphqlQuery, { + headers: { + 'Content-Type': 'application/json' + } + }) + + if (!profileData) { + profileData = { + __typename: response.data.data.profile.__typename, + trailheadStats: response.data.data.profile.trailheadStats + } } - }) - return response.data - } catch (error) { - console.error('Error fetching data: ', error) - return null + allEdges = allEdges.concat(response.data.data.profile.earnedAwards.edges) + pageInfo = response.data.data.profile.earnedAwards.pageInfo + + if (!pageInfo.hasNextPage) { + break + } + } catch (error) { + console.error('Error fetching data: ', error) + return { + data: { + profile: { + ...profileData, + earnedAwards: { edges: allEdges, pageInfo } + } + } + } + } + } + + return { + data: { + profile: { ...profileData, earnedAwards: { edges: allEdges, pageInfo } } + } } } async function fetchTrailblazerEventBadgesInfo(trailheadUsername) { const endpoint = 'https://profile.api.trailhead.com/graphql' + let allEdges = [] + let pageInfo = null + let profileData = null - const graphqlQuery = { - query: GET_TRAILBLAZER_BADGES, - variables: { - count: 100, - after: null, - filter: 'EVENT', - hasSlug: true, - slug: trailheadUsername + while (true) { + const graphqlQuery = { + query: GET_TRAILBLAZER_BADGES, + variables: { + count: 100, + after: pageInfo ? pageInfo.endCursor : null, + filter: 'EVENT', + hasSlug: true, + slug: trailheadUsername + } } - } - try { - const response = await axios.post(endpoint, graphqlQuery, { - headers: { - 'Content-Type': 'application/json' + try { + const response = await axios.post(endpoint, graphqlQuery, { + headers: { + 'Content-Type': 'application/json' + } + }) + + if (!profileData) { + profileData = { + __typename: response.data.data.profile.__typename, + trailheadStats: response.data.data.profile.trailheadStats + } } - }) - return response.data - } catch (error) { - console.error('Error fetching data: ', error) - return null + allEdges = allEdges.concat(response.data.data.profile.earnedAwards.edges) + pageInfo = response.data.data.profile.earnedAwards.pageInfo + + if (!pageInfo.hasNextPage) { + break + } + } catch (error) { + console.error('Error fetching data: ', error) + return { + data: { + profile: { + ...profileData, + earnedAwards: { edges: allEdges, pageInfo } + } + } + } + } + } + + return { + data: { + profile: { ...profileData, earnedAwards: { edges: allEdges, pageInfo } } + } } } async function fetchTrailblazerCertifsInfo(trailheadUsername) { const endpoint = 'https://profile.api.trailhead.com/graphql' + let allCertifications = [] + let profileData = null const graphqlQuery = { query: GET_TRAILBLAZER_CERTIFS, variables: { hasSlug: true, - slug: trailheadUsername + slug: trailheadUsername, + count: 100 } } @@ -134,15 +226,55 @@ async function fetchTrailblazerCertifsInfo(trailheadUsername) { } }) - return response.data + if (!profileData) { + profileData = { + __typename: response.data.data.profile.__typename, + id: response.data.data.profile.id, + credential: { + messages: response.data.data.profile.credential.messages, + messagesOnly: response.data.data.profile.credential.messagesOnly, + brands: response.data.data.profile.credential.brands + } + } + } + + allCertifications = allCertifications.concat( + response.data.data.profile.credential.certifications + ) } catch (error) { console.error('Error fetching data: ', error) - return null + profileData = profileData || {} + profileData.credential = profileData.credential || {} + return { + data: { + profile: { + ...profileData, + credential: { + ...profileData.credential, + certifications: allCertifications + } + } + } + } + } + + return { + data: { + profile: { + ...profileData, + credential: { + ...profileData.credential, + certifications: allCertifications + } + } + } } } async function fetchTrailblazerSkillsInfo(trailheadUsername) { const endpoint = 'https://profile.api.trailhead.com/graphql' + let allSkills = [] + let profileData = null const graphqlQuery = { query: GET_TRAILBLAZER_SKILLS, @@ -159,35 +291,80 @@ async function fetchTrailblazerSkillsInfo(trailheadUsername) { } }) - return response.data + if (!profileData) { + profileData = { + __typename: response.data.data.profile.__typename, + id: response.data.data.profile.id + } + } + + allSkills = allSkills.concat(response.data.data.profile.earnedSkills) } catch (error) { console.error('Error fetching data: ', error) - return null + return { + data: { + profile: { + ...profileData, + earnedSkills: allSkills + } + } + } + } + + return { + data: { + profile: { + ...profileData, + earnedSkills: allSkills + } + } } } async function fetchTrailblazerEarnedStampsInfo(trailheadUsername) { const endpoint = 'https://mobile.api.trailhead.com/graphql' + let allEdges = [] + let pageInfo = null - const graphqlQuery = { - query: GET_TRAILBLAZER_EARNED_STAMPS, - variables: { - slug: trailheadUsername, - first: 10 + while (true) { + const graphqlQuery = { + query: GET_TRAILBLAZER_EARNED_STAMPS, + variables: { + first: 100, + count: 100, + after: pageInfo ? pageInfo.endCursor : null, + hasSlug: true, + slug: trailheadUsername + } } - } - try { - const response = await axios.post(endpoint, graphqlQuery, { - headers: { - 'Content-Type': 'application/json' + try { + const response = await axios.post(endpoint, graphqlQuery, { + headers: { + 'Content-Type': 'application/json' + } + }) + + allEdges = allEdges.concat(response.data.data.earnedStamps.edges) + pageInfo = response.data.data.earnedStamps.pageInfo + + if (!pageInfo.hasNextPage) { + break } - }) + } catch (error) { + console.error('Error fetching data: ', error) + return { + data: { + earnedStamps: { edges: allEdges, pageInfo } + } + } + } + } - return response.data - } catch (error) { - console.error('Error fetching data: ', error) - return null + return { + data: { + earnedStamps: { edges: allEdges, pageInfo } + } } } From fa635fef14dc83a2d1aab9f7cb988f63fa9bb063 Mon Sep 17 00:00:00 2001 From: Nathan Abondance Date: Sun, 29 Sep 2024 17:07:39 +0200 Subject: [PATCH 02/10] refactor(core): getStats is factorized --- badges/coverage.svg | 2 +- dist/index.js | 468 ++++++++++++++++++-------------------------- src/getStats.js | 468 ++++++++++++++++++-------------------------- src/rankinfo.js | 24 +++ 4 files changed, 401 insertions(+), 561 deletions(-) create mode 100644 src/rankinfo.js diff --git a/badges/coverage.svg b/badges/coverage.svg index f8e0486..f181026 100644 --- a/badges/coverage.svg +++ b/badges/coverage.svg @@ -1 +1 @@ -Coverage: 94.95%Coverage94.95% \ No newline at end of file +Coverage: 91.24%Coverage91.24% \ No newline at end of file diff --git a/dist/index.js b/dist/index.js index c351826..e15b2e8 100644 --- a/dist/index.js +++ b/dist/index.js @@ -292232,209 +292232,232 @@ const { GET_TRAILBLAZER_EARNED_STAMPS } = __nccwpck_require__(37056) -async function fetchTrailblazerRankInfo(trailheadUsername) { - const endpoint = 'https://profile.api.trailhead.com/graphql' +// Define the extractData function for each specific function +const extractRankData = (response, profileData, allEdges, pageInfo) => { + if (!profileData) { + profileData = { + __typename: response.data.data.profile.__typename, + trailheadStats: response.data.data.profile.trailheadStats + } + } - const graphqlQuery = { - query: GET_TRAILBLAZER_RANK, - variables: { - hasSlug: true, - slug: trailheadUsername + return { + newProfileData: profileData, + newAllEdges: allEdges, + newPageInfo: pageInfo + } +} + +const extractBadgesData = (response, profileData, allEdges, pageInfo) => { + if (!profileData) { + profileData = { + __typename: response.data.data.profile.__typename, + trailheadStats: response.data.data.profile.trailheadStats } } - try { - const response = await axios.post(endpoint, graphqlQuery, { - headers: { - 'Content-Type': 'application/json' - } - }) + allEdges = allEdges.concat(response.data.data.profile.earnedAwards.edges) + pageInfo = response.data.data.profile.earnedAwards.pageInfo - return response.data - } catch (error) { - console.error('Error fetching data: ', error) - return null + return { + newProfileData: profileData, + newAllEdges: allEdges, + newPageInfo: pageInfo } } -async function fetchTrailblazerBadgesInfo(trailheadUsername) { - const endpoint = 'https://profile.api.trailhead.com/graphql' - let allEdges = [] - let pageInfo = null - let profileData = null - - while (true) { - const graphqlQuery = { - query: GET_TRAILBLAZER_BADGES, - variables: { - count: 100, - after: pageInfo ? pageInfo.endCursor : null, - hasSlug: true, - slug: trailheadUsername +const extractCertifsData = (response, profileData, allEdges, pageInfo) => { + if (!profileData) { + profileData = { + __typename: response.data.data.profile.__typename, + id: response.data.data.profile.id, + credential: { + messages: response.data.data.profile.credential.messages, + messagesOnly: response.data.data.profile.credential.messagesOnly, + brands: response.data.data.profile.credential.brands } } + } - try { - const response = await axios.post(endpoint, graphqlQuery, { - headers: { - 'Content-Type': 'application/json' - } - }) - - if (!profileData) { - profileData = { - __typename: response.data.data.profile.__typename, - trailheadStats: response.data.data.profile.trailheadStats - } - } + allEdges = allEdges.concat( + response.data.data.profile.credential.certifications + ) - allEdges = allEdges.concat(response.data.data.profile.earnedAwards.edges) - pageInfo = response.data.data.profile.earnedAwards.pageInfo + return { + newProfileData: profileData, + newAllEdges: allEdges, + newPageInfo: pageInfo + } +} - if (!pageInfo.hasNextPage) { - break - } - } catch (error) { - console.error('Error fetching data: ', error) - return { - data: { - profile: { - ...profileData, - earnedAwards: { edges: allEdges, pageInfo } - } - } - } +const extractSkillsData = (response, profileData, allEdges, pageInfo) => { + if (!profileData) { + profileData = { + __typename: response.data.data.profile.__typename, + id: response.data.data.profile.id } } + allEdges = allEdges.concat(response.data.data.profile.earnedSkills) + return { - data: { - profile: { ...profileData, earnedAwards: { edges: allEdges, pageInfo } } - } + newProfileData: profileData, + newAllEdges: allEdges, + newPageInfo: pageInfo } } -async function fetchTrailblazerSuperBadgesInfo(trailheadUsername) { - const endpoint = 'https://profile.api.trailhead.com/graphql' - let allEdges = [] - let pageInfo = null - let profileData = null +const extractStampsData = (response, profileData, allEdges, pageInfo) => { + allEdges = allEdges.concat(response.data.data.earnedStamps.edges) + pageInfo = response.data.data.earnedStamps.pageInfo - while (true) { - const graphqlQuery = { - query: GET_TRAILBLAZER_BADGES, - variables: { - count: 100, - after: pageInfo ? pageInfo.endCursor : null, - filter: 'SUPERBADGE', - hasSlug: true, - slug: trailheadUsername + return { + newProfileData: profileData, + newAllEdges: allEdges, + newPageInfo: pageInfo + } +} + +// Define the factorized fetchData function +async function fetchData( + endpoint, + graphqlQuery, + extractData, + profileData = null, + allEdges = [], + pageInfo = null, + maxPage = 1000 // maximum number of pages to fetch +) { + if (maxPage <= 0) { + return { + data: { + profile: { + ...profileData, + earnedAwards: { edges: allEdges, pageInfo } + } } } + } + let response + let retries = 5 + + while (retries > 0) { try { - const response = await axios.post(endpoint, graphqlQuery, { + response = await axios.post(endpoint, graphqlQuery, { headers: { 'Content-Type': 'application/json' } }) - - if (!profileData) { - profileData = { - __typename: response.data.data.profile.__typename, - trailheadStats: response.data.data.profile.trailheadStats - } - } - - allEdges = allEdges.concat(response.data.data.profile.earnedAwards.edges) - pageInfo = response.data.data.profile.earnedAwards.pageInfo - - if (!pageInfo.hasNextPage) { - break - } + break } catch (error) { - console.error('Error fetching data: ', error) - return { - data: { - profile: { - ...profileData, - earnedAwards: { edges: allEdges, pageInfo } - } - } + if (error.response && error.response.status === 429 && retries > 0) { + await new Promise(resolve => setTimeout(resolve, 1000)) // wait for 1 second + retries-- + } else { + throw error } } } - return { - data: { - profile: { ...profileData, earnedAwards: { edges: allEdges, pageInfo } } - } - } + const { + data: { profile }, + errors + } = response.data + const { newProfileData, newAllEdges, newPageInfo } = extractData( + profile, + allEdges, + pageInfo + ) + + if (errors) { + console.error(errors) + throw new Error('Error fetching data') + } + + return await fetchData( + endpoint, + graphqlQuery, + extractData, + newProfileData, + newAllEdges, + newPageInfo, + maxPage - 1 + ) } -async function fetchTrailblazerEventBadgesInfo(trailheadUsername) { +// Use the fetchData function in each specific function +async function fetchTrailblazerRankInfo(trailheadUsername) { + console.log('fetchTrailblazerRankInfo') const endpoint = 'https://profile.api.trailhead.com/graphql' - let allEdges = [] - let pageInfo = null - let profileData = null - - while (true) { - const graphqlQuery = { - query: GET_TRAILBLAZER_BADGES, - variables: { - count: 100, - after: pageInfo ? pageInfo.endCursor : null, - filter: 'EVENT', - hasSlug: true, - slug: trailheadUsername - } + const graphqlQuery = { + query: GET_TRAILBLAZER_RANK, + variables: { + count: 100, + after: null, + filter: null, + hasSlug: true, + slug: trailheadUsername } + } - try { - const response = await axios.post(endpoint, graphqlQuery, { - headers: { - 'Content-Type': 'application/json' - } - }) + return await fetchData(endpoint, graphqlQuery, extractRankData, (maxPage = 1)) +} - if (!profileData) { - profileData = { - __typename: response.data.data.profile.__typename, - trailheadStats: response.data.data.profile.trailheadStats - } - } +async function fetchTrailblazerBadgesInfo(trailheadUsername) { + console.log('fetchTrailblazerBadgesInfo') + const endpoint = 'https://profile.api.trailhead.com/graphql' + const graphqlQuery = { + query: GET_TRAILBLAZER_BADGES, + variables: { + count: 100, + after: null, + filter: null, + hasSlug: true, + slug: trailheadUsername + } + } - allEdges = allEdges.concat(response.data.data.profile.earnedAwards.edges) - pageInfo = response.data.data.profile.earnedAwards.pageInfo + return await fetchData(endpoint, graphqlQuery, extractBadgesData) +} - if (!pageInfo.hasNextPage) { - break - } - } catch (error) { - console.error('Error fetching data: ', error) - return { - data: { - profile: { - ...profileData, - earnedAwards: { edges: allEdges, pageInfo } - } - } - } +async function fetchTrailblazerSuperBadgesInfo(trailheadUsername) { + console.log('fetchTrailblazerSuperBadgesInfo') + const endpoint = 'https://profile.api.trailhead.com/graphql' + const graphqlQuery = { + query: GET_TRAILBLAZER_BADGES, + variables: { + count: 100, + after: null, + filter: 'SUPERBADGE', + hasSlug: true, + slug: trailheadUsername } } - return { - data: { - profile: { ...profileData, earnedAwards: { edges: allEdges, pageInfo } } + return await fetchData(endpoint, graphqlQuery, extractBadgesData) +} + +async function fetchTrailblazerEventBadgesInfo(trailheadUsername) { + console.log('fetchTrailblazerEventBadgesInfo') + const endpoint = 'https://profile.api.trailhead.com/graphql' + const graphqlQuery = { + query: GET_TRAILBLAZER_BADGES, + variables: { + count: 100, + after: null, + filter: 'EVENT', + hasSlug: true, + slug: trailheadUsername } } + + return await fetchData(endpoint, graphqlQuery, extractBadgesData) } async function fetchTrailblazerCertifsInfo(trailheadUsername) { + console.log('fetchTrailblazerCertifsInfo') const endpoint = 'https://profile.api.trailhead.com/graphql' - let allCertifications = [] - let profileData = null - const graphqlQuery = { query: GET_TRAILBLAZER_CERTIFS, variables: { @@ -292444,63 +292467,12 @@ async function fetchTrailblazerCertifsInfo(trailheadUsername) { } } - try { - const response = await axios.post(endpoint, graphqlQuery, { - headers: { - 'Content-Type': 'application/json' - } - }) - - if (!profileData) { - profileData = { - __typename: response.data.data.profile.__typename, - id: response.data.data.profile.id, - credential: { - messages: response.data.data.profile.credential.messages, - messagesOnly: response.data.data.profile.credential.messagesOnly, - brands: response.data.data.profile.credential.brands - } - } - } - - allCertifications = allCertifications.concat( - response.data.data.profile.credential.certifications - ) - } catch (error) { - console.error('Error fetching data: ', error) - profileData = profileData || {} - profileData.credential = profileData.credential || {} - return { - data: { - profile: { - ...profileData, - credential: { - ...profileData.credential, - certifications: allCertifications - } - } - } - } - } - - return { - data: { - profile: { - ...profileData, - credential: { - ...profileData.credential, - certifications: allCertifications - } - } - } - } + return await fetchData(endpoint, graphqlQuery, extractCertifsData) } async function fetchTrailblazerSkillsInfo(trailheadUsername) { + console.log('fetchTrailblazerSkillsInfo') const endpoint = 'https://profile.api.trailhead.com/graphql' - let allSkills = [] - let profileData = null - const graphqlQuery = { query: GET_TRAILBLAZER_SKILLS, variables: { @@ -292509,88 +292481,24 @@ async function fetchTrailblazerSkillsInfo(trailheadUsername) { } } - try { - const response = await axios.post(endpoint, graphqlQuery, { - headers: { - 'Content-Type': 'application/json' - } - }) - - if (!profileData) { - profileData = { - __typename: response.data.data.profile.__typename, - id: response.data.data.profile.id - } - } - - allSkills = allSkills.concat(response.data.data.profile.earnedSkills) - } catch (error) { - console.error('Error fetching data: ', error) - return { - data: { - profile: { - ...profileData, - earnedSkills: allSkills - } - } - } - } - - return { - data: { - profile: { - ...profileData, - earnedSkills: allSkills - } - } - } + return await fetchData(endpoint, graphqlQuery, extractSkillsData) } async function fetchTrailblazerEarnedStampsInfo(trailheadUsername) { + console.log('fetchTrailblazerEarnedStampsInfo') const endpoint = 'https://mobile.api.trailhead.com/graphql' - let allEdges = [] - let pageInfo = null - - while (true) { - const graphqlQuery = { - query: GET_TRAILBLAZER_EARNED_STAMPS, - variables: { - first: 100, - count: 100, - after: pageInfo ? pageInfo.endCursor : null, - hasSlug: true, - slug: trailheadUsername - } - } - - try { - const response = await axios.post(endpoint, graphqlQuery, { - headers: { - 'Content-Type': 'application/json' - } - }) - - allEdges = allEdges.concat(response.data.data.earnedStamps.edges) - pageInfo = response.data.data.earnedStamps.pageInfo - - if (!pageInfo.hasNextPage) { - break - } - } catch (error) { - console.error('Error fetching data: ', error) - return { - data: { - earnedStamps: { edges: allEdges, pageInfo } - } - } + const graphqlQuery = { + query: GET_TRAILBLAZER_EARNED_STAMPS, + variables: { + first: 100, + count: 100, + after: null, + hasSlug: true, + slug: trailheadUsername } } - return { - data: { - earnedStamps: { edges: allEdges, pageInfo } - } - } + return await fetchData(endpoint, graphqlQuery, extractStampsData) } module.exports = { diff --git a/src/getStats.js b/src/getStats.js index 8649c7c..43058cd 100644 --- a/src/getStats.js +++ b/src/getStats.js @@ -7,365 +7,273 @@ const { GET_TRAILBLAZER_EARNED_STAMPS } = require('./queries') -async function fetchTrailblazerRankInfo(trailheadUsername) { - const endpoint = 'https://profile.api.trailhead.com/graphql' - - const graphqlQuery = { - query: GET_TRAILBLAZER_RANK, - variables: { - hasSlug: true, - slug: trailheadUsername +// Define the extractData function for each specific function +const extractRankData = (response, profileData, allEdges, pageInfo) => { + if (!profileData) { + profileData = { + __typename: response.data.data.profile.__typename, + trailheadStats: response.data.data.profile.trailheadStats } } - try { - const response = await axios.post(endpoint, graphqlQuery, { - headers: { - 'Content-Type': 'application/json' - } - }) - - return response.data - } catch (error) { - console.error('Error fetching data: ', error) - return null + return { + newProfileData: profileData, + newAllEdges: allEdges, + newPageInfo: pageInfo } } -async function fetchTrailblazerBadgesInfo(trailheadUsername) { - const endpoint = 'https://profile.api.trailhead.com/graphql' - let allEdges = [] - let pageInfo = null - let profileData = null - - while (true) { - const graphqlQuery = { - query: GET_TRAILBLAZER_BADGES, - variables: { - count: 100, - after: pageInfo ? pageInfo.endCursor : null, - hasSlug: true, - slug: trailheadUsername - } +const extractBadgesData = (response, profileData, allEdges, pageInfo) => { + if (!profileData) { + profileData = { + __typename: response.data.data.profile.__typename, + trailheadStats: response.data.data.profile.trailheadStats } + } - try { - const response = await axios.post(endpoint, graphqlQuery, { - headers: { - 'Content-Type': 'application/json' - } - }) - - if (!profileData) { - profileData = { - __typename: response.data.data.profile.__typename, - trailheadStats: response.data.data.profile.trailheadStats - } - } + allEdges = allEdges.concat(response.data.data.profile.earnedAwards.edges) + pageInfo = response.data.data.profile.earnedAwards.pageInfo - allEdges = allEdges.concat(response.data.data.profile.earnedAwards.edges) - pageInfo = response.data.data.profile.earnedAwards.pageInfo + return { + newProfileData: profileData, + newAllEdges: allEdges, + newPageInfo: pageInfo + } +} - if (!pageInfo.hasNextPage) { - break - } - } catch (error) { - console.error('Error fetching data: ', error) - return { - data: { - profile: { - ...profileData, - earnedAwards: { edges: allEdges, pageInfo } - } - } +const extractCertifsData = (response, profileData, allEdges, pageInfo) => { + if (!profileData) { + profileData = { + __typename: response.data.data.profile.__typename, + id: response.data.data.profile.id, + credential: { + messages: response.data.data.profile.credential.messages, + messagesOnly: response.data.data.profile.credential.messagesOnly, + brands: response.data.data.profile.credential.brands } } } + allEdges = allEdges.concat( + response.data.data.profile.credential.certifications + ) + return { - data: { - profile: { ...profileData, earnedAwards: { edges: allEdges, pageInfo } } - } + newProfileData: profileData, + newAllEdges: allEdges, + newPageInfo: pageInfo } } -async function fetchTrailblazerSuperBadgesInfo(trailheadUsername) { - const endpoint = 'https://profile.api.trailhead.com/graphql' - let allEdges = [] - let pageInfo = null - let profileData = null - - while (true) { - const graphqlQuery = { - query: GET_TRAILBLAZER_BADGES, - variables: { - count: 100, - after: pageInfo ? pageInfo.endCursor : null, - filter: 'SUPERBADGE', - hasSlug: true, - slug: trailheadUsername - } +const extractSkillsData = (response, profileData, allEdges, pageInfo) => { + if (!profileData) { + profileData = { + __typename: response.data.data.profile.__typename, + id: response.data.data.profile.id } + } - try { - const response = await axios.post(endpoint, graphqlQuery, { - headers: { - 'Content-Type': 'application/json' - } - }) - - if (!profileData) { - profileData = { - __typename: response.data.data.profile.__typename, - trailheadStats: response.data.data.profile.trailheadStats - } - } - - allEdges = allEdges.concat(response.data.data.profile.earnedAwards.edges) - pageInfo = response.data.data.profile.earnedAwards.pageInfo + allEdges = allEdges.concat(response.data.data.profile.earnedSkills) - if (!pageInfo.hasNextPage) { - break - } - } catch (error) { - console.error('Error fetching data: ', error) - return { - data: { - profile: { - ...profileData, - earnedAwards: { edges: allEdges, pageInfo } - } - } - } - } + return { + newProfileData: profileData, + newAllEdges: allEdges, + newPageInfo: pageInfo } +} + +const extractStampsData = (response, profileData, allEdges, pageInfo) => { + allEdges = allEdges.concat(response.data.data.earnedStamps.edges) + pageInfo = response.data.data.earnedStamps.pageInfo return { - data: { - profile: { ...profileData, earnedAwards: { edges: allEdges, pageInfo } } - } + newProfileData: profileData, + newAllEdges: allEdges, + newPageInfo: pageInfo } } -async function fetchTrailblazerEventBadgesInfo(trailheadUsername) { - const endpoint = 'https://profile.api.trailhead.com/graphql' - let allEdges = [] - let pageInfo = null - let profileData = null - - while (true) { - const graphqlQuery = { - query: GET_TRAILBLAZER_BADGES, - variables: { - count: 100, - after: pageInfo ? pageInfo.endCursor : null, - filter: 'EVENT', - hasSlug: true, - slug: trailheadUsername +// Define the factorized fetchData function +async function fetchData( + endpoint, + graphqlQuery, + extractData, + profileData = null, + allEdges = [], + pageInfo = null, + maxPage = 1000 // maximum number of pages to fetch +) { + if (maxPage <= 0) { + return { + data: { + profile: { + ...profileData, + earnedAwards: { edges: allEdges, pageInfo } + } } } + } + let response + let retries = 5 + + while (retries > 0) { try { - const response = await axios.post(endpoint, graphqlQuery, { + response = await axios.post(endpoint, graphqlQuery, { headers: { 'Content-Type': 'application/json' } }) - - if (!profileData) { - profileData = { - __typename: response.data.data.profile.__typename, - trailheadStats: response.data.data.profile.trailheadStats - } - } - - allEdges = allEdges.concat(response.data.data.profile.earnedAwards.edges) - pageInfo = response.data.data.profile.earnedAwards.pageInfo - - if (!pageInfo.hasNextPage) { - break - } + break } catch (error) { - console.error('Error fetching data: ', error) - return { - data: { - profile: { - ...profileData, - earnedAwards: { edges: allEdges, pageInfo } - } - } + if (error.response && error.response.status === 429 && retries > 0) { + await new Promise(resolve => setTimeout(resolve, 1000)) // wait for 1 second + retries-- + } else { + throw error } } } - return { - data: { - profile: { ...profileData, earnedAwards: { edges: allEdges, pageInfo } } - } + const { + data: { profile }, + errors + } = response.data + const { newProfileData, newAllEdges, newPageInfo } = extractData( + profile, + allEdges, + pageInfo + ) + + if (errors) { + console.error(errors) + throw new Error('Error fetching data') } + + return await fetchData( + endpoint, + graphqlQuery, + extractData, + newProfileData, + newAllEdges, + newPageInfo, + maxPage - 1 + ) } -async function fetchTrailblazerCertifsInfo(trailheadUsername) { +// Use the fetchData function in each specific function +async function fetchTrailblazerRankInfo(trailheadUsername) { + console.log('fetchTrailblazerRankInfo') const endpoint = 'https://profile.api.trailhead.com/graphql' - let allCertifications = [] - let profileData = null - const graphqlQuery = { - query: GET_TRAILBLAZER_CERTIFS, + query: GET_TRAILBLAZER_RANK, variables: { + count: 100, + after: null, + filter: null, hasSlug: true, - slug: trailheadUsername, - count: 100 + slug: trailheadUsername } } - try { - const response = await axios.post(endpoint, graphqlQuery, { - headers: { - 'Content-Type': 'application/json' - } - }) - - if (!profileData) { - profileData = { - __typename: response.data.data.profile.__typename, - id: response.data.data.profile.id, - credential: { - messages: response.data.data.profile.credential.messages, - messagesOnly: response.data.data.profile.credential.messagesOnly, - brands: response.data.data.profile.credential.brands - } - } - } + return await fetchData(endpoint, graphqlQuery, extractRankData, (maxPage = 1)) +} - allCertifications = allCertifications.concat( - response.data.data.profile.credential.certifications - ) - } catch (error) { - console.error('Error fetching data: ', error) - profileData = profileData || {} - profileData.credential = profileData.credential || {} - return { - data: { - profile: { - ...profileData, - credential: { - ...profileData.credential, - certifications: allCertifications - } - } - } +async function fetchTrailblazerBadgesInfo(trailheadUsername) { + console.log('fetchTrailblazerBadgesInfo') + const endpoint = 'https://profile.api.trailhead.com/graphql' + const graphqlQuery = { + query: GET_TRAILBLAZER_BADGES, + variables: { + count: 100, + after: null, + filter: null, + hasSlug: true, + slug: trailheadUsername } } - return { - data: { - profile: { - ...profileData, - credential: { - ...profileData.credential, - certifications: allCertifications - } - } - } - } + return await fetchData(endpoint, graphqlQuery, extractBadgesData) } -async function fetchTrailblazerSkillsInfo(trailheadUsername) { +async function fetchTrailblazerSuperBadgesInfo(trailheadUsername) { + console.log('fetchTrailblazerSuperBadgesInfo') const endpoint = 'https://profile.api.trailhead.com/graphql' - let allSkills = [] - let profileData = null - const graphqlQuery = { - query: GET_TRAILBLAZER_SKILLS, + query: GET_TRAILBLAZER_BADGES, variables: { + count: 100, + after: null, + filter: 'SUPERBADGE', hasSlug: true, slug: trailheadUsername } } - try { - const response = await axios.post(endpoint, graphqlQuery, { - headers: { - 'Content-Type': 'application/json' - } - }) + return await fetchData(endpoint, graphqlQuery, extractBadgesData) +} - if (!profileData) { - profileData = { - __typename: response.data.data.profile.__typename, - id: response.data.data.profile.id - } +async function fetchTrailblazerEventBadgesInfo(trailheadUsername) { + console.log('fetchTrailblazerEventBadgesInfo') + const endpoint = 'https://profile.api.trailhead.com/graphql' + const graphqlQuery = { + query: GET_TRAILBLAZER_BADGES, + variables: { + count: 100, + after: null, + filter: 'EVENT', + hasSlug: true, + slug: trailheadUsername } + } - allSkills = allSkills.concat(response.data.data.profile.earnedSkills) - } catch (error) { - console.error('Error fetching data: ', error) - return { - data: { - profile: { - ...profileData, - earnedSkills: allSkills - } - } + return await fetchData(endpoint, graphqlQuery, extractBadgesData) +} + +async function fetchTrailblazerCertifsInfo(trailheadUsername) { + console.log('fetchTrailblazerCertifsInfo') + const endpoint = 'https://profile.api.trailhead.com/graphql' + const graphqlQuery = { + query: GET_TRAILBLAZER_CERTIFS, + variables: { + hasSlug: true, + slug: trailheadUsername, + count: 100 } } - return { - data: { - profile: { - ...profileData, - earnedSkills: allSkills - } + return await fetchData(endpoint, graphqlQuery, extractCertifsData) +} + +async function fetchTrailblazerSkillsInfo(trailheadUsername) { + console.log('fetchTrailblazerSkillsInfo') + const endpoint = 'https://profile.api.trailhead.com/graphql' + const graphqlQuery = { + query: GET_TRAILBLAZER_SKILLS, + variables: { + hasSlug: true, + slug: trailheadUsername } } + + return await fetchData(endpoint, graphqlQuery, extractSkillsData) } async function fetchTrailblazerEarnedStampsInfo(trailheadUsername) { + console.log('fetchTrailblazerEarnedStampsInfo') const endpoint = 'https://mobile.api.trailhead.com/graphql' - let allEdges = [] - let pageInfo = null - - while (true) { - const graphqlQuery = { - query: GET_TRAILBLAZER_EARNED_STAMPS, - variables: { - first: 100, - count: 100, - after: pageInfo ? pageInfo.endCursor : null, - hasSlug: true, - slug: trailheadUsername - } - } - - try { - const response = await axios.post(endpoint, graphqlQuery, { - headers: { - 'Content-Type': 'application/json' - } - }) - - allEdges = allEdges.concat(response.data.data.earnedStamps.edges) - pageInfo = response.data.data.earnedStamps.pageInfo - - if (!pageInfo.hasNextPage) { - break - } - } catch (error) { - console.error('Error fetching data: ', error) - return { - data: { - earnedStamps: { edges: allEdges, pageInfo } - } - } + const graphqlQuery = { + query: GET_TRAILBLAZER_EARNED_STAMPS, + variables: { + first: 100, + count: 100, + after: null, + hasSlug: true, + slug: trailheadUsername } } - return { - data: { - earnedStamps: { edges: allEdges, pageInfo } - } - } + return await fetchData(endpoint, graphqlQuery, extractStampsData) } module.exports = { diff --git a/src/rankinfo.js b/src/rankinfo.js new file mode 100644 index 0000000..33257b2 --- /dev/null +++ b/src/rankinfo.js @@ -0,0 +1,24 @@ +async function fetchTrailblazerRankInfo(trailheadUsername) { + const endpoint = 'https://profile.api.trailhead.com/graphql' + + const graphqlQuery = { + query: GET_TRAILBLAZER_RANK, + variables: { + hasSlug: true, + slug: trailheadUsername + } + } + + try { + const response = await axios.post(endpoint, graphqlQuery, { + headers: { + 'Content-Type': 'application/json' + } + }) + + return response.data + } catch (error) { + console.error('Error fetching data: ', error) + return null + } +} From 80e2f1fd47541a0503b151c27b2f0c5ce22e7e7e Mon Sep 17 00:00:00 2001 From: Nathan Abondance Date: Sun, 29 Sep 2024 17:59:28 +0200 Subject: [PATCH 03/10] dd --- badges/coverage.svg | 2 +- dist/index.js | 3 ++- src/getStats.js | 3 ++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/badges/coverage.svg b/badges/coverage.svg index f181026..a75522c 100644 --- a/badges/coverage.svg +++ b/badges/coverage.svg @@ -1 +1 @@ -Coverage: 91.24%Coverage91.24% \ No newline at end of file +Coverage: 91.06%Coverage91.06% \ No newline at end of file diff --git a/dist/index.js b/dist/index.js index e15b2e8..5d0d4f1 100644 --- a/dist/index.js +++ b/dist/index.js @@ -292329,11 +292329,12 @@ async function fetchData( maxPage = 1000 // maximum number of pages to fetch ) { if (maxPage <= 0) { + const { dataModel } = extractData(profileData, allEdges, pageInfo) return { data: { profile: { ...profileData, - earnedAwards: { edges: allEdges, pageInfo } + ...dataModel } } } diff --git a/src/getStats.js b/src/getStats.js index 43058cd..2e27ff9 100644 --- a/src/getStats.js +++ b/src/getStats.js @@ -104,11 +104,12 @@ async function fetchData( maxPage = 1000 // maximum number of pages to fetch ) { if (maxPage <= 0) { + const { dataModel } = extractData(profileData, allEdges, pageInfo) return { data: { profile: { ...profileData, - earnedAwards: { edges: allEdges, pageInfo } + ...dataModel } } } From e5dc424124107496fc38f322f63e412333f3e65d Mon Sep 17 00:00:00 2001 From: Nathan Abondance Date: Sun, 29 Sep 2024 18:01:21 +0200 Subject: [PATCH 04/10] ddd --- badges/coverage.svg | 2 +- dist/index.js | 2 ++ src/getStats.js | 2 ++ 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/badges/coverage.svg b/badges/coverage.svg index a75522c..0fa4b81 100644 --- a/badges/coverage.svg +++ b/badges/coverage.svg @@ -1 +1 @@ -Coverage: 91.06%Coverage91.06% \ No newline at end of file +Coverage: 90.71%Coverage90.71% \ No newline at end of file diff --git a/dist/index.js b/dist/index.js index 5d0d4f1..8ac3114 100644 --- a/dist/index.js +++ b/dist/index.js @@ -292330,6 +292330,8 @@ async function fetchData( ) { if (maxPage <= 0) { const { dataModel } = extractData(profileData, allEdges, pageInfo) + console.log('fetchData: maxPage reached') + console.log('dataModel:' + dataModel) return { data: { profile: { diff --git a/src/getStats.js b/src/getStats.js index 2e27ff9..5dd9a76 100644 --- a/src/getStats.js +++ b/src/getStats.js @@ -105,6 +105,8 @@ async function fetchData( ) { if (maxPage <= 0) { const { dataModel } = extractData(profileData, allEdges, pageInfo) + console.log('fetchData: maxPage reached') + console.log('dataModel:' + dataModel) return { data: { profile: { From a8d36a607f08dcbf9d50b700a60629424fe8ca9f Mon Sep 17 00:00:00 2001 From: Nathan Abondance Date: Sun, 29 Sep 2024 18:03:59 +0200 Subject: [PATCH 05/10] simpler --- src/getStats.js | 100 +++++++++++++++++++----------------------------- 1 file changed, 39 insertions(+), 61 deletions(-) diff --git a/src/getStats.js b/src/getStats.js index 5dd9a76..9af4c9c 100644 --- a/src/getStats.js +++ b/src/getStats.js @@ -100,73 +100,57 @@ async function fetchData( extractData, profileData = null, allEdges = [], - pageInfo = null, - maxPage = 1000 // maximum number of pages to fetch + pageInfo = null ) { - if (maxPage <= 0) { - const { dataModel } = extractData(profileData, allEdges, pageInfo) - console.log('fetchData: maxPage reached') - console.log('dataModel:' + dataModel) - return { - data: { - profile: { - ...profileData, - ...dataModel + try { + const response = await axios.post(endpoint, graphqlQuery, { + headers: { + 'Content-Type': 'application/json' + } + }) + + const { newProfileData, newAllEdges, newPageInfo } = extractData( + response, + profileData, + allEdges, + pageInfo + ) + + if (newPageInfo != undefined && !newPageInfo.hasNextPage) { + console.log('newPageInfo: ', newPageInfo) + return { + data: { + profile: { + ...newProfileData, + earnedAwards: { edges: newAllEdges, pageInfo: newPageInfo } + } } } } - } - - let response - let retries = 5 - while (retries > 0) { - try { - response = await axios.post(endpoint, graphqlQuery, { - headers: { - 'Content-Type': 'application/json' + return await fetchData( + endpoint, + graphqlQuery, + extractData, + newProfileData, + newAllEdges, + newPageInfo + ) + } catch (error) { + console.error('Error fetching data: ', error) + return { + data: { + profile: { + ...profileData, + earnedAwards: { edges: allEdges, pageInfo } } - }) - break - } catch (error) { - if (error.response && error.response.status === 429 && retries > 0) { - await new Promise(resolve => setTimeout(resolve, 1000)) // wait for 1 second - retries-- - } else { - throw error } } } - - const { - data: { profile }, - errors - } = response.data - const { newProfileData, newAllEdges, newPageInfo } = extractData( - profile, - allEdges, - pageInfo - ) - - if (errors) { - console.error(errors) - throw new Error('Error fetching data') - } - - return await fetchData( - endpoint, - graphqlQuery, - extractData, - newProfileData, - newAllEdges, - newPageInfo, - maxPage - 1 - ) } // Use the fetchData function in each specific function async function fetchTrailblazerRankInfo(trailheadUsername) { - console.log('fetchTrailblazerRankInfo') const endpoint = 'https://profile.api.trailhead.com/graphql' const graphqlQuery = { query: GET_TRAILBLAZER_RANK, @@ -179,11 +163,10 @@ async function fetchTrailblazerRankInfo(trailheadUsername) { } } - return await fetchData(endpoint, graphqlQuery, extractRankData, (maxPage = 1)) + return await fetchData(endpoint, graphqlQuery, extractRankData) } async function fetchTrailblazerBadgesInfo(trailheadUsername) { - console.log('fetchTrailblazerBadgesInfo') const endpoint = 'https://profile.api.trailhead.com/graphql' const graphqlQuery = { query: GET_TRAILBLAZER_BADGES, @@ -200,7 +183,6 @@ async function fetchTrailblazerBadgesInfo(trailheadUsername) { } async function fetchTrailblazerSuperBadgesInfo(trailheadUsername) { - console.log('fetchTrailblazerSuperBadgesInfo') const endpoint = 'https://profile.api.trailhead.com/graphql' const graphqlQuery = { query: GET_TRAILBLAZER_BADGES, @@ -217,7 +199,6 @@ async function fetchTrailblazerSuperBadgesInfo(trailheadUsername) { } async function fetchTrailblazerEventBadgesInfo(trailheadUsername) { - console.log('fetchTrailblazerEventBadgesInfo') const endpoint = 'https://profile.api.trailhead.com/graphql' const graphqlQuery = { query: GET_TRAILBLAZER_BADGES, @@ -234,7 +215,6 @@ async function fetchTrailblazerEventBadgesInfo(trailheadUsername) { } async function fetchTrailblazerCertifsInfo(trailheadUsername) { - console.log('fetchTrailblazerCertifsInfo') const endpoint = 'https://profile.api.trailhead.com/graphql' const graphqlQuery = { query: GET_TRAILBLAZER_CERTIFS, @@ -249,7 +229,6 @@ async function fetchTrailblazerCertifsInfo(trailheadUsername) { } async function fetchTrailblazerSkillsInfo(trailheadUsername) { - console.log('fetchTrailblazerSkillsInfo') const endpoint = 'https://profile.api.trailhead.com/graphql' const graphqlQuery = { query: GET_TRAILBLAZER_SKILLS, @@ -263,7 +242,6 @@ async function fetchTrailblazerSkillsInfo(trailheadUsername) { } async function fetchTrailblazerEarnedStampsInfo(trailheadUsername) { - console.log('fetchTrailblazerEarnedStampsInfo') const endpoint = 'https://mobile.api.trailhead.com/graphql' const graphqlQuery = { query: GET_TRAILBLAZER_EARNED_STAMPS, From eedb5400530a3b71844ad46de4934aab8aefa0ff Mon Sep 17 00:00:00 2001 From: Nathan Abondance Date: Sun, 29 Sep 2024 18:04:31 +0200 Subject: [PATCH 06/10] log --- src/getStats.js | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/getStats.js b/src/getStats.js index 9af4c9c..ddfcfc2 100644 --- a/src/getStats.js +++ b/src/getStats.js @@ -151,6 +151,7 @@ async function fetchData( // Use the fetchData function in each specific function async function fetchTrailblazerRankInfo(trailheadUsername) { + console.log('fetchTrailblazerRankInfo') const endpoint = 'https://profile.api.trailhead.com/graphql' const graphqlQuery = { query: GET_TRAILBLAZER_RANK, @@ -167,6 +168,7 @@ async function fetchTrailblazerRankInfo(trailheadUsername) { } async function fetchTrailblazerBadgesInfo(trailheadUsername) { + console.log('fetchTrailblazerBadgesInfo') const endpoint = 'https://profile.api.trailhead.com/graphql' const graphqlQuery = { query: GET_TRAILBLAZER_BADGES, @@ -183,6 +185,7 @@ async function fetchTrailblazerBadgesInfo(trailheadUsername) { } async function fetchTrailblazerSuperBadgesInfo(trailheadUsername) { + console.log('fetchTrailblazerSuperBadgesInfo') const endpoint = 'https://profile.api.trailhead.com/graphql' const graphqlQuery = { query: GET_TRAILBLAZER_BADGES, @@ -199,6 +202,7 @@ async function fetchTrailblazerSuperBadgesInfo(trailheadUsername) { } async function fetchTrailblazerEventBadgesInfo(trailheadUsername) { + console.log('fetchTrailblazerEventBadgesInfo') const endpoint = 'https://profile.api.trailhead.com/graphql' const graphqlQuery = { query: GET_TRAILBLAZER_BADGES, @@ -215,6 +219,7 @@ async function fetchTrailblazerEventBadgesInfo(trailheadUsername) { } async function fetchTrailblazerCertifsInfo(trailheadUsername) { + console.log('fetchTrailblazerCertifsInfo') const endpoint = 'https://profile.api.trailhead.com/graphql' const graphqlQuery = { query: GET_TRAILBLAZER_CERTIFS, @@ -229,6 +234,7 @@ async function fetchTrailblazerCertifsInfo(trailheadUsername) { } async function fetchTrailblazerSkillsInfo(trailheadUsername) { + console.log('fetchTrailblazerSkillsInfo') const endpoint = 'https://profile.api.trailhead.com/graphql' const graphqlQuery = { query: GET_TRAILBLAZER_SKILLS, @@ -242,6 +248,7 @@ async function fetchTrailblazerSkillsInfo(trailheadUsername) { } async function fetchTrailblazerEarnedStampsInfo(trailheadUsername) { + console.log('fetchTrailblazerEarnedStampsInfo') const endpoint = 'https://mobile.api.trailhead.com/graphql' const graphqlQuery = { query: GET_TRAILBLAZER_EARNED_STAMPS, From 0d1be2da8d6c1ba82f9d11924e4bccad32b86a64 Mon Sep 17 00:00:00 2001 From: Nathan Abondance Date: Sun, 29 Sep 2024 18:26:03 +0200 Subject: [PATCH 07/10] more log --- dist/index.js | 100 ++++++++++++++++++++++-------------------------- src/getStats.js | 7 ++++ 2 files changed, 53 insertions(+), 54 deletions(-) diff --git a/dist/index.js b/dist/index.js index 8ac3114..54b12c1 100644 --- a/dist/index.js +++ b/dist/index.js @@ -292325,68 +292325,60 @@ async function fetchData( extractData, profileData = null, allEdges = [], - pageInfo = null, - maxPage = 1000 // maximum number of pages to fetch + pageInfo = null ) { - if (maxPage <= 0) { - const { dataModel } = extractData(profileData, allEdges, pageInfo) - console.log('fetchData: maxPage reached') - console.log('dataModel:' + dataModel) - return { - data: { - profile: { - ...profileData, - ...dataModel + console.log('fetchData') + console.log('endpoint: ', endpoint) + console.log('graphqlQuery: ', graphqlQuery) + console.log('profileData: ', profileData) + console.log('allEdges: ', allEdges) + console.log('pageInfo: ', pageInfo) + + try { + const response = await axios.post(endpoint, graphqlQuery, { + headers: { + 'Content-Type': 'application/json' + } + }) + + const { newProfileData, newAllEdges, newPageInfo } = extractData( + response, + profileData, + allEdges, + pageInfo + ) + + if (newPageInfo != undefined && !newPageInfo.hasNextPage) { + console.log('newPageInfo: ', newPageInfo) + return { + data: { + profile: { + ...newProfileData, + earnedAwards: { edges: newAllEdges, pageInfo: newPageInfo } + } } } } - } - - let response - let retries = 5 - while (retries > 0) { - try { - response = await axios.post(endpoint, graphqlQuery, { - headers: { - 'Content-Type': 'application/json' + return await fetchData( + endpoint, + graphqlQuery, + extractData, + newProfileData, + newAllEdges, + newPageInfo + ) + } catch (error) { + console.error('Error fetching data: ', error) + return { + data: { + profile: { + ...profileData, + earnedAwards: { edges: allEdges, pageInfo } } - }) - break - } catch (error) { - if (error.response && error.response.status === 429 && retries > 0) { - await new Promise(resolve => setTimeout(resolve, 1000)) // wait for 1 second - retries-- - } else { - throw error } } } - - const { - data: { profile }, - errors - } = response.data - const { newProfileData, newAllEdges, newPageInfo } = extractData( - profile, - allEdges, - pageInfo - ) - - if (errors) { - console.error(errors) - throw new Error('Error fetching data') - } - - return await fetchData( - endpoint, - graphqlQuery, - extractData, - newProfileData, - newAllEdges, - newPageInfo, - maxPage - 1 - ) } // Use the fetchData function in each specific function @@ -292404,7 +292396,7 @@ async function fetchTrailblazerRankInfo(trailheadUsername) { } } - return await fetchData(endpoint, graphqlQuery, extractRankData, (maxPage = 1)) + return await fetchData(endpoint, graphqlQuery, extractRankData) } async function fetchTrailblazerBadgesInfo(trailheadUsername) { diff --git a/src/getStats.js b/src/getStats.js index ddfcfc2..137888c 100644 --- a/src/getStats.js +++ b/src/getStats.js @@ -102,6 +102,13 @@ async function fetchData( allEdges = [], pageInfo = null ) { + console.log('fetchData') + console.log('endpoint: ', endpoint) + console.log('graphqlQuery: ', graphqlQuery) + console.log('profileData: ', profileData) + console.log('allEdges: ', allEdges) + console.log('pageInfo: ', pageInfo) + try { const response = await axios.post(endpoint, graphqlQuery, { headers: { From 98cabf378e49fafd64860a6733eff5d9abbc47d8 Mon Sep 17 00:00:00 2001 From: Nathan Abondance Date: Sun, 29 Sep 2024 18:30:07 +0200 Subject: [PATCH 08/10] maxpage --- dist/index.js | 17 +++++++++++++++++ src/getStats.js | 17 +++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/dist/index.js b/dist/index.js index 54b12c1..5642734 100644 --- a/dist/index.js +++ b/dist/index.js @@ -292323,6 +292323,8 @@ async function fetchData( endpoint, graphqlQuery, extractData, + maxPage = 100, + currentPage = 1, profileData = null, allEdges = [], pageInfo = null @@ -292330,10 +292332,23 @@ async function fetchData( console.log('fetchData') console.log('endpoint: ', endpoint) console.log('graphqlQuery: ', graphqlQuery) + console.log('maxPage: ', maxPage) + console.log('currentPage: ', currentPage) console.log('profileData: ', profileData) console.log('allEdges: ', allEdges) console.log('pageInfo: ', pageInfo) + if (currentPage > maxPage) { + return { + data: { + profile: { + ...profileData, + earnedAwards: { edges: allEdges, pageInfo } + } + } + } + } + try { const response = await axios.post(endpoint, graphqlQuery, { headers: { @@ -292364,6 +292379,8 @@ async function fetchData( endpoint, graphqlQuery, extractData, + maxPage, + currentPage + 1, newProfileData, newAllEdges, newPageInfo diff --git a/src/getStats.js b/src/getStats.js index 137888c..00c5770 100644 --- a/src/getStats.js +++ b/src/getStats.js @@ -98,6 +98,8 @@ async function fetchData( endpoint, graphqlQuery, extractData, + maxPage = 100, + currentPage = 1, profileData = null, allEdges = [], pageInfo = null @@ -105,10 +107,23 @@ async function fetchData( console.log('fetchData') console.log('endpoint: ', endpoint) console.log('graphqlQuery: ', graphqlQuery) + console.log('maxPage: ', maxPage) + console.log('currentPage: ', currentPage) console.log('profileData: ', profileData) console.log('allEdges: ', allEdges) console.log('pageInfo: ', pageInfo) + if (currentPage > maxPage) { + return { + data: { + profile: { + ...profileData, + earnedAwards: { edges: allEdges, pageInfo } + } + } + } + } + try { const response = await axios.post(endpoint, graphqlQuery, { headers: { @@ -139,6 +154,8 @@ async function fetchData( endpoint, graphqlQuery, extractData, + maxPage, + currentPage + 1, newProfileData, newAllEdges, newPageInfo From 5e8a896f08f004bbefc0bdad3ec27fc0cc8716a9 Mon Sep 17 00:00:00 2001 From: Nathan Abondance Date: Sun, 29 Sep 2024 18:39:38 +0200 Subject: [PATCH 09/10] retry 429 --- dist/index.js | 48 +++++++++++++++++++++++++++++++++++++----------- src/getStats.js | 48 +++++++++++++++++++++++++++++++++++++----------- 2 files changed, 74 insertions(+), 22 deletions(-) diff --git a/dist/index.js b/dist/index.js index 5642734..5093e8f 100644 --- a/dist/index.js +++ b/dist/index.js @@ -292327,7 +292327,9 @@ async function fetchData( currentPage = 1, profileData = null, allEdges = [], - pageInfo = null + pageInfo = null, + maxRetry = 3, + currentRetry = 0 ) { console.log('fetchData') console.log('endpoint: ', endpoint) @@ -292383,15 +292385,39 @@ async function fetchData( currentPage + 1, newProfileData, newAllEdges, - newPageInfo + newPageInfo, + maxRetry, + currentRetry ) } catch (error) { console.error('Error fetching data: ', error) - return { - data: { - profile: { - ...profileData, - earnedAwards: { edges: allEdges, pageInfo } + if (error.response && error.response.status === 429) { + if (currentRetry < maxRetry) { + console.log('Too many requests, waiting for 1 minute before retrying...') + await new Promise(resolve => setTimeout(resolve, 60000)) // wait for 1 minute + return await fetchData( + endpoint, + graphqlQuery, + extractData, + maxPage, + currentPage, + profileData, + allEdges, + pageInfo, + maxRetry, + currentRetry + 1 + ) + } else { + console.error('Max retry limit reached') + throw error + } + } else { + return { + data: { + profile: { + ...profileData, + earnedAwards: { edges: allEdges, pageInfo } + } } } } @@ -292413,7 +292439,7 @@ async function fetchTrailblazerRankInfo(trailheadUsername) { } } - return await fetchData(endpoint, graphqlQuery, extractRankData) + return await fetchData(endpoint, graphqlQuery, extractRankData, 2) } async function fetchTrailblazerBadgesInfo(trailheadUsername) { @@ -292479,7 +292505,7 @@ async function fetchTrailblazerCertifsInfo(trailheadUsername) { } } - return await fetchData(endpoint, graphqlQuery, extractCertifsData) + return await fetchData(endpoint, graphqlQuery, extractCertifsData, 5) } async function fetchTrailblazerSkillsInfo(trailheadUsername) { @@ -292493,7 +292519,7 @@ async function fetchTrailblazerSkillsInfo(trailheadUsername) { } } - return await fetchData(endpoint, graphqlQuery, extractSkillsData) + return await fetchData(endpoint, graphqlQuery, extractSkillsData, 5) } async function fetchTrailblazerEarnedStampsInfo(trailheadUsername) { @@ -292510,7 +292536,7 @@ async function fetchTrailblazerEarnedStampsInfo(trailheadUsername) { } } - return await fetchData(endpoint, graphqlQuery, extractStampsData) + return await fetchData(endpoint, graphqlQuery, extractStampsData, 5) } module.exports = { diff --git a/src/getStats.js b/src/getStats.js index 00c5770..a8d1622 100644 --- a/src/getStats.js +++ b/src/getStats.js @@ -102,7 +102,9 @@ async function fetchData( currentPage = 1, profileData = null, allEdges = [], - pageInfo = null + pageInfo = null, + maxRetry = 3, + currentRetry = 0 ) { console.log('fetchData') console.log('endpoint: ', endpoint) @@ -158,15 +160,39 @@ async function fetchData( currentPage + 1, newProfileData, newAllEdges, - newPageInfo + newPageInfo, + maxRetry, + currentRetry ) } catch (error) { console.error('Error fetching data: ', error) - return { - data: { - profile: { - ...profileData, - earnedAwards: { edges: allEdges, pageInfo } + if (error.response && error.response.status === 429) { + if (currentRetry < maxRetry) { + console.log('Too many requests, waiting for 1 minute before retrying...') + await new Promise(resolve => setTimeout(resolve, 60000)) // wait for 1 minute + return await fetchData( + endpoint, + graphqlQuery, + extractData, + maxPage, + currentPage, + profileData, + allEdges, + pageInfo, + maxRetry, + currentRetry + 1 + ) + } else { + console.error('Max retry limit reached') + throw error + } + } else { + return { + data: { + profile: { + ...profileData, + earnedAwards: { edges: allEdges, pageInfo } + } } } } @@ -188,7 +214,7 @@ async function fetchTrailblazerRankInfo(trailheadUsername) { } } - return await fetchData(endpoint, graphqlQuery, extractRankData) + return await fetchData(endpoint, graphqlQuery, extractRankData, 2) } async function fetchTrailblazerBadgesInfo(trailheadUsername) { @@ -254,7 +280,7 @@ async function fetchTrailblazerCertifsInfo(trailheadUsername) { } } - return await fetchData(endpoint, graphqlQuery, extractCertifsData) + return await fetchData(endpoint, graphqlQuery, extractCertifsData, 5) } async function fetchTrailblazerSkillsInfo(trailheadUsername) { @@ -268,7 +294,7 @@ async function fetchTrailblazerSkillsInfo(trailheadUsername) { } } - return await fetchData(endpoint, graphqlQuery, extractSkillsData) + return await fetchData(endpoint, graphqlQuery, extractSkillsData, 5) } async function fetchTrailblazerEarnedStampsInfo(trailheadUsername) { @@ -285,7 +311,7 @@ async function fetchTrailblazerEarnedStampsInfo(trailheadUsername) { } } - return await fetchData(endpoint, graphqlQuery, extractStampsData) + return await fetchData(endpoint, graphqlQuery, extractStampsData, 5) } module.exports = { From c68446a16ed886a17d130fcea2321ad6f5bf6c15 Mon Sep 17 00:00:00 2001 From: Nathan Abondance Date: Sun, 29 Sep 2024 20:19:55 +0200 Subject: [PATCH 10/10] reduce logs --- dist/index.js | 14 +++++++------- src/getStats.js | 14 +++++++------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/dist/index.js b/dist/index.js index 5093e8f..048312b 100644 --- a/dist/index.js +++ b/dist/index.js @@ -292332,13 +292332,13 @@ async function fetchData( currentRetry = 0 ) { console.log('fetchData') - console.log('endpoint: ', endpoint) - console.log('graphqlQuery: ', graphqlQuery) - console.log('maxPage: ', maxPage) - console.log('currentPage: ', currentPage) - console.log('profileData: ', profileData) - console.log('allEdges: ', allEdges) - console.log('pageInfo: ', pageInfo) + // console.log('endpoint: ', endpoint) + // console.log('graphqlQuery: ', graphqlQuery) + // console.log('maxPage: ', maxPage) + // console.log('currentPage: ', currentPage) + // console.log('profileData: ', profileData) + // console.log('allEdges: ', allEdges) + // console.log('pageInfo: ', pageInfo) if (currentPage > maxPage) { return { diff --git a/src/getStats.js b/src/getStats.js index a8d1622..44fd42f 100644 --- a/src/getStats.js +++ b/src/getStats.js @@ -107,13 +107,13 @@ async function fetchData( currentRetry = 0 ) { console.log('fetchData') - console.log('endpoint: ', endpoint) - console.log('graphqlQuery: ', graphqlQuery) - console.log('maxPage: ', maxPage) - console.log('currentPage: ', currentPage) - console.log('profileData: ', profileData) - console.log('allEdges: ', allEdges) - console.log('pageInfo: ', pageInfo) + // console.log('endpoint: ', endpoint) + // console.log('graphqlQuery: ', graphqlQuery) + // console.log('maxPage: ', maxPage) + // console.log('currentPage: ', currentPage) + // console.log('profileData: ', profileData) + // console.log('allEdges: ', allEdges) + // console.log('pageInfo: ', pageInfo) if (currentPage > maxPage) { return {