diff --git a/src/core.js b/src/core.js index a5c82ce..2d663e5 100644 --- a/src/core.js +++ b/src/core.js @@ -14,10 +14,37 @@ const START_ENHANCING_MEMBER = 'START_ENHANCING_MEMBER' const ENHANCING_MEMBER_DONE = 'ENHANCING_MEMBER_DONE' const ORGANIZATION_LOADED = 'ORGANIZATION_LOADED' -async function* generateOrganizationData(githubOrganization) { - const members = await getMembersByOrganization(githubOrganization) +async function* generateOrganizationData(githubOrganizations) { + let members = [] + const membersByOrganisation = {} + + for (githubOrganization of githubOrganizations) { + const organizationMembers = await getMembersByOrganization( + githubOrganization, + ) + membersByOrganisation[githubOrganization] = organizationMembers + members = [...members, ...organizationMembers] + } + + members = filterNonUniqueBy(members, (a, b) => a.login === b.login) yield { type: MEMBERS_LOADED, value: members } + for (githubOrganization of githubOrganizations) { + const organizationRepositories = await getRepositoriesByOrganization( + githubOrganization, + ) + const organization = { + name: githubOrganization, + members: membersByOrganisation[githubOrganization], + repositories: organizationRepositories, + } + yield { + type: ORGANIZATION_LOADED, + value: organization, + name: githubOrganization, + } + } + for (member of members) { yield { type: START_ENHANCING_MEMBER, value: member.login } @@ -29,10 +56,27 @@ async function* generateOrganizationData(githubOrganization) { for (repository of repositories) { await sleep(25) - const contributors = await getRepositoryContributors( - repository.owner.login, - repository.name, - ) + + let attempts = 0 + let contributors + + do { + if (attempts > 0) { + console.log('Retry fetch contributors for ', repository.name) + await sleep(1000) + } + contributors = await getRepositoryContributors( + repository.owner.login, + repository.name, + ) + attempts++ + } while (!Array.isArray(contributors) && attempts < 3) + + if (!Array.isArray(contributors)) { + console.log('Too much retry, set empty value: ', repository.name) + contributors = [] + } + repository.contributors = contributors.map( ({ weeks, ...contributor }) => contributor, ) @@ -43,11 +87,11 @@ async function* generateOrganizationData(githubOrganization) { value: { ...member, repositories, contributionsCollection }, } } - - const organization = await getRepositoriesByOrganization(githubOrganization) - yield { type: ORGANIZATION_LOADED, value: organization } } +const filterNonUniqueBy = (arr, fn) => + arr.filter((v, i) => arr.every((x, j) => (i === j) === fn(v, x, i, j))) + module.exports = { generateOrganizationData, MEMBERS_LOADED, diff --git a/src/index.js b/src/index.js index 5182ba4..1b871da 100644 --- a/src/index.js +++ b/src/index.js @@ -28,14 +28,14 @@ require('yargs') }, }, async function(argv) { - const githubOrganization = argv.organization + const githubOrganizations = !Array.isArray(argv.organization) ? [argv.organization] : argv.organization const githubId = process.env.GITHUB_ID const githubToken = process.env.GITHUB_OAUTH await createDataFolder() - let bar - const generator = generateOrganizationData(githubOrganization) + let bar + const generator = generateOrganizationData(githubOrganizations) for await (const event of generator) { switch (event.type) { @@ -57,7 +57,7 @@ require('yargs') writeMember(event.value) break case ORGANIZATION_LOADED: - writeOrganization(event.value) + writeOrganization(event.name, event.value) break } } diff --git a/src/stats.js b/src/stats.js index 9450fd4..9f3b821 100644 --- a/src/stats.js +++ b/src/stats.js @@ -5,22 +5,26 @@ const chalk = require('chalk') const dataFolder = '../data' + const organizationsFolder = path.join(__dirname, dataFolder, 'organizations') const statsFile = 'stats.json' const generateFile = !!process.argv[2] const stats = {} // Get the Gihub organization based on the .env values or the organization.json - stats.organization = - process.env.GITHUB_ORGA || - JSON.parse( - fs.readFileSync(path.join(__dirname, dataFolder, 'organization.json')), + if (process.env.GITHUB_ORGA) { + stats.organization = process.env.GITHUB_ORGA + } else { + const firstOrganizationFile = fs.readdirSync(organizationsFolder)[0] + console.log(path.join(organizationsFolder, firstOrganizationFile)) + stats.organization = JSON.parse( + fs.readFileSync(path.join(organizationsFolder, firstOrganizationFile)), )[0].owner.login + } const members = fs .readdirSync(path.join(__dirname, dataFolder)) .filter( - file => - !['members.json', 'organization.json', 'stats.json'].includes(file), + file => !['members.json', 'organizations', 'stats.json'].includes(file), ) .map(file => JSON.parse(fs.readFileSync(path.join(__dirname, dataFolder, file))), @@ -74,9 +78,18 @@ .slice(0, 10) .map(([repo, count]) => ({ repo, count })) - const organizationRepositories = JSON.parse( - fs.readFileSync(path.join(__dirname, dataFolder, 'organization.json')), - ) + const organizationRepositories = fs + .readdirSync(organizationsFolder) + .map(file => + JSON.parse( + fs.readFileSync( + path.join(__dirname, dataFolder, 'organizations', file), + ), + ), + ) + .map(organization => organization.repositories) + .flatMap(repositories => repositories) + stats.totalOrganizationRepositories = organizationRepositories.length stats.topOrganizationRepositories = organizationRepositories .reduce((acc, next) => { diff --git a/src/utils.js b/src/utils.js index 60a8319..b84405b 100644 --- a/src/utils.js +++ b/src/utils.js @@ -20,6 +20,7 @@ async function createDataFolder() { const folderPath = path.join(__dirname, rootDir) const alreadyExists = await existsAsync(folderPath) if (!alreadyExists) return mkdirAsync(folderPath) + .then(() => mkdirAsync(path.join(folderPath, 'organizations'))) } async function writeMember(member) { @@ -38,9 +39,9 @@ async function writeMembers(members) { return members } -async function writeOrganization(organization) { +async function writeOrganization(name, organization) { await writeFileAsync( - path.join(__dirname, rootDir, 'organization.json'), + path.join(__dirname, rootDir, 'organizations', `${name}.json`), prettyJson(organization), ) return organization