diff --git a/appFn.ts b/appFn.ts index 09fa4e2..9a2fe7e 100644 --- a/appFn.ts +++ b/appFn.ts @@ -1,3 +1,4 @@ +import { setTimeout } from 'timers/promises'; import { ApplicationFunction, Probot } from 'probot'; import { throttleAll } from 'promise-throttle-all'; import getMetaData from 'metadata-scraper'; @@ -7,8 +8,7 @@ import { getPlaylistIdFromUrl } from './getPlaylistIdFromUrl'; type ReviewEvent = 'REQUEST_CHANGES' | 'COMMENT' | 'APPROVE'; const appFn: ApplicationFunction = (app: Probot, { getRouter }) => { - const router = getRouter!('/ping'); - router.get('/pong', (req, res) => res.sendStatus(200)); + getRouter!('/ping').get('/pong', (_, res) => res.sendStatus(200)); app.on( ['pull_request.opened', 'pull_request.synchronize'], @@ -64,16 +64,13 @@ const appFn: ApplicationFunction = (app: Probot, { getRouter }) => { ReturnType >['data']; - const sleep = (ms: number) => - new Promise((resolve) => setTimeout(resolve, ms)); - const prFiles: PRFileArray = []; let page = 1; - let loadingPages = true; + let isLoadingPages = true; let timeToRateLimitReset = 0; - while (loadingPages) { - await sleep(timeToRateLimitReset); + while (isLoadingPages) { + await setTimeout(timeToRateLimitReset); const { data, headers } = await octokit.pulls.listFiles({ ...workingRepo, @@ -90,11 +87,9 @@ const appFn: ApplicationFunction = (app: Probot, { getRouter }) => { : (Number(headers['x-ratelimit-reset']) || now) - now; if (headers.link?.includes(`rel=\"next\"`)) page++; - else loadingPages = false; + else isLoadingPages = false; } - console.log('Total entries: ' + prFiles.length); - const filesToVerify = prFiles.filter( ({ status, filename }) => filename.startsWith(registryDirectoryPath) && @@ -103,6 +98,12 @@ const appFn: ApplicationFunction = (app: Probot, { getRouter }) => { if (filesToVerify.length === 0) return; + console.log('Entries to validate: ' + filesToVerify.length); + + let numEntriesBeforeCooldown = 3; + let numProcessedEntries = 0; + let cooldownTimeout = 1500; + const playlistSearchResults = await throttleAll( 1, filesToVerify.map(({ filename }) => async () => { @@ -114,12 +115,18 @@ const appFn: ApplicationFunction = (app: Probot, { getRouter }) => { ? filenameWithoutRegistryPath : `https://open.spotify.com/playlist/${filenameWithoutRegistryPath}`; + if ( + numProcessedEntries > 0 && + numProcessedEntries % numEntriesBeforeCooldown === 0 + ) + await setTimeout(cooldownTimeout); + const spotifyResponse = await fetch(url); const expectedStatusCodes = [200, 400, 404]; if (!expectedStatusCodes.includes(spotifyResponse.status)) throw new Error( - `Received ${spotifyResponse.status} status code from playlist page response` + `Received ${spotifyResponse.status} status code from ${url}` ); const found = spotifyResponse.status === 200; @@ -150,7 +157,7 @@ const appFn: ApplicationFunction = (app: Probot, { getRouter }) => { if (!playlistAuthorResponse.ok) throw new Error( - `Received ${playlistAuthorResponse.status} status code from author page response` + `Received ${playlistAuthorResponse.status} status code from ${authorUrl}` ); const authorPageHtml = await playlistAuthorResponse.text(); @@ -169,6 +176,8 @@ const appFn: ApplicationFunction = (app: Probot, { getRouter }) => { details = playlistMeta.join(' ยท '); } + numProcessedEntries++; + return { filename: filenameWithoutRegistryPath, found, @@ -283,7 +292,7 @@ const appFn: ApplicationFunction = (app: Probot, { getRouter }) => { await upsertReview( undefined, 'COMMENT', - `Something went wrong while validating new entries! @${workingRepo.owner} should handle it shortly...` + `Something went wrong while validating new entries! @${workingRepo.owner} should handle it shortly.` ); } } diff --git a/package-lock.json b/package-lock.json index f9680e8..fd170b0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,6 +14,7 @@ "promise-throttle-all": "^1.1.1" }, "devDependencies": { + "@types/node": "^20.14.11", "ts-node-dev": "^2.0.0", "typescript": "^4.8.3" }, @@ -922,9 +923,13 @@ "integrity": "sha512-Y4XFY5VJAuw0FgAqPNd6NNoV44jbq9Bz2L7Rh/J6jLTiHBSBJa9fxqQIvkIld4GsoDOcCbvzOUAbLPsSKKg+uA==" }, "node_modules/@types/node": { - "version": "18.7.16", - "resolved": "https://registry.npmjs.org/@types/node/-/node-18.7.16.tgz", - "integrity": "sha512-EQHhixfu+mkqHMZl1R2Ovuvn47PUw18azMJOTwSZr9/fhzHNGXAJ0ma0dayRVchprpCj0Kc1K1xKoWaATWF1qg==" + "version": "20.14.11", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.14.11.tgz", + "integrity": "sha512-kprQpL8MMeszbz6ojB5/tU8PLN4kesnN8Gjzw349rDlNgsSzg90lAVj3llK99Dh7JON+t9AuscPPFW6mPbTnSA==", + "license": "MIT", + "dependencies": { + "undici-types": "~5.26.4" + } }, "node_modules/@types/pino": { "version": "6.3.12", @@ -1199,12 +1204,13 @@ } }, "node_modules/braces": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz", - "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==", + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", + "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", "dev": true, + "license": "MIT", "dependencies": { - "fill-range": "^7.0.1" + "fill-range": "^7.1.1" }, "engines": { "node": ">=8" @@ -1737,10 +1743,11 @@ } }, "node_modules/fill-range": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", - "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==", + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", + "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", "dev": true, + "license": "MIT", "dependencies": { "to-regex-range": "^5.0.1" }, @@ -2224,6 +2231,7 @@ "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", "dev": true, + "license": "MIT", "engines": { "node": ">=0.12.0" } @@ -3358,6 +3366,7 @@ "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", "dev": true, + "license": "MIT", "dependencies": { "is-number": "^7.0.0" }, @@ -3535,6 +3544,12 @@ "node": ">=0.8.0" } }, + "node_modules/undici-types": { + "version": "5.26.5", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", + "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", + "license": "MIT" + }, "node_modules/universal-github-app-jwt": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/universal-github-app-jwt/-/universal-github-app-jwt-1.1.1.tgz", @@ -4473,9 +4488,12 @@ "integrity": "sha512-Y4XFY5VJAuw0FgAqPNd6NNoV44jbq9Bz2L7Rh/J6jLTiHBSBJa9fxqQIvkIld4GsoDOcCbvzOUAbLPsSKKg+uA==" }, "@types/node": { - "version": "18.7.16", - "resolved": "https://registry.npmjs.org/@types/node/-/node-18.7.16.tgz", - "integrity": "sha512-EQHhixfu+mkqHMZl1R2Ovuvn47PUw18azMJOTwSZr9/fhzHNGXAJ0ma0dayRVchprpCj0Kc1K1xKoWaATWF1qg==" + "version": "20.14.11", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.14.11.tgz", + "integrity": "sha512-kprQpL8MMeszbz6ojB5/tU8PLN4kesnN8Gjzw349rDlNgsSzg90lAVj3llK99Dh7JON+t9AuscPPFW6mPbTnSA==", + "requires": { + "undici-types": "~5.26.4" + } }, "@types/pino": { "version": "6.3.12", @@ -4707,12 +4725,12 @@ } }, "braces": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz", - "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==", + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", + "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", "dev": true, "requires": { - "fill-range": "^7.0.1" + "fill-range": "^7.1.1" } }, "btoa-lite": { @@ -5113,9 +5131,9 @@ } }, "fill-range": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", - "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==", + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", + "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", "dev": true, "requires": { "to-regex-range": "^5.0.1" @@ -6449,6 +6467,11 @@ "integrity": "sha512-aTeNPVmgIMPpm1cxXr2Q/nEbvkmV8yq66F3om7X3P/cvOXQ0TMQ64Wk63iyT1gPlmdmGzjGpyLh1f3y8MZWXGg==", "optional": true }, + "undici-types": { + "version": "5.26.5", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", + "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==" + }, "universal-github-app-jwt": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/universal-github-app-jwt/-/universal-github-app-jwt-1.1.1.tgz", diff --git a/package.json b/package.json index 357be9f..d286c7e 100644 --- a/package.json +++ b/package.json @@ -25,6 +25,7 @@ "node": ">=18" }, "devDependencies": { + "@types/node": "^20.14.11", "ts-node-dev": "^2.0.0", "typescript": "^4.8.3" }