maciejpedzich/playlist-entry-validator
1
0
Fork 0
mirror of https://github.com/maciejpedzich/playlist-entry-validator.git synced 2024-11-27 16:05:48 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity

Implement throttling for processing entries

Browse Source
This commit is contained in:
Maciej Pędzich 2024-07-18 22:39:57 +02:00
parent 179ca09937
commit 59706765c9
Signed by: maciejpedzich
GPG Key ID: CE4A303D84882F0D
3 changed files with 67 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
appFn.ts
View File

@ -1,3 +1,4 @@
import { setTimeout } from 'timers/promises';
import { ApplicationFunction, Probot } from 'probot'; import { ApplicationFunction, Probot } from 'probot';
import { throttleAll } from 'promise-throttle-all'; import { throttleAll } from 'promise-throttle-all';
import getMetaData from 'metadata-scraper'; import getMetaData from 'metadata-scraper';
@ -7,8 +8,7 @@ import { getPlaylistIdFromUrl } from './getPlaylistIdFromUrl';
type ReviewEvent = 'REQUEST_CHANGES' | 'COMMENT' | 'APPROVE'; type ReviewEvent = 'REQUEST_CHANGES' | 'COMMENT' | 'APPROVE';
const appFn: ApplicationFunction = (app: Probot, { getRouter }) => { const appFn: ApplicationFunction = (app: Probot, { getRouter }) => {
const router = getRouter!('/ping'); getRouter!('/ping').get('/pong', (_, res) => res.sendStatus(200));
router.get('/pong', (req, res) => res.sendStatus(200));
app.on( app.on(
['pull_request.opened', 'pull_request.synchronize'], ['pull_request.opened', 'pull_request.synchronize'],
@ -64,16 +64,13 @@ const appFn: ApplicationFunction = (app: Probot, { getRouter }) => {
ReturnType<typeof octokit.pulls.listFiles> ReturnType<typeof octokit.pulls.listFiles>
>['data']; >['data'];
const sleep = (ms: number) =>
new Promise((resolve) => setTimeout(resolve, ms));
const prFiles: PRFileArray = []; const prFiles: PRFileArray = [];
let page = 1; let page = 1;
let loadingPages = true; let isLoadingPages = true;
let timeToRateLimitReset = 0; let timeToRateLimitReset = 0;
while (loadingPages) { while (isLoadingPages) {
await sleep(timeToRateLimitReset); await setTimeout(timeToRateLimitReset);
const { data, headers } = await octokit.pulls.listFiles({ const { data, headers } = await octokit.pulls.listFiles({
...workingRepo, ...workingRepo,
@ -90,11 +87,9 @@ const appFn: ApplicationFunction = (app: Probot, { getRouter }) => {
: (Number(headers['x-ratelimit-reset']) || now) - now; : (Number(headers['x-ratelimit-reset']) || now) - now;
if (headers.link?.includes(`rel=\"next\"`)) page++; if (headers.link?.includes(`rel=\"next\"`)) page++;
else loadingPages = false; else isLoadingPages = false;
} }
console.log('Total entries: ' + prFiles.length);
const filesToVerify = prFiles.filter( const filesToVerify = prFiles.filter(
({ status, filename }) => ({ status, filename }) =>
filename.startsWith(registryDirectoryPath) && filename.startsWith(registryDirectoryPath) &&
@ -103,6 +98,12 @@ const appFn: ApplicationFunction = (app: Probot, { getRouter }) => {
if (filesToVerify.length === 0) return; if (filesToVerify.length === 0) return;
console.log('Entries to validate: ' + filesToVerify.length);
let numEntriesBeforeCooldown = 3;
let numProcessedEntries = 0;
let cooldownTimeout = 1500;
const playlistSearchResults = await throttleAll( const playlistSearchResults = await throttleAll(
1, 1,
filesToVerify.map(({ filename }) => async () => { filesToVerify.map(({ filename }) => async () => {
@ -114,12 +115,18 @@ const appFn: ApplicationFunction = (app: Probot, { getRouter }) => {
? filenameWithoutRegistryPath ? filenameWithoutRegistryPath
: `https://open.spotify.com/playlist/${filenameWithoutRegistryPath}`; : `https://open.spotify.com/playlist/${filenameWithoutRegistryPath}`;
if (
numProcessedEntries > 0 &&
numProcessedEntries % numEntriesBeforeCooldown === 0
)
await setTimeout(cooldownTimeout);
const spotifyResponse = await fetch(url); const spotifyResponse = await fetch(url);
const expectedStatusCodes = [200, 400, 404]; const expectedStatusCodes = [200, 400, 404];
if (!expectedStatusCodes.includes(spotifyResponse.status)) if (!expectedStatusCodes.includes(spotifyResponse.status))
throw new Error( throw new Error(
`Received ${spotifyResponse.status} status code from playlist page response` `Received ${spotifyResponse.status} status code from ${url}`
); );
const found = spotifyResponse.status === 200; const found = spotifyResponse.status === 200;
@ -150,7 +157,7 @@ const appFn: ApplicationFunction = (app: Probot, { getRouter }) => {
if (!playlistAuthorResponse.ok) if (!playlistAuthorResponse.ok)
throw new Error( throw new Error(
`Received ${playlistAuthorResponse.status} status code from author page response` `Received ${playlistAuthorResponse.status} status code from ${authorUrl}`
); );
const authorPageHtml = await playlistAuthorResponse.text(); const authorPageHtml = await playlistAuthorResponse.text();
@ -169,6 +176,8 @@ const appFn: ApplicationFunction = (app: Probot, { getRouter }) => {
details = playlistMeta.join(' · '); details = playlistMeta.join(' · ');
} }
numProcessedEntries++;
return { return {
filename: filenameWithoutRegistryPath, filename: filenameWithoutRegistryPath,
found, found,
@ -283,7 +292,7 @@ const appFn: ApplicationFunction = (app: Probot, { getRouter }) => {
await upsertReview( await upsertReview(
undefined, undefined,
'COMMENT', 'COMMENT',
`Something went wrong while validating new entries! @${workingRepo.owner} should handle it shortly...` `Something went wrong while validating new entries! @${workingRepo.owner} should handle it shortly.`
); );
} }
} }

63
package-lock.json generated
View File

@ -14,6 +14,7 @@
"promise-throttle-all": "^1.1.1" "promise-throttle-all": "^1.1.1"
}, },
"devDependencies": { "devDependencies": {
"@types/node": "^20.14.11",
"ts-node-dev": "^2.0.0", "ts-node-dev": "^2.0.0",
"typescript": "^4.8.3" "typescript": "^4.8.3"
}, },
@ -922,9 +923,13 @@
"integrity": "sha512-Y4XFY5VJAuw0FgAqPNd6NNoV44jbq9Bz2L7Rh/J6jLTiHBSBJa9fxqQIvkIld4GsoDOcCbvzOUAbLPsSKKg+uA==" "integrity": "sha512-Y4XFY5VJAuw0FgAqPNd6NNoV44jbq9Bz2L7Rh/J6jLTiHBSBJa9fxqQIvkIld4GsoDOcCbvzOUAbLPsSKKg+uA=="
}, },
"node_modules/@types/node": { "node_modules/@types/node": {
"version": "18.7.16", "version": "20.14.11",
"resolved": "https://registry.npmjs.org/@types/node/-/node-18.7.16.tgz", "resolved": "https://registry.npmjs.org/@types/node/-/node-20.14.11.tgz",
"integrity": "sha512-EQHhixfu+mkqHMZl1R2Ovuvn47PUw18azMJOTwSZr9/fhzHNGXAJ0ma0dayRVchprpCj0Kc1K1xKoWaATWF1qg==" "integrity": "sha512-kprQpL8MMeszbz6ojB5/tU8PLN4kesnN8Gjzw349rDlNgsSzg90lAVj3llK99Dh7JON+t9AuscPPFW6mPbTnSA==",
"license": "MIT",
"dependencies": {
"undici-types": "~5.26.4"
}
}, },
"node_modules/@types/pino": { "node_modules/@types/pino": {
"version": "6.3.12", "version": "6.3.12",
@ -1199,12 +1204,13 @@
} }
}, },
"node_modules/braces": { "node_modules/braces": {
"version": "3.0.2", "version": "3.0.3",
"resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz", "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz",
"integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==", "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==",
"dev": true, "dev": true,
"license": "MIT",
"dependencies": { "dependencies": {
"fill-range": "^7.0.1" "fill-range": "^7.1.1"
}, },
"engines": { "engines": {
"node": ">=8" "node": ">=8"
@ -1737,10 +1743,11 @@
} }
}, },
"node_modules/fill-range": { "node_modules/fill-range": {
"version": "7.0.1", "version": "7.1.1",
"resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz",
"integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==", "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==",
"dev": true, "dev": true,
"license": "MIT",
"dependencies": { "dependencies": {
"to-regex-range": "^5.0.1" "to-regex-range": "^5.0.1"
}, },
@ -2224,6 +2231,7 @@
"resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz",
"integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==",
"dev": true, "dev": true,
"license": "MIT",
"engines": { "engines": {
"node": ">=0.12.0" "node": ">=0.12.0"
} }
@ -3358,6 +3366,7 @@
"resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz",
"integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==",
"dev": true, "dev": true,
"license": "MIT",
"dependencies": { "dependencies": {
"is-number": "^7.0.0" "is-number": "^7.0.0"
}, },
@ -3535,6 +3544,12 @@
"node": ">=0.8.0" "node": ">=0.8.0"
} }
}, },
"node_modules/undici-types": {
"version": "5.26.5",
"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz",
"integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==",
"license": "MIT"
},
"node_modules/universal-github-app-jwt": { "node_modules/universal-github-app-jwt": {
"version": "1.1.1", "version": "1.1.1",
"resolved": "https://registry.npmjs.org/universal-github-app-jwt/-/universal-github-app-jwt-1.1.1.tgz", "resolved": "https://registry.npmjs.org/universal-github-app-jwt/-/universal-github-app-jwt-1.1.1.tgz",
@ -4473,9 +4488,12 @@
"integrity": "sha512-Y4XFY5VJAuw0FgAqPNd6NNoV44jbq9Bz2L7Rh/J6jLTiHBSBJa9fxqQIvkIld4GsoDOcCbvzOUAbLPsSKKg+uA==" "integrity": "sha512-Y4XFY5VJAuw0FgAqPNd6NNoV44jbq9Bz2L7Rh/J6jLTiHBSBJa9fxqQIvkIld4GsoDOcCbvzOUAbLPsSKKg+uA=="
}, },
"@types/node": { "@types/node": {
"version": "18.7.16", "version": "20.14.11",
"resolved": "https://registry.npmjs.org/@types/node/-/node-18.7.16.tgz", "resolved": "https://registry.npmjs.org/@types/node/-/node-20.14.11.tgz",
"integrity": "sha512-EQHhixfu+mkqHMZl1R2Ovuvn47PUw18azMJOTwSZr9/fhzHNGXAJ0ma0dayRVchprpCj0Kc1K1xKoWaATWF1qg==" "integrity": "sha512-kprQpL8MMeszbz6ojB5/tU8PLN4kesnN8Gjzw349rDlNgsSzg90lAVj3llK99Dh7JON+t9AuscPPFW6mPbTnSA==",
"requires": {
"undici-types": "~5.26.4"
}
}, },
"@types/pino": { "@types/pino": {
"version": "6.3.12", "version": "6.3.12",
@ -4707,12 +4725,12 @@
} }
}, },
"braces": { "braces": {
"version": "3.0.2", "version": "3.0.3",
"resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz", "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz",
"integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==", "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==",
"dev": true, "dev": true,
"requires": { "requires": {
"fill-range": "^7.0.1" "fill-range": "^7.1.1"
} }
}, },
"btoa-lite": { "btoa-lite": {
@ -5113,9 +5131,9 @@
} }
}, },
"fill-range": { "fill-range": {
"version": "7.0.1", "version": "7.1.1",
"resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz",
"integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==", "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==",
"dev": true, "dev": true,
"requires": { "requires": {
"to-regex-range": "^5.0.1" "to-regex-range": "^5.0.1"
@ -6449,6 +6467,11 @@
"integrity": "sha512-aTeNPVmgIMPpm1cxXr2Q/nEbvkmV8yq66F3om7X3P/cvOXQ0TMQ64Wk63iyT1gPlmdmGzjGpyLh1f3y8MZWXGg==", "integrity": "sha512-aTeNPVmgIMPpm1cxXr2Q/nEbvkmV8yq66F3om7X3P/cvOXQ0TMQ64Wk63iyT1gPlmdmGzjGpyLh1f3y8MZWXGg==",
"optional": true "optional": true
}, },
"undici-types": {
"version": "5.26.5",
"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz",
"integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA=="
},
"universal-github-app-jwt": { "universal-github-app-jwt": {
"version": "1.1.1", "version": "1.1.1",
"resolved": "https://registry.npmjs.org/universal-github-app-jwt/-/universal-github-app-jwt-1.1.1.tgz", "resolved": "https://registry.npmjs.org/universal-github-app-jwt/-/universal-github-app-jwt-1.1.1.tgz",

1
package.json
View File

@ -25,6 +25,7 @@
"node": ">=18" "node": ">=18"
}, },
"devDependencies": { "devDependencies": {
"@types/node": "^20.14.11",
"ts-node-dev": "^2.0.0", "ts-node-dev": "^2.0.0",
"typescript": "^4.8.3" "typescript": "^4.8.3"
} }