mirror of
https://github.com/maciejpedzich/playlist-entry-validator.git
synced 2024-09-19 18:46:21 +02:00
Implement throttling for processing entries
This commit is contained in:
parent
179ca09937
commit
59706765c9
GPG Key ID:
CE4A303D84882F0D
37
appFn.ts
37
appFn.ts
@ -1,3 +1,4 @@
|
||||
import { setTimeout } from 'timers/promises';
|
||||
import { ApplicationFunction, Probot } from 'probot';
|
||||
import { throttleAll } from 'promise-throttle-all';
|
||||
import getMetaData from 'metadata-scraper';
|
||||
@ -7,8 +8,7 @@ import { getPlaylistIdFromUrl } from './getPlaylistIdFromUrl';
|
||||
type ReviewEvent = 'REQUEST_CHANGES' | 'COMMENT' | 'APPROVE';
|
||||
|
||||
const appFn: ApplicationFunction = (app: Probot, { getRouter }) => {
|
||||
const router = getRouter!('/ping');
|
||||
router.get('/pong', (req, res) => res.sendStatus(200));
|
||||
getRouter!('/ping').get('/pong', (_, res) => res.sendStatus(200));
|
||||
|
||||
app.on(
|
||||
['pull_request.opened', 'pull_request.synchronize'],
|
||||
@ -64,16 +64,13 @@ const appFn: ApplicationFunction = (app: Probot, { getRouter }) => {
|
||||
ReturnType<typeof octokit.pulls.listFiles>
|
||||
>['data'];
|
||||
|
||||
const sleep = (ms: number) =>
|
||||
new Promise((resolve) => setTimeout(resolve, ms));
|
||||
|
||||
const prFiles: PRFileArray = [];
|
||||
let page = 1;
|
||||
let loadingPages = true;
|
||||
let isLoadingPages = true;
|
||||
let timeToRateLimitReset = 0;
|
||||
|
||||
while (loadingPages) {
|
||||
await sleep(timeToRateLimitReset);
|
||||
while (isLoadingPages) {
|
||||
await setTimeout(timeToRateLimitReset);
|
||||
|
||||
const { data, headers } = await octokit.pulls.listFiles({
|
||||
...workingRepo,
|
||||
@ -90,11 +87,9 @@ const appFn: ApplicationFunction = (app: Probot, { getRouter }) => {
|
||||
: (Number(headers['x-ratelimit-reset']) || now) - now;
|
||||
|
||||
if (headers.link?.includes(`rel=\"next\"`)) page++;
|
||||
else loadingPages = false;
|
||||
else isLoadingPages = false;
|
||||
}
|
||||
|
||||
console.log('Total entries: ' + prFiles.length);
|
||||
|
||||
const filesToVerify = prFiles.filter(
|
||||
({ status, filename }) =>
|
||||
filename.startsWith(registryDirectoryPath) &&
|
||||
@ -103,6 +98,12 @@ const appFn: ApplicationFunction = (app: Probot, { getRouter }) => {
|
||||
|
||||
if (filesToVerify.length === 0) return;
|
||||
|
||||
console.log('Entries to validate: ' + filesToVerify.length);
|
||||
|
||||
let numEntriesBeforeCooldown = 3;
|
||||
let numProcessedEntries = 0;
|
||||
let cooldownTimeout = 1500;
|
||||
|
||||
const playlistSearchResults = await throttleAll(
|
||||
1,
|
||||
filesToVerify.map(({ filename }) => async () => {
|
||||
@ -114,12 +115,18 @@ const appFn: ApplicationFunction = (app: Probot, { getRouter }) => {
|
||||
? filenameWithoutRegistryPath
|
||||
: `https://open.spotify.com/playlist/${filenameWithoutRegistryPath}`;
|
||||
|
||||
if (
|
||||
numProcessedEntries > 0 &&
|
||||
numProcessedEntries % numEntriesBeforeCooldown === 0
|
||||
)
|
||||
await setTimeout(cooldownTimeout);
|
||||
|
||||
const spotifyResponse = await fetch(url);
|
||||
const expectedStatusCodes = [200, 400, 404];
|
||||
|
||||
if (!expectedStatusCodes.includes(spotifyResponse.status))
|
||||
throw new Error(
|
||||
`Received ${spotifyResponse.status} status code from playlist page response`
|
||||
`Received ${spotifyResponse.status} status code from ${url}`
|
||||
);
|
||||
|
||||
const found = spotifyResponse.status === 200;
|
||||
@ -150,7 +157,7 @@ const appFn: ApplicationFunction = (app: Probot, { getRouter }) => {
|
||||
|
||||
if (!playlistAuthorResponse.ok)
|
||||
throw new Error(
|
||||
`Received ${playlistAuthorResponse.status} status code from author page response`
|
||||
`Received ${playlistAuthorResponse.status} status code from ${authorUrl}`
|
||||
);
|
||||
|
||||
const authorPageHtml = await playlistAuthorResponse.text();
|
||||
@ -169,6 +176,8 @@ const appFn: ApplicationFunction = (app: Probot, { getRouter }) => {
|
||||
details = playlistMeta.join(' · ');
|
||||
}
|
||||
|
||||
numProcessedEntries++;
|
||||
|
||||
return {
|
||||
filename: filenameWithoutRegistryPath,
|
||||
found,
|
||||
@ -283,7 +292,7 @@ const appFn: ApplicationFunction = (app: Probot, { getRouter }) => {
|
||||
await upsertReview(
|
||||
undefined,
|
||||
'COMMENT',
|
||||
`Something went wrong while validating new entries! @${workingRepo.owner} should handle it shortly...`
|
||||
`Something went wrong while validating new entries! @${workingRepo.owner} should handle it shortly.`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
63
package-lock.json
generated
63
package-lock.json
generated
@ -14,6 +14,7 @@
|
||||
"promise-throttle-all": "^1.1.1"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^20.14.11",
|
||||
"ts-node-dev": "^2.0.0",
|
||||
"typescript": "^4.8.3"
|
||||
},
|
||||
@ -922,9 +923,13 @@
|
||||
"integrity": "sha512-Y4XFY5VJAuw0FgAqPNd6NNoV44jbq9Bz2L7Rh/J6jLTiHBSBJa9fxqQIvkIld4GsoDOcCbvzOUAbLPsSKKg+uA=="
|
||||
},
|
||||
"node_modules/@types/node": {
|
||||
"version": "18.7.16",
|
||||
"resolved": "https://registry.npmjs.org/@types/node/-/node-18.7.16.tgz",
|
||||
"integrity": "sha512-EQHhixfu+mkqHMZl1R2Ovuvn47PUw18azMJOTwSZr9/fhzHNGXAJ0ma0dayRVchprpCj0Kc1K1xKoWaATWF1qg=="
|
||||
"version": "20.14.11",
|
||||
"resolved": "https://registry.npmjs.org/@types/node/-/node-20.14.11.tgz",
|
||||
"integrity": "sha512-kprQpL8MMeszbz6ojB5/tU8PLN4kesnN8Gjzw349rDlNgsSzg90lAVj3llK99Dh7JON+t9AuscPPFW6mPbTnSA==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"undici-types": "~5.26.4"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/pino": {
|
||||
"version": "6.3.12",
|
||||
@ -1199,12 +1204,13 @@
|
||||
}
|
||||
},
|
||||
"node_modules/braces": {
|
||||
"version": "3.0.2",
|
||||
"resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz",
|
||||
"integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==",
|
||||
"version": "3.0.3",
|
||||
"resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz",
|
||||
"integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"fill-range": "^7.0.1"
|
||||
"fill-range": "^7.1.1"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=8"
|
||||
@ -1737,10 +1743,11 @@
|
||||
}
|
||||
},
|
||||
"node_modules/fill-range": {
|
||||
"version": "7.0.1",
|
||||
"resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz",
|
||||
"integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==",
|
||||
"version": "7.1.1",
|
||||
"resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz",
|
||||
"integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"to-regex-range": "^5.0.1"
|
||||
},
|
||||
@ -2224,6 +2231,7 @@
|
||||
"resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz",
|
||||
"integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=0.12.0"
|
||||
}
|
||||
@ -3358,6 +3366,7 @@
|
||||
"resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz",
|
||||
"integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"is-number": "^7.0.0"
|
||||
},
|
||||
@ -3535,6 +3544,12 @@
|
||||
"node": ">=0.8.0"
|
||||
}
|
||||
},
|
||||
"node_modules/undici-types": {
|
||||
"version": "5.26.5",
|
||||
"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz",
|
||||
"integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/universal-github-app-jwt": {
|
||||
"version": "1.1.1",
|
||||
"resolved": "https://registry.npmjs.org/universal-github-app-jwt/-/universal-github-app-jwt-1.1.1.tgz",
|
||||
@ -4473,9 +4488,12 @@
|
||||
"integrity": "sha512-Y4XFY5VJAuw0FgAqPNd6NNoV44jbq9Bz2L7Rh/J6jLTiHBSBJa9fxqQIvkIld4GsoDOcCbvzOUAbLPsSKKg+uA=="
|
||||
},
|
||||
"@types/node": {
|
||||
"version": "18.7.16",
|
||||
"resolved": "https://registry.npmjs.org/@types/node/-/node-18.7.16.tgz",
|
||||
"integrity": "sha512-EQHhixfu+mkqHMZl1R2Ovuvn47PUw18azMJOTwSZr9/fhzHNGXAJ0ma0dayRVchprpCj0Kc1K1xKoWaATWF1qg=="
|
||||
"version": "20.14.11",
|
||||
"resolved": "https://registry.npmjs.org/@types/node/-/node-20.14.11.tgz",
|
||||
"integrity": "sha512-kprQpL8MMeszbz6ojB5/tU8PLN4kesnN8Gjzw349rDlNgsSzg90lAVj3llK99Dh7JON+t9AuscPPFW6mPbTnSA==",
|
||||
"requires": {
|
||||
"undici-types": "~5.26.4"
|
||||
}
|
||||
},
|
||||
"@types/pino": {
|
||||
"version": "6.3.12",
|
||||
@ -4707,12 +4725,12 @@
|
||||
}
|
||||
},
|
||||
"braces": {
|
||||
"version": "3.0.2",
|
||||
"resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz",
|
||||
"integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==",
|
||||
"version": "3.0.3",
|
||||
"resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz",
|
||||
"integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"fill-range": "^7.0.1"
|
||||
"fill-range": "^7.1.1"
|
||||
}
|
||||
},
|
||||
"btoa-lite": {
|
||||
@ -5113,9 +5131,9 @@
|
||||
}
|
||||
},
|
||||
"fill-range": {
|
||||
"version": "7.0.1",
|
||||
"resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz",
|
||||
"integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==",
|
||||
"version": "7.1.1",
|
||||
"resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz",
|
||||
"integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"to-regex-range": "^5.0.1"
|
||||
@ -6449,6 +6467,11 @@
|
||||
"integrity": "sha512-aTeNPVmgIMPpm1cxXr2Q/nEbvkmV8yq66F3om7X3P/cvOXQ0TMQ64Wk63iyT1gPlmdmGzjGpyLh1f3y8MZWXGg==",
|
||||
"optional": true
|
||||
},
|
||||
"undici-types": {
|
||||
"version": "5.26.5",
|
||||
"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz",
|
||||
"integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA=="
|
||||
},
|
||||
"universal-github-app-jwt": {
|
||||
"version": "1.1.1",
|
||||
"resolved": "https://registry.npmjs.org/universal-github-app-jwt/-/universal-github-app-jwt-1.1.1.tgz",
|
||||
|
||||
@ -25,6 +25,7 @@
|
||||
"node": ">=18"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^20.14.11",
|
||||
"ts-node-dev": "^2.0.0",
|
||||
"typescript": "^4.8.3"
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user