From e64142f4364ea8a4835cb85eddcc920033bfb04e Mon Sep 17 00:00:00 2001 From: Kato Twofold Date: Tue, 2 Nov 2021 23:05:49 +0200 Subject: [PATCH] + Work on InkBunny + CLI --- src/cli.ts | 28 ++++++++++++ src/helper/requestManager.ts | 2 +- src/module/inkbunny.ts | 83 ++++++++++++++++++++++++++++++++++++ src/test.ts | 8 +++- src/test/inkbunny.ts | 23 ++++++++++ src/test/rule34xxx.ts | 24 +++++++---- 6 files changed, 158 insertions(+), 10 deletions(-) create mode 100644 src/cli.ts create mode 100644 src/module/inkbunny.ts create mode 100644 src/test/inkbunny.ts diff --git a/src/cli.ts b/src/cli.ts new file mode 100644 index 0000000..7eb6e16 --- /dev/null +++ b/src/cli.ts @@ -0,0 +1,28 @@ +import {runTest as runRule34XXX} from "./test/rule34xxx"; + +var url: URL; + +try { + url = new URL(process.argv[2]); +} catch ( err ) { + console.error(err); + process.exit(99); +} + +let pageCount: number = 30; +if ( process.argv.length >= 4 ) { + if ( !isNaN((process.argv[3] as any)) ) { + pageCount = Number.parseInt(process.argv[3]); + } +} + +main(); + +async function main() { + const results = await runRule34XXX(process.argv[2], pageCount, true); + + process.stdout.write(String(results)); + process.stdout.end(); + process.exit(0); +} + diff --git a/src/helper/requestManager.ts b/src/helper/requestManager.ts index 4e875d3..a244e9c 100644 --- a/src/helper/requestManager.ts +++ b/src/helper/requestManager.ts @@ -5,7 +5,7 @@ export function getPageContents(url: string): Promise> { + + // Check if the provided link is valid + if (!this.checkURLBase(url)) { + throw new Error(`Invalid url provided`); + } + + // Initialize the page contents here + let pageContents: string = null; + + // Send out the request to grab the contents of the post + try { + if (this.verbose) { + console.error(`Sniffing page...`); + } + // Send out the initial Axios request to fetch the data from the page + await getPageContents(url) + .then(request => { + if (request.status < 200 || request.status > 299) { + this.logs.push({ + msg: `Invalid response code[${request.status}]`, + type: LogType.ERROR, + err: null, + data: null, + ts: new Date() + }); + throw new Error(`Invalid response code[${request.status}]`); + } + + pageContents = (request.data as string); + }) + } catch (err) { + // "Handle" the error so that it's in the above .catch + this.logs.push({ + msg: `[Error]::getPostsFromPage::`, + type: LogType.ERROR, + err: (err as Error), + data: null, + ts: new Date() + }); + throw err; + } + + // Process the page's posts with cheerio + const $ = cheerio.load((pageContents as string)); + + // Define the post List + const postList: Array = []; + + // Workaround I guess + let self = this; + + // Go through all of the posts + $(`.widget_imageFromSubmission a`).each(function () { + const href = $(this).attr(`href`); + if (`${href}`.length >= 4) + postList.push(`${self.domain}/${href}`); + }); + + if (this.verbose) { + console.error(`Found ${postList.length} posts`); + } + + return postList; + } + + +} \ No newline at end of file diff --git a/src/test.ts b/src/test.ts index b52d4b2..f993078 100644 --- a/src/test.ts +++ b/src/test.ts @@ -1,6 +1,12 @@ import {runTest as runRule34XXX} from "./test/rule34xxx"; +import {runTest as inkbunnytest} from "./test/inkbunny"; console.log(`Testing Rule34.xxx`); // Running the rule34 test with "Most popular" page -runRule34XXX(`https://rule34.xxx/index.php?page=post&s=list&tags=-male%2fmale+-furry+-yaoi+-male_focus+-male_only+-anthro+-3d+-cum+-my_little_pony+-vore+-unknown_species+-fur+-animal`, 50); \ No newline at end of file +runRule34XXX(`https://rule34.xxx/index.php?page=post&s=list&tags=-male%2fmale+-furry+-yaoi+-male_focus+-male_only+-anthro+-3d+-cum+-my_little_pony+-vore+-unknown_species+-fur+-animal`, 50); + +// ! +runRule34XXX(`https://rule34.xxx/index.php?page=post&s=list&tags=superbusty`, 50); + +// inkbunnytest(); \ No newline at end of file diff --git a/src/test/inkbunny.ts b/src/test/inkbunny.ts new file mode 100644 index 0000000..7c32dfe --- /dev/null +++ b/src/test/inkbunny.ts @@ -0,0 +1,23 @@ +// This is the test file for the library, different tests are ran in here. +import { InkBunny } from "../module/inkbunny"; +import {Post} from "../type/generic"; +import * as fs from "fs/promises"; + + +export async function runTest(startingPage: string = `https://inkbunny.net/gallery/CyanCapsule/1/7413c06e38`, pages: number = 10) { + + const bunny = new InkBunny(); + bunny.verbose = true; + + let postLinks: Array = []; + await bunny.getPostsFromPage(`https://inkbunny.net/gallery/CyanCapsule/1/7413c06e38`) + .then( res => { + postLinks = res; + }) + .catch( err => { + console.error(err); + }); + + console.log(postLinks); + +} \ No newline at end of file diff --git a/src/test/rule34xxx.ts b/src/test/rule34xxx.ts index d288923..0e10d8e 100644 --- a/src/test/rule34xxx.ts +++ b/src/test/rule34xxx.ts @@ -3,10 +3,12 @@ import {Rule34xxx} from "../module/rule34xxx"; import {Post} from "../type/generic"; import * as fs from "fs/promises"; -export async function runTest(startingPage: string = `https://rule34.xxx/index.php?page=post&s=list&tags=sort%3Ascore%3Adesc+id%3A%3E4563063&pid=252`, pages: number = 10) { +export async function runTest(startingPage: string = `https://rule34.xxx/index.php?page=post&s=list&tags=sort%3Ascore%3Adesc+id%3A%3E4563063&pid=252`, pages: number = 10, doReturn: boolean = false) { // Initialize the rule34 module const r34: Rule34xxx = new Rule34xxx(); - r34.verbose = true; + if ( !doReturn ) { + r34.verbose = true; + } // Run the get post Details function let pageList: Array; @@ -63,18 +65,24 @@ export async function runTest(startingPage: string = `https://rule34.xxx/index.p console.error(`err: `, err); }) .finally(() => { - console.log(`[${i}/${postLinks.length}][${(i/postLinks.length * 100).toFixed(2)}%] Scrapping...`); + if ( r34.verbose ) { + console.log(`[${i}/${postLinks.length}][${(i/postLinks.length * 100).toFixed(2)}%] Scrapping...`); + } }) } - console.log(`Done!`); - // Reverse sort them postList = postList.reverse(); - await fs.writeFile(`./export/r34xxx_pageList_example.json`, JSON.stringify(pageList, null, 4)); - await fs.writeFile(`./export/r34xxx_postLinks_example.json`, JSON.stringify(postLinks, null, 4)); - await fs.writeFile(`./export/r34xxx_postList_example.json`, JSON.stringify(postList, null, 4)); + if ( doReturn ) { + return JSON.stringify(postList, null, 2); + } else { + console.log(`Done!`); + // await fs.writeFile(`./export/r34xxx_pageList_example.json`, JSON.stringify(pageList, null, 4)); + // await fs.writeFile(`./export/r34xxx_postLinks_example.json`, JSON.stringify(postLinks, null, 4)); + await fs.writeFile(`./export/postList_${new Date().getTime()}.json`, JSON.stringify(postList, null, 4)); + } + // Display results console.log({