kato
/
HentaiSnatcher
Archived
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

* Various Changes

This commit is contained in:
Daniel Legt 2021-10-31 23:46:18 +02:00
parent e9a2f3d488
commit 9acbd6b93f
4 changed files with 715 additions and 697 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
package-lock.json generated
View File

@ -1,5 +1,5 @@
{ {
"name": "post_scrapper", "name": "HentaiSnatcher",
"lockfileVersion": 2, "lockfileVersion": 2,
"requires": true, "requires": true,
"packages": { "packages": {

15
src/module/twitter.ts Normal file
View File

@ -0,0 +1,15 @@
import { Post, Tag, LogEntry, LogType } from "../type/generic";
import { Scrapper } from "../class/Scrapper";
import { getPageContents } from "../helper/requestManager";
import * as cheerio from 'cheerio';
export class Rule34xxx extends Scrapper {
constructor() {
// Set the domain base of the current Scrapper as "twitter.com"
super("https://twitter.com");
}
}

2
src/test.ts
View File

@ -3,4 +3,4 @@ import {runTest as runRule34XXX} from "./test/rule34xxx";
console.log(`Testing Rule34.xxx`); console.log(`Testing Rule34.xxx`);
// Running the rule34 test with "Most popular" page // Running the rule34 test with "Most popular" page
runRule34XXX(`https://rule34.xxx/index.php?page=post&s=list&tags=cyancapsule`, 30); runRule34XXX(`https://rule34.xxx/index.php?page=post&s=list&tags=-male%2fmale+-furry+-yaoi+-male_focus+-male_only+-anthro+-3d+-cum+-my_little_pony+-vore+-unknown_species+-fur+-animal`, 50);

7
src/test/rule34xxx.ts
View File

@ -34,7 +34,7 @@ export async function runTest(startingPage: string = `https://rule34.xxx/index.p
/** /**
* The definitive list of posts * The definitive list of posts
*/ */
const postList: Array<Post> = []; let postList: Array<Post> = [];
/** /**
* The amount of posts to fetch per cycle * The amount of posts to fetch per cycle
@ -54,7 +54,7 @@ export async function runTest(startingPage: string = `https://rule34.xxx/index.p
.then( result => { .then( result => {
// Append the results to the postList // Append the results to the postList
for ( let p of result ) { for ( let p of result ) {
if ( (p as any).value !== null ) { if ( (p as any).value != null ) {
postList.push(((p as any).value as Post)); postList.push(((p as any).value as Post));
} }
} }
@ -69,6 +69,9 @@ export async function runTest(startingPage: string = `https://rule34.xxx/index.p
console.log(`Done!`); console.log(`Done!`);
// Reverse sort them
postList = postList.reverse();
await fs.writeFile(`./export/r34xxx_pageList_example.json`, JSON.stringify(pageList, null, 4)); await fs.writeFile(`./export/r34xxx_pageList_example.json`, JSON.stringify(pageList, null, 4));
await fs.writeFile(`./export/r34xxx_postLinks_example.json`, JSON.stringify(postLinks, null, 4)); await fs.writeFile(`./export/r34xxx_postLinks_example.json`, JSON.stringify(postLinks, null, 4));
await fs.writeFile(`./export/r34xxx_postList_example.json`, JSON.stringify(postList, null, 4)); await fs.writeFile(`./export/r34xxx_postList_example.json`, JSON.stringify(postList, null, 4));