From df3560cc6f80fa3f283b7bed62bc69c3a9469e63 Mon Sep 17 00:00:00 2001 From: wenpeng Date: Wed, 11 Sep 2024 23:13:20 +0800 Subject: [PATCH] =?UTF-8?q?=E6=A0=BC=E5=BC=8F=E5=8C=96=E5=B7=A5=E5=85=B7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .prettierrc | 13 ++++++++++ examples/zhihu.js | 64 +++++++++++++++++++++++++++++++++------------- package.json | 2 +- utils/userUtils.js | 22 ++++++++-------- 4 files changed, 71 insertions(+), 30 deletions(-) create mode 100644 .prettierrc diff --git a/.prettierrc b/.prettierrc new file mode 100644 index 0000000..02d8a0d --- /dev/null +++ b/.prettierrc @@ -0,0 +1,13 @@ +{ + "printWidth": 100, + "tabWidth": 4, + "semi": true, + "singleQuote": false, + "quoteProps": "consistent", + "trailingComma": "es5", + "bracketSpacing": true, + "arrowParens": "avoid", + "proseWrap": "never", + "endOfLine": "lf", + "embeddedLanguageFormatting": "off" +} \ No newline at end of file diff --git a/examples/zhihu.js b/examples/zhihu.js index 6407a11..27773f8 100644 --- a/examples/zhihu.js +++ b/examples/zhihu.js @@ -1,19 +1,26 @@ -import { launch } from 'puppeteer'; -import 'dotenv/config.js' -import { injectionCookie } from '../utils/userUtils.js'; -import fs from 'fs'; +import { launch } from "puppeteer"; +import "dotenv/config.js"; +import { injectionCookie } from "../utils/userUtils.js"; +import fs from "fs"; +import dotenv from "dotenv"; +dotenv.config(); (async () => { - const browser = await launch({ headless: false, args: [`--window-size=1920,1080`], defaultViewport: { width: 1920, height: 1080 } }); + const browser = await launch({ + headless: false, + args: [`--window-size=1920,1080`], + defaultViewport: { width: 1920, height: 1080 }, + }); const page = await browser.newPage(); + page.on("console", msg => console.log("PAGE LOG", msg.text())); + page.on("error", error => console.error("PAGE error", error)); // 注入cookie - const webUrl = 'https://www.zhihu.com/hot'; - const cookieFileUrl = './data/zhihu.json'; + const webUrl = "https://www.zhihu.com/hot"; + const cookieFileUrl = "./data/zhihu.json"; await injectionCookie(page, webUrl, cookieFileUrl); - const hotItems = await page.$$('.HotList-list .HotItem'); - console.log("========"); + const hotItems = await page.$$(".HotList-list .HotItem"); const hotText = new Array(); @@ -21,18 +28,39 @@ import fs from 'fs'; if (Object.hasOwnProperty.call(hotItems, hot)) { const element = hotItems[hot]; - const h2Element = await element.$('h2'); - const pElement = await element.$('p'); + const h2Element = await element.$("h2"); + const pElement = await element.$("p"); - const title = h2Element != null ? await h2Element.evaluate(e => e.textContent) : ''; - const excerpt = pElement != null ? await pElement.evaluate(e => e.textContent) : ''; - - hotText.push({ title, excerpt }); + const title = h2Element != null ? await h2Element.evaluate(e => e.textContent) : ""; + const excerpt = pElement != null ? await pElement.evaluate(e => e.textContent) : ""; + const createTime = Date.now(); + hotText.push({ title, excerpt, createTime }); } } - console.log("end..."); - fs.writeFileSync('./data/zhihuData.json', JSON.stringify(hotText, null, 2)); + fs.writeFileSync("./data/zhihuData.json", JSON.stringify(hotText, null, 2)); + + const url = process.env.URL; + console.log(process.env.URL); + + try { + const response = await fetch(url, { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify(hotText, null, 2), + }); + + if (!response.ok) { + throw new Error(`Http error! status : ${response.status}`); + } + + const responseData = await response.json(); + console.log("Repsonse data", responseData); + } catch (error) { + console.error(error); + } await browser.close(); -})(); \ No newline at end of file +})(); diff --git a/package.json b/package.json index 3d6cedf..0dcb59f 100644 --- a/package.json +++ b/package.json @@ -4,4 +4,4 @@ "dotenv": "^16.4.5", "puppeteer": "^22.13.1" } -} +} \ No newline at end of file diff --git a/utils/userUtils.js b/utils/userUtils.js index 6347fc2..9de59f4 100644 --- a/utils/userUtils.js +++ b/utils/userUtils.js @@ -1,38 +1,38 @@ -import fs from 'fs'; +import fs from "fs"; /** * 设置页面的LocalStorage - * @param {*} page + * @param {*} page * @param {String} file 用户信息 */ export async function setLocalStorage(page, file) { - const outputLocalStorge = fs.readFileSync(file, 'utf-8'); + const outputLocalStorge = fs.readFileSync(file, "utf-8"); let localStorageTest = JSON.parse(outputLocalStorge); // 设置页面的localStorage await page.evaluateOnNewDocument(localStorageTest => { localStorage.clear(); - localStorage.setItem('merchant_access_token', localStorageTest.merchant_access_token); - localStorage.setItem('merchant_refresh_token', localStorageTest.merchant_refresh_token); - localStorage.setItem('merchant_login_data', localStorageTest.merchant_login_data); - localStorage.setItem('merchant_metadata_data', localStorageTest.merchant_metadata_data); + localStorage.setItem("merchant_access_token", localStorageTest.merchant_access_token); + localStorage.setItem("merchant_refresh_token", localStorageTest.merchant_refresh_token); + localStorage.setItem("merchant_login_data", localStorageTest.merchant_login_data); + localStorage.setItem("merchant_metadata_data", localStorageTest.merchant_metadata_data); }, localStorageTest); - console.log('set success'); + console.log("set success"); } /** - * @param {Page} page + * @param {Page} page * @param {String} webUrl 网站 * @param {String} cookieFile cookie地址Url */ export async function injectionCookie(page, webUrl, cookieFile) { try { await page.goto(webUrl); - const cookies = JSON.parse(fs.readFileSync(cookieFile, 'utf-8')); + const cookies = JSON.parse(fs.readFileSync(cookieFile, "utf-8")); await page.setCookie(...cookies); await page.goto(webUrl); } catch (error) { console.error(error); } -} \ No newline at end of file +}