rsgl/puppeteer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compare commits

base: rsgl:e422b908dc5c05a48e04899d04cbfd13073c39d0
Branches Tags
rsgl:main
rsgl:zhihu
...
compare: rsgl:f531b1a30644dd53ffb76a7eb350b3f8f5de6c58
Branches Tags
rsgl:main
rsgl:zhihu

17 Commits
e422b908dc ... f531b1a306

Author SHA1 Message Date
wenpeng
f531b1a306 333 2024-09-12 22:58:00 +08:00
wenpeng
3f5415fb6e Merge branch 'main' into zhihu 2024-09-12 22:56:44 +08:00
wenpeng
e193fe077e add READEME 2024-09-12 22:56:13 +08:00
wenpeng
050aae791b Merge branch 'zhihu' of https://github.com/LingandRX/puppeteer into zhihu 2024-09-12 22:52:11 +08:00
wenpeng
7c17cb5244 fix 2024-09-12 22:52:06 +08:00
wenpeng
5042f80954 格式化工具 2024-09-12 22:52:06 +08:00
rsgltzyd
57e16fa7d9 fix 2024-09-12 22:52:06 +08:00
rsgltzyd
a29ea55085 add injectionCookie 
fix
 2024-09-12 22:52:05 +08:00
rsgltzyd
ec8a8a6474 fix 2024-09-12 22:52:05 +08:00
rsgltzyd
ece0c181b0 fix 2024-09-12 22:51:57 +08:00
wenpeng
188d115a26 fix 2024-09-11 23:14:15 +08:00
wenpeng
df3560cc6f 格式化工具 2024-09-11 23:13:20 +08:00
rsgltzyd
f553840132 fix 2024-07-30 22:18:39 +08:00
rsgltzyd
a8e451a8c2 add injectionCookie 
fix
 2024-07-30 21:23:35 +08:00
rsgltzyd
7ff9cc663f fix 2024-07-30 20:58:55 +08:00
rsgltzyd
f49aa6531d fix 2024-07-29 22:17:43 +08:00
rsgltzyd
198ba79187 fix 2024-07-28 18:45:34 +08:00
13 changed files with 111 additions and 422 deletions
Show Stats Expand all files Collapse all files

2
.gitignore vendored
View File

@ -3,3 +3,5 @@ node_modules/
*.png *.png
businessData.json businessData.json
data/user.json data/user.json
data/zhihu.json
data/zhihuData.json

13
.prettierrc Normal file
View File

@ -0,0 +1,13 @@
{
"printWidth": 100,
"tabWidth": 4,
"semi": true,
"singleQuote": false,
"quoteProps": "consistent",
"trailingComma": "es5",
"bracketSpacing": true,
"arrowParens": "avoid",
"proseWrap": "never",
"endOfLine": "lf",
"embeddedLanguageFormatting": "off"
}

24
BusinessDataAnalysisSheet.js
View File

@ -1,24 +0,0 @@
import { launch } from 'puppeteer';
import 'dotenv/config.js'
(async () => {
const browser = await launch({ headless: true, args: [`--window-size=1920,1080`], defaultViewport: { width: 1920, height: 1080 } });
const page = await browser.newPage();
await page.setViewport({ width: 1920, height: 1080 });
await page.goto('https://dev-dmp.meiguanjia.net/report/businessData');
console.log(current);
console.log(process.env.ACCOUNT);
const item = await page.$eval('.item.cur', node=>node.innerHTML);
console.log(item);
// const arcoMessage = await page.$eval('.arco-message .arco-message-content', node => node.innerHTML);
// await page.screenshot({ path: 'resetPassword.png' });
console.log(arcoMessage);
page.on('console', msg => {
console.log('PAGE:', msg.text())
})
await browser.close();
})();

3
README.md Normal file
View File

@ -0,0 +1,3 @@
111
333

207
data/zhihu.json
View File

@ -1,207 +0,0 @@
[
{
"domain": ".zhihu.com",
"expirationDate": 1722947938,
"hostOnly": false,
"httpOnly": false,
"name": "__zse_ck",
"path": "/",
"sameSite": "unspecified",
"secure": false,
"session": false,
"storeId": "0",
"value": "001_4GAjdIPApDlfoktrmr9xjL8uAYQHAcICZZSRr3DIWK2bqp4Bq/CuLHYwdYNdl/eklSstaHYiP0AJTNUBgHqtoJU+7WwKhbIWdceQ/pukPfHRIozZ3/AKy6Yb6zaKg3tC",
"id": 1
},
{
"domain": ".zhihu.com",
"expirationDate": 1756903138.09559,
"hostOnly": false,
"httpOnly": false,
"name": "_xsrf",
"path": "/",
"sameSite": "unspecified",
"secure": false,
"session": false,
"storeId": "0",
"value": "Vtmi0jwDJyVfhUfdHTVeLR16MLbuoqcX",
"id": 2
},
{
"domain": ".zhihu.com",
"expirationDate": 1751639986.575218,
"hostOnly": false,
"httpOnly": false,
"name": "_zap",
"path": "/",
"sameSite": "unspecified",
"secure": false,
"session": false,
"storeId": "0",
"value": "08b6a9a6-ffc7-4ee8-90a8-f7cba4165388",
"id": 3
},
{
"domain": ".zhihu.com",
"expirationDate": 1751639987.056828,
"hostOnly": false,
"httpOnly": false,
"name": "d_c0",
"path": "/",
"sameSite": "unspecified",
"secure": false,
"session": false,
"storeId": "0",
"value": "ACAfoAGQshiPTkU5aXH6DAjwlFi2TYokPrU=|1717079988",
"id": 4
},
{
"domain": ".zhihu.com",
"hostOnly": false,
"httpOnly": false,
"name": "Hm_lpvt_98beee57fd2ef70ccdd5ca52b9740c49",
"path": "/",
"sameSite": "unspecified",
"secure": false,
"session": true,
"storeId": "0",
"value": "1722343511",
"id": 5
},
{
"domain": ".zhihu.com",
"expirationDate": 1753879511,
"hostOnly": false,
"httpOnly": false,
"name": "Hm_lvt_98beee57fd2ef70ccdd5ca52b9740c49",
"path": "/",
"sameSite": "unspecified",
"secure": false,
"session": false,
"storeId": "0",
"value": "1722082618,1722130658,1722151750,1722343139",
"id": 6
},
{
"domain": ".zhihu.com",
"hostOnly": false,
"httpOnly": false,
"name": "HMACCOUNT",
"path": "/",
"sameSite": "unspecified",
"secure": false,
"session": true,
"storeId": "0",
"value": "2D7CA8E84CD24E76",
"id": 7
},
{
"domain": ".zhihu.com",
"expirationDate": 1724935511,
"hostOnly": false,
"httpOnly": false,
"name": "tst",
"path": "/",
"sameSite": "unspecified",
"secure": false,
"session": false,
"storeId": "0",
"value": "h",
"id": 8
},
{
"domain": ".zhihu.com",
"expirationDate": 1737025782.444365,
"hostOnly": false,
"httpOnly": true,
"name": "z_c0",
"path": "/",
"sameSite": "unspecified",
"secure": true,
"session": false,
"storeId": "0",
"value": "2|1:0|10:1721473784|4:z_c0|80:MS4xTG9IdkJBQUFBQUFtQUFBQVlBSlZUZmpvaUdlZkxyaVRRb0NuNGw1b2JvZ0ZqNDA0eW9NUHV3PT0=|92f4651e6a267edb95be6b11b839e2546d91a0970768db8301bcac39a6eecac2",
"id": 9
},
{
"domain": "www.zhihu.com",
"expirationDate": 1748615987,
"hostOnly": true,
"httpOnly": false,
"name": "__snaker__id",
"path": "/",
"sameSite": "unspecified",
"secure": false,
"session": false,
"storeId": "0",
"value": "fD9Cyd25ImUPIyJH",
"id": 10
},
{
"domain": "www.zhihu.com",
"expirationDate": 1722347111.96829,
"hostOnly": true,
"httpOnly": false,
"name": "BEC",
"path": "/",
"sameSite": "unspecified",
"secure": false,
"session": false,
"storeId": "0",
"value": "6ff32b60f55255af78892ba1e551063a",
"id": 11
},
{
"domain": "www.zhihu.com",
"hostOnly": true,
"httpOnly": false,
"name": "JOID",
"path": "/",
"sameSite": "unspecified",
"secure": false,
"session": true,
"storeId": "0",
"value": "W1wUA0wmJxgWn4dVLS4UAgCbHuQ9XEhbYPT4ZX9dYGtS9sQVS3TDtnmcj1ErNsTNn23GyAfDQcki_FGhxhtiuBY=",
"id": 12
},
{
"domain": "www.zhihu.com",
"hostOnly": true,
"httpOnly": false,
"name": "osd",
"path": "/",
"sameSite": "unspecified",
"secure": false,
"session": true,
"storeId": "0",
"value": "U1AVBksuKxkTmI9ZLCsTCgyaG-M1UEleZ_z0ZHpaaGdT88MdR3XGsXGQjlQsPsjMmmrOxAbGRsEu_VSmzhdjvRE=",
"id": 13
},
{
"domain": "www.zhihu.com",
"expirationDate": 1751639999.367131,
"hostOnly": true,
"httpOnly": false,
"name": "q_c1",
"path": "/",
"sameSite": "unspecified",
"secure": false,
"session": false,
"storeId": "0",
"value": "6755bb9e74ff466faa6b2d0ba240faf9|1717080000000|1717080000000",
"id": 14
},
{
"domain": "www.zhihu.com",
"hostOnly": true,
"httpOnly": false,
"name": "SESSIONID",
"path": "/",
"sameSite": "unspecified",
"secure": false,
"session": true,
"storeId": "0",
"value": "wjBqer5SuJvEnMMcxO4MqOg0HjznYy14sqbhB66uWk1",
"id": 15
}
]

38
example.js
View File

@ -1,38 +0,0 @@
import { launch } from 'puppeteer';
import 'dotenv/config.js'
(async () => {
const browser = await launch({ headless: true, args: [`--window-size=1920,1080`], defaultViewport: { width: 1920, height: 1080 } });
const page = await browser.newPage();
await page.setViewport({ width: 1920, height: 1080 });
const dev = 'https://dev-dmp.meiguanjia.net/login';
const stging = 'https://boss.aizhb.net/login';
let current = '';
if (process.env.PROFILE === 'dev') {
current = dev;
} else if (process.env.PROFILE === 'stging') {
current = stging;
}
console.log(current);
console.log(process.env.ACCOUNT);
await page.goto(current);
await page.locator('#userName input').fill(process.env.ACCOUNT);
await page.locator('.forgot_box').click();
await page.waitForSelector('.modify_main');
await page.locator('#digitalCode input').fill('1');
await page.locator('#smsCode input').fill(process.env.SMS);
await page.locator('#newPassWord input').fill(process.env.PASSWORD);
await page.locator('#confirmPassWord input').fill(process.env.PASSWORD);
await page.locator('[type="submit"]').click();
await page.waitForSelector('.arco-message .arco-message-content', { visible: true });
const arcoMessage = await page.$eval('.arco-message .arco-message-content', node => node.innerHTML);
await page.screenshot({ path: 'resetPassword.png' });
console.log(arcoMessage);
page.on('console', msg => {
console.log('PAGE:', msg.text())
})
await browser.close();
})();

43
example2.js
View File

@ -1,43 +0,0 @@
import { launch } from 'puppeteer';
import fs from 'fs';
import 'dotenv/config.js'
import { setLocalStorage } from './utils/userUtils.js';
import { getDate, cleanType, changeTimeDimension } from './utils/businessDataUtil.js';
(async () => {
const browser = await launch({ headless: false, args: [`--window-size=1920,1080`], defaultViewport: { width: 1920, height: 1080 } });
const page = await browser.newPage();
setLocalStorage(page, '.\\data\\user.json');
await page.goto('https://dev-dmp.meiguanjia.net/report/businessData');
// await page.goto('https://boss.aizhb.net/report/businessData');
await page.waitForSelector('.menu-list-second');
const childElements = await page.$$('.menu-list-second .menu-list-second-item');
await new Promise(r => setTimeout(r, 1000));
await childElements[1].click();
const str = await page.evaluate(node => node.textContent.trim(), childElements[1]);
console.log(str);
await changeTimeDimension(page, '周报');
const file = './data/businessData.json';
fs.writeFileSync(file, '', 'utf-8');
await getDate(page, fs, file);
console.log('========');
await cleanType(page, '.arco-picker-start-time', '2024-06');
await getDate(page, fs, file);
console.log('========');
await cleanType(page, '.arco-picker-start-time', '2024-05');
await getDate(page, fs, file);
console.log('========');
await cleanType(page, '.arco-picker-start-time', '2024-04');
await getDate(page, fs, file);
console.log('========');
await browser.close();
})();

66
examples/zhihu.js Normal file
View File

@ -0,0 +1,66 @@
import { launch } from "puppeteer";
import "dotenv/config.js";
import { injectionCookie } from "../utils/userUtils.js";
import fs from "fs";
import dotenv from "dotenv";
dotenv.config();
(async () => {
const browser = await launch({
headless: false,
args: [`--window-size=1920,1080`],
defaultViewport: { width: 1920, height: 1080 },
});
const page = await browser.newPage();
page.on("console", msg => console.log("PAGE LOG", msg.text()));
page.on("error", error => console.error("PAGE error", error));
// 注入cookie
const webUrl = "https://www.zhihu.com/hot";
const cookieFileUrl = "./data/zhihu.json";
await injectionCookie(page, webUrl, cookieFileUrl);
const hotItems = await page.$$(".HotList-list .HotItem");
const hotText = new Array();
for (const hot in hotItems) {
if (Object.hasOwnProperty.call(hotItems, hot)) {
const element = hotItems[hot];
const h2Element = await element.$("h2");
const pElement = await element.$("p");
const title = h2Element != null ? await h2Element.evaluate(e => e.textContent) : "";
const excerpt = pElement != null ? await pElement.evaluate(e => e.textContent) : "";
const createTime = Date.now();
hotText.push({ title, excerpt, createTime });
}
}
fs.writeFileSync("./data/zhihuData.json", JSON.stringify(hotText, null, 2));
const url = process.env.URL;
console.log(process.env.URL);
try {
const response = await fetch(url, {
method: "POST",
headers: {
"Content-Type": "application/json",
},
body: JSON.stringify(hotText, null, 2),
});
if (!response.ok) {
throw new Error(`Http error! status : ${response.status}`);
}
const responseData = await response.json();
console.log("Repsonse data", responseData);
} catch (error) {
console.error(error);
}
await browser.close();
})();

29
login.js
View File

@ -1,29 +0,0 @@
import { launch } from 'puppeteer';
import fs from 'fs';
import 'dotenv/config.js';
(async () => {
const browser = await launch({ headless: false, args: [`--window-size=1920,1080`], defaultViewport: { width: 1920, height: 1080 } });
const page = await browser.newPage();
await page.goto('https://dev-dmp.meiguanjia.net/login');
// await page.goto('https://boss.aizhb.net/login');
await page.locator('#userName input').fill(process.env.ACCOUNT);
await page.locator('#passWord input').fill(process.env.PASSWORD);
await page.locator('[type="submit"]').click();
await page.waitForSelector('.arco-message .arco-message-content', { visible: true });
const arcoMessage = await page.$eval('.arco-message .arco-message-content', node => node.innerHTML);
console.log(arcoMessage);
await new Promise(r => setTimeout(r, 1000));
const localStorage = await page.evaluate(() => Object.assign({}, window.localStorage));
const importStorage = JSON.stringify(localStorage);
const file = './data/user.json';
if (!fs.existsSync('./data/')) {
fs.mkdirSync('./data/');
console.log('文件夹不存在，创建文件夹');
}
fs.writeFileSync(file, importStorage, 'utf-8');
await browser.close();
})();

2
package.json
View File

@ -4,4 +4,4 @@
"dotenv": "^16.4.5", "dotenv": "^16.4.5",
"puppeteer": "^22.13.1" "puppeteer": "^22.13.1"
} }
} }

17
uselogin.js
View File

@ -1,17 +0,0 @@
import { launch } from 'puppeteer';
import fs from 'fs';
import 'dotenv/config.js';
import { setLocalStorage } from './utils/userUtils.js';
(async () => {
const browser = await launch({ headless: false, devtools: true, args: [`--window-size=1920,1080`], defaultViewport: { width: 1920, height: 1080 } });
const page = await browser.newPage();
await page.goto('https://dev-dmp.meiguanjia.net/login');
await setLocalStorage(page, 'user.json');
await page.goto('https://dev-dmp.meiguanjia.net/report/businessData');
await browser.close();
})();

55
utils/businessDataUtil.js
View File

@ -1,55 +0,0 @@
export async function getDate(page, fs, file) {
await new Promise(r => setTimeout(r, 2000));
await page.waitForSelector('.ai_custome');
const aiCustome = await page.$$('.ai_custome .ai_custome_item .amount');
let ai = new Set();
for (const e of aiCustome) {
let str = await page.evaluate(element => element.textContent, e);
ai.add(str);
}
let importStorage = '\n' + JSON.stringify(Array.from(ai), null, 2) + '\n';
fs.appendFileSync(file, importStorage, 'utf-8');
const elementHandle = await page.waitForSelector('.bi_warp iframe');
const frame = await elementHandle.contentFrame();
await frame.waitForSelector('#label-content');
const items = await frame.$$('#label-content'); // Use $$ to get all matching elements
let its = new Set();
if (items.length > 0) { // Check if items array is not empty
for (const e of items) {
let str = await frame.evaluate(element => element.textContent, e); // Use frame.evaluate() here
its.add(str.replace(/\s+/g, ' '));
}
} else {
console.log('#label-content not found in the iframe');
}
importStorage = JSON.stringify(Array.from(its), null, 2) + '\n';
fs.appendFileSync(file, importStorage, 'utf-8');
}
export async function cleanType(page, element, timeContent) {
const time = await page.$(element);
await time.click({ clickCount: 3 });
await time.type(timeContent, { delay: 100 });
}
export async function changeTimeDimension(page, timeDimension) {
let rb = await page.$('.arco-space-item .arco-select-view-value');
let s = await page.evaluate(node => node.textContent.trim(), rb);
console.log(s);
await rb.click();
const timeMap = new Map();
rb = await page.$$('.arco-select-dropdown-list .arco-select-option');
for (const iterator of rb) {
s = await page.evaluate(node => node.textContent.trim(), iterator)
console.log(s);
timeMap.set(s, iterator);
}
console.log('===');
console.log(timeMap);
await timeMap.get(timeDimension).click();
}

34
utils/userUtils.js
View File

@ -1,20 +1,38 @@
import fs from 'fs'; import fs from "fs";
/** /**
* 设置页面的LocalStorage * 设置页面的LocalStorage
* @param {*} page * @param {import('puppeteer').Page} page
* @param {String} file 用户信息 * @param {String} file 用户信息
*/ */
export async function setLocalStorage(page, file) { export async function setLocalStorage(page, file) {
const outputLocalStorge = fs.readFileSync(file, 'utf-8'); const outputLocalStorge = fs.readFileSync(file, "utf-8");
let localStorageTest = JSON.parse(outputLocalStorge); let localStorageTest = JSON.parse(outputLocalStorge);
// 设置页面的localStorage // 设置页面的localStorage
await page.evaluateOnNewDocument(localStorageTest => { await page.evaluateOnNewDocument(localStorageTest => {
localStorage.clear(); localStorage.clear();
localStorage.setItem('merchant_access_token', localStorageTest.merchant_access_token); localStorage.setItem("merchant_access_token", localStorageTest.merchant_access_token);
localStorage.setItem('merchant_refresh_token', localStorageTest.merchant_refresh_token); localStorage.setItem("merchant_refresh_token", localStorageTest.merchant_refresh_token);
localStorage.setItem('merchant_login_data', localStorageTest.merchant_login_data); localStorage.setItem("merchant_login_data", localStorageTest.merchant_login_data);
localStorage.setItem('merchant_metadata_data', localStorageTest.merchant_metadata_data); localStorage.setItem("merchant_metadata_data", localStorageTest.merchant_metadata_data);
}, localStorageTest); }, localStorageTest);
}
console.log("set success");
}
/**
* @param {Page} page
* @param {String} webUrl 网站
* @param {String} cookieFile cookie地址Url
*/
export async function injectionCookie(page, webUrl, cookieFile) {
try {
await page.goto(webUrl);
const cookies = JSON.parse(fs.readFileSync(cookieFile, "utf-8"));
await page.setCookie(...cookies);
await page.goto(webUrl);
} catch (error) {
console.error(error);
}
}