rsgl/puppeteer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compare commits

base: rsgl:f531b1a30644dd53ffb76a7eb350b3f8f5de6c58
Branches Tags
rsgl:main
rsgl:zhihu
..
compare: rsgl:e422b908dc5c05a48e04899d04cbfd13073c39d0
Branches Tags
rsgl:main
rsgl:zhihu

No commits in common. "f531b1a30644dd53ffb76a7eb350b3f8f5de6c58" and "e422b908dc5c05a48e04899d04cbfd13073c39d0" have entirely different histories.
f531b1a306 ... e422b908dc

13 changed files with 422 additions and 111 deletions
Show Stats Expand all files Collapse all files

2
.gitignore vendored
View File

@ -3,5 +3,3 @@ node_modules/
*.png *.png
businessData.json businessData.json
data/user.json data/user.json
data/zhihu.json
data/zhihuData.json

13
.prettierrc
View File

@ -1,13 +0,0 @@
{
"printWidth": 100,
"tabWidth": 4,
"semi": true,
"singleQuote": false,
"quoteProps": "consistent",
"trailingComma": "es5",
"bracketSpacing": true,
"arrowParens": "avoid",
"proseWrap": "never",
"endOfLine": "lf",
"embeddedLanguageFormatting": "off"
}

24
BusinessDataAnalysisSheet.js Normal file
View File

@ -0,0 +1,24 @@
import { launch } from 'puppeteer';
import 'dotenv/config.js'
(async () => {
const browser = await launch({ headless: true, args: [`--window-size=1920,1080`], defaultViewport: { width: 1920, height: 1080 } });
const page = await browser.newPage();
await page.setViewport({ width: 1920, height: 1080 });
await page.goto('https://dev-dmp.meiguanjia.net/report/businessData');
console.log(current);
console.log(process.env.ACCOUNT);
const item = await page.$eval('.item.cur', node=>node.innerHTML);
console.log(item);
// const arcoMessage = await page.$eval('.arco-message .arco-message-content', node => node.innerHTML);
// await page.screenshot({ path: 'resetPassword.png' });
console.log(arcoMessage);
page.on('console', msg => {
console.log('PAGE:', msg.text())
})
await browser.close();
})();

3
README.md
View File

@ -1,3 +0,0 @@
111
333

207
data/zhihu.json Normal file
View File

@ -0,0 +1,207 @@
[
{
"domain": ".zhihu.com",
"expirationDate": 1722947938,
"hostOnly": false,
"httpOnly": false,
"name": "__zse_ck",
"path": "/",
"sameSite": "unspecified",
"secure": false,
"session": false,
"storeId": "0",
"value": "001_4GAjdIPApDlfoktrmr9xjL8uAYQHAcICZZSRr3DIWK2bqp4Bq/CuLHYwdYNdl/eklSstaHYiP0AJTNUBgHqtoJU+7WwKhbIWdceQ/pukPfHRIozZ3/AKy6Yb6zaKg3tC",
"id": 1
},
{
"domain": ".zhihu.com",
"expirationDate": 1756903138.09559,
"hostOnly": false,
"httpOnly": false,
"name": "_xsrf",
"path": "/",
"sameSite": "unspecified",
"secure": false,
"session": false,
"storeId": "0",
"value": "Vtmi0jwDJyVfhUfdHTVeLR16MLbuoqcX",
"id": 2
},
{
"domain": ".zhihu.com",
"expirationDate": 1751639986.575218,
"hostOnly": false,
"httpOnly": false,
"name": "_zap",
"path": "/",
"sameSite": "unspecified",
"secure": false,
"session": false,
"storeId": "0",
"value": "08b6a9a6-ffc7-4ee8-90a8-f7cba4165388",
"id": 3
},
{
"domain": ".zhihu.com",
"expirationDate": 1751639987.056828,
"hostOnly": false,
"httpOnly": false,
"name": "d_c0",
"path": "/",
"sameSite": "unspecified",
"secure": false,
"session": false,
"storeId": "0",
"value": "ACAfoAGQshiPTkU5aXH6DAjwlFi2TYokPrU=|1717079988",
"id": 4
},
{
"domain": ".zhihu.com",
"hostOnly": false,
"httpOnly": false,
"name": "Hm_lpvt_98beee57fd2ef70ccdd5ca52b9740c49",
"path": "/",
"sameSite": "unspecified",
"secure": false,
"session": true,
"storeId": "0",
"value": "1722343511",
"id": 5
},
{
"domain": ".zhihu.com",
"expirationDate": 1753879511,
"hostOnly": false,
"httpOnly": false,
"name": "Hm_lvt_98beee57fd2ef70ccdd5ca52b9740c49",
"path": "/",
"sameSite": "unspecified",
"secure": false,
"session": false,
"storeId": "0",
"value": "1722082618,1722130658,1722151750,1722343139",
"id": 6
},
{
"domain": ".zhihu.com",
"hostOnly": false,
"httpOnly": false,
"name": "HMACCOUNT",
"path": "/",
"sameSite": "unspecified",
"secure": false,
"session": true,
"storeId": "0",
"value": "2D7CA8E84CD24E76",
"id": 7
},
{
"domain": ".zhihu.com",
"expirationDate": 1724935511,
"hostOnly": false,
"httpOnly": false,
"name": "tst",
"path": "/",
"sameSite": "unspecified",
"secure": false,
"session": false,
"storeId": "0",
"value": "h",
"id": 8
},
{
"domain": ".zhihu.com",
"expirationDate": 1737025782.444365,
"hostOnly": false,
"httpOnly": true,
"name": "z_c0",
"path": "/",
"sameSite": "unspecified",
"secure": true,
"session": false,
"storeId": "0",
"value": "2|1:0|10:1721473784|4:z_c0|80:MS4xTG9IdkJBQUFBQUFtQUFBQVlBSlZUZmpvaUdlZkxyaVRRb0NuNGw1b2JvZ0ZqNDA0eW9NUHV3PT0=|92f4651e6a267edb95be6b11b839e2546d91a0970768db8301bcac39a6eecac2",
"id": 9
},
{
"domain": "www.zhihu.com",
"expirationDate": 1748615987,
"hostOnly": true,
"httpOnly": false,
"name": "__snaker__id",
"path": "/",
"sameSite": "unspecified",
"secure": false,
"session": false,
"storeId": "0",
"value": "fD9Cyd25ImUPIyJH",
"id": 10
},
{
"domain": "www.zhihu.com",
"expirationDate": 1722347111.96829,
"hostOnly": true,
"httpOnly": false,
"name": "BEC",
"path": "/",
"sameSite": "unspecified",
"secure": false,
"session": false,
"storeId": "0",
"value": "6ff32b60f55255af78892ba1e551063a",
"id": 11
},
{
"domain": "www.zhihu.com",
"hostOnly": true,
"httpOnly": false,
"name": "JOID",
"path": "/",
"sameSite": "unspecified",
"secure": false,
"session": true,
"storeId": "0",
"value": "W1wUA0wmJxgWn4dVLS4UAgCbHuQ9XEhbYPT4ZX9dYGtS9sQVS3TDtnmcj1ErNsTNn23GyAfDQcki_FGhxhtiuBY=",
"id": 12
},
{
"domain": "www.zhihu.com",
"hostOnly": true,
"httpOnly": false,
"name": "osd",
"path": "/",
"sameSite": "unspecified",
"secure": false,
"session": true,
"storeId": "0",
"value": "U1AVBksuKxkTmI9ZLCsTCgyaG-M1UEleZ_z0ZHpaaGdT88MdR3XGsXGQjlQsPsjMmmrOxAbGRsEu_VSmzhdjvRE=",
"id": 13
},
{
"domain": "www.zhihu.com",
"expirationDate": 1751639999.367131,
"hostOnly": true,
"httpOnly": false,
"name": "q_c1",
"path": "/",
"sameSite": "unspecified",
"secure": false,
"session": false,
"storeId": "0",
"value": "6755bb9e74ff466faa6b2d0ba240faf9|1717080000000|1717080000000",
"id": 14
},
{
"domain": "www.zhihu.com",
"hostOnly": true,
"httpOnly": false,
"name": "SESSIONID",
"path": "/",
"sameSite": "unspecified",
"secure": false,
"session": true,
"storeId": "0",
"value": "wjBqer5SuJvEnMMcxO4MqOg0HjznYy14sqbhB66uWk1",
"id": 15
}
]

38
example.js Normal file
View File

@ -0,0 +1,38 @@
import { launch } from 'puppeteer';
import 'dotenv/config.js'
(async () => {
const browser = await launch({ headless: true, args: [`--window-size=1920,1080`], defaultViewport: { width: 1920, height: 1080 } });
const page = await browser.newPage();
await page.setViewport({ width: 1920, height: 1080 });
const dev = 'https://dev-dmp.meiguanjia.net/login';
const stging = 'https://boss.aizhb.net/login';
let current = '';
if (process.env.PROFILE === 'dev') {
current = dev;
} else if (process.env.PROFILE === 'stging') {
current = stging;
}
console.log(current);
console.log(process.env.ACCOUNT);
await page.goto(current);
await page.locator('#userName input').fill(process.env.ACCOUNT);
await page.locator('.forgot_box').click();
await page.waitForSelector('.modify_main');
await page.locator('#digitalCode input').fill('1');
await page.locator('#smsCode input').fill(process.env.SMS);
await page.locator('#newPassWord input').fill(process.env.PASSWORD);
await page.locator('#confirmPassWord input').fill(process.env.PASSWORD);
await page.locator('[type="submit"]').click();
await page.waitForSelector('.arco-message .arco-message-content', { visible: true });
const arcoMessage = await page.$eval('.arco-message .arco-message-content', node => node.innerHTML);
await page.screenshot({ path: 'resetPassword.png' });
console.log(arcoMessage);
page.on('console', msg => {
console.log('PAGE:', msg.text())
})
await browser.close();
})();

43
example2.js Normal file
View File

@ -0,0 +1,43 @@
import { launch } from 'puppeteer';
import fs from 'fs';
import 'dotenv/config.js'
import { setLocalStorage } from './utils/userUtils.js';
import { getDate, cleanType, changeTimeDimension } from './utils/businessDataUtil.js';
(async () => {
const browser = await launch({ headless: false, args: [`--window-size=1920,1080`], defaultViewport: { width: 1920, height: 1080 } });
const page = await browser.newPage();
setLocalStorage(page, '.\\data\\user.json');
await page.goto('https://dev-dmp.meiguanjia.net/report/businessData');
// await page.goto('https://boss.aizhb.net/report/businessData');
await page.waitForSelector('.menu-list-second');
const childElements = await page.$$('.menu-list-second .menu-list-second-item');
await new Promise(r => setTimeout(r, 1000));
await childElements[1].click();
const str = await page.evaluate(node => node.textContent.trim(), childElements[1]);
console.log(str);
await changeTimeDimension(page, '周报');
const file = './data/businessData.json';
fs.writeFileSync(file, '', 'utf-8');
await getDate(page, fs, file);
console.log('========');
await cleanType(page, '.arco-picker-start-time', '2024-06');
await getDate(page, fs, file);
console.log('========');
await cleanType(page, '.arco-picker-start-time', '2024-05');
await getDate(page, fs, file);
console.log('========');
await cleanType(page, '.arco-picker-start-time', '2024-04');
await getDate(page, fs, file);
console.log('========');
await browser.close();
})();

66
examples/zhihu.js
View File

@ -1,66 +0,0 @@
import { launch } from "puppeteer";
import "dotenv/config.js";
import { injectionCookie } from "../utils/userUtils.js";
import fs from "fs";
import dotenv from "dotenv";
dotenv.config();
(async () => {
const browser = await launch({
headless: false,
args: [`--window-size=1920,1080`],
defaultViewport: { width: 1920, height: 1080 },
});
const page = await browser.newPage();
page.on("console", msg => console.log("PAGE LOG", msg.text()));
page.on("error", error => console.error("PAGE error", error));
// 注入cookie
const webUrl = "https://www.zhihu.com/hot";
const cookieFileUrl = "./data/zhihu.json";
await injectionCookie(page, webUrl, cookieFileUrl);
const hotItems = await page.$$(".HotList-list .HotItem");
const hotText = new Array();
for (const hot in hotItems) {
if (Object.hasOwnProperty.call(hotItems, hot)) {
const element = hotItems[hot];
const h2Element = await element.$("h2");
const pElement = await element.$("p");
const title = h2Element != null ? await h2Element.evaluate(e => e.textContent) : "";
const excerpt = pElement != null ? await pElement.evaluate(e => e.textContent) : "";
const createTime = Date.now();
hotText.push({ title, excerpt, createTime });
}
}
fs.writeFileSync("./data/zhihuData.json", JSON.stringify(hotText, null, 2));
const url = process.env.URL;
console.log(process.env.URL);
try {
const response = await fetch(url, {
method: "POST",
headers: {
"Content-Type": "application/json",
},
body: JSON.stringify(hotText, null, 2),
});
if (!response.ok) {
throw new Error(`Http error! status : ${response.status}`);
}
const responseData = await response.json();
console.log("Repsonse data", responseData);
} catch (error) {
console.error(error);
}
await browser.close();
})();

29
login.js Normal file
View File

@ -0,0 +1,29 @@
import { launch } from 'puppeteer';
import fs from 'fs';
import 'dotenv/config.js';
(async () => {
const browser = await launch({ headless: false, args: [`--window-size=1920,1080`], defaultViewport: { width: 1920, height: 1080 } });
const page = await browser.newPage();
await page.goto('https://dev-dmp.meiguanjia.net/login');
// await page.goto('https://boss.aizhb.net/login');
await page.locator('#userName input').fill(process.env.ACCOUNT);
await page.locator('#passWord input').fill(process.env.PASSWORD);
await page.locator('[type="submit"]').click();
await page.waitForSelector('.arco-message .arco-message-content', { visible: true });
const arcoMessage = await page.$eval('.arco-message .arco-message-content', node => node.innerHTML);
console.log(arcoMessage);
await new Promise(r => setTimeout(r, 1000));
const localStorage = await page.evaluate(() => Object.assign({}, window.localStorage));
const importStorage = JSON.stringify(localStorage);
const file = './data/user.json';
if (!fs.existsSync('./data/')) {
fs.mkdirSync('./data/');
console.log('文件夹不存在，创建文件夹');
}
fs.writeFileSync(file, importStorage, 'utf-8');
await browser.close();
})();

2
package.json
View File

@ -4,4 +4,4 @@
"dotenv": "^16.4.5", "dotenv": "^16.4.5",
"puppeteer": "^22.13.1" "puppeteer": "^22.13.1"
} }
} }

17
uselogin.js Normal file
View File

@ -0,0 +1,17 @@
import { launch } from 'puppeteer';
import fs from 'fs';
import 'dotenv/config.js';
import { setLocalStorage } from './utils/userUtils.js';
(async () => {
const browser = await launch({ headless: false, devtools: true, args: [`--window-size=1920,1080`], defaultViewport: { width: 1920, height: 1080 } });
const page = await browser.newPage();
await page.goto('https://dev-dmp.meiguanjia.net/login');
await setLocalStorage(page, 'user.json');
await page.goto('https://dev-dmp.meiguanjia.net/report/businessData');
await browser.close();
})();

55
utils/businessDataUtil.js Normal file
View File

@ -0,0 +1,55 @@
export async function getDate(page, fs, file) {
await new Promise(r => setTimeout(r, 2000));
await page.waitForSelector('.ai_custome');
const aiCustome = await page.$$('.ai_custome .ai_custome_item .amount');
let ai = new Set();
for (const e of aiCustome) {
let str = await page.evaluate(element => element.textContent, e);
ai.add(str);
}
let importStorage = '\n' + JSON.stringify(Array.from(ai), null, 2) + '\n';
fs.appendFileSync(file, importStorage, 'utf-8');
const elementHandle = await page.waitForSelector('.bi_warp iframe');
const frame = await elementHandle.contentFrame();
await frame.waitForSelector('#label-content');
const items = await frame.$$('#label-content'); // Use $$ to get all matching elements
let its = new Set();
if (items.length > 0) { // Check if items array is not empty
for (const e of items) {
let str = await frame.evaluate(element => element.textContent, e); // Use frame.evaluate() here
its.add(str.replace(/\s+/g, ' '));
}
} else {
console.log('#label-content not found in the iframe');
}
importStorage = JSON.stringify(Array.from(its), null, 2) + '\n';
fs.appendFileSync(file, importStorage, 'utf-8');
}
export async function cleanType(page, element, timeContent) {
const time = await page.$(element);
await time.click({ clickCount: 3 });
await time.type(timeContent, { delay: 100 });
}
export async function changeTimeDimension(page, timeDimension) {
let rb = await page.$('.arco-space-item .arco-select-view-value');
let s = await page.evaluate(node => node.textContent.trim(), rb);
console.log(s);
await rb.click();
const timeMap = new Map();
rb = await page.$$('.arco-select-dropdown-list .arco-select-option');
for (const iterator of rb) {
s = await page.evaluate(node => node.textContent.trim(), iterator)
console.log(s);
timeMap.set(s, iterator);
}
console.log('===');
console.log(timeMap);
await timeMap.get(timeDimension).click();
}

34
utils/userUtils.js
View File

@ -1,38 +1,20 @@
import fs from "fs"; import fs from 'fs';
/** /**
* 设置页面的LocalStorage * 设置页面的LocalStorage
* @param {import('puppeteer').Page} page * @param {*} page
* @param {String} file 用户信息 * @param {String} file 用户信息
*/ */
export async function setLocalStorage(page, file) { export async function setLocalStorage(page, file) {
const outputLocalStorge = fs.readFileSync(file, "utf-8"); const outputLocalStorge = fs.readFileSync(file, 'utf-8');
let localStorageTest = JSON.parse(outputLocalStorge); let localStorageTest = JSON.parse(outputLocalStorge);
// 设置页面的localStorage // 设置页面的localStorage
await page.evaluateOnNewDocument(localStorageTest => { await page.evaluateOnNewDocument(localStorageTest => {
localStorage.clear(); localStorage.clear();
localStorage.setItem("merchant_access_token", localStorageTest.merchant_access_token); localStorage.setItem('merchant_access_token', localStorageTest.merchant_access_token);
localStorage.setItem("merchant_refresh_token", localStorageTest.merchant_refresh_token); localStorage.setItem('merchant_refresh_token', localStorageTest.merchant_refresh_token);
localStorage.setItem("merchant_login_data", localStorageTest.merchant_login_data); localStorage.setItem('merchant_login_data', localStorageTest.merchant_login_data);
localStorage.setItem("merchant_metadata_data", localStorageTest.merchant_metadata_data); localStorage.setItem('merchant_metadata_data', localStorageTest.merchant_metadata_data);
}, localStorageTest); }, localStorageTest);
}
console.log("set success");
}
/**
* @param {Page} page
* @param {String} webUrl 网站
* @param {String} cookieFile cookie地址Url
*/
export async function injectionCookie(page, webUrl, cookieFile) {
try {
await page.goto(webUrl);
const cookies = JSON.parse(fs.readFileSync(cookieFile, "utf-8"));
await page.setCookie(...cookies);
await page.goto(webUrl);
} catch (error) {
console.error(error);
}
}