333

fix

.gitignore

@@ -3,3 +3,5 @@ node_modules/
 *.png
 businessData.json
 data/user.json
+data/zhihu.json
+data/zhihuData.json

.prettierrc

@@ -0,0 +1,13 @@
+{
+  "printWidth": 100,
+  "tabWidth": 4,
+  "semi": true,
+  "singleQuote": false,
+  "quoteProps": "consistent",
+  "trailingComma": "es5",
+  "bracketSpacing": true,
+  "arrowParens": "avoid",
+  "proseWrap": "never",
+  "endOfLine": "lf",
+  "embeddedLanguageFormatting": "off"
+}

BusinessDataAnalysisSheet.js

@@ -1,24 +0,0 @@
-import { launch } from 'puppeteer';
-import 'dotenv/config.js'
-
-(async () => {
-    const browser = await launch({ headless: true, args: [`--window-size=1920,1080`], defaultViewport: { width: 1920, height: 1080 } });
-    const page = await browser.newPage();
-    await page.setViewport({ width: 1920, height: 1080 });
-    await page.goto('https://dev-dmp.meiguanjia.net/report/businessData');
-
-    console.log(current);
-    console.log(process.env.ACCOUNT);
-
-    const item = await page.$eval('.item.cur', node=>node.innerHTML);
-    console.log(item);
-    // const arcoMessage = await page.$eval('.arco-message .arco-message-content', node => node.innerHTML);
-    // await page.screenshot({ path: 'resetPassword.png' });
-    console.log(arcoMessage);
-
-    page.on('console', msg => {
-        console.log('PAGE:', msg.text())
-    })
-
-    await browser.close();
-})();

README.md

@@ -0,0 +1,3 @@
+111
+
+333

data/zhihu.json

@@ -1,207 +0,0 @@
-[
-    {
-        "domain": ".zhihu.com",
-        "expirationDate": 1722947938,
-        "hostOnly": false,
-        "httpOnly": false,
-        "name": "__zse_ck",
-        "path": "/",
-        "sameSite": "unspecified",
-        "secure": false,
-        "session": false,
-        "storeId": "0",
-        "value": "001_4GAjdIPApDlfoktrmr9xjL8uAYQHAcICZZSRr3DIWK2bqp4Bq/CuLHYwdYNdl/eklSstaHYiP0AJTNUBgHqtoJU+7WwKhbIWdceQ/pukPfHRIozZ3/AKy6Yb6zaKg3tC",
-        "id": 1
-    },
-    {
-        "domain": ".zhihu.com",
-        "expirationDate": 1756903138.09559,
-        "hostOnly": false,
-        "httpOnly": false,
-        "name": "_xsrf",
-        "path": "/",
-        "sameSite": "unspecified",
-        "secure": false,
-        "session": false,
-        "storeId": "0",
-        "value": "Vtmi0jwDJyVfhUfdHTVeLR16MLbuoqcX",
-        "id": 2
-    },
-    {
-        "domain": ".zhihu.com",
-        "expirationDate": 1751639986.575218,
-        "hostOnly": false,
-        "httpOnly": false,
-        "name": "_zap",
-        "path": "/",
-        "sameSite": "unspecified",
-        "secure": false,
-        "session": false,
-        "storeId": "0",
-        "value": "08b6a9a6-ffc7-4ee8-90a8-f7cba4165388",
-        "id": 3
-    },
-    {
-        "domain": ".zhihu.com",
-        "expirationDate": 1751639987.056828,
-        "hostOnly": false,
-        "httpOnly": false,
-        "name": "d_c0",
-        "path": "/",
-        "sameSite": "unspecified",
-        "secure": false,
-        "session": false,
-        "storeId": "0",
-        "value": "ACAfoAGQshiPTkU5aXH6DAjwlFi2TYokPrU=|1717079988",
-        "id": 4
-    },
-    {
-        "domain": ".zhihu.com",
-        "hostOnly": false,
-        "httpOnly": false,
-        "name": "Hm_lpvt_98beee57fd2ef70ccdd5ca52b9740c49",
-        "path": "/",
-        "sameSite": "unspecified",
-        "secure": false,
-        "session": true,
-        "storeId": "0",
-        "value": "1722343511",
-        "id": 5
-    },
-    {
-        "domain": ".zhihu.com",
-        "expirationDate": 1753879511,
-        "hostOnly": false,
-        "httpOnly": false,
-        "name": "Hm_lvt_98beee57fd2ef70ccdd5ca52b9740c49",
-        "path": "/",
-        "sameSite": "unspecified",
-        "secure": false,
-        "session": false,
-        "storeId": "0",
-        "value": "1722082618,1722130658,1722151750,1722343139",
-        "id": 6
-    },
-    {
-        "domain": ".zhihu.com",
-        "hostOnly": false,
-        "httpOnly": false,
-        "name": "HMACCOUNT",
-        "path": "/",
-        "sameSite": "unspecified",
-        "secure": false,
-        "session": true,
-        "storeId": "0",
-        "value": "2D7CA8E84CD24E76",
-        "id": 7
-    },
-    {
-        "domain": ".zhihu.com",
-        "expirationDate": 1724935511,
-        "hostOnly": false,
-        "httpOnly": false,
-        "name": "tst",
-        "path": "/",
-        "sameSite": "unspecified",
-        "secure": false,
-        "session": false,
-        "storeId": "0",
-        "value": "h",
-        "id": 8
-    },
-    {
-        "domain": ".zhihu.com",
-        "expirationDate": 1737025782.444365,
-        "hostOnly": false,
-        "httpOnly": true,
-        "name": "z_c0",
-        "path": "/",
-        "sameSite": "unspecified",
-        "secure": true,
-        "session": false,
-        "storeId": "0",
-        "value": "2|1:0|10:1721473784|4:z_c0|80:MS4xTG9IdkJBQUFBQUFtQUFBQVlBSlZUZmpvaUdlZkxyaVRRb0NuNGw1b2JvZ0ZqNDA0eW9NUHV3PT0=|92f4651e6a267edb95be6b11b839e2546d91a0970768db8301bcac39a6eecac2",
-        "id": 9
-    },
-    {
-        "domain": "www.zhihu.com",
-        "expirationDate": 1748615987,
-        "hostOnly": true,
-        "httpOnly": false,
-        "name": "__snaker__id",
-        "path": "/",
-        "sameSite": "unspecified",
-        "secure": false,
-        "session": false,
-        "storeId": "0",
-        "value": "fD9Cyd25ImUPIyJH",
-        "id": 10
-    },
-    {
-        "domain": "www.zhihu.com",
-        "expirationDate": 1722347111.96829,
-        "hostOnly": true,
-        "httpOnly": false,
-        "name": "BEC",
-        "path": "/",
-        "sameSite": "unspecified",
-        "secure": false,
-        "session": false,
-        "storeId": "0",
-        "value": "6ff32b60f55255af78892ba1e551063a",
-        "id": 11
-    },
-    {
-        "domain": "www.zhihu.com",
-        "hostOnly": true,
-        "httpOnly": false,
-        "name": "JOID",
-        "path": "/",
-        "sameSite": "unspecified",
-        "secure": false,
-        "session": true,
-        "storeId": "0",
-        "value": "W1wUA0wmJxgWn4dVLS4UAgCbHuQ9XEhbYPT4ZX9dYGtS9sQVS3TDtnmcj1ErNsTNn23GyAfDQcki_FGhxhtiuBY=",
-        "id": 12
-    },
-    {
-        "domain": "www.zhihu.com",
-        "hostOnly": true,
-        "httpOnly": false,
-        "name": "osd",
-        "path": "/",
-        "sameSite": "unspecified",
-        "secure": false,
-        "session": true,
-        "storeId": "0",
-        "value": "U1AVBksuKxkTmI9ZLCsTCgyaG-M1UEleZ_z0ZHpaaGdT88MdR3XGsXGQjlQsPsjMmmrOxAbGRsEu_VSmzhdjvRE=",
-        "id": 13
-    },
-    {
-        "domain": "www.zhihu.com",
-        "expirationDate": 1751639999.367131,
-        "hostOnly": true,
-        "httpOnly": false,
-        "name": "q_c1",
-        "path": "/",
-        "sameSite": "unspecified",
-        "secure": false,
-        "session": false,
-        "storeId": "0",
-        "value": "6755bb9e74ff466faa6b2d0ba240faf9|1717080000000|1717080000000",
-        "id": 14
-    },
-    {
-        "domain": "www.zhihu.com",
-        "hostOnly": true,
-        "httpOnly": false,
-        "name": "SESSIONID",
-        "path": "/",
-        "sameSite": "unspecified",
-        "secure": false,
-        "session": true,
-        "storeId": "0",
-        "value": "wjBqer5SuJvEnMMcxO4MqOg0HjznYy14sqbhB66uWk1",
-        "id": 15
-    }
-    ]

example.js

@@ -1,38 +0,0 @@
-import { launch } from 'puppeteer';
-import 'dotenv/config.js'
-
-(async () => {
-    const browser = await launch({ headless: true, args: [`--window-size=1920,1080`], defaultViewport: { width: 1920, height: 1080 } });
-    const page = await browser.newPage();
-    await page.setViewport({ width: 1920, height: 1080 });
-    const dev = 'https://dev-dmp.meiguanjia.net/login';
-    const stging = 'https://boss.aizhb.net/login';
-    let current = '';
-    if (process.env.PROFILE === 'dev') {
-        current = dev;
-    } else if (process.env.PROFILE === 'stging') {
-        current = stging;
-    }
-
-    console.log(current);
-    console.log(process.env.ACCOUNT);
-    await page.goto(current);
-    await page.locator('#userName input').fill(process.env.ACCOUNT);
-    await page.locator('.forgot_box').click();
-    await page.waitForSelector('.modify_main');
-    await page.locator('#digitalCode input').fill('1');
-    await page.locator('#smsCode input').fill(process.env.SMS);
-    await page.locator('#newPassWord input').fill(process.env.PASSWORD);
-    await page.locator('#confirmPassWord input').fill(process.env.PASSWORD);
-    await page.locator('[type="submit"]').click();
-    await page.waitForSelector('.arco-message .arco-message-content', { visible: true });
-    const arcoMessage = await page.$eval('.arco-message .arco-message-content', node => node.innerHTML);
-    await page.screenshot({ path: 'resetPassword.png' });
-    console.log(arcoMessage);
-
-    page.on('console', msg => {
-        console.log('PAGE:', msg.text())
-      })
-
-    await browser.close();
-})();

example2.js

@@ -1,43 +0,0 @@
-import { launch } from 'puppeteer';
-import fs from 'fs';
-import 'dotenv/config.js'
-import { setLocalStorage } from './utils/userUtils.js';
-import { getDate, cleanType, changeTimeDimension } from './utils/businessDataUtil.js';
-
-(async () => {
-    const browser = await launch({ headless: false, args: [`--window-size=1920,1080`], defaultViewport: { width: 1920, height: 1080 } });
-    const page = await browser.newPage();
-    setLocalStorage(page, '.\\data\\user.json');
-    await page.goto('https://dev-dmp.meiguanjia.net/report/businessData');
-    // await page.goto('https://boss.aizhb.net/report/businessData');
-
-    await page.waitForSelector('.menu-list-second');
-    const childElements = await page.$$('.menu-list-second .menu-list-second-item');
-    await new Promise(r => setTimeout(r, 1000));
-    await childElements[1].click();
-    const str = await page.evaluate(node => node.textContent.trim(), childElements[1]);
-    console.log(str);
-
-    await changeTimeDimension(page, '周报');
-
-    const file = './data/businessData.json';
-
-    fs.writeFileSync(file, '', 'utf-8');
-
-    await getDate(page, fs, file);
-    console.log('========');
-
-    await cleanType(page, '.arco-picker-start-time', '2024-06');
-    await getDate(page, fs, file);
-    console.log('========');
-
-    await cleanType(page, '.arco-picker-start-time', '2024-05');
-    await getDate(page, fs, file);
-    console.log('========');
-
-    await cleanType(page, '.arco-picker-start-time', '2024-04');
-    await getDate(page, fs, file);
-    console.log('========');
-
-    await browser.close();
-})();

examples/zhihu.js

@@ -0,0 +1,66 @@
+import { launch } from "puppeteer";
+import "dotenv/config.js";
+import { injectionCookie } from "../utils/userUtils.js";
+import fs from "fs";
+import dotenv from "dotenv";
+dotenv.config();
+
+(async () => {
+    const browser = await launch({
+        headless: false,
+        args: [`--window-size=1920,1080`],
+        defaultViewport: { width: 1920, height: 1080 },
+    });
+    const page = await browser.newPage();
+    page.on("console", msg => console.log("PAGE LOG", msg.text()));
+    page.on("error", error => console.error("PAGE error", error));
+
+    // 注入cookie
+    const webUrl = "https://www.zhihu.com/hot";
+    const cookieFileUrl = "./data/zhihu.json";
+    await injectionCookie(page, webUrl, cookieFileUrl);
+
+    const hotItems = await page.$$(".HotList-list .HotItem");
+
+    const hotText = new Array();
+
+    for (const hot in hotItems) {
+        if (Object.hasOwnProperty.call(hotItems, hot)) {
+            const element = hotItems[hot];
+
+            const h2Element = await element.$("h2");
+            const pElement = await element.$("p");
+
+            const title = h2Element != null ? await h2Element.evaluate(e => e.textContent) : "";
+            const excerpt = pElement != null ? await pElement.evaluate(e => e.textContent) : "";
+            const createTime = Date.now();
+            hotText.push({ title, excerpt, createTime });
+        }
+    }
+
+    fs.writeFileSync("./data/zhihuData.json", JSON.stringify(hotText, null, 2));
+
+    const url = process.env.URL;
+    console.log(process.env.URL);
+
+    try {
+        const response = await fetch(url, {
+            method: "POST",
+            headers: {
+                "Content-Type": "application/json",
+            },
+            body: JSON.stringify(hotText, null, 2),
+        });
+
+        if (!response.ok) {
+            throw new Error(`Http error! status : ${response.status}`);
+        }
+
+        const responseData = await response.json();
+        console.log("Repsonse data", responseData);
+    } catch (error) {
+        console.error(error);
+    }
+
+    await browser.close();
+})();

login.js

@@ -1,29 +0,0 @@
-import { launch } from 'puppeteer';
-import fs from 'fs';
-import 'dotenv/config.js';
-
-(async () => {
-    const browser = await launch({ headless: false, args: [`--window-size=1920,1080`], defaultViewport: { width: 1920, height: 1080 } });
-    const page = await browser.newPage();
-    await page.goto('https://dev-dmp.meiguanjia.net/login');
-    // await page.goto('https://boss.aizhb.net/login');
-    await page.locator('#userName input').fill(process.env.ACCOUNT);
-    await page.locator('#passWord input').fill(process.env.PASSWORD);
-    await page.locator('[type="submit"]').click();
-
-    await page.waitForSelector('.arco-message .arco-message-content', { visible: true });
-    const arcoMessage = await page.$eval('.arco-message .arco-message-content', node => node.innerHTML);
-    console.log(arcoMessage);
-
-    await new Promise(r => setTimeout(r, 1000));
-    const localStorage = await page.evaluate(() => Object.assign({}, window.localStorage));
-
-    const importStorage = JSON.stringify(localStorage);
-    const file = './data/user.json';
-    if (!fs.existsSync('./data/')) {
-        fs.mkdirSync('./data/');
-        console.log('文件夹不存在，创建文件夹');
-    }
-    fs.writeFileSync(file, importStorage, 'utf-8');
-    await browser.close();
-})();

package.json

@@ -4,4 +4,4 @@
     "dotenv": "^16.4.5",
     "puppeteer": "^22.13.1"
   }
-}
+}

uselogin.js

@@ -1,17 +0,0 @@
-import { launch } from 'puppeteer';
-import fs from 'fs';
-import 'dotenv/config.js';
-
-import { setLocalStorage } from './utils/userUtils.js';
-
-(async () => {
-    const browser = await launch({ headless: false, devtools: true, args: [`--window-size=1920,1080`], defaultViewport: { width: 1920, height: 1080 } });
-    const page = await browser.newPage();
-    await page.goto('https://dev-dmp.meiguanjia.net/login');
-
-    await setLocalStorage(page, 'user.json');
-
-    await page.goto('https://dev-dmp.meiguanjia.net/report/businessData');
-
-    await browser.close();
-})();

utils/businessDataUtil.js

@@ -1,55 +0,0 @@
-export async function getDate(page, fs, file) {
-    await new Promise(r => setTimeout(r, 2000));
-    await page.waitForSelector('.ai_custome');
-    const aiCustome = await page.$$('.ai_custome .ai_custome_item .amount');
-    let ai = new Set();
-    for (const e of aiCustome) {
-        let str = await page.evaluate(element => element.textContent, e);
-        ai.add(str);
-    }
-
-    let importStorage = '\n' + JSON.stringify(Array.from(ai), null, 2) + '\n';
-    fs.appendFileSync(file, importStorage, 'utf-8');
-
-    const elementHandle = await page.waitForSelector('.bi_warp iframe');
-    const frame = await elementHandle.contentFrame();
-    await frame.waitForSelector('#label-content');
-    const items = await frame.$$('#label-content'); // Use $$ to get all matching elements
-    let its = new Set();
-    if (items.length > 0) { // Check if items array is not empty
-        for (const e of items) {
-            let str = await frame.evaluate(element => element.textContent, e); // Use frame.evaluate() here
-            its.add(str.replace(/\s+/g, ' '));
-        }
-    } else {
-        console.log('#label-content not found in the iframe');
-    }
-
-    importStorage = JSON.stringify(Array.from(its), null, 2) + '\n';
-    fs.appendFileSync(file, importStorage, 'utf-8');
-}
-
-export async function cleanType(page, element, timeContent) {
-    const time = await page.$(element);
-    await time.click({ clickCount: 3 });
-    await time.type(timeContent, { delay: 100 });
-}
-
-export async function changeTimeDimension(page, timeDimension) {
-    let rb = await page.$('.arco-space-item .arco-select-view-value');
-    let s = await page.evaluate(node => node.textContent.trim(), rb);
-    console.log(s);
-
-    await rb.click();
-    const timeMap = new Map();
-    rb = await page.$$('.arco-select-dropdown-list .arco-select-option');
-    for (const iterator of rb) {
-        s = await page.evaluate(node => node.textContent.trim(), iterator)
-        console.log(s);
-        timeMap.set(s, iterator);
-    }
-    console.log('===');
-    console.log(timeMap);
-
-    await timeMap.get(timeDimension).click();
-}

utils/userUtils.js

@@ -1,20 +1,38 @@
-import fs from 'fs';
+import fs from "fs";
 
 /**
  * 设置页面的LocalStorage
- * @param {*} page 
+ * @param {import('puppeteer').Page} page
  * @param {String} file 用户信息
  */
 export async function setLocalStorage(page, file) {
-    const outputLocalStorge = fs.readFileSync(file, 'utf-8');
+    const outputLocalStorge = fs.readFileSync(file, "utf-8");
     let localStorageTest = JSON.parse(outputLocalStorge);
 
     // 设置页面的localStorage
     await page.evaluateOnNewDocument(localStorageTest => {
         localStorage.clear();
-        localStorage.setItem('merchant_access_token', localStorageTest.merchant_access_token);
-        localStorage.setItem('merchant_refresh_token', localStorageTest.merchant_refresh_token);
-        localStorage.setItem('merchant_login_data', localStorageTest.merchant_login_data);
-        localStorage.setItem('merchant_metadata_data', localStorageTest.merchant_metadata_data);
+        localStorage.setItem("merchant_access_token", localStorageTest.merchant_access_token);
+        localStorage.setItem("merchant_refresh_token", localStorageTest.merchant_refresh_token);
+        localStorage.setItem("merchant_login_data", localStorageTest.merchant_login_data);
+        localStorage.setItem("merchant_metadata_data", localStorageTest.merchant_metadata_data);
     }, localStorageTest);
-}
+
+    console.log("set success");
+}
+
+/**
+ * @param {Page} page
+ * @param {String} webUrl 网站
+ * @param {String} cookieFile cookie地址Url
+ */
+export async function injectionCookie(page, webUrl, cookieFile) {
+    try {
+        await page.goto(webUrl);
+        const cookies = JSON.parse(fs.readFileSync(cookieFile, "utf-8"));
+        await page.setCookie(...cookies);
+        await page.goto(webUrl);
+    } catch (error) {
+        console.error(error);
+    }
+}