Node爬虫实战

这几天对 Node 编写小爬虫进行了学习，这里记录一下实战的代码。

爬取网页信息

let axios = require('axios')
let { fsRead, fsWrite, fsDir } = require('./rw')
let httpUrl = 'https://www.1905.com/vod/list/n_1_t_1/o3p1.html'
function req(link) {
  return new Promise(function (resolve, reject) {
    axios.get(link).then(function (res) {
      resolve(res.data)
    })
  })
}
// 获取起始页面的所有分类地址
async function getClassUrl(link) {
  let data = await req(link)
  // 解析HTML内容
  let reg = /<span class="search-index-L">类型(.*?)<div class="grid-12x">/gis
  let result = reg.exec(data)[1]
  // console.log(result);
  let reg1 = /<a href="javascript\:void\(0\);" onclick="location\.href='(.*?)';return false;" .*?>(.*?)<\/a>/gis
  let arr = []

  while ((result2 = reg1.exec(result))) {
    if (result2[2] != '全部') {
      let obj = {
        className: result2[2],
        link: result2[1],
      }
      arr.push(obj)
      await fsDir('./movies/' + result2[2])
      getMovie(result2[1], result2[2])
    }
  }
  // console.log(arr);
}
// 获取分类里的电影连接
// 根据电影连接获取电影的详细信息
// 通过分类获取页面中的连接
async function getMovie(link, movieType) {
  let data = await req(link)
  let reg = /<a class="pic-pack-outer" target="\_blank" href="(.*?)" .*?>/gis
  var res9
  var arrList = []
  while ((res9 = reg.exec(data))) {
    arrList.push(res9[1])
    parsePage(res9[1], movieType)
  }
  // console.log("分类：" + movieType);
  // console.log(arrList);
}

async function parsePage(url, movieType) {
  // console.log(url);
  let data = await req(url)
  let reg = /<h1 class="playerBox-info-name playerBox-info-cnName">(.*?)<\/h1>.*?id="playerBoxIntroCon">(.*?)<a .*?导演.*?data-hrefexp="fr=vodplay\_ypzl\_dy">(.*?)<\/a>/gis
  let res3 = reg.exec(data)
  console.log(res3[1])
  let movie = {
    name: res3[1],
    brief: res3[2],
    daoyan: res3[3],
    movieUrl: url,
    movieType,
  }
  let str = JSON.stringify(movie)
  fsWrite('./movies/' + movieType + '/' + res3[1] + 'json', str)
}
getClassUrl(httpUrl)

爬取表情包

const cheerio = require('cheerio')
const axios = require('axios')
const fs = require('fs')
const url = require('url')
const path = require('path')
// 将延迟函数封装成promise对象（防止请求速度过快下载失败）
function wait(millSeconds) {
  return new Promise(function (resolve, reject) {
    setTimeout(() => {
      resolve('成功执行延迟函数，延迟时间：' + millSeconds)
    }, millSeconds)
  })
}
// 获取HTML文档的内容
// 获取页面总数
async function pageNum(link) {
  let res = await axios.get(link)
  let $ = cheerio.load(res.data)
  let btnLength = $('.pagination li').length
  let allNum = $('.pagination li')
    .eq(btnLength - 2)
    .find('a')
    .text()
  return allNum
}
// 获取页面
async function getListPage(pageNum) {
  let httpUrl = `https://www.doutula.com/article/list/?page=${pageNum}`
  let res = await axios.get(httpUrl)
  let $ = cheerio.load(res.data)
  $('#home .col-sm-9>a').each((i, element) => {
    let pageUrl = $(element).attr('href')
    let title = $(element).find('.random_title').text()
    let reg = /(.*?)\d/gis
    title = reg.exec(title)[1]
    fs.mkdir('./img/' + title, function (err) {
      if (err) {
        console.log(err)
      } else {
        console.log('创建：' + './img/' + title)
      }
    })
    parsePage(pageUrl, title)
  })
}
// 进入表情包页面
async function parsePage(link, title) {
  let res = await axios.get(link)
  let $ = cheerio.load(res.data)
  $('.pic-content img').each((i, element) => {
    let imgUrl = $(element).attr('src')
    let b = url.parse(imgUrl)
    let name = path.parse(b.pathname)
    // 创建路径名字
    let filePath = `./img/${title}/${name.base}/`
    // 创建写入流
    let ws = fs.createWriteStream(filePath.trim())
    axios.get(imgUrl, { responseType: 'stream' }).then(function (res) {
      res.data.pipe(ws)
      console.log('正在下载表情：' + filePath)
      // 监听事件，关闭写入流
      res.data.on('close', () => ws.close())
    })
  })
}
// 开始爬取所有页面
async function spider(link) {
  let allPageNum = await pageNum(link)
  for (let i = 1; i <= allPageNum; i++) {
    await wait(4000 * i) // 每个页面延迟3秒
    getListPage(i)
  }
}
spider('https://www.doutula.com/article/list/?page=1')

爬取音乐

const axios = require('axios')
const fs = require('fs')
const path = require('path')
// 目标：下载音乐
// 1.获取音乐相关的信息，通过信息获取下载地址
// 2.通过获取音乐列表获取音乐信息
// 3.通过音乐的分类页获取音乐列表
async function getPage(num) {
  let httpUrl = 'http://www.app-echo.com/api/recommend/sound-day?page=' + num
  let res = await axios.get(httpUrl)
  // console.log(res.data.list);
  let list = res.data.list
  list.forEach(function (item, i) {
    let title = item.sound.name
    let musicUrl = item.sound.source
    let fileName = path.parse(musicUrl).name
    let content = `${title},${musicUrl}.${fileName}\n`
    fs.writeFile('music.txt', content, { flag: 'a' }, function () {
      // console.log('写入完成:'+ title);
    })
    // console.log(path.parse(musicUrl));
    download(musicUrl, fileName)
  })
}
async function download(link, fileName) {
  let res = await axios.get(link, { responseType: 'stream' })
  let ws = fs.createWriteStream('./music/' + fileName + '.mp3')
  console.log(res.data)
  res.data.pipe(ws)
  res.data.on('close', function () {
    ws.close()
  })
}
// 爬一页意思一下就行了
getPage(1)