电影天堂爬虫代码示例

---

代码展示

# 首先获取网站首页html内容，过程省略
# 为了防止反爬，将html保存于本地
f = open('dy2018.html', 'r', encoding='utf-8')
html = f.read()
f.close()

# 将html以标题进行分类
op = re.compile(
    r'<div class="co_area2"(?P<item>.*?)\s*</div>\s*</div>',
    re.S)
item = re.findall(op, html)

# 创建总列表用于存储
sum_list = []

# 遍历分类子元素
for i in item:
    op = re.compile(
        r'.*?<div class="title_all">.*?<span style="float:left;">(?P<title_name>.*?)</span>.*?<ul>(?P<ul_content>.*?)</ul>',
        re.S)
    matches = re.findall(op, i)[0]

# 清洗title
re_title = r'(?:<a [^>]*>([^<]+)</a>|([^<]+))'
title = re.findall(re_title, matches[0])[0]
if title[0]:
    title = title[0]
else:
    title = title[1]

# 清洗ul_content
to_content_list = r'<li>(.*?)</li>'
content_list = re.findall(to_content_list, matches[1])
re_content = r"<a href='(?P<link>.*?)' title=\"(?P<name>.*?)\">"

# 创建列表接收数据
movie_list = []

# 将re_content中的li提取为title和详情页链接link
for content in content_list:
    content = re.findall(re_content, content)
    if content:
        content = content[0]
        link = "https://www.dy2018.com/" + content[0]
        movie_title = content[1]

#添加headers防止反爬
 headers = {略}
 req = requests.get(link, headers=headers)
 req.encoding = 'gbk'

# 获取电影详情页下载地址
link_list = re.findall(r'<td style="WORD-WRAP: break-word" bgcolor="#fdfddf"><a href="(.*?)&', req.text)
movie_list.append({
    "movie_title": movie_title,
    "link": link_list
})

    # 为列表添加子元素
    sum_list.append(
        {
            "title": title,
            "movie_list": movie_list
        })
# 代码存储至本地
f = open('dy2018.json', 'w', encoding='utf-8')
f.write(json.dumps(sum_list, ensure_ascii=False))

---

结果展示

[
  {
    "title": "2024新片精品",
    "movie_list": [
      {
        "movie_title": "2024年国产6.4分犯罪片《浴火之路》HD国语中英双字",
        "link": [
          "magnet:?xt=urn:btih:57eb67973d2dd0e711b989ca6255d9a621b232af"
        ]
      },
      {
        "movie_title": "2019年泰国6.2分喜剧片《友情以上》HD泰语中字",
        "link": [
          "magnet:?xt=urn:btih:632853d4479bb0e060bbe57a09dcd91275c94529"
        ]
      },
      {
        "movie_title": "2024年日本8.5分动画片《蓦然回首》HD日语中字",
        "link": [
          "magnet:?xt=urn:btih:e657d8c9e0db74f2ca3d39c2fbb210eb721f8dc7"
        ]
      },
      {
......