CVPR2020の論文を毎日自動で送ってくれるslack botを作ってみた！

1 経緯
2 プログラムの手順
3 スクレイピング
4 要約の和訳
5 Adversarial attacks の排除
6 slack に自動配信
7 完成した slack bot
8 感想

経緯

最近はCVPR2020の論文の要約を読んで、面白そうな論文はmendeleyにストックするという作業をやっています。その作業が結構大変です。理由は以下の通りです。

採択されている論文が多い (1,500ぐらいだったような)
あんまり興味がない論文が結構ある。Adversarial attacks の防御論文多すぎ(笑)。
わからない英単語が結構あるので、google 翻訳でその都度調べる

このような理由でどうにか作業をもっと簡単にしたいと思い、以下のようなシステムを作ることにしました。

プログラムの手順

スクレイピングで CVPR2020 の公式サイトから論文のタイトル、要約、pdf のリンクを取ってくる。
要約を和訳する。
要約の中で、あまり興味がない内容の単語が入っている場合は別にする。今回は Adversarial attacks 系の論文。
論文のタイトル、要約、要約を和訳したもの、pdf のリンクを1時間おきに slack のチャンネルに自動で配信する。

手順ごとに自分の実装を説明していきます。

スクレイピング

def main():
    headers = {
        "User-Agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:47.0) Gecko/20100101 Firefox/47.0"
    }
    url_name = "https://openaccess.thecvf.com/CVPR2020"
    url = requests.get(url_name, headers)
    soup = BeautifulSoup(url.content, "html.parser")
    elems = soup.find_all("dt")
    for elem in tqdm(elems):
        url = elem.next.next.attrs['href']
        ori_link = "https://openaccess.thecvf.com/"
        contents_url = ori_link + url
        paper_link = requests.get(contents_url, headers)
        paper_soup = BeautifulSoup(paper_link.content, "html.parser")
        title = paper_soup.find('div', id="papertitle").text
        abstract = paper_soup.find('div', id="abstract").text

def main():

headers = {

"User-Agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:47.0) Gecko/20100101 Firefox/47.0"

}

url_name = "https://openaccess.thecvf.com/CVPR2020"

url = requests.get(url_name, headers)

soup = BeautifulSoup(url.content, "html.parser")

elems = soup.find_all("dt")

for elem in tqdm(elems):

url = elem.next.next.attrs['href']

ori_link = "https://openaccess.thecvf.com/"

contents_url = ori_link + url

paper_link = requests.get(contents_url, headers)

paper_soup = BeautifulSoup(paper_link.content, "html.parser")

title = paper_soup.find('div', id="papertitle").text

abstract = paper_soup.find('div', id="abstract").text

beautfulsoup を使っています。CVPR の論文はサイト上で以下のような感じで並んでいます。

それぞれのタイトルをクリックするとその論文の詳細ページに飛びます。

このページから論文のタイトル、要約、pdf のリンク(下のリンク)をスクレイピングします。

要約の和訳

abstract_ja = translator.translate(abstract, src='en', dest='ja')
abstract_ja = str(abstract_ja.text)

1 2	abstract_ja = translator.translate(abstract, src='en', dest='ja') abstract_ja = str(abstract_ja.text)

googletrans を使いました。こちらを参考にしました。

Adversarial attacks の排除

def sort_class(abstract):
    sentence = abstract
    adversarial_1 = re.search("adversarial attacks", sentence)
    adversarial_2 = re.search("Adversarial Attacks", sentence)
    adversarial_3 = re.search("adversarial examples", sentence)
    adversarial_4 = re.search("Adversarial Examples", sentence)
    adversarial_5 = re.search("adversarial robustness", sentence)
    adversarial_6 = re.search("Adversarial Robustness", sentence)
    adversarial_7 = re.search("adversarial noise", sentence)
    adversarial_8 = re.search("Adversarial Noise", sentence)
    if adversarial_1 != None or adversarial_2 != None or adversarial_3 != None or adversarial_4 != None or adversarial_5 != None or adversarial_6 != None or adversarial_7 != None or adversarial_8 != None:
        class_name = "adver"
        return class_name
    else:
        class_name = "other"
        return class_name

def sort_class(abstract):

sentence = abstract

adversarial_1 = re.search("adversarial attacks", sentence)

adversarial_2 = re.search("Adversarial Attacks", sentence)

adversarial_3 = re.search("adversarial examples", sentence)

adversarial_4 = re.search("Adversarial Examples", sentence)

adversarial_5 = re.search("adversarial robustness", sentence)

adversarial_6 = re.search("Adversarial Robustness", sentence)

adversarial_7 = re.search("adversarial noise", sentence)

adversarial_8 = re.search("Adversarial Noise", sentence)

if adversarial_1 != None or adversarial_2 != None or adversarial_3 != None or adversarial_4 != None or adversarial_5 != None or adversarial_6 != None or adversarial_7 != None or adversarial_8 != None:

class_name = "adver"

return class_name

else:

class_name = "other"

return class_name

ちゃんと正規表現をやればもっと簡単にできると思います。今回は馬鹿正直に候補を全て書きました。adversarial attacks の論文は別のチャンネルに上げるようにします。

slack に自動配信

slackweb を使いました。使い方をこちらを参考にしました。コード全体を下にまとめます。

import re
import time
import slackweb
import requests
from tqdm import tqdm
from bs4 import BeautifulSoup
from googletrans import Translator
translator = Translator()

def sort_class(abstract):
    sentence = abstract
    adversarial_1 = re.search("adversarial attacks", sentence)
    adversarial_2 = re.search("Adversarial Attacks", sentence)
    adversarial_3 = re.search("adversarial examples", sentence)
    adversarial_4 = re.search("Adversarial Examples", sentence)
    adversarial_5 = re.search("adversarial robustness", sentence)
    adversarial_6 = re.search("Adversarial Robustness", sentence)
    adversarial_7 = re.search("adversarial noise", sentence)
    adversarial_8 = re.search("Adversarial Noise", sentence)
    if adversarial_1 != None or adversarial_2 != None or adversarial_3 != None or adversarial_4 != None or adversarial_5 != None or adversarial_6 != None or adversarial_7 != None or adversarial_8 != None:
        class_name = "adver"
        return class_name
    else:
        class_name = "other"
        return class_name

def main():
    headers = {
        "User-Agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:47.0) Gecko/20100101 Firefox/47.0"
    }
    url_name = "https://openaccess.thecvf.com/CVPR2020"
    url = requests.get(url_name, headers)
    soup = BeautifulSoup(url.content, "html.parser")
    elems = soup.find_all("dt")
    for elem in tqdm(elems):
        url = elem.next.next.attrs['href']
        ori_link = "https://openaccess.thecvf.com/"
        contents_url = ori_link + url
        paper_link = requests.get(contents_url, headers)
        paper_soup = BeautifulSoup(paper_link.content, "html.parser")
        title = paper_soup.find('div', id="papertitle").text
        abstract = paper_soup.find('div', id="abstract").text
        abstract_ja = translator.translate(abstract, src='en', dest='ja')
        abstract_ja = str(abstract_ja.text)
        pdf = paper_soup.find("dd").find("a").attrs['href'][6:]
        pdf_link = ori_link + pdf
        class_name = sort_class(abstract)

        attachments_title = []
        attachments_contents = []
        paper_title = {"title": title,
                "text": pdf_link}
        paper_contents = {"title": abstract,
                "text": abstract_ja}
        attachments_title.append(paper_title)
        attachments_contents.append(paper_contents)
        if class_name == "adver":
            slack = slackweb.Slack(url="YOUR_TOKEN")
            slack.notify(attachments=attachments_title)
            slack.notify(attachments=attachments_contents)
        else:
            slack = slackweb.Slack(url="YOUR_TOKEN")
            slack.notify(attachments=attachments_title)
            slack.notify(attachments=attachments_contents)

        time.sleep(3600)

if __name__ == "__main__":
    main()

import re

import time

import slackweb

import requests

from tqdm import tqdm

from bs4 import BeautifulSoup

from googletrans import Translator

translator = Translator()

def sort_class(abstract):

sentence = abstract

adversarial_1 = re.search("adversarial attacks", sentence)

adversarial_2 = re.search("Adversarial Attacks", sentence)

adversarial_3 = re.search("adversarial examples", sentence)

adversarial_4 = re.search("Adversarial Examples", sentence)

adversarial_5 = re.search("adversarial robustness", sentence)

adversarial_6 = re.search("Adversarial Robustness", sentence)

adversarial_7 = re.search("adversarial noise", sentence)

adversarial_8 = re.search("Adversarial Noise", sentence)

class_name = "adver"

return class_name

else:

class_name = "other"

return class_name

def main():

headers = {

"User-Agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:47.0) Gecko/20100101 Firefox/47.0"

}

url_name = "https://openaccess.thecvf.com/CVPR2020"

url = requests.get(url_name, headers)

soup = BeautifulSoup(url.content, "html.parser")

elems = soup.find_all("dt")

for elem in tqdm(elems):

url = elem.next.next.attrs['href']

ori_link = "https://openaccess.thecvf.com/"

contents_url = ori_link + url

paper_link = requests.get(contents_url, headers)

paper_soup = BeautifulSoup(paper_link.content, "html.parser")

title = paper_soup.find('div', id="papertitle").text

abstract = paper_soup.find('div', id="abstract").text

abstract_ja = translator.translate(abstract, src='en', dest='ja')

abstract_ja = str(abstract_ja.text)

pdf = paper_soup.find("dd").find("a").attrs['href'][6:]

pdf_link = ori_link + pdf

class_name = sort_class(abstract)

attachments_title = []

attachments_contents = []

paper_title = {"title": title,

"text": pdf_link}

paper_contents = {"title": abstract,

"text": abstract_ja}

attachments_title.append(paper_title)

attachments_contents.append(paper_contents)

if class_name == "adver":

slack = slackweb.Slack(url="YOUR_TOKEN")

slack.notify(attachments=attachments_title)

slack.notify(attachments=attachments_contents)

else:

slack = slackweb.Slack(url="YOUR_TOKEN")

slack.notify(attachments=attachments_title)

slack.notify(attachments=attachments_contents)

time.sleep(3600)

if __name__ == "__main__":

main()

完成した slack bot

論文のタイトル、pdf のリンク先、要約、要約を和訳したものの順で並んでいます。これが1時間ごとに自動で配信されます。heroku にデプロイしました。

感想

スクレイピングめっちゃ楽しかったです。ちょっとハマりそうです(笑)。

本記事は qiita にも投稿されております。
元記事：https://qiita.com/Hiroaki-K4/items/31557f87b014bfdfd5c9