42 lines
1.4 KiB
Python
42 lines
1.4 KiB
Python
import scrapy
|
|
from bs4 import BeautifulSoup
|
|
import requests
|
|
from selenium import webdriver
|
|
import html5lib
|
|
import re
|
|
import sys
|
|
|
|
#Def the Uptodate URI for use
|
|
up_search_url = "https://www.uptodate.com/contents/search?search="
|
|
up_api_url = "https://www.uptodate.com/services/app/contents/search/2/json?&language=en&max=10&search="
|
|
up_prefix_url = "https://www.uptodate.com"
|
|
|
|
def do_uptodate_search_with_gecko(key_word):
|
|
print(up_search_url + key_word)
|
|
driver = webdriver.Firefox()
|
|
driver.get(up_search_url + key_word)
|
|
html = driver.page_source
|
|
soup = BeautifulSoup(html, 'html.parser')
|
|
articles_links = soup.find_all(".search-results") #Still unable to catch the wanted result
|
|
for links in articles_links:
|
|
print(links)
|
|
|
|
def do_uptodate_search_with_headless(key_word):
|
|
print(up_search_url + key_word)
|
|
option = webdriver.ChromeOptions()
|
|
option.add_argument('headless')
|
|
driver = webdriver.Chrome(chrome_options=option)
|
|
driver.get(up_search_url + key_word)
|
|
html = driver.page_source
|
|
soup = BeautifulSoup(html, 'html.parser')
|
|
articles_links = soup.select("#search-results-container")
|
|
print(articles_links)
|
|
|
|
def do_uptodate_search_with_uptodate_api(key_word):
|
|
search_results = requests.get(up_api_url + key_word)
|
|
print(search_results.json())
|
|
|
|
if __name__ == '__main__':
|
|
key_word = input("Please enter your keyword: ")
|
|
do_uptodate_search_with_uptodate_api(key_word)
|