Uptodatecrawler/Main.py

import scrapy
from bs4 import BeautifulSoup
import requests
from selenium import webdriver
import html5lib
import re
import sys

#Def the Uptodate URI for use
up_search_url = "https://www.uptodate.com/contents/search?search="
up_api_url = "https://www.uptodate.com/services/app/contents/search/2/json?&language=en&max=10&search="
up_prefix_url = "https://www.uptodate.com"

def do_uptodate_search_with_gecko(key_word):
    print(up_search_url + key_word)
    driver = webdriver.Firefox()
    driver.get(up_search_url + key_word)
    html = driver.page_source
    soup = BeautifulSoup(html, 'html.parser')
    articles_links = soup.find_all(".search-results")  #Still unable to catch the wanted result
    for links in articles_links:
        print(links)

def do_uptodate_search_with_headless(key_word):
    print(up_search_url + key_word)
    option = webdriver.ChromeOptions()
    option.add_argument('headless')
    driver = webdriver.Chrome(chrome_options=option)
    driver.get(up_search_url + key_word)
    html = driver.page_source
    soup = BeautifulSoup(html, 'html.parser')
    articles_links = soup.select("#search-results-container")
    print(articles_links)

def do_uptodate_search_with_uptodate_api(key_word):
    search_results = requests.get(up_api_url + key_word)
    print(search_results.json())

if __name__ == '__main__':
    key_word = input("Please enter your keyword: ")
    do_uptodate_search_with_uptodate_api(key_word)