import scrapy
from bs4 import BeautifulSoup
import requests
from selenium import webdriver
import html5lib
import re
import sys


up_search_url = "https://www.uptodate.com/contents/search?search="

def do_uptodate_search_with_gecko(key_word):
    print(up_search_url + key_word)
    driver = webdriver.Firefox()
    driver.get(up_search_url + key_word)
    html = driver.page_source
    soup = BeautifulSoup(html, 'html.parser')
    articles_links = soup.find_all(".search-results")  #Still unable to catch the wanted result
    for links in articles_links:
        print(links)

def do_uptodate_search(key_word):
    print(up_search_url + key_word)
    driver = webdriver.Firefox()
    driver.get(up_search_url + key_word)
    html = driver.page_source
    soup = BeautifulSoup(html, 'html.parser')
    print(html)
    articles_links = soup.find_all("div.search-result")

    for link in articles_links:
        print(link)

if __name__ == '__main__':
    key_word = input("Please enter your keyword: ")
    do_uptodate_search_with_gecko(key_word)