search function with selenium gecko
This commit is contained in:
parent
edc7e217ca
commit
edd4843f16
36
Main.py
36
Main.py
@ -0,0 +1,36 @@
|
|||||||
|
import scrapy
|
||||||
|
from bs4 import BeautifulSoup
|
||||||
|
import requests
|
||||||
|
from selenium import webdriver
|
||||||
|
import html5lib
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
|
|
||||||
|
|
||||||
|
up_search_url = "https://www.uptodate.com/contents/search?search="
|
||||||
|
|
||||||
|
def do_uptodate_search_with_gecko(key_word):
|
||||||
|
print(up_search_url + key_word)
|
||||||
|
driver = webdriver.Firefox()
|
||||||
|
driver.get(up_search_url + key_word)
|
||||||
|
html = driver.page_source
|
||||||
|
soup = BeautifulSoup(html, 'html.parser')
|
||||||
|
articles_links = soup.find_all(".search-results") #Still unable to catch the wanted result
|
||||||
|
for links in articles_links:
|
||||||
|
print(links)
|
||||||
|
|
||||||
|
def do_uptodate_search(key_word):
|
||||||
|
print(up_search_url + key_word)
|
||||||
|
driver = webdriver.Firefox()
|
||||||
|
driver.get(up_search_url + key_word)
|
||||||
|
html = driver.page_source
|
||||||
|
soup = BeautifulSoup(html, 'html.parser')
|
||||||
|
print(html)
|
||||||
|
articles_links = soup.find_all("div.search-result")
|
||||||
|
|
||||||
|
for link in articles_links:
|
||||||
|
print(link)
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
key_word = input("Please enter your keyword: ")
|
||||||
|
do_uptodate_search_with_gecko(key_word)
|
Loading…
Reference in New Issue
Block a user