Develop Ali Express Scraper in Python with Scraper API

def parse(url):
title = ''
price = ''
image = ''
store_name = ''
record = {}
headers = {
'authority': 'www.aliexpress.com',
'pragma': 'no-cache',
'cache-control': 'no-cache',
'sec-ch-ua': '" Not A;Brand";v="99", "Chromium";v="96", "Google Chrome";v="96"',
'sec-ch-ua-mobile': '?0',
'sec-ch-ua-platform': '"macOS"',
'dnt': '1',
'upgrade-insecure-requests': '1',
'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.55 Safari/537.36',
'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
'sec-fetch-site': 'none',
'sec-fetch-mode': 'navigate',
'sec-fetch-user': '?1',
'sec-fetch-dest': 'document',
'accept-language': 'en-US,en;q=0.9,ur;q=0.8,zh-CN;q=0.7,zh;q=0.6',
}

with open('API_KEY.txt', encoding='utf8') as f:
API_KEY = f.read()

payload = {'api_key': API_KEY, 'url': url, 'render': 'true', 'country_code': 'pk', 'keep_headers': 'true'}

r = requests.get('http://api.scraperapi.com', params=payload, timeout=60, headers=headers)

if r.status_code == 200:
html = r.text.strip()
soup = BeautifulSoup(html, 'lxml')
title_section = soup.select('.product-title-text')
if title_section:
title = title_section[0].text.strip()

price_section = soup.select('.uniform-banner-box-price')
if price_section:
price = price_section[0].text.strip()

image_section = soup.select('.image-viewer img')
if image_section:
image = image_section[0]['src']

store_section = soup.select('.shop-name a')
if store_section:
store_name = store_section[0].text.strip()
record = {'title': title, 'price': price, 'image': image, 'store': store_name}
return record

Conclusion

So in this post, you learned how to scrape e-commerce sites like AliExpress to get the desired data by using Scraper API. You do not have to worry about Proxy IPs either nor do you have to pay hundreds of dollars, especially when you are an individual or working in a startup. Companies spend 100s of dollars on a monthly basis just for the proxy IPs.

--

--

Get the Medium app

A button that says 'Download on the App Store', and if clicked it will lead you to the iOS App store
A button that says 'Get it on, Google Play', and if clicked it will lead you to the Google Play store