Updated to handle case where a "data:image/webp" is returned instead of a regular url + transform the url to a high resolution image url

tducret · tducret · commit 62f58bbda864 · 2019-01-11T22:32:56.000+01:00
diff --git a/amazonscraper/client.py b/amazonscraper/client.py
@@ -232,13 +232,19 @@ def _get_products(self, keywords="", search_url="", max_product_nb=100):
 
                     # Get image before url and asin
                     css_selector = css_selector_dict.get("img", "")
-                    url_product_soup = product.select(css_selector)
-                    if url_product_soup:
-                        url = urljoin(
-                            self.base_url,
-                            url_product_soup[0].get('src'))
-                        proper_url = url.split("/ref=")[0]
-                        product_dict['img'] = proper_url
+                    img_product_soup = product.select(css_selector)
+                    if img_product_soup:
+                        img_url = img_product_soup[0].get('src')
+                        # Check if it is not a base64 formatted image
+                        if "data:image/webp" in img_url:
+                            img_url = img_product_soup[0].get(
+                                'data-search-image-source-set',
+                                '').split(' ')[0]
+
+                        if img_url != '':
+                            img_url = _get_high_res_img_url(img_url=img_url)
+
+                        product_dict['img'] = img_url
 
                     css_selector = css_selector_dict.get("url", "")
                     url_product_soup = product.select(css_selector)
@@ -272,11 +278,28 @@ def _get_products(self, keywords="", search_url="", max_product_nb=100):
 
 
 def _css_select(soup, css_selector):
-        """ Returns the content of the element pointed by the CSS selector,
-        or an empty string if not found """
-        selection = soup.select(css_selector)
-        retour = ""
-        if len(selection) > 0:
-            if hasattr(selection[0], 'text'):
-                retour = selection[0].text.strip()
-        return retour
+    """ Returns the content of the element pointed by the CSS selector,
+    or an empty string if not found """
+    selection = soup.select(css_selector)
+    retour = ""
+    if len(selection) > 0:
+        if hasattr(selection[0], 'text'):
+            retour = selection[0].text.strip()
+    return retour
+
+def _get_high_res_img_url(img_url):
+    """ Returns a modified url pointing to the high resolution version of
+    the image
+    >>> print(_get_high_res_img_url("https://images-na.ssl-images-amazon.com/\
+images/I/513gErH1dML._AC_SX236_SY340_FMwebp_QL65_.jpg"))
+    https://images-na.ssl-images-amazon.com/\
+images/I/513gErH1dML.jpg
+    >>> print(_get_high_res_img_url("https://images-na.ssl-images-amazon.com/\
+images/I/51F48HFHq6L._AC_SX118_SY170_QL70_.jpg"))
+    https://images-na.ssl-images-amazon.com/\
+images/I/51F48HFHq6L.jpg
+    """
+    high_res_url = img_url.split("._")[0] + ".jpg"
+    return high_res_url
+
+
