diff --git a/crawler.py b/crawler.py index e697677..6e1f22b 100644 --- a/crawler.py +++ b/crawler.py @@ -31,6 +31,18 @@ def save_to_redis(article): redis_db.set(key, json.dumps(article), ex=3600*24*7) return True +def url_shorten(url): + api_url = "https://s.tunpok.com/api/v2/links" + api_key = "HJAs1~1sjd5mX66_Ydg8~9oSuqgkd75VeLhjjni6" + headers = { + 'x-api-key': api_key, + } + resp = requests.post(api_url, headers=headers, json={"target": url}) + if resp.status_code == 201: + return resp.json()['link'] + else: + return url + def crawler(): # get article list in html div class name = "acon cl" home_url = 'https://www.chiphell.com/' @@ -108,7 +120,7 @@ def crawler(): 'date': date, 'category': category, 'content': content, - 'url': url, + 'url': url_shorten(url), 'article_id': article_id }