From a4c7f762162d842ad0ff926a10c66119a4cce435 Mon Sep 17 00:00:00 2001
From: Ching <loooching@gmail.com>
Date: Tue, 16 Apr 2024 14:12:58 +0800
Subject: [PATCH] refactor: Add URL shortening functionality

---
 crawler.py | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/crawler.py b/crawler.py
index e697677..6e1f22b 100644
--- a/crawler.py
+++ b/crawler.py
@@ -31,6 +31,18 @@ def save_to_redis(article):
     redis_db.set(key, json.dumps(article), ex=3600*24*7)
     return True
 
+def url_shorten(url):
+  api_url = "https://s.tunpok.com/api/v2/links"
+  api_key = "HJAs1~1sjd5mX66_Ydg8~9oSuqgkd75VeLhjjni6"
+  headers = {
+    'x-api-key': api_key,
+  }
+  resp = requests.post(api_url, headers=headers, json={"target": url})
+  if resp.status_code == 201:
+    return resp.json()['link']
+  else:
+    return url
+
 def crawler():
   # get article list in html div class name = "acon cl"
   home_url = 'https://www.chiphell.com/'
@@ -108,7 +120,7 @@ def crawler():
       'date': date,
       'category': category,
       'content': content,
-      'url': url,
+      'url': url_shorten(url),
       'article_id': article_id
     }