Feature/crawl skincare

crawlers/skincare/crawl_brand_skincare.py

@@ -128,15 +128,14 @@ def get_brand_product_detail_info(sb, goods_no: str) -> dict:
     except Exception as e:
         log.warning(f"[get_brand_product_detail_info] 총 리뷰수 파싱 실패: {e}")
         total_review = 0
-
     # 리뷰평점
     try:
         review_score = soup.select_one("#repReview b")
         review_score = float(review_score.text.strip())
         log.info(f"[get_brand_product_detail_info] 리뷰평점: {review_score}")
     except Exception as e:
         log.warning(f"[get_brand_product_detail_info] 리뷰평점 파싱 실패: {e}")
-        review_score = None
+        review_score = ""
 
     # 리뷰 분포 기본값
     pctOf5 = pctOf4 = pctOf3 = pctOf2 = pctOf1 = None
@@ -164,9 +163,10 @@ def get_brand_product_detail_info(sb, goods_no: str) -> dict:
             except Exception:
                 total_comment = ""
                 log.warning("[get_brand_product_detail_info] 대표 코멘트 추출 실패")
-
         except Exception as e:
-            log.warning(f"[get_brand_product_detail_info] 리뷰 정보 없음: {e}")
+            log.warning(f"[get_brand_product_detail_info] 리뷰 정보 수집 실패: {e}")
+    else:
+        log.warning("[get_product_detail_info] 리뷰 정보 없음: 리뷰 수가 0건 입니다.")
 
     # === 상세스펙(구매정보) 추출 ===
     # 구매정보 탭 클릭
@@ -190,7 +190,7 @@ def get_detail_info(soup, title):
                     dt_text = dt.text.strip()
                     dd_text = dd.text.strip()
                     if title in dt_text:
-                        log.info(f"[get_brand_product_detail_info] {title} 추출: {dd_text}")
+                        log.info(f"[get_brand_product_detail_info] {title} 추출 성공!")
                         return dd_text
         except Exception as e:
             log.warning(f"[get_brand_product_detail_info] 상세 정보 파싱 실패 ({title}): {e}")
@@ -228,7 +228,6 @@ def get_detail_info(soup, title):
     for title, key in spec_map.items():
         detail_spec[key] = get_detail_info(soup, title)
 
-    log.info("[get_brand_product_detail_info] 최종 데이터 리턴")
     return {
         "category": category,
         "totalComment": total_comment,

crawlers/skincare/crawl_rank_skincare.py

@@ -9,6 +9,7 @@
 from bs4 import BeautifulSoup
 from airflow.utils.log.logging_mixin import LoggingMixin
 
+
 def get_top100_skincare() -> tuple:
     log = LoggingMixin().log
     log.info("[get_top100_skincare] 시작")
@@ -21,12 +22,12 @@ def get_top100_skincare() -> tuple:
     )
 
     driver = webdriver.Chrome(
+        #service=Service(ChromeDriverManager().install()), options=chrome_options
         service=Service("/usr/local/bin/chromedriver"), options=chrome_options
     )
 
     # 올리브영 스킨케어 랭킹 페이지 열기
     url = "https://www.oliveyoung.co.kr/store/main/getBestList.do?dispCatNo=900000100100001&fltDispCatNo=10000010001&pageIdx=1&rowsPerPage=8&t_page=%EB%9E%AD%ED%82%B9&t_click=%ED%8C%90%EB%A7%A4%EB%9E%AD%ED%82%B9_%EC%8A%A4%ED%82%A8%EC%BC%80%EC%96%B4"
-    log.info(f"[get_top100_skincare] URL 오픈: {url}")
     driver.get(url)
 
     # 페이지 로딩 대기
@@ -149,8 +150,8 @@ def get_product_detail_info(sb, goods_no: str) -> dict:
     log = LoggingMixin().log
     url = f"https://www.oliveyoung.co.kr/store/goods/getGoodsDetail.do?goodsNo={goods_no}"
     log.info(f"[get_product_detail_info] 시작: goods_no={goods_no}")
-    sb.uc_open_with_reconnect(url, reconnect_time=5)
-    log.info(f"[get_product_detail_info] URL 오픈: {url}")
+    #sb.uc_open_with_reconnect(url, reconnect_time=5)  # 속도 더 느림
+    sb.open(url)
     time.sleep(1)
     html = sb.driver.page_source
     soup = BeautifulSoup(html, 'html.parser')
@@ -175,6 +176,7 @@ def get_product_detail_info(sb, goods_no: str) -> dict:
     pctOf5 = pctOf4 = pctOf3 = pctOf2 = pctOf1 = None
 
     # 리뷰가 1건 이상 있을 때만 리뷰탭 클릭 및 분포 수집
+    total_comment = ""
     if total_review > 0:
         try:
             sb.click("a.goods_reputation")
@@ -199,11 +201,10 @@ def get_product_detail_info(sb, goods_no: str) -> dict:
             except Exception:
                 total_comment = ""
                 log.warning("[get_product_detail_info] 대표 코멘트 추출 실패")
-
         except Exception as e:
-            log.warning(f"[get_product_detail_info] 리뷰 정보 없음: {e}")
-
-
+            log.warning(f"[get_product_detail_info] 리뷰 정보 수집 실패: {e}")
+    else:
+        log.warning("[get_product_detail_info] 리뷰 정보 없음: 리뷰 수가 0건 입니다.")
 
     # === 상세스펙(구매정보) 추출 ===
     # 구매정보 탭 클릭
@@ -227,7 +228,7 @@ def get_detail_info(soup, title):
                     dt_text = dt.text.strip()
                     dd_text = dd.text.strip()
                     if title in dt_text:
-                        log.info(f"[get_product_detail_info] {title} 추출: {dd_text}")
+                        log.info(f"[get_product_detail_info] {title} 추출 성공!")
                         return dd_text
         except Exception as e:
             log.warning(f"[get_product_detail_info] 상세 정보 파싱 실패 ({title}): {e}")