
    AHjlJ                        d Z ddlZddlZddlZddlmZ ddlmZ ddlm	Z	 ddl
mZmZmZmZmZ ddlmZ ddlmZ dd	lmZmZ d
dlmZ d
dlmZmZmZmZ d
dlmZm Z  d
dl!m"Z"m#Z#m$Z$m%Z% d
dl&m'Z'm(Z(m)Z)m*Z*m+Z+ dgZ, ejZ                  e.      Z/ ejZ                  d      ja                  ejb                          ejd                  d      Z3 ejd                  d      Z4 ejd                  d      Z5 ejd                  d      Z6 ejd                  d      Z7 ejd                  dejp                        Z9h dZ:h dZ;h dZ<h dZ=h dZ>ddhZ?h dZ@d d!hZAd"d#hZBh d$ZCd%d&d'd(d(d(d)d*ZDd+d,hZEg d-ZFd.eGd/eGfd0ZHd1eGd2eIeG   d/eGdz  fd3ZJd4ed5ed/efd6ZKd4ed/eLeGeGdz  f   fd7ZMd4ed/efd8ZNdLd4ed9eOe   d:ePd/eGdz  fd;ZQd4ed/eReGeGdz  eGdz  f   fd<ZSd4ed/eGdz  fd=ZTd4ed/eGdz  fd>ZUdMd4ed?eGdz  d/eGdz  fd@ZVd4ed/eGdz  fdAZWdBeGd4ed/eOeG   fdCZXdNdDedEeYd/eGdz  fdFZZd4ed/eGdz  fdGZ[	 	 	 	 dOdHeeGz  d?eGdz  dIeLeGe	f   dz  dJeYd2eIeG   dz  d/efdKZ\y)PzH
Module bundling all functions needed to scrape metadata from webpages.
    N)deepcopy)unescape)Any)extract_domainget_base_urlis_valid_urlnormalize_urlvalidate_url)	find_date)XPath)HtmlElementtostring   )prune_unwanted_nodes)extract_jsonextract_json_parse_errornormalize_authorsnormalize_json)Documentset_date_params)HTML_STRIP_TAGSline_processing	load_htmltrim)AUTHOR_DISCARD_XPATHSAUTHOR_XPATHSCATEGORIES_XPATHSTAGS_XPATHSTITLE_XPATHSr   htmldatez$https?://(?:www\.|w[0-9]+\.)?([^/]+)z("(?:\\"|[^"])*")|\su5   ^(.+)?\s+[–•·—|⁄*⋆~‹«<›»>:-]\s+(.+)$z["\']z=/(by-nc-nd|by-nc-sa|by-nc|by-nd|by-sa|by|zero)/([1-9]\.[0-9])zT(cc|creative commons) (by-nc-nd|by-nc-sa|by-nc|by-nd|by-sa|by|zero) ?([1-9]\.[0-9])?>   
dc.creator
dc:creatordcsext.authoratc-metaauthordc.creator.autparsely-authordcterms.creatorsailthru.authordcterms.creator.autshareaholic:article_author_namebylauthorauthorscreator	rbauthorscitation_authorarticle:author>   dc.descriptiondc:descriptiondcterms.abstractdcterms.descriptiontwitter:descriptionsailthru.descriptiondescription>
   dc.publisherdc:publisherdcterms.publishersailthru.publisher	copyright	publisher	rbpubnamecitation_journal_titletwitter:sitearticle:publisher>   parsely-tagsdcterms.subjectshareaholic:keywordstagskeywordscitation_keywords>   dc.titledcterms.titleparsely-titletwitter:titlesailthru.titleshareaholic:titletitlerbtitlefb_titleheadlinecitation_title	rbmainurltwitter:url>   twitter:imagetwitter:image:srcimageog:imageog:image:urlog:image:secure_urlr,   r1   rA   zapplication-name>   
http-equivcharsetpropertyrO   r8   sitenamerX   pagetype)zog:titlezog:descriptionzog:site_namerY   rZ   r[   zog:typez	og:authorzog:article:author)z.//head//link[@rel="canonical"]z.//head//basez6.//head//link[@rel="alternate"][@hreflang="x-default"]rF   returnc                     t        t        |             }|syt        j                  d|      } dj	                  t        d| j                  d                  S )z!Remove special characters of tags z, N)r   r   CLEAN_META_TAGSsubjoinfiltersplit)rF   trimmeds     L/root/tools/cai/cai_env/lib/python3.12/site-packages/trafilatura/metadata.pynormalize_tagsrk      sG    8D>"GsG,D99VD$**T"2344    r-   author_blacklistc                 0   |D ch c]  }|j                          }}| j                  d      D cg c]3  }|j                         j                         |vs$|j                         5 }}|r dj                  |      j                  d      S yc c}w c c}w )z:Check if the authors string correspond to expected values.;z; N)lowerrh   striprf   )r-   rm   ar,   new_authorss        rj   check_authorsrt      s}    +;<a	<<07c0BuffllnFZFZF\dtFt6<<>uKuyy%++D11	 =us   B%BBtreemetadatac                 ,   | j                  d      D ]Z  }|j                  st        t        j	                  d|j                              }	 t        j                  |      }t        ||      }\ |S # t
        j                  $ r t        ||      }Y w xY w)z,Parse and extract metadata from JSON-LD datazK.//script[@type="application/ld+json" or @type="application/settings+json"]z\1)
xpathtextr   JSON_MINIFYre   jsonloadsr   JSONDecodeErrorr   )ru   rv   elemelement_textschemas        rj   extract_meta_jsonr      s    

hi Hyy%kooeTYY&GH	HZZ-F#FH5HH O ## 	H/hGH	Hs   !A11BBc                 N   t         j                  d      }| j                  d      D ]{  }|j                  d      |j                  d      }}|s(|j	                         r9|t
        v r||t
        |   <   N|dk(  rt        |      r||d<   d|t        v smt        d|      |d<   } |S )	zESearch meta tags following the OpenGraph guidelines (https://ogp.me/))rO   r,   urlr8   r_   rX   r`   z+.//head/meta[starts-with(@property, "og:")]r^   contentzog:urlr   Nr,   )	dictfromkeysrx   getisspaceOG_PROPERTIESr   	OG_AUTHORr   )ru   resultr~   property_namer   s        rj   extract_opengraphr      s    ]]efF 

HI 	D!%*!5txx	7Jw7??,-7>}]34(*|G/D 'u)+#4T7#Cx 	D Mrl   c           	      B   t               j                  t        |             }t        |j                  |j
                  |j                  |j                  |j                  |j                  f      r|S g d}}| j                  d      D ]  t        j                  dj                  dd            j                         }|s;dj                  v rj                  dd      j!                         }|j#                  d      r{|dk(  r|j%                  t'        |             |t(        v rt+        |j
                  |      |_        |dk(  r|j                  xs ||_        |t,        v s|j                  xs ||_	        d	j                  v rj                  d	d      j!                         }|t.        v rt+        |j
                  |      |_        M|t0        v r|j                  xs ||_        l|t2        v r|j                  xs ||_        |t4        v r|j                  xs ||_        |t,        v r|j                  xs ||_	        |t6        v sd
|v r|}|dk(  r |j                  st9        |      r	||_        |t:        v s|j%                  t'        |             $dj                  v r|j                  dd      j!                         }|dk(  rt+        |j
                  |      |_        t|dk(  r|j                  xs ||_        |dk(  s|j                  xs ||_        t        fdt<        D              st>        jA                  dtC        dd      j                                 |j                  xs ||_        ||_"        |S )z)Search meta tags for relevant informationNz.//head/meta[@content]rc   r   r^   og:zarticle:tagrB   nameztwitter:app:namerU   itempropr,   r8   rR   c              3   :   K   | ]  }|j                   v  y wN)attrib).0keyr~   s     rj   	<genexpr>zexamine_meta.<locals>.<genexpr>  s     >CDKK'>s   zunknown attribute: %sFunicode)pretty_printencoding)#r   	from_dictr   allrO   r,   r   r8   r_   rX   iterfindr   re   r   rq   r   rp   
startswithappendrk   PROPERTY_AUTHORr   METANAME_IMAGEMETANAME_AUTHORMETANAME_TITLEMETANAME_DESCRIPTIONMETANAME_PUBLISHERTWITTER_ATTRSr   METANAME_TAG
EXTRA_METALOGGERdebugr   rF   )	ru   rv   rF   backup_sitenamecontent_attrproperty_attr	name_attritemprop_attrr~   s	           @rj   examine_metar      s    z##$5d$;<H NNOOLL  NN	
	 /D 67 A&**2txx	2/FGMMO $ HHZ4::<M''.-N<89/1"3HOO\"R"55$,$5$5$E!.0!)!?<t{{",224IO+"3HOO\"Rn,!)!?<22'/';';'K|$00$,$5$5$E!n,!)!?<m+/AY/N".m+HLL\R^E_+l*N<894;;& HHZ4::<M("3HOO\"R-/'/';';'K|$*,!)!?< >:>>LL'EIFLLN}AH !))<_HHMOrl   expressions	len_limitc                    |D ]  } ||       }|D ]J  }t        dj                  |j                                     }|s.dt        |      cxk  r|k  sCn F|c c S  t        |      dkD  sit        j                  d|t        |              y)zExtract meta information    r   z#more than one invalid result: %s %sN)r   rf   itertextlenr   r   )ru   r   r   
expressionresultsr~   r   s          rj   extract_metainfor   ,  s     " Z
T" 	D388DMMO45G1s7|7i7	 w<!LL>
CPWLYZ rl   c                     d}| j                  d      }|;t        |j                               }t        j	                  |      x}r||d   |d   fS t
        j                  d       |ddfS )z2Extract text segments out of main <title> element.rc   z.//head//titleNr   r   zno main title found)findr   text_contentHTMLTITLE_REGEXmatchr   r   )ru   rO   title_elementr   s       rj   examine_title_elementr   ;  sv     EII./M ]//12#))%0050%(E!H,,
LL&'$rl   c                    | j                  d      }t        |      dk(  r t        |d   j                               }|r|S t	        | t
              xs d}|r|S t        |       \  }}}||fD ]  }|sd|vs|c S  |rt        |d   j                               S 	 t        | j                  d      d   j                               }|xs dS # t        $ r t        j                  d       Y &w xY w)	zExtract the document titlez.//h1r   r   rc   .z.//h2zno h2 title foundN)findallr   r   r   r   r   r   rx   
IndexErrorr   r   )ru   
h1_resultsrO   firstsecondts         rj   extract_titler   I  s     g&J
:!Z]//12LT<06BE06E5&V_ AH JqM..011*TZZ(+88:; =D  *()*s   +C C,+C,c                 x    t        t        |       t              }t        |t        d      }|rt        d|      }|S )zExtract the document author(s)x   )r   N)r   r   r   r   r   r   )ru   subtreer,   s      rj   extract_authorr   e  s5    "8D>3HIGg}DF"40Mrl   default_urlc                    t         D ]6  }| j                  |      }||j                  j                  d      nd}|s6 n r|j	                  d      r| j                  d      D ]n  }|j                  d      xs |j                  d      xs d}|j	                  d      s|j	                  d	      sNt        |j                  d
         }|si||z   } n |rt        |      \  }}|r|rt        |      nd}|xs |S )z'Extract the URL from the canonical linkNhref/z.//head//meta[@content]r   r^   rc   r   ztwitter:r   )	URL_SELECTORSr   r   r   r   r   r   r
   r	   )	ru   r   selectorelementr   attrtypebase_urlvalidation_result
parsed_urls	            rj   extract_urlr   o  s    ! ))H%,3,?gnn  (T	 s~~c"}}%>? 	G{{6*Kgkk*.EKH""5)X-@-@-L'y(AB"S.C	 (4S(9%:+<mJ'QU+rl   c                 B    t        |       ^}}t        d |D        d      S )z=Extract the name of a site from the main title (if it exists)c              3   0   K   | ]  }|sd |v s|  yw)r   N )r   parts     rj   r   z#extract_sitename.<locals>.<genexpr>  s     @$4C4K@s   N)r   next)ru   _partss      rj   extract_sitenamer     s#    %d+IA@%@$GGrl   metatypec                    g }d| j                  d      z   dz   | dk(  rt        nt        }|D ]&  }|j                  fd ||      D               |s& n | dk(  r6|s4|j	                  d      D ]   }|j                  |j                  d          " t        j                  d |D              D cg c]  }|s|	 c}S c c}w )	z!Find category and tag informationr   yz(?:y|ies|s)?/categoryc              3      K   | ]8  }t        j                  |j                  d          s'|j                          : yw)r   N)researchr   r   )r   r~   regexprs     rj   r   z#extract_catstags.<locals>.<genexpr>  s5     pt		RY[_[f[fgm[nHot((*ps
   (AAzR.//head//meta[@property="article:section" or contains(@name, "subject")][@content]r   c              3   8   K   | ]  }|st        |        y wr   )r   )r   xs     rj   r   z#extract_catstags.<locals>.<genexpr>  s     $NAA_Q%7$Ns   )	rstripr   r   extendrx   r   r   r   r   )r   ru   r   xpath_expressioncatexprr   rr   s          @rj   extract_catstagsr     s    GHOOC((?:G,4
,B(# pwt}pp
 :gzz"vw 	6GNN7>>)45	6 }}$N$NNT!RSATTTs   0B?8B?r   strictc                    t         j                  | j                  dd            }|rd|d   j                          d|d    S t	        | j                               }|r"|rt        j                  |      }|r|d   S dS |S y)	zkProbe a link for identifiable free license cues.
    Parse the href attribute first and then the link text.r   rc   zCC r   r   r   r   N)LICENSE_REGEXr   r   upperr   r   TEXT_LICENSE_REGEX)r   r   r   ry   s       rj   parse_license_elementr     s       VR!89EU1X^^%&aaz22$$&'D&--d3E$58.$.rl   c                     | j                  d      D ]  }t        |d      }||c S  | j                  d      D ]  }t        |d      }||c S  y)z:Search the HTML code for license information and parse it.z.//a[@rel="license"][@href]F)r   Nz[.//footer//a[@href]|.//div[contains(@class, "footer") or contains(@id, "footer")]//a[@href]T)r   r   rx   )ru   r   r   s      rj   extract_licenser     sk     << => &wu=M
 ::{| &wt<M rl   filecontentdate_config	extensivec                 n   |xs
 t               }|ri |n
t        |      }t        |       }|
t               S t	        |      }|j
                  rd|j
                  vrd|_        	 t        ||      }|j                  st        |      |_
        |j
                  r|rt        |j
                  |      |_        |j
                  st        |      |_        |j
                  r|rt        |j
                  |      |_        |j                  st        ||      |_        |j                  rt!        |j                  d      |_        |j                  |d<   t%        |fi ||_        |j(                  st+        |      |_        |j(                  rt-        |j(                  t.              r|j(                  d   |_        n4t-        |j(                  t0              rt3        |j(                        |_        |j(                  j5                  d      |_        |j(                  rd	|j(                  vrt|j(                  d   j7                         sW|j(                  j                         |_        n7|j                  r+t8        j;                  |j                        }|r
|d
   |_        |j<                  st?        d|      |_        |j@                  st?        d|      |_         tC        |      |_"        |jG                  d      |_$        |jK                          |S # t        $ r!}t        j                  d|       Y d}~d}~ww xY w)aR  Main process for metadata extraction.

    Args:
        filecontent: HTML code as string or parsed tree.
        default_url: Previously known URL of the downloaded document.
        date_config: Provide extraction parameters to htmldate as dict().
        extensive: Use extensive search for date extraction.
        author_blacklist: Provide a blacklist of Author Names as set() to filter out authors.

    Returns:
        A trafilatura.settings.Document containing the extracted metadata information.
        The Document class has .as_dict() method that will return a copy as a dict.
    Nr   z%error in JSON metadata extraction: %sT)fastr   r   @r   r   r   tagmax_date)&setr   r   r   r   r,   r   	Exceptionr   warningrO   r   rt   r   r   r   r   hostnamer   dater_   r   
isinstancelistr   strlstripisupperMETA_URLr   
categoriesr   rF   r   licenser   filedateclean_and_trim)	r   r   r   r   rm   ru   rv   errmymatchs	            rj   extract_metadatar    s   * (035%0/[/oi6PK [!D|z D!H 3hoo5E$T84
 >>&t, +'9IJ??(.+'9IJ <<"45 ||*8<<dC "Kd2k2HM ,T2h''. ( 1 1! 4H))40 #H$5$5 6H$--44S9H,=,=!=hFWFWXYFZFbFbFd ( 1 1 7 7 9H	... '
H .z4@ ==(5 't,H $
3HOC  E>DDEs   $L
 
	L4L//L4)   r   )F)NNTN)]__doc__r{   loggingr   copyr   htmlr   typingr   courlanr   r   r   r	   r
   r    r   
lxml.etreer   	lxml.htmlr   r   htmlprocessingr   json_metadatar   r   r   r   settingsr   r   utilsr   r   r   r   xpathsr   r   r   r   r   __all__	getLogger__name__r   setLevelWARNINGcompiler  rz   r   rd   r   Ir   r   r   r   r   r   METANAME_URLr   r   r   r   r   r   r   r
  rk   r  rt   r   r   r   r   r	  intr   tupler   r   r   r   r   r   boolr   r   r  r   rl   rj   <module>r.     s     	       + 0  0 D D  ,			8	$   *  & &w 72::=>bjj01"**UV"**X&

[\RZZ[DD 
&   ]+ -.!34 3
 #" -.	~5 5 53 #c( sTz K 8  K DcDj,A *]{ ]x ]@; T%[ S [^ae[e 

3d
C$J&' d
 8 t k d
 cDj 6H; H3: HUs U+ U$s) U*;  t $+ #* " #)-(,hs"hth c3h$&h 	h
 #hoh hrl   